00000400:00000100:12.0F:1619677458.718401:0:19943:0:(lib-msg.c:698:lnet_attempt_msg_resend()) msg 10.51.3.43@o2ib3->10.0.10.103@o2ib7 exceeded retry count 0
00000400:00000100:5.0F:1619677458.741437:0:19935:0:(lib-msg.c:698:lnet_attempt_msg_resend()) msg 10.51.2.27@o2ib3->10.0.10.107@o2ib7 exceeded retry count 0
00000400:00000100:12.0:1619677458.742618:0:19943:0:(lib-msg.c:698:lnet_attempt_msg_resend()) msg 10.51.14.12@o2ib3->10.0.10.111@o2ib7 exceeded retry count 0
00000800:00000200:5.2F:1619677468.325481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.327557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.327596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.330926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.2F:1619677468.331373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2c600] (21)++
00000800:00000200:12.0:1619677468.331551:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.331554:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.331555:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.331558:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0F:1619677468.331560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.331564:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677468.331575:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677468.331579:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677468.331580:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.331581:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677468.331581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.331582:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677468.331651:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.331653:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.331654:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.331656:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.331657:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.331659:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.331922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.335136:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.335138:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.335139:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.335140:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.335143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.335144:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.336219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.336963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.337001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.340970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.341168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677468.341209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:5.2:1619677468.341912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.344040:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.344042:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677468.344044:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677468.344047:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677468.344048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677468.344049:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677468.344050:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96535514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677468.344052:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677468.344053:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677468.344054:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677468.344057:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677468.344059:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677468.344062:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677468.344063:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:12.0:1619677468.344064:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:12.0:1619677468.344066:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:12.0:1619677468.344066:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.344067:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.344068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.345907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.347181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.347223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.350220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.352491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.354543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.357818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.357857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.359319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.360639:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.360643:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.360647:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.360647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.360651:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.360654:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677468.360657:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96535420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.360660:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.360662:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.360664:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677468.360669:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.360673:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677468.360677:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.360679:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677468.360680:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677468.360682:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677468.360683:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.360684:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.360685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677468.362372:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.362374:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.362375:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.362377:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.362379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.362380:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.362944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.363665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.363873:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.363877:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677468.363879:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : GET - for me
00000800:00000200:12.0:1619677468.363880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:13.0:1619677468.363882:0:19942:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.103@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:13.0:1619677468.363884:0:19942:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.103@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:13.0:1619677468.363886:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.363887:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:13.0:1619677468.363888:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:13.0:1619677468.363893:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:13.0:1619677468.363895:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.363898:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.363899:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.363899:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.363901:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.363901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.363915:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.363917:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.363918:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.363920:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:13.0:1619677468.363921:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.363922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.367742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.367781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.368523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.369573:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.369576:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.369578:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.369580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.369581:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.369583:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677468.369584:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96535369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.369586:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.369587:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.369587:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677468.369591:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.369592:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677468.369594:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.369595:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:13.0:1619677468.369595:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:13.0:1619677468.369597:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:13.0:1619677468.369597:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.369602:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677468.369603:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677468.369604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.369617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677468.369619:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677468.369620:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.369621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677468.371784:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.371786:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.371787:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.371788:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.371791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.371792:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677468.371803:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677468.371806:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677468.371806:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.371807:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677468.371808:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.371809:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.372768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.373127:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.373130:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.373132:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677468.373134:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677468.373134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.373136:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:13.0:1619677468.373138:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96535351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.373139:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.373140:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.373141:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677468.373144:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.373146:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677468.373148:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.373149:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:13.0:1619677468.373149:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677468.373151:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.373152:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.373152:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.373153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677468.373164:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.373166:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.373167:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.373168:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.373170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.373171:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.373829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.374776:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.374779:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.374781:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677468.374783:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677468.374783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.374784:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:13.0:1619677468.374785:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96535339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.374786:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.374787:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.374788:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677468.374792:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.374794:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677468.374796:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.374796:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:13.0:1619677468.374797:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677468.374799:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.374799:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.374800:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.374800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677468.374811:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.374814:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.374814:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.374816:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.374818:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677468.374818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:5.2:1619677468.377091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.377130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.379172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.381468:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.381471:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.381472:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.381473:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.381475:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.381476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.381893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.384498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.385210:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.385212:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.385213:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.385215:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.385217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.385222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.386182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.386220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.389831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.390961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.2:1619677468.393396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:5.2:1619677468.394708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.395453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.395493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.399711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.400446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.404379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.406986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.407025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.411424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.412285:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.412288:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.412289:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.412292:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:5.2:1619677468.412297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.412303:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.412304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677468.412305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.412307:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677468.412308:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)++
00000800:00000200:13.0:1619677468.412309:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (24)--
00000800:00000200:13.0:1619677468.412309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677468.412310:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.412311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.415883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.417612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.417651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.420193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.422912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.424683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.424686:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.424687:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.424688:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.424691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.424692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.425073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.428261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.428314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.429400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.433458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.434377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.438885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.438923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.439696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.440245:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.440248:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677468.440250:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677468.440252:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.440253:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:13.0:1619677468.440254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677468.440255:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96534979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677468.440256:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677468.440258:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677468.440258:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677468.440262:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677468.440264:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677468.440267:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677468.440268:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:12.0:1619677468.440272:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:12.0:1619677468.440274:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:12.0:1619677468.440275:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.440276:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.440276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.442066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677468.442106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677468.442111:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.442113:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.442115:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.442116:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.442118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.442119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677468.442603:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.442605:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.442607:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.442608:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.442610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.442611:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.442620:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.442622:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.442623:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.442625:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.442626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.442627:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.444164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.445145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.447302:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.447306:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.447310:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.447311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.447313:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.447316:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677468.447318:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96534940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.447321:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.447322:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.447324:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677468.447329:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.447332:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677468.447336:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe0d80d00] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.447337:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb719600] (20)++
00000800:00000200:13.0:1619677468.447338:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb719600] (21)++
00000800:00000200:13.0:1619677468.447341:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb719600] (22)--
00000800:00000200:13.0:1619677468.447342:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.447344:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677468.447346:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677468.447347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.447362:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677468.447365:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677468.447366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.447367:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.448904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677468.449572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.449611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.450322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.454121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.455725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.459109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.459157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.461044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.461820:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.461822:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.461823:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.461824:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.461826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.461827:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677468.462899:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.462901:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.462902:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.462903:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.462905:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.462906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.463465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.466904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.468278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.468325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.472261:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.472264:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.472266:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.472268:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.472269:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.472270:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677468.472272:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96534785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.472273:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.472274:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.472275:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677468.472278:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.472279:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677468.472282:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.472283:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:13.0:1619677468.472283:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:13.0:1619677468.472286:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:13.0:1619677468.472286:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.472287:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.472287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.472772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.473479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.473591:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.473594:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677468.473596:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677468.473598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677468.473598:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.473600:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677468.473602:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96534773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677468.473603:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677468.473604:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677468.473605:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677468.473609:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677468.473610:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677468.473613:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677468.473614:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:12.0:1619677468.473615:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:12.0:1619677468.473616:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:12.0:1619677468.473617:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.473618:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.473618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677468.474896:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.474898:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.474901:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.474902:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.474904:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.474906:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677468.474907:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96534767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.474908:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.474909:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.474910:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:13.0:1619677468.474914:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.474915:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677468.474919:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.474919:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:13.0:1619677468.474920:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:13.0:1619677468.474923:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:13.0:1619677468.474923:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.474924:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.474924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.478009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.482358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.482395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.483981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.487254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.491639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.491897:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.491900:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.491902:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.491904:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.491907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.491909:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677468.491926:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677468.491930:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677468.491931:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.491932:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677468.491934:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.491936:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.493238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677468.494546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.494587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.496506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.496644:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677468.496649:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677468.496652:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.111@o2ib7 : GET - for me
00000800:00000200:12.0:1619677468.496653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:13.0:1619677468.496656:0:19942:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.111@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:13.0:1619677468.496659:0:19942:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.111@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:13.0:1619677468.496661:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677468.496662:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:13.0:1619677468.496665:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:13.0:1619677468.496672:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : REPLY try# 0
00000800:00000200:13.0:1619677468.496675:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677468.496677:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.496679:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677468.496680:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.496682:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677468.496682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.496694:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.496696:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.496697:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.496699:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: REPLY: OK
00000800:00000200:12.0:1619677468.496701:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.496702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.500870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.505098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.507009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.507051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.511393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.511788:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.511790:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.511791:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.511792:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.511795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.511795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:5.2:1619677468.515839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.517061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.520685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.520723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.525026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.525699:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.525702:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.525703:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.525704:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.525707:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.525707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:5.2:1619677468.527580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.527874:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.527877:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.527877:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.527879:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.527881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.527882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.529776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.534097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.534137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.537797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.538545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.539772:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.539775:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.539777:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.539779:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.539780:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.539781:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677468.539783:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96534365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.539784:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.539785:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.539786:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677468.539789:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.539791:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677468.539793:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.539793:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:13.0:1619677468.539795:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:13.0:1619677468.539796:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:13.0:1619677468.539797:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.539798:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677468.539800:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677468.539800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.539814:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677468.539816:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677468.539817:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.539818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677468.542980:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.542982:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677468.542986:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677468.542987:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.542989:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.542989:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.542990:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.542992:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677468.542993:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:12.0:1619677468.542994:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96534340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677468.542996:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677468.542998:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677468.542999:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677468.543003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.543005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677468.543005:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.543007:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677468.543008:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677468.543010:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.543011:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.543012:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.543013:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.543013:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677468.543014:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677468.543015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.543015:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:12.0:1619677468.543017:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:12.0:1619677468.543018:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.543020:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.543020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.543238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.547563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.547602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.549059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.555531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.559226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.559961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.560001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.564208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.564330:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.564333:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.564335:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677468.564338:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677468.564338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.564340:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677468.564341:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96534201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.564342:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.564343:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.564344:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677468.564348:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.564349:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677468.564352:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.564353:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:13.0:1619677468.564354:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:13.0:1619677468.564355:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:13.0:1619677468.564357:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.564357:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.564358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.568977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.571104:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.571107:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.571109:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677468.571112:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677468.571112:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.571116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.571261:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.571264:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.571265:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.571267:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.571269:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.571276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677468.571304:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.571308:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.571309:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677468.571312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:12.0:1619677468.571312:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.571317:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.571443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.573245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.573283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.577539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.577571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:13.0:1619677468.579693:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.579696:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.579697:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.579699:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.579700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.579702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677468.579714:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677468.579716:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677468.579717:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.579718:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677468.579718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.579719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.581960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.583779:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.583781:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.583782:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.583784:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.583786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.583787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.584647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.585785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.585825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.586101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.587632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.587680:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.587683:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.587685:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.587687:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.587688:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.587691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.589372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.590979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.591018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.594287:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.594289:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.594290:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.594291:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.594296:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.594299:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.594952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.596665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.596711:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.596714:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.596715:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.596717:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.596719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.596720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.596728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.596731:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.596732:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.596733:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.596733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.596734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.600132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.600948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.600988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.604479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.605261:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.605263:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.605264:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.605265:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.605267:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.605268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.607250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.607299:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.607302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.607303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.607305:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.607308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.607310:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.608579:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.608582:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.608585:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677468.608587:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677468.608587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.608589:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677468.608590:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96533881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.608592:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.608593:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.608594:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677468.608597:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.608599:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677468.608602:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.608602:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:13.0:1619677468.608603:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:13.0:1619677468.608605:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:13.0:1619677468.608605:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.608606:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.608607:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.608773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.611944:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.611946:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677468.611948:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677468.611951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677468.611951:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.611953:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677468.611954:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96533863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677468.611955:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677468.611956:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677468.611957:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677468.611960:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677468.611961:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677468.611963:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677468.611964:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:12.0:1619677468.611964:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677468.611966:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.611967:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.611968:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677468.611969:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677468.611969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.611979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.611981:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.611982:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.611983:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.611983:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677468.611983:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677468.611984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.611985:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.611986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677468.611987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.613509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.613557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.614840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.617838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.617953:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.617955:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.617956:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.617957:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.617960:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.617961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677468.617977:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677468.617979:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677468.617980:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.617980:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677468.617981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.617982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:24.2:1619677468.618371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2c600] (21)++
00000800:00000200:5.2:1619677468.620811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.620856:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.620859:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.620860:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.620861:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.620864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.620864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.622144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.622186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.622190:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.622193:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.622194:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.622196:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.622198:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.622199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.622232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.622235:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.622236:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.622238:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.622239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.622241:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.626916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.627661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.627714:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.627717:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.627718:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.627720:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.627722:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.627723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.627724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.627724:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677468.627725:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.627725:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.627726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.631189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.632319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.632360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.632366:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.632369:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.632370:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.632371:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.632374:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.632374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:5.2:1619677468.635516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.636788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.637348:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.637351:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.637351:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.637353:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.637355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.637356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.640575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.640621:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.640624:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.640625:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.640626:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.640628:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.640629:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.641348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.641390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677468.641395:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.641398:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.641399:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.641401:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.641403:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.641404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.643736:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.643738:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.643739:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.643740:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.643743:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.643744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677468.644696:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.644698:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.644700:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677468.644702:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677468.644703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.644704:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677468.644705:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96533661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.644706:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.644707:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.644709:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677468.644712:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.644713:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677468.644717:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.644718:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:13.0:1619677468.644718:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:13.0:1619677468.644720:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:13.0:1619677468.644721:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.644722:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.644722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.645113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.646032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.646909:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.646912:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.646913:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.646914:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.646917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677468.646917:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.646933:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677468.646936:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677468.646937:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.646937:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677468.646938:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.646939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.649927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.650871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.650914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.654274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.655194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.655240:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.655243:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.655244:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.655245:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.655248:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.655248:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.655257:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677468.655260:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677468.655261:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.655261:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.655262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.655263:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.657461:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.657464:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.657465:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.657468:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.657470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.657471:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.659042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.659977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.660027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.663397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.664352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.668353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.669268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.669317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677468.669324:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.669328:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.669329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.669331:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.669333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.669335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.669943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677468.669986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:13.0:1619677468.671341:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.671344:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.671346:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.671348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.671349:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.671350:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677468.671352:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96533499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.671353:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.671354:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.671355:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677468.671358:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.671360:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677468.671362:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe0d80d00] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.671363:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb719600] (20)++
00000800:00000200:13.0:1619677468.671364:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb719600] (21)++
00000800:00000200:13.0:1619677468.671365:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb719600] (22)--
00000800:00000200:13.0:1619677468.671366:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.671367:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.671367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.672728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.673637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.673683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.673685:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.673686:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.673688:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.673690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.673691:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.677577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.678411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.678454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.678460:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.678463:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.678464:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.678466:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.678468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.678468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.678476:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677468.678479:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677468.678480:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.678481:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.678483:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.678483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:5.2:1619677468.682487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.683287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.687013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.687059:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.687062:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.687063:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.687064:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.687067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.687067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.687946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.687991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.687995:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.687999:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.688000:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.688003:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.688005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.688007:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.691880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.691927:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.691931:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.691932:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.691936:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.691938:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.691940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.692827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.692877:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.692879:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.692880:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.692882:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.692884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.692885:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.692893:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677468.692896:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677468.692897:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.692897:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.692898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.692900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:23.0F:1619677468.695302:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:5.2:1619677468.696616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.697405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.697445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.699766:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.699770:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.699773:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.699775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.699776:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.699778:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677468.699781:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96533323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.699783:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.699785:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.699786:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677468.699792:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.699795:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677468.699799:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.699800:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:13.0:1619677468.699801:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:13.0:1619677468.699803:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:13.0:1619677468.699804:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.699805:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.699806:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.701022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.701068:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.701071:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.701072:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.701074:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.701075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.701076:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.701176:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.701178:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.701180:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.701182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.701183:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.701184:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677468.701185:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96533312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.701186:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.701187:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.701188:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677468.701191:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.701193:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677468.701195:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.701196:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677468.701197:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677468.701199:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677468.701199:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.701201:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677468.701202:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677468.701203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.701216:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677468.701218:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677468.701218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.701219:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.701988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.702034:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.702037:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.702038:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.702039:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.702041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.702042:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.705836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.705882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.705884:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.705885:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.705887:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.705889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.705890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.706424:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.706426:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.706427:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.706429:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.706431:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.706432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.706433:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.706434:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.706434:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.706434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.706435:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.706435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.706436:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.706807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.706848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.706852:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.706854:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.706855:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.706857:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.706858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.706859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.710708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.710757:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.710760:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.710761:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.710764:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.710765:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.710767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.711511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.715611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.715657:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.715660:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.715661:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.715663:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.715665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.715665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:5.2:1619677468.716409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.716450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.720506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.720558:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.720560:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.720561:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.720563:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.720565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.720566:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.720575:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.720578:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.720579:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.720579:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.720580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.720581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.721828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.721874:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.721877:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.721878:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.721879:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.721881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.721882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.724840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.724886:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.724888:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.724889:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.724891:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.724893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.724894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.726744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.726787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677468.728645:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.728649:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677468.728653:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677468.728656:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677468.728657:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.728659:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677468.728661:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96533155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677468.728664:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677468.728666:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677468.728668:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677468.728673:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677468.728676:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677468.728680:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677468.728681:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:12.0:1619677468.728682:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:12.0:1619677468.728685:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:12.0:1619677468.728686:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.728687:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.728688:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.729606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.729652:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.729654:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.729655:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.729657:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.729659:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.729660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.731093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.734466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.734513:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.734515:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.734516:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.734518:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.734520:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.734521:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.734531:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.734534:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.734535:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.734535:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.734536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.734537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.735920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.735961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677468.735965:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.735968:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.735969:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.735971:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.735973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.735973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.739420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.739469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.739471:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.739472:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.739474:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.739476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.739477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.740529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.741560:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.741563:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.741564:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.741566:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.741568:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.741569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.741994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.742041:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.742044:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.742045:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.742047:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.742048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.742049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.742062:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.742065:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.742066:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.742066:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.742067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.742068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.742765:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.742768:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.742769:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.742770:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.742773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.742774:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.744918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.744959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.745959:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.745962:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.745963:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.745965:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.745966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.745968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.746101:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.746103:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.746104:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.746106:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.746107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.746113:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.746871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.746918:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.746922:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.746922:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.746924:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.746926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.746927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.749213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.749263:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.749265:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677468.749267:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677468.749270:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677468.749270:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.749271:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677468.749273:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96533016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677468.749274:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677468.749275:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677468.749276:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677468.749279:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677468.749280:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677468.749284:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677468.749285:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:12.0:1619677468.749285:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:12.0:1619677468.749287:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:12.0:1619677468.749288:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.749289:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.749289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.751681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.751725:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.751728:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.751729:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.751731:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.751732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.751733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.751744:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677468.751746:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677468.751747:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.751748:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.751748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.751750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.753967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.754008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.755766:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.755769:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.755770:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.755772:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.755773:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.755782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.756338:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.756341:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.756342:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.756343:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.756345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.756347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.756465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.756509:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.756512:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.756513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.756514:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.756516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.756517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.757525:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.757528:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.757529:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.757531:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.757533:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.757534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.757655:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.757658:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.757659:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.757661:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.757662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.757669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.758389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.759313:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.759316:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677468.759318:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677468.759320:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677468.759320:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.759322:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677468.759324:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96532939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677468.759325:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677468.759326:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677468.759327:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677468.759330:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677468.759332:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677468.759334:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677468.759335:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:12.0:1619677468.759335:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:12.0:1619677468.759337:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:12.0:1619677468.759337:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.759338:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.759339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.761275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.761328:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.761331:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.761332:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.761333:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.761335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.761336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.761349:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677468.761351:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677468.761352:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.761353:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.761354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.761355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.763284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.763329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.766289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.766344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.766347:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.766348:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.766350:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.766352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.766352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:5.2:1619677468.768124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.771133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.771184:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.771187:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.771188:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.771191:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.771194:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.771195:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.772621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.772661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.772665:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.772668:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.772670:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.772671:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.772673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.772674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.774629:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.774631:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677468.774633:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677468.774635:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677468.774636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677468.774637:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:12.0:1619677468.774638:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96532858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677468.774639:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677468.774640:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677468.774641:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:12.0:1619677468.774644:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677468.774646:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677468.774649:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677468.774650:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:12.0:1619677468.774650:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:12.0:1619677468.774652:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:12.0:1619677468.774653:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.774654:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677468.774655:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677468.774656:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.774670:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677468.774672:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677468.774673:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.774674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.776095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.776140:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.776143:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.776144:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.776146:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.776147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.776148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.776159:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677468.776161:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677468.776162:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.776163:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.776164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.776165:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.776438:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.776441:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.776443:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677468.776445:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677468.776445:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.776447:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:13.0:1619677468.776448:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96532849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.776450:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.776451:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.776451:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:13.0:1619677468.776454:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.776456:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677468.776460:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.776461:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:13.0:1619677468.776461:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:13.0:1619677468.776463:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:13.0:1619677468.776464:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.776465:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.776465:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.777477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.780762:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.780766:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677468.780769:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677468.780772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677468.780773:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.780775:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:12.0:1619677468.780777:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96532826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677468.780779:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677468.780781:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677468.780783:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:12.0:1619677468.780788:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677468.780790:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677468.780794:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677468.780795:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:12.0:1619677468.780796:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:12.0:1619677468.780798:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:12.0:1619677468.780799:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.780800:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.780801:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.781602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.781808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07dd000] (20)++
00000800:00000200:13.0:1619677468.781863:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.781865:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.781866:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677468.781866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07dd000] (21)++
00000400:00000200:13.0:1619677468.781868:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.781870:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.781871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.781882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07dd000] (21)++
00000800:00000200:5.2:1619677468.781927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07dd000] (22)++
00000800:00000200:12.0:1619677468.782015:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (24)++
00000800:00000200:12.0:1619677468.782018:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (24)--
00000400:00000200:12.0:1619677468.782019:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677468.782022:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)++
00000400:00000200:12.0:1619677468.782022:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.782023:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (24)--
00000400:00000200:13.0:1619677468.782024:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.782025:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.782031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677468.782032:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.782033:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677468.782034:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677468.782034:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.782035:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.782035:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.782036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.782136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07dd000] (20)++
00000800:00000200:5.2:1619677468.782178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07dd000] (21)++
00000800:00000200:5.2:1619677468.782587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.782628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.786522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.787405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.787454:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.787456:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.787457:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.787459:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.787461:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.787462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.788294:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.788301:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677468.788302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:12.0:1619677468.788303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.788304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.788307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.790862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.792001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.792038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.792044:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.792047:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.792048:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.792049:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.792051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.792052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.792442:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.792444:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.792445:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.792447:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.792449:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.792450:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.795681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.796763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.796808:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.796811:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.796812:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.796814:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.796817:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.796818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.796827:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.796830:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.796831:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.796832:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.796832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.796833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.799979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677468.799987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.801343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.801382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.804819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.806114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.809109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.810783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.810829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.810836:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.810840:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.810841:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.810844:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.810847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.810848:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.813965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.815671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.815716:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.815719:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.815720:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.815721:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.815723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.815724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.818349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.820429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.820470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.820476:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.820478:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.820480:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.820481:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.820483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.820484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.822975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.825269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.825320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.825324:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.825325:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677468.825329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677468.825329:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.825333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.825341:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.825344:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.825345:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.825345:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.825346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.825347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.827225:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.827227:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.827228:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.827230:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.827232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.827233:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677468.827261:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677468.827263:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677468.827264:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.827265:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677468.827266:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.827267:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.827772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.830566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.830608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.830613:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.830616:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.830617:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.830619:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.830620:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.830621:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.831559:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.831562:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.831562:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.831564:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.831566:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.831567:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.831600:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.831602:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.831603:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.831604:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.831606:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.831607:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.832168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.2:1619677468.834350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2c600] (21)++
00000800:00000200:24.2:1619677468.834371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2c600] (20)++
00000800:00000200:5.2:1619677468.835461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.835510:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.835512:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.835513:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.835515:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.835517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.835518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.837075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.838420:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.838422:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.838423:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.838426:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.838427:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.838429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.838438:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677468.838441:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677468.838442:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.838443:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.838443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.838444:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.840902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.840943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677468.840948:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.840950:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.840951:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.840953:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.840955:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.840956:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.841868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.845806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.846784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.850700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.850746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.851586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.851631:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.851634:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.851635:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.851636:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.851638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.851640:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.853356:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.853359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.853360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.853362:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.853363:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.853365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.853487:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.853490:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.853491:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.853494:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.853494:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.853498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.855089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.856465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.858839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.858842:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.858843:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.858845:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.858848:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.858849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.859966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.860004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.861667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.861710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:12.0:1619677468.861714:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.861716:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.861717:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.861719:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.861721:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.861722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.861739:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.861742:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.861743:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.861744:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.861744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.861745:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.864300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.866480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.866531:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.866533:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.866534:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.866536:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.866538:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.866539:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.867740:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.867743:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.867743:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.867745:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.867747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.867748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677468.867759:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677468.867761:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677468.867762:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.867763:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677468.867763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.867765:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.869150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.869189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.871773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.871819:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.871822:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.871823:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.871825:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.871827:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.871828:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.873478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.877082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.877131:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.877134:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.877135:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.877138:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.877140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.877142:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.878400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.878441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677468.878447:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.878450:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.878451:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.878453:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.878455:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.878455:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.878456:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.878457:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.878457:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.878458:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (24)--
00000800:00000200:13.0:1619677468.878458:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.878458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (24)--
00000800:00000200:12.0:1619677468.878459:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.879923:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.879926:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.879929:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.879930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.879931:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.879933:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677468.879934:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96532235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.879935:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.879936:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.879937:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677468.879940:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.879941:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677468.879944:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.879945:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677468.879945:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677468.879947:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677468.879948:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.879948:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.879949:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.882372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.882418:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.882421:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.882422:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.882424:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.882426:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.882427:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.883185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.885988:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.885992:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.885994:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.885996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.885997:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.885998:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677468.885999:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96532203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.886000:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.886001:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.886002:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677468.886005:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.886007:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677468.886009:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.886010:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677468.886011:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677468.886012:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677468.886013:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.886014:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677468.886016:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677468.886016:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.886030:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677468.886032:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677468.886033:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.886034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.887237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.887284:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.887286:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.887287:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.887289:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.887291:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.887292:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.888018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.888055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.888061:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.888064:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.888065:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.888067:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.888068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.888070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.891748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.892917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.894416:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.894419:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.894420:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.894421:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.894423:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.894424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.894436:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.894439:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.894440:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.894440:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.894441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.894442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.896680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.898039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.898077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677468.901556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.903171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.903227:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.903229:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.903230:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.903232:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.903234:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.903235:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.905887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.906325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07dd000] (20)++
00000800:00000200:5.2:1619677468.908118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.908153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.908159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.908162:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.908163:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.908165:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.908167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.908168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.910702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.913148:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.913151:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.913152:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.913154:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.913156:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.913157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.913524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.913573:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.913575:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.913576:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.913578:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.913580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.913581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.913591:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677468.913594:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677468.913595:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.913595:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.913596:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.913597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.915613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.916529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:13.0:1619677468.916794:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.916796:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.916797:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.916798:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.916800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.916801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.918322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.918364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677468.918369:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.918371:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.918373:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.918374:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.918376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.918377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.920540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.922286:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.922288:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.922291:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.922293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.922301:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.922303:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677468.922304:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96532007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.922305:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.922306:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.922307:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677468.922310:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.922312:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677468.922315:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.922316:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:13.0:1619677468.922316:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:13.0:1619677468.922318:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:13.0:1619677468.922319:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.922320:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.922320:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.923113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.923157:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.923160:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.923161:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.923163:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.923164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.923166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.924834:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.924836:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.924837:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.924838:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.924841:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.924841:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.924887:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.924890:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.924890:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.924892:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.924894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.924895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.924909:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.924911:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677468.924912:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.924913:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.924915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.924916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.925393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.925986:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.925989:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.925991:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677468.925994:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677468.925994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.925995:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677468.925996:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96531975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.925998:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.925999:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.926000:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677468.926003:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.926004:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677468.926007:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.926008:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:13.0:1619677468.926008:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:13.0:1619677468.926010:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:13.0:1619677468.926011:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.926011:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.926012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677468.927715:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.927719:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677468.927723:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677468.927726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677468.927727:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.927731:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677468.927733:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96531969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677468.927737:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677468.927739:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677468.927741:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677468.927749:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677468.927753:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677468.927757:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677468.927759:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:12.0:1619677468.927760:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:12.0:1619677468.927763:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:12.0:1619677468.927764:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677468.927765:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.927766:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.927919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.927967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.927974:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.927977:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.927979:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.927981:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.927982:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.927985:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.927985:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.927987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.927987:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.927988:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.927989:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.929792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.931290:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.931293:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:12.0:1619677468.931298:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.931300:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677468.931302:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.931303:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677468.931305:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96531946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.931306:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.931307:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.931308:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677468.931311:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.931312:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677468.931315:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.931316:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:13.0:1619677468.931316:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:13.0:1619677468.931318:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:13.0:1619677468.931319:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.931320:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677468.931322:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677468.931322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.931336:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677468.931338:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677468.931339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677468.931340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677468.932149:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.932151:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.932152:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.932153:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.932155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.932156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:24.2:1619677468.932591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2c600] (21)++
00000800:00000200:5.2:1619677468.932701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.932751:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.932754:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.932755:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.932757:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.932758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.932760:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:24.2:1619677468.932916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:12.0:1619677468.933416:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.933419:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.933420:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.933422:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.933424:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.933425:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.934629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.937564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.937605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.937613:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.937616:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.937617:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.937620:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.937622:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.937624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.937625:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.937626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.937626:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.937627:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.937628:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.939497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.940562:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.940565:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.940569:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.940571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.940572:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.940574:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677468.940576:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96531880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.940579:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.940580:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.940582:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677468.940587:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.940590:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677468.940594:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.940595:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:13.0:1619677468.940596:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:13.0:1619677468.940598:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:13.0:1619677468.940599:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.940600:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.940601:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677468.942383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.942433:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.942436:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.942436:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.942438:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.942440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.942442:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.942717:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.942720:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.942722:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.942723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.942725:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.942726:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677468.942728:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96531868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.942729:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.942730:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.942731:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677468.942734:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.942736:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677468.942738:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.942739:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:13.0:1619677468.942740:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:13.0:1619677468.942741:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:13.0:1619677468.942742:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.942743:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.942743:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677468.943722:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.943724:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.943725:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.943727:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.943729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.943730:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.944321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.947185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.947228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.947233:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.947235:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.947237:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.947238:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.947240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.947241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.949137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.952032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.952075:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.952078:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.952078:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.952080:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.952082:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.952083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.952095:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.952098:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.952099:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.952100:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.952101:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.952102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.954463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.956871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.956908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677468.956915:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.956917:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.956918:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.956920:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.956922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.956922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.959307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.960362:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.960365:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.960365:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.960367:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.960369:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.960370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.961731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.961776:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.961778:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.961779:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.961781:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.961783:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.961784:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.964180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.966551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.966591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.966597:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.966600:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.966601:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.966603:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.966605:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.966606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.966606:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.966607:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.966607:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.966608:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.966608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.968511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.971804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.971815:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.971819:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677468.971822:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677468.971822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677468.971824:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.971826:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677468.971828:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96531710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677468.971829:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677468.971831:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677468.971832:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677468.971839:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677468.971842:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677468.971844:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677468.971845:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:13.0:1619677468.971845:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:13.0:1619677468.971847:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:13.0:1619677468.971847:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677468.971848:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677468.971849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677468.971864:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.971866:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.971867:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.971869:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.971870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.971872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.973281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.976674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.976713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.976721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.976724:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.976725:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.976728:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.976729:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.976732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.976948:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677468.976951:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677468.976951:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.976953:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.976954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677468.976956:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677468.976968:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677468.976971:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677468.976972:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677468.976973:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677468.976974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677468.976975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677468.978084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.979214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677468.979254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:5.2:1619677468.981461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.981507:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.981510:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.981511:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.981512:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.981515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.981517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.983226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.983272:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.983275:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.983276:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.983277:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.983279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.983280:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.983290:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677468.983292:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677468.983298:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.983299:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.983299:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677468.983300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.986432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf61e00] (20)++
00000800:00000200:5.2:1619677468.986455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf61e00] (21)++
00000800:00000200:5.2:1619677468.986471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf61e00] (21)++
00000800:00000200:5.2:1619677468.986913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.986956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677468.986963:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.986965:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.986966:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.986968:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.986970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.986972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.988073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.988119:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.988122:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.988122:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.988124:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.988126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.988127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.992232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677468.992276:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.992279:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.992280:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.992281:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.992283:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.992284:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.993086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677468.993325:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.993328:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.993329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.993330:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.993333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.993333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.993342:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.993345:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.993346:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.993347:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.993347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.993348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:24.2:1619677468.995753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:13.0:1619677468.995803:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.995805:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.995806:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.995808:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.995810:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.995811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677468.996013:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.996016:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.996017:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677468.996019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677468.996019:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.996024:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.996156:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.996159:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677468.996160:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677468.996162:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677468.996162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.996164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.997054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677468.997097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677468.997102:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677468.997105:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677468.997106:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677468.997107:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677468.997109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677468.997110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677468.997123:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677468.997126:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677468.997127:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677468.997127:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677468.997128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677468.997129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677468.998043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.001833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.002828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.006701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.006740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.007681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.011565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.011756:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.011759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.011760:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.011762:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.011764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.011766:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.012762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.016019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.016073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.017798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.017843:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.017845:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.017846:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.017848:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.017850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.017851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.020854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.020949:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.020952:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.020953:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.020955:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.020959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.020961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.023246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.023298:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.023301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.023302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.023303:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.023306:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.023307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.025211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.025248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.028521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.028574:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.028577:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.028578:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.028580:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.028581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.028582:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.028593:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.028596:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.028597:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.028598:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.028598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.028600:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.029366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.030060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.034307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.038079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.038139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.038915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.040947:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.040950:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.040952:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.040954:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.040954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.040956:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677469.040957:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96531325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.040958:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.040959:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.040960:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677469.040963:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.040965:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677469.040968:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.040969:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677469.040969:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677469.040971:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677469.040971:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.040973:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677469.040974:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677469.040975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.041014:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677469.041016:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677469.041017:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.041018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.043026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.043802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.047377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.047417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.048720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.048769:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.048772:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.048773:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.048776:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.048777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.048780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.050308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.050356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.050358:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.050359:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.050361:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.050362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.050364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.052243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.052481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.052530:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.052533:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.052534:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.052536:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.052538:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.052539:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.053493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.053537:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.053540:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.053541:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.053542:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.053544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.053546:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.054531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.054574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677469.054578:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.054581:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.054582:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.054583:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.054586:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.054587:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677469.054587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.054588:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.054588:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.054589:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (24)--
00000800:00000200:12.0:1619677469.054589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.054590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.054591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.056650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.056700:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.056702:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.056703:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.056706:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.056707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.056709:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.057114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.057154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.057731:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.057733:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.057735:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.057738:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.057738:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.057739:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677469.057741:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96531230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.057742:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.057743:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.057744:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677469.057748:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.057749:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677469.057752:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.057753:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:12.0:1619677469.057754:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:12.0:1619677469.057756:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:12.0:1619677469.057757:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.057757:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.057758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.058313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.058358:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.058361:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.058362:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.058364:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.058365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.058367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.060049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.060097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.060099:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.060100:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.060102:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.060104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.060105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.061490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.062993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.063039:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.063042:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.063043:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.063045:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.063047:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.063048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.063058:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.063061:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.063061:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.063062:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.063063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.063064:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.065038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.065084:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.065087:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.065088:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.065091:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.065091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.065095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.065882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.065922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.067808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.067857:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.067859:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.067860:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.067862:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.067865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.067866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.069785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.069831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677469.069835:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.069838:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.069839:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.069840:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.069843:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.069844:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677469.069853:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677469.069856:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.069857:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.069858:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.069859:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.069860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.070694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.072694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.072739:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.072742:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.072743:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.072745:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.072746:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.072747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.072913:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.072915:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.072918:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677469.072919:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.072920:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.072922:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677469.072923:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96531147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.072925:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.072926:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.072927:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677469.072930:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.072932:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677469.072934:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.072938:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:13.0:1619677469.072939:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:13.0:1619677469.072941:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:13.0:1619677469.072941:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.072942:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.072942:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677469.073469:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.073471:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.073472:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.073474:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.073476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.073477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.074082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.074127:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.074130:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.074131:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.074132:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.074133:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.074135:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.075001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.075043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.075392:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.075394:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.075395:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.075396:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.075398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.075399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.075709:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.075712:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.075714:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.075716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.075717:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.075718:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677469.075719:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96531130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.075720:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.075721:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.075722:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677469.075725:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.075727:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677469.075730:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.075731:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:12.0:1619677469.075731:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:12.0:1619677469.075733:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:12.0:1619677469.075734:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.075734:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.075735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.077506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.077553:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.077556:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.077556:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.077558:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.077560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.077561:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.077569:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.077572:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.077573:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.077573:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.077574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.077575:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.078163:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.078165:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.078166:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.078167:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.078169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.078170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.078777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.078826:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.078829:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.078830:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.078832:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.078834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.078835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677469.078843:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677469.078846:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.078846:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.078847:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.078848:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.078848:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.079070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:13.0:1619677469.079783:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.079786:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.079788:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677469.079790:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.079791:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.079793:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677469.079794:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96531106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.079796:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.079797:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.079798:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:13.0:1619677469.079802:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.079805:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677469.079807:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.079808:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:13.0:1619677469.079808:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:13.0:1619677469.079811:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:13.0:1619677469.079811:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.079813:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677469.079814:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677469.079815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.079829:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677469.079831:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677469.079832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.079833:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.079979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.080121:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.080123:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.080124:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.080126:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.080127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.080129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.080146:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.080148:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.080149:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.080151:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.080153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.080153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.080250:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.080253:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.080254:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.080256:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.080257:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.080262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.082323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.082369:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.082372:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.082373:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.082374:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.082376:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.082377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.083495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.083545:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.083548:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.083549:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.083551:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.083553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.083553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.084578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.084618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.087124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.087172:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.087176:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.087177:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.087180:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.087181:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.087185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.087188:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.087190:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.087190:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.087191:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.087193:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.087193:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.088237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.088279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677469.088284:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.088287:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.088288:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.088289:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.088291:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.088292:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.089474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.090844:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.090846:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.090847:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.090849:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.090851:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.090852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.090865:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677469.090868:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677469.090869:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.090870:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.090871:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.090872:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.091963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.092009:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.092011:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.092012:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.092014:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.092016:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.092017:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.092561:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.092564:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.092565:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.092566:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.092568:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.092569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.093087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.093139:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.093142:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.093142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.093144:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.093145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.093147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.094306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.094353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.096733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.096791:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.096795:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.096796:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.096799:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.096800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.096802:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.097872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.097919:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.097922:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.097923:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.097924:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.097926:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.097927:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.098666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.101541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.101587:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.101590:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.101592:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.101594:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.101595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.101598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.102269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.102321:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.102323:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.102324:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.102326:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.102327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.102329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677469.102342:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677469.102345:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.102346:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.102346:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.102348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.102349:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.103576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.103617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.106418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.106471:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.106474:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.106476:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.106478:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.106480:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.106482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.106483:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.106484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.106484:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.106485:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.106486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.107346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.107388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677469.107404:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.107406:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.107407:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.107409:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.107410:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.107412:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.107890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.110481:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.110484:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.110486:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.110487:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.110489:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.110490:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.111282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.111331:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.111334:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.111335:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.111337:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.111338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.111339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.112199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.112245:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.112247:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.112248:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.112250:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.112251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.112253:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.112791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.112832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.116221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.116265:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.116268:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.116269:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.116270:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.116272:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.116273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.116436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.116483:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.116486:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.116487:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.116488:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.116490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.116491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677469.116499:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677469.116502:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.116503:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.116503:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.116504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.116506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.117721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.121127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.121172:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.121175:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.121176:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.121177:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.121179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.121180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.121210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.121257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.121259:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.121260:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.121262:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.121263:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.121265:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.122647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.122688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.125908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.125946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677469.125955:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.125958:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.125960:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.125962:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.125965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.125966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.126017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.126062:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.126065:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.126066:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.126068:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.126069:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.126070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.126083:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.126085:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.126086:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.126087:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.126087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.126089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.127220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.127268:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.127270:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.127271:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.127273:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.127274:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.127275:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.130980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.131039:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.131043:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.131044:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.131047:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.131048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.131051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.131127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.131174:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.131177:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.131179:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.131181:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.131183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.131185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.132259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.132304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.132305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.132308:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.132309:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.132311:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.132313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.132314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.135322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.135374:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.135378:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.135379:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.135382:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.135384:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.135385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.135386:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.135387:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.135388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.135389:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.135390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.135453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.135509:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.135512:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.135514:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.135517:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.135518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.135520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.135531:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.135535:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.135536:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.135537:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.135538:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.135540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.136447:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.136450:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.136452:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.136454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.136454:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.136456:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677469.136457:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96530807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.136458:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.136459:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.136460:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677469.136463:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.136465:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677469.136467:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.136468:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:12.0:1619677469.136469:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:12.0:1619677469.136471:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:12.0:1619677469.136471:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.136472:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.136472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.136586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.140013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.140060:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.140063:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.140065:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.140067:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.140067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.140070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.140349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.140395:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.140398:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.140398:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.140400:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.140402:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.140403:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.141538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.141579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.141584:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.141586:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.141587:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.141589:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.141591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.141592:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.144729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.144771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677469.144775:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.144778:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.144779:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.144780:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.144782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.144783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.145281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.145334:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.145337:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.145338:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.145341:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.145341:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.145344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.146273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.149407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.149456:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.149459:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.149460:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.149461:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.149464:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.149464:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.150244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.150301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.150303:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.150304:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.150306:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.150307:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.150309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.150309:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.150310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.150310:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.150310:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.150311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.151107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.151145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.151409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:13.0:1619677469.152440:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.152442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.152443:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.152457:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.152459:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[3] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.152461:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)++
00000800:00000200:13.0:1619677469.152463:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (24)--
00000800:00000200:12.0:1619677469.152463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (24)--
00000800:00000200:13.0:1619677469.152464:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.152479:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.152482:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.152482:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.152484:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.152486:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.152486:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.153651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.153698:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.153701:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.153701:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.153703:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.153705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.153707:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.155157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.155206:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.155209:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.155210:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.155212:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.155214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.155215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.155504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.155552:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.155554:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.155555:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.155557:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.155559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.155560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.155984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.159797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.160622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.160663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.160669:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.160672:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.160673:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.160675:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.160676:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.160677:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.164671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.165081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.165118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.165120:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.165123:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.165125:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.165125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:5.2:1619677469.165125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000400:00000200:13.0:1619677469.165126:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:12.0:1619677469.165127:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000400:00000200:13.0:1619677469.165128:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96530677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677469.165128:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.165129:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.165129:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.165130:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.165131:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677469.165131:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.165134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:13.0:1619677469.165135:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.165135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677469.165137:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677469.165139:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.165140:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:13.0:1619677469.165141:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:13.0:1619677469.165142:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:13.0:1619677469.165143:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.165143:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.165146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677469.165147:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677469.165149:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.165150:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.165151:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.165151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.165153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.165472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.165520:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.165523:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.165524:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.165525:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.165527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.165528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.166170:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.166173:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.166173:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.166175:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.166177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.166177:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.167237:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.167240:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.167241:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.167243:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.167244:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.167245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.167257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.167260:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.167261:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.167262:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.167262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.167264:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.169124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.170221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.170261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.170266:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.170269:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.170270:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.170271:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.170273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.170275:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.173974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.174465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.174511:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.174514:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.174514:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.174516:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.174518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.174519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.175094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.175139:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.175142:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.175142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.175144:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.175146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.175147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.178388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.179741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.179782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.179787:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.179789:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.179791:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.179792:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.179794:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.179795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.182717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.182766:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.182768:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.182769:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.182771:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.182773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.182774:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.183376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.186259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.186308:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.186310:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.186311:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.186313:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.186316:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.186317:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.186322:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.186323:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.186324:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.186325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.186327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.189735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.189796:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.189799:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.189800:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.189802:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.189805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.189806:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.190749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.190799:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.190802:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.190803:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.190804:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.190806:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.190807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677469.190817:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677469.190821:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.190821:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.190823:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.190824:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.190825:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.191603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.191644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.191662:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.191665:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.191667:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.191669:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.191669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.191671:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677469.191672:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96530532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.191674:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.191675:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.191676:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677469.191679:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.191680:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677469.191683:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.191684:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677469.191685:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677469.191686:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677469.191687:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.191688:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.191688:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677469.193141:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.193144:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.193146:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.193148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.193148:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.193150:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677469.193152:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96530527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.193153:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.193154:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.193155:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677469.193158:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.193159:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677469.193162:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.193163:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:12.0:1619677469.193163:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:12.0:1619677469.193165:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:12.0:1619677469.193166:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.193167:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677469.193168:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677469.193169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.193182:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677469.193184:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677469.193185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.193186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.194658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.194702:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.194705:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.194706:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.194707:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.194709:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.194710:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.196088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.196134:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.196137:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.196138:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.196139:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.196141:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.196142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.198592:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.198595:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.198595:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.198597:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.198599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.198600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.198612:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.198615:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.198616:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.198617:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.198617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.198619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:24.2:1619677469.198913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677469.198936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:5.2:1619677469.199518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.199566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677469.199570:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.199573:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.199574:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.199575:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.199577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.199578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.199615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.199658:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.199661:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.199662:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.199663:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.199665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.199666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.200970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.201011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.201016:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.201019:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.201020:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.201022:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.201023:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.201025:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.202056:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.202058:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.202059:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.202060:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.202063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.202064:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.204499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.204556:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.204559:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.204560:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.204561:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.204564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.204565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.204576:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.204579:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.204580:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.204580:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.204581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.204582:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.205773:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.205775:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.205776:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.205778:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.205780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.205781:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.205903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.205947:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.205950:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.205951:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.205953:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.205954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.205955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.209374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.209424:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.209426:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.209427:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.209429:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.209431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.209432:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.209706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.209709:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.209711:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.209713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.209714:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.209715:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677469.209716:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96530430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.209718:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.209719:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.209720:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677469.209723:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.209725:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677469.209727:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.209729:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:12.0:1619677469.209729:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:12.0:1619677469.209731:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:12.0:1619677469.209731:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.209732:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.209733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.210412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.210451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.214230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.214279:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.214282:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.214283:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.214286:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.214287:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.214288:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.214289:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)++
00000800:00000200:13.0:1619677469.214290:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (24)--
00000800:00000200:12.0:1619677469.214290:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (24)--
00000800:00000200:12.0:1619677469.214291:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.214292:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.215317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.216039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.216083:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.216086:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.216087:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.216088:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.216091:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.216091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.219133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.219178:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.219181:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.219182:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.219183:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.219186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.219187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.219339:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a4607a5400] (21)++
00000800:00000200:13.0:1619677469.219342:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.17@o2ib3
00000400:00000200:13.0:1619677469.219346:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.6.17@o2ib3 : GET - for me
00000800:00000200:12.0:1619677469.219348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a4607a5400] (22)--
00000400:00000200:13.0:1619677469.219350:0:19942:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.6.17@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:13.0:1619677469.219353:0:19942:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.6.17@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:13.0:1619677469.219356:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a4607a5400] (21)++
00000800:00000200:13.0:1619677469.219356:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a4607a5400] (22)--
00000400:00000200:13.0:1619677469.219359:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.6.17@o2ib3 local destination
00000800:00000200:13.0:1619677469.219361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a4607a5400] (21)--
00000800:00000200:5.2:1619677469.219923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.219963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.219969:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.219973:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.219974:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.219976:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.219978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.219980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.220057:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.220060:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.220061:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.220064:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.220066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.220067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.221240:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.221243:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.221243:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.221245:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.221247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.221248:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677469.221251:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.221251:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.221252:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.221253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.221254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.224002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.224047:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.224051:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.224052:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.224055:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.224057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.224058:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.224511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.224562:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.224565:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.224566:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.224568:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.224570:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.224571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.224755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.224800:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.224803:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.224804:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.224806:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.224807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.224808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.224816:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.224819:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.224820:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.224821:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.224821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.224822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.225753:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.225755:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.225756:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.225758:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.225759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.225760:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.226396:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.226399:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.226400:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.226402:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.226404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.226404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.228887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.228933:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.228936:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.228937:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.228938:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.228940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.228941:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.229567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:12.0:1619677469.231472:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.231475:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.231477:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.231480:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.231480:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.231481:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677469.231482:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96530318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.231484:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.231485:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.231486:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677469.231488:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.231490:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677469.231492:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.231493:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:12.0:1619677469.231494:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677469.231496:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.231496:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.231497:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.231497:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677469.231508:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.231510:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.231511:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.231512:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.231514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.231515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.232012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.232055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.233843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.233851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.233895:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.233897:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.233897:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000400:00000200:13.0:1619677469.233898:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.233899:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.233899:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.233900:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:12.0:1619677469.233901:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.233903:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.233904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.233905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.233906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.233918:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.233920:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.233921:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.233922:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.233922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.233924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.236838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.238684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.238729:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.238731:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.238732:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.238734:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.238736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.238737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.241665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.241704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.243163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.243209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677469.243215:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.243217:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.243219:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.243220:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.243223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.243224:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677469.243233:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677469.243236:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.243236:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.243237:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.243238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.243239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.243544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.243591:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.243595:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.243596:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.243599:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.243601:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.243602:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.246016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.248368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.248414:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.248417:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.248417:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.248419:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.248422:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.248423:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.250797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.250836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.252720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.252769:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.252772:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.252772:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.252774:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.252777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.252778:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.253240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.253285:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.253288:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.253289:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.253290:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.253292:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.253300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.253301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.253303:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.253303:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.253303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.253304:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.253305:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.255109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.258283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.258338:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.258341:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.258342:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.258343:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.258345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.258346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.259031:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.259033:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.259034:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.259035:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.259037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.259038:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.261000:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.261002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.261003:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.261005:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.261007:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.261008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:24.2:1619677469.262150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2c600] (21)++
00000800:00000200:24.2:1619677469.262266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2c600] (21)++
00000800:00000200:24.2:1619677469.262308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:13.0:1619677469.262360:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.262363:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.262364:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.262367:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.262368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.262370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.262451:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.262454:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.262455:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677469.262457:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000400:00000200:12.0:1619677469.262458:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.262459:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.262460:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)++
00000800:00000200:13.0:1619677469.262461:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (24)--
00000800:00000200:13.0:1619677469.262461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.262462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.262466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.263067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.263107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.263112:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.263115:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.263116:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.263117:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.263119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.263120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.263158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.263206:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.263209:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.263210:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.263212:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.263213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.263214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.263885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.267703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.268529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.268584:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.268587:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.268588:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.268589:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.268592:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.268593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.268605:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.268607:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.268608:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.268609:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.268609:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.268610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.272106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.272146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.273248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.276984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.276995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.278184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.278359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.278405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.279733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.281493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.281533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.282842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.286414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.287533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677469.287574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:5.2:1619677469.287694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.287741:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.287743:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.287744:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.287746:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.287748:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.287749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.288612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.290825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.290862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.292031:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.292034:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.292035:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.292036:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.292038:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.292039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.292084:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.292086:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.292087:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.292089:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.292090:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.292092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.292225:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.292228:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.292229:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677469.292231:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:12.0:1619677469.292231:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.292234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.292358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.292403:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.292405:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.292406:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.292408:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.292410:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.292411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.292421:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.292424:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.292425:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.292426:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.292426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.292428:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.293145:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.293148:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.293149:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.293150:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.293151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.293153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.293705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.295756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.297263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.297315:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.297317:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.297318:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.297320:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.297322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.297323:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.300111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.300153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.301963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.302008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.302162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.302222:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.302225:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.302226:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.302227:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.302229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.302230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.305010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.307067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.307112:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.307115:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.307116:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.307118:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.307120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.307121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.307931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.309313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.309354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.311896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.311943:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.311946:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.311947:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.311948:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.311951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.311952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.311960:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.311963:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.311964:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.311965:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.311965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.311966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.313562:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.313565:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.313567:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.313569:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.313569:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.313571:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677469.313572:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96529888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.313573:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.313574:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.313575:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677469.313581:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.313583:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677469.313586:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.313586:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:13.0:1619677469.313587:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:13.0:1619677469.313589:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:13.0:1619677469.313589:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.313590:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.313591:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.314155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.315844:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.315847:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.315848:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.315850:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.315851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.315853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.317067:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.317070:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.317072:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.317074:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.317074:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.317076:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677469.317077:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96529868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.317079:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.317080:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.317081:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677469.317084:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.317085:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677469.317088:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.317089:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:13.0:1619677469.317089:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:13.0:1619677469.317091:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:13.0:1619677469.317092:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.317093:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677469.317095:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677469.317095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.317108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677469.317111:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677469.317111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.317112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.318289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.318643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.319810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.319848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.323266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.323558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.324420:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.324424:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.324425:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.324428:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.324430:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.324431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.324432:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.324433:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.324433:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.324434:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677469.324435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.324692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.324740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.324743:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.324745:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677469.324747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677469.324747:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.324751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.328279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.328336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.328503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.330374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.330415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.330420:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.330423:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.330424:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.330426:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.330429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.330431:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.332664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.332953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.336630:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.336633:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.336635:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.336637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.336638:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.336639:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677469.336641:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96529770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.336642:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.336643:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.336644:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677469.336646:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.336648:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677469.336650:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.336651:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:12.0:1619677469.336652:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:12.0:1619677469.336653:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:12.0:1619677469.336654:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.336655:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.336655:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.337490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.337542:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.337545:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.337546:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.337548:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.337549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.337551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.337560:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.337563:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.337564:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.337564:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.337565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.337566:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.337599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.341261:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.341264:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.341264:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.341266:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.341268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.341269:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.341773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.342519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.342587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.342597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.342839:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.342841:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.342842:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.342843:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.342845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.342846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.346291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.347472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.347518:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.347521:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.347522:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.347523:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.347525:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.347526:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.347608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.347654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.351220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.352559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.352574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.352600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.354737:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.354741:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.354745:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.354747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.354748:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.354750:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677469.354753:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96529655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.354755:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.354756:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.354758:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677469.354763:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.354766:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677469.354770:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.354771:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:12.0:1619677469.354772:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:12.0:1619677469.354774:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:12.0:1619677469.354775:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.354776:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.354777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.356174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.357464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.357509:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.357512:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.357513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.357515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.357518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.357519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.357525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.359543:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.359545:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.359546:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.359548:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.359550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.359552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.360601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.362595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.362691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.362731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.362738:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.362741:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.362743:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.362745:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.362746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.362749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.363869:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.363871:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677469.363874:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : GET - for me
00000800:00000200:13.0:1619677469.363876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677469.363877:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.103@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677469.363879:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.103@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677469.363880:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.363881:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677469.363882:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:12.0:1619677469.363886:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677469.363887:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.363889:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.363890:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.363891:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.363892:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.363893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.363904:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.363907:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.363907:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.363909:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:12.0:1619677469.363911:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.363912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.365513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.367120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.367156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.367561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.367606:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.367609:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.367610:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.367612:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.367614:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.367614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.369863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.372066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.372935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.372976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.372980:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.372983:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.372984:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.372985:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.372987:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.372988:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.374713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.377089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.377707:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.377709:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.377710:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.377712:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.377714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.377715:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.377723:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.377726:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.377727:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.377728:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.377728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.377730:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.377771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.377832:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.377835:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.377836:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.377837:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.377838:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.377840:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.379149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.379196:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.379199:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.379200:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.379201:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.379203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.379204:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.382095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.383246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.383287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.383298:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.383301:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.383302:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.383304:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.383307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.383308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.384400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.384452:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.384455:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.384455:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.384457:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.384460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.384461:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.384465:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.384465:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.384466:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.384466:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.384467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.387149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.387189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.387436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677469.388533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.389287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.391552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.393515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.393555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.394233:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.394235:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.394236:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.394238:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.394240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.394241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.394253:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677469.394256:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677469.394256:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.394257:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.394258:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.394259:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.394611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.394666:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.394669:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.394670:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.394671:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.394674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.394674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:24.2:1619677469.394864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2c600] (21)++
00000800:00000200:24.2:1619677469.394887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2c600] (20)++
00000800:00000200:13.0:1619677469.394952:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.394955:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.394956:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.394957:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.394959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.394963:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.396507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.398354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.399566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.401531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.402811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.402851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.402988:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.402991:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.402992:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.402993:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.402996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.402997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.404725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.404771:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.404774:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.404775:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.404776:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.404779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.404780:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.404790:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.404793:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.404794:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.404795:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.404796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.404797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.406446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.406488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.407613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.409351:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.409354:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.409356:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.409358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.409359:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.409361:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677469.409362:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96529315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.409364:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.409365:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.409366:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677469.409369:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.409371:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677469.409374:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.409375:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:12.0:1619677469.409375:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:12.0:1619677469.409377:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:12.0:1619677469.409378:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.409378:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.409379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.410100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.410145:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.410148:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.410148:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.410150:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.410151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.410153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.411416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.412480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.412520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.415004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.415051:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.415054:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.415055:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.415058:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.415059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.415061:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.416352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.416881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.420375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.420423:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.420426:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.420427:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.420430:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.420430:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.420433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.421278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.421830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.421871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.421876:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.421879:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.421880:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.421881:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.421883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.421884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.421892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.421895:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.421896:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.421896:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.421897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.421898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.423994:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.423996:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.423997:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.423999:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.424001:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.424002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.425383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.425430:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.425433:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.425434:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.425436:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.425438:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.425439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.426204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.426248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.426807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.426853:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.426855:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.426856:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.426858:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.426860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.426861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.430343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.430388:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.430391:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.430392:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.430393:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.430395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.430396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.430404:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.430407:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.430408:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.430408:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.430409:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.430410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.430637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.431473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.431511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.431710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:12.0:1619677469.433838:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.433841:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.433842:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.433845:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.433846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.433848:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.435550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.435837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.435885:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.435888:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.435888:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.435890:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.435892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.435893:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.436656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.436699:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.436702:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.436703:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.436704:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.436706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.436707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.438210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.440751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.440799:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.440801:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.440802:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.440804:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.440806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.440807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.440817:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.440820:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.440821:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.440822:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.440822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.440824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.441583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.441626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.441631:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.441634:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.441635:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.441637:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.441639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.441639:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.442420:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.442422:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.442423:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.442425:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.442427:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.442428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.443290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.443334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.445292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.446303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.446358:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.446361:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.446362:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.446364:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.446366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.446367:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.448264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.449783:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.449787:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.449790:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677469.449791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.449794:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.449796:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677469.449798:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96529094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.449800:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.449802:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.449803:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677469.449808:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.449811:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677469.449815:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.449816:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:13.0:1619677469.449817:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:13.0:1619677469.449820:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:13.0:1619677469.449821:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.449823:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677469.449825:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677469.449826:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.449839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677469.449841:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677469.449842:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.449843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677469.449978:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.449980:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.449981:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.449982:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.449985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.449986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.450206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.451223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.451263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.451267:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.451270:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.451270:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.451272:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.451274:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.451275:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.451288:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.451290:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.451291:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.451292:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.451293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.451304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.453236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.455108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.456141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.456667:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.456670:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.456671:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.456673:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.456674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.456675:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.458176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.459664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.461008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.461046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.463123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.463164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.464668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.465893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.465938:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.465942:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.465943:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.465946:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.465948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.465949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.468028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.469563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.470720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.470761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.470767:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.470770:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.470771:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.470773:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.470775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.470776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.473054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.474457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.475018:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.475021:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.475022:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.475023:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.475027:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.475028:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.475607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.476453:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.476456:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.476458:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677469.476460:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677469.476461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.476462:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:12.0:1619677469.476464:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96528943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.476465:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.476466:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.476467:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:12.0:1619677469.476470:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.476472:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:12.0:1619677469.476475:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0980] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.476476:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0000] (20)++
00000800:00000200:12.0:1619677469.476477:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0000] (21)++
00000800:00000200:12.0:1619677469.476479:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0000] (22)--
00000800:00000200:12.0:1619677469.476480:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.476481:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.476481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.478206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.478891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.2:1619677469.479171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:5.2:1619677469.480263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.480308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.480314:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.480317:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.480318:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.480320:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.480322:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.480323:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.480331:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.480334:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.480335:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.480336:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.480336:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.480338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:24.2:1619677469.481045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:13.0:1619677469.481101:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.481104:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.481105:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.481106:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.481108:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.481114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.481243:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)++
00000800:00000200:12.0:1619677469.481245:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (24)--
00000400:00000200:12.0:1619677469.481246:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.481248:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.481250:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)++
00000800:00000200:13.0:1619677469.481251:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (24)--
00000400:00000200:13.0:1619677469.481251:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.481252:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.481253:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.481253:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.481256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.481272:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677469.481274:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677469.481275:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.481276:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.481276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.481278:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.483749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.483790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.483804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.485222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.488305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.488711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.490045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.490085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.490093:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.490096:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.490097:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.490100:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.490102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.490104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.491329:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.491331:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.491332:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.491333:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.491335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.491337:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.493108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.493179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.494982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.495028:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.495032:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.495033:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.495036:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.495038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.495040:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.495960:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677469.495962:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677469.495965:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.111@o2ib7 : GET - for me
00000800:00000200:13.0:1619677469.495966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:12.0:1619677469.495967:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.111@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677469.495969:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.111@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677469.495971:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677469.495971:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:12.0:1619677469.495973:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677469.495977:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677469.495979:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677469.495981:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.495981:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677469.495982:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.495983:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.495984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.495995:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.495998:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.495998:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.496000:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: REPLY: OK
00000800:00000200:13.0:1619677469.496002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.496002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:5.2:1619677469.498071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.498078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.499923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.499965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.499972:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.499975:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.499976:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.499979:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.499980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.499983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.499999:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.500002:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.500003:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.500003:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.500004:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.500005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.502498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.503097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.503142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.504897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.504945:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.504949:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.504950:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.504953:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.504953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.504956:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.505694:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.505697:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.505699:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677469.505702:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677469.505703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.505704:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677469.505705:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96528776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.505707:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.505708:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.505708:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677469.505712:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.505713:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677469.505716:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.505717:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:12.0:1619677469.505718:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:12.0:1619677469.505719:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:12.0:1619677469.505720:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.505720:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.505721:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.507544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.508034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.511787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.511827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.512574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.512958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.515356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.515360:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.515364:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677469.515365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.515368:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.515370:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677469.515373:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96528721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.515375:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.515376:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.515378:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677469.515384:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.515386:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677469.515390:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.515391:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:13.0:1619677469.515392:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:13.0:1619677469.515394:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:13.0:1619677469.515395:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.515396:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.515397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.516183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.516232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.516235:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.516236:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.516237:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.516238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.516240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.516951:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.516953:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.516954:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.516956:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.516958:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.516959:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.517268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.517318:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.517321:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.517322:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.517323:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.517325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.517326:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.517339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.517342:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.517343:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.517344:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.517344:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.517346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.517900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.519940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.519981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.519986:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.519989:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.519990:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.519992:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.519995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.519996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.521243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.522843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.522883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.523238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.523283:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.523285:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.523286:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.523288:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.523290:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.523290:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:24.2:1619677469.524985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2c600] (21)++
00000800:00000200:12.0:1619677469.525054:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.525057:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.525058:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.525060:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.525062:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.525064:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.525120:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.525123:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.525124:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.525127:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.525127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.525134:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.525188:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.525190:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.525192:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677469.525194:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:12.0:1619677469.525196:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.525200:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677469.525391:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.525394:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.525395:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.525397:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.525399:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.525400:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677469.525401:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.525401:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)++
00000800:00000200:13.0:1619677469.525402:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.525402:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.525403:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.525920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.527166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.527206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.527211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.527214:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.527215:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.527217:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.527218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.527220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.527231:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.527234:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.527235:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.527235:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.527236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.527238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.527773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.530331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.531125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.532747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.535308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.535862:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.535865:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.535867:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677469.535869:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.535870:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.535871:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677469.535873:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96528593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.535874:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.535875:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.535876:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677469.535879:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.535881:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677469.535883:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.535884:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677469.535885:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677469.535886:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677469.535887:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.535889:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677469.535890:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677469.535891:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.535904:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677469.535906:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677469.535907:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.535908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.536674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.536718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.537678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.537923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677469.540311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.541349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.541399:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.541402:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.541403:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.541406:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.541407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.541409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.541579:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.541581:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.541582:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.541584:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.541586:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.541587:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.542641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.542693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.544759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.545994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.546034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.546039:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.546042:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.546043:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.546044:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.546046:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.546048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.548174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.548738:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.548740:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.548743:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677469.548745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.548746:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.548747:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677469.548748:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96528512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.548750:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.548751:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.548752:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677469.548754:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.548756:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677469.548759:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.548759:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:13.0:1619677469.548760:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:13.0:1619677469.548761:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:13.0:1619677469.548762:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.548763:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.548763:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.549697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.551141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.551184:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.551187:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.551188:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.551190:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.551192:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.551193:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.553138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.554230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.555995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.556034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.556041:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.556044:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.556046:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.556048:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.556050:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.556051:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.556052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.556052:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)++
00000800:00000200:12.0:1619677469.556053:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.556054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.556055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.558085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.559177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.560822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.560870:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.560873:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.560874:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.560877:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.560879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.560880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.563032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.563076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.563550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.566177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.566218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.566223:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.566226:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.566227:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.566229:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.566231:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.566232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.567967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.567996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.572115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.572164:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.572168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.572169:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.572171:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.572172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.572175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.572493:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.572496:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.572498:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.572501:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.572502:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677469.572502:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.572504:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96528375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.572505:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.572506:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.572507:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677469.572510:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.572511:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677469.572514:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.572515:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677469.572515:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677469.572517:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677469.572518:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.572518:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.572519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.573518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.577013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.577061:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.577064:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.577065:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.577068:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.577069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.577071:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.577081:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.577083:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.577084:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.577085:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.577085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.577086:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.578560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.580735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.580777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.580781:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.580784:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.580785:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.580787:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.580790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.580791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.580859:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.580862:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.580863:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.580865:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.580866:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.580867:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.582121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.582168:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.582170:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.582171:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.582173:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.582175:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.582176:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.583559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.583599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.586170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.586218:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.586222:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.586223:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.586226:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.586228:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.586229:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.586940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.586986:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.586989:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.586990:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.586991:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.586993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.586994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.587004:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.587007:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.587007:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.587008:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.587009:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.587010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.588460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.591338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.591378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.591785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677469.591826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:5.2:1619677469.592169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.592216:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.592218:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.592219:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.592221:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.592223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.592224:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.593398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.596257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.596303:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.596306:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.596308:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.596310:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.596311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.596314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.597040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.597086:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.597089:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.597089:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.597091:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.597093:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.597094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.598329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.600638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.600680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.602028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.602073:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.602076:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.602077:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.602078:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.602080:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.602081:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.602091:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.602094:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.602095:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.602096:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.602096:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.602097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.603329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.603372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.605668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.606930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.606978:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.606980:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.606981:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.606983:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.606984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.606986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.608263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.610068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.610109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.612019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.612068:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.612071:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.612072:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.612073:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.612075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.612077:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.613223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.613581:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.613583:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.613585:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.613588:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.613588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.613589:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677469.613591:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96528114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.613592:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.613593:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.613594:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677469.613597:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.613599:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677469.613602:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.613602:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:13.0:1619677469.613603:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:13.0:1619677469.613605:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:13.0:1619677469.613605:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.613606:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.613607:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.614428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.616857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.616905:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.616908:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.616910:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.616912:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.616913:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.616916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.617605:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.617608:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.617609:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.617610:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.617613:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.617616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.618176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.619334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.619373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.622152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.622202:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.622205:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.622206:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.622209:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.622211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.622212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.622213:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.622214:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.622214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.622216:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.622217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.623080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.623120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.623738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.626951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.626998:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.627002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.627003:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677469.627006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677469.627006:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.627009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.628062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.628863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.628902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.632329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.632376:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.632378:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.632379:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.632381:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.632383:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.632384:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.633162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.633508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.633554:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.633557:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.633558:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.633560:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.633561:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.633562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.634760:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.634763:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.634765:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.634768:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.634768:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.634769:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677469.634771:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96527988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.634772:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.634773:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.634774:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677469.634777:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.634778:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677469.634781:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.634782:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (23)++
00000800:00000200:13.0:1619677469.634783:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (24)++
00000800:00000200:13.0:1619677469.634785:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (25)--
00000800:00000200:13.0:1619677469.634785:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.634787:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677469.634788:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677469.634789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.634802:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677469.634805:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677469.634806:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.634807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677469.636789:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.636791:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.636793:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.636796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.636796:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.636798:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677469.636799:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96527976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.636800:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.636801:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.636802:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677469.636805:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.636806:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677469.636810:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.636810:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:12.0:1619677469.636811:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:12.0:1619677469.636813:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:12.0:1619677469.636813:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.636814:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.636814:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.637241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.637287:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.637289:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.637290:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.637292:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.637293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.637304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.637305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.637307:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.637308:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.637308:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.637308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.637309:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.637680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.638453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.638492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.638498:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.638500:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.638501:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.638503:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.638505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.638506:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.642165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.642209:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.642212:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.642212:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.642214:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.642216:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.642217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.642616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.642660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.643181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.643225:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.643228:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.643229:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.643230:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.643232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.643233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.643717:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.643719:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.643722:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677469.643724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.643725:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.643726:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677469.643728:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96527934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.643729:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.643730:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.643731:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677469.643735:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.643737:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677469.643739:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.643740:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:13.0:1619677469.643741:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:13.0:1619677469.643742:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:13.0:1619677469.643743:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.643744:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.643744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677469.645792:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.645795:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.645797:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.645800:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.645800:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.645802:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677469.645803:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96527922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.645804:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.645805:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.645806:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677469.645810:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.645812:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677469.645814:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.645815:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:12.0:1619677469.645816:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:12.0:1619677469.645818:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:12.0:1619677469.645818:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.645819:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.645820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.647546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.647567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.647592:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.647595:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.647596:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.647597:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.647599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.647600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.647610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.647613:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.647614:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.647615:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.647615:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.647617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.648320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.648362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.648367:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.648370:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.648371:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.648373:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.648374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.648375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.648412:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.648415:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.648417:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.648419:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.648420:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.648421:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:12.0:1619677469.648423:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96527899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.648424:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.648425:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.648426:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:12.0:1619677469.648430:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.648431:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677469.648434:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.648435:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:12.0:1619677469.648435:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:12.0:1619677469.648437:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:12.0:1619677469.648437:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.648438:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677469.648440:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677469.648441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.648454:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677469.648456:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677469.648460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.648461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.652027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.652515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.652942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.652986:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.652989:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.652990:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.652992:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.652995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.652996:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.656526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.657483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.657676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.657713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.657862:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.657865:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.657865:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.657867:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.657869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.657870:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.658333:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.658336:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.658337:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.658340:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.658340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.658344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.658357:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677469.658360:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677469.658361:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.658362:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.658362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.658363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.661484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.662442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.662484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.662650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.665902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.666439:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.666442:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.666443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.666445:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.666447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.666448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.666456:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.666459:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.666460:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.666460:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.666461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.666463:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.667407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.667582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.667622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.667627:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.667629:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.667631:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.667633:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.667634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.667636:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.670763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.672402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.672530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.672576:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.672578:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.672579:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.672581:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.672583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.672584:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.675147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.677334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.677374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.677381:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:5.2:1619677469.677381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.677385:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.677386:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.677388:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.677391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.677393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.680056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.681237:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.681239:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.681240:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.681241:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.681243:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.681244:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.682131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.682147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:13.0:1619677469.682181:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.682184:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.682185:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.682187:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.682188:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.682189:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.682201:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.682204:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.682205:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.682206:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.682207:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.682208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.682283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.682331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.684482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.686928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.686969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.686976:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.686979:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.686979:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.686981:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.686983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.686984:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.687318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.689412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.691727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.691775:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.691778:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.691780:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.691782:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.691785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.691786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.692321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.694344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.695670:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.695673:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.695675:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.695677:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.695677:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.695679:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677469.695680:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96527593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.695682:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.695683:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.695684:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677469.695687:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.695689:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677469.695691:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.695692:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:13.0:1619677469.695692:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:13.0:1619677469.695694:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:13.0:1619677469.695694:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.695695:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.695696:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677469.695899:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.695902:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.695902:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.695904:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.695906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.695907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.696501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.696539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.696545:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.696548:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.696549:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.696550:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.696553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.696553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.696565:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.696568:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.696569:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.696569:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.696570:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.696572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.697309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.698731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.701742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.701801:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.701804:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.701805:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.701808:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.701809:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.701811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.702287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.702329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.703768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.705004:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.705006:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.705009:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.705011:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.705011:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.705012:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677469.705013:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96527536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.705015:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.705016:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.705017:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677469.705019:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.705021:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677469.705024:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.705025:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677469.705025:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677469.705027:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677469.705027:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.705028:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.705029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.706647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.706687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.706692:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.706694:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.706696:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.706697:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.706699:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.706700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.707227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.708690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.711521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.711568:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.711572:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.711573:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.711576:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.711576:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.711579:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.712172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.713551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.716787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.716844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.716850:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.716853:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.716854:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.716857:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.716860:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.716862:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.716863:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.716864:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.716865:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.716865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.716866:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.716867:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.716868:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.717118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677469.717189:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.717191:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.717192:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.717194:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.717196:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.717196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.718137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.721737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.721785:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.721789:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.721790:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.721792:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.721793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.721796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.722122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.722164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.723119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.723164:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.723167:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.723168:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.723170:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.723172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.723172:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.726665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.726706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.726711:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.726714:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.726714:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.726716:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.726718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.726719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.727094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.728010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.728056:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.728059:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.728060:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.728062:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.728064:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.728065:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.728076:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.728079:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.728079:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.728080:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.728081:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.728082:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.732061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.732118:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.732121:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.732122:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.732125:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.732126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.732128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.732646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.732883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.734521:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.734523:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.734525:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.734528:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.734528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.734530:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677469.734531:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96527345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.734532:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.734533:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.734534:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677469.734538:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.734539:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677469.734542:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.734542:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:13.0:1619677469.734543:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:13.0:1619677469.734545:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:13.0:1619677469.734545:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.734546:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.734547:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.736928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.736968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.736973:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.736976:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.736977:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.736979:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.736980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.736982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.737591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.737868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.739395:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.739398:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.739400:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.739402:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.739402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.739404:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677469.739405:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96527314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.739407:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.739408:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.739409:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677469.739412:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.739414:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677469.739416:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.739417:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:13.0:1619677469.739417:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:13.0:1619677469.739419:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:13.0:1619677469.739419:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.739420:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677469.739422:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677469.739422:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.739435:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677469.739438:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677469.739439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.739440:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.741804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.741849:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.741851:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.741852:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.741854:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.741856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.741857:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.741930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677469.742597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.742642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.742819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.747196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.747233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.747239:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.747241:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.747242:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.747244:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.747246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.747247:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.747255:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.747258:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.747259:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.747260:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.747260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.747262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.747530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.748034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.748080:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.748082:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.748083:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.748085:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.748086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.748088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.751340:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.751343:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.751346:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677469.751348:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677469.751349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.751350:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677469.751351:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96527244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.751352:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.751354:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.751354:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677469.751358:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.751360:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677469.751363:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.751363:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:12.0:1619677469.751364:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:12.0:1619677469.751366:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:12.0:1619677469.751366:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.751367:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.751368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.751737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.752514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.752702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.755130:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.755134:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.755137:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677469.755138:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.755141:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.755143:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677469.755145:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96527218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.755148:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.755149:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.755151:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677469.755156:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.755159:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677469.755163:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.755164:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677469.755165:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677469.755168:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677469.755169:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.755170:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.755171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.756748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.756790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.757494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.757911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.757954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.757957:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.757957:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.757959:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.757961:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.757962:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.761966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.762020:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.762024:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.762025:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.762028:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.762028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.762032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.762494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.762535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.762761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.762812:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.762815:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.762816:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.762817:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.762819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.762821:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.762828:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.762831:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.762832:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.762833:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.762833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.762835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.766821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.766863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.766869:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.766872:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.766872:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.766874:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.766877:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.766878:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.767460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.767615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.767661:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.767664:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.767665:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.767667:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.767669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.767670:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.771937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.772431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.772754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.772807:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.772810:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.772811:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.772813:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.772815:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.772816:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.776928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.776968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.777375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:24.2:1619677469.777501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677469.777535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:5.2:1619677469.777673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.777875:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (24)++
00000800:00000200:12.0:1619677469.777878:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (24)--
00000400:00000200:12.0:1619677469.777878:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.777880:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.777882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.777883:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.777885:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.777886:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.777887:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.777888:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.777889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.777890:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.777891:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.777891:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.777892:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.777892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.781813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.781860:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.781864:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.781865:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.781868:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.781868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.781871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.782323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.782366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.782585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.782631:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.782635:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.782635:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.782637:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.782639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.782640:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.786694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.786733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.786739:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.786742:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.786742:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.786744:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.786746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.786747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.786757:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.786760:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.786760:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.786761:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.786762:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.786762:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.787206:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.787209:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.787209:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.787211:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.787213:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.787215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.787257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.787472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.791052:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.791055:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.791056:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.791059:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.791060:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.791063:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.791547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.791591:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.791594:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.791595:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.791597:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.791599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.791599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.792261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.792390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.796458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.796498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.796504:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.796507:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.796508:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.796510:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.796512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.796513:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.796522:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.796525:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.796526:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.796526:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.796527:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.796528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.797405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.797453:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.797455:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.797456:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.797458:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.797460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.797460:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.799763:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.799765:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.799766:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.799767:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.799769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.799773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.799794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.801280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.801330:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.801333:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.801334:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.801336:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.801338:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.801339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.802346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.802401:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.802404:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.802405:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.802407:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.802409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.802410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.804741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.804783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.806290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.806350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.806355:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.806359:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.806360:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.806363:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.806364:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.806365:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.806366:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)++
00000800:00000200:13.0:1619677469.806367:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (24)--
00000800:00000200:12.0:1619677469.806367:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.806368:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.806369:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.807149:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.807152:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.807153:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.807154:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.807157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.807158:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.807174:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677469.807179:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677469.807180:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.807182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.807182:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.807183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.807341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.807386:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.807389:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.807390:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.807392:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.807393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.807394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.809699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.811211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677469.811272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.811323:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.811326:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.811327:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.811328:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.811331:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.811332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.812484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.815333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.816095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.816136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.816141:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.816143:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.816145:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.816146:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.816148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.816149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.816506:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.816509:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.816512:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.816514:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.816514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.816516:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677469.816518:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96526794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.816519:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.816520:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.816521:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677469.816525:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.816527:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677469.816530:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.816531:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:13.0:1619677469.816531:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:13.0:1619677469.816533:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:13.0:1619677469.816534:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.816535:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.816535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.817565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.817609:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.817612:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.817613:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.817614:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.817616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.817618:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.817628:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.817631:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.817631:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.817632:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.817632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.817633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.820260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.821244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.821290:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.821300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.821304:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677469.821305:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.821308:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.821311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.822462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.822510:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.822512:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.822513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.822515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.822517:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.822518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.825201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.825241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.826130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.826169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.826175:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.826177:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.826178:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.826180:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.826182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.826183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.827287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.827339:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.827342:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.827343:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.827345:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.827346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.827347:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.827356:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.827358:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.827359:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.827360:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.827361:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.827362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.828959:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.828961:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.828963:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.828966:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.828966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.828967:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677469.828968:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96526704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.828970:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.828970:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.828971:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677469.828974:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.828976:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677469.828979:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.828980:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677469.828980:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677469.828982:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677469.828982:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.828984:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677469.828985:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677469.828986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.829034:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677469.829036:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677469.829037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.829038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.829606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.831015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.831060:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.831062:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.831063:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.831065:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.831067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.831068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.832261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.834546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.835854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.835894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.835899:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.835903:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.835903:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.835905:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.835908:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.835909:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.837217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.839506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.840711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.840756:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.840758:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.840759:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.840761:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.840763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.840764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.842052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.842097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.842100:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.842101:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.842102:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.842105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.842105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.842118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.842120:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.842122:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.842122:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.842123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.842124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.844510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.844553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.846074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.846114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.846121:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.846124:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.846126:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.846128:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.846129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.846131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.846271:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.846273:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.846274:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.846276:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.846278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.846279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.847465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.2:1619677469.848165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:5.2:1619677469.849410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.850920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.850962:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.850965:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.850966:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.850968:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.850970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.850970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:5.2:1619677469.852597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.854318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.855759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.855798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.855804:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.855807:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.855808:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.855809:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.855811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.855812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.855824:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.855827:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.855827:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.855828:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.855829:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.855830:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.857525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.859319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.860595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.860642:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.860644:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.860645:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.860647:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.860649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.860650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.862269:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.862271:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.862272:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.862274:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.862276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.862277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.862661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.864277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.864325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.865916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.865962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.865965:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.865968:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.865969:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.865971:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.865973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.865973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.867592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.869286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.870782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.870826:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.870829:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.870830:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.870832:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.870834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.870835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.870851:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.870854:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.870855:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.870856:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.870856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.870857:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.872524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.872740:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.872743:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.872744:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.872746:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.872747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.872749:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.872798:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.872800:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.872801:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677469.872804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:12.0:1619677469.872804:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.872814:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.874264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.875598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.875638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.875643:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.875645:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.875646:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.875648:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.875650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.875651:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.877424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.879207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.880392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.880440:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.880443:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.880444:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.880447:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.880448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.880451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.882382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.884212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.884252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.885171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.885210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.885215:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.885218:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.885219:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.885221:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.885223:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.885224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.885234:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.885237:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.885237:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.885238:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.885239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.885240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.886769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.889046:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.889049:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.889049:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.889051:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.889053:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.889054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.889197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.890528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.890572:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.890575:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.890576:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.890578:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.890579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.890580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.891130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:5.2:1619677469.891170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:12.0:1619677469.891662:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.891665:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.891667:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.891669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.891669:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.891670:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677469.891672:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96526269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.891673:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.891674:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.891675:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677469.891678:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.891679:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677469.891681:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.891682:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:12.0:1619677469.891683:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677469.891684:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.891685:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.891685:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.891686:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677469.891696:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.891699:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.891699:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.891701:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.891702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.891703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.891906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.891947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6000] (20)++
00000800:00000200:13.0:1619677469.891951:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.891953:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.891954:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.891957:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.891957:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.891960:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.891970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6000] (21)++
00000800:00000200:5.2:1619677469.892006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6000] (21)++
00000800:00000200:5.2:1619677469.892236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6000] (20)++
00000800:00000200:12.0:1619677469.893265:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.893268:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.893269:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.893270:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.893272:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.893273:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.894132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677469.894543:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.894545:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.894548:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.894550:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.894550:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.894551:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677469.894553:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96526250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.894554:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.894555:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.894556:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677469.894560:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.894561:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677469.894564:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.894565:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:13.0:1619677469.894565:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:13.0:1619677469.894567:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:13.0:1619677469.894568:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.894568:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.894569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.895417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.895456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.895461:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.895464:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.895465:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.895467:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.895468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.895470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.895480:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.895483:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.895484:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.895485:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.895485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.895487:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.897103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.897148:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.897151:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.897152:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.897153:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.897155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.897156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.899125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.900278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.900332:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.900334:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.900335:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.900337:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.900339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.900340:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.901323:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.901326:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.901327:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.901328:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.901331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.901331:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.901781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.904045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.904087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.905137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.905175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.905181:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.905184:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.905185:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.905186:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.905189:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.905189:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.905201:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.905204:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.905205:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.905205:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.905206:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.905207:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.906461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.909015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.910063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.910109:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.910111:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.910112:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.910114:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.910116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.910117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.911432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.914043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.914921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.914961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.914966:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.914968:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.914970:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.914971:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.914973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.914974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.916336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.916382:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.916385:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.916385:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.916387:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.916389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.916390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.919009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.919808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.919866:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.919869:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.919869:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.919871:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.919873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.919874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.919891:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.919894:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.919895:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.919895:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.919896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.919897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.921258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.921304:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.921307:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.921308:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.921309:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.921311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.921313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.923913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6000] (20)++
00000800:00000200:5.2:1619677469.923955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6000] (20)++
00000800:00000200:5.2:1619677469.923988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6000] (20)++
00000800:00000200:5.2:1619677469.924020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6000] (21)++
00000800:00000200:5.2:1619677469.924032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.924074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.924112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6000] (20)++
00000800:00000200:5.2:1619677469.924158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6000] (20)++
00000800:00000200:13.0:1619677469.924159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.924162:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.924163:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.924165:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.924167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.924168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.924181:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677469.924183:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677469.924184:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.924185:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.924186:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677469.924187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.924302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6000] (20)++
00000800:00000200:5.2:1619677469.924338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6000] (21)++
00000800:00000200:5.2:1619677469.924785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.924824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.924830:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.924833:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.924834:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.924835:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.924837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.924838:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.925847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.928943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.929694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.929740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.929743:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.929744:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.929745:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.929747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.929749:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.930933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.933557:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.933559:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.933560:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.933561:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.933564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.933565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.933574:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.933578:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.933579:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.933580:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.933581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.933583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.933862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.934530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.934571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677469.934577:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.934580:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.934581:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.934583:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.934585:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.934586:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.935579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.935624:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.935627:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.935628:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.935630:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.935631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.935632:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.938815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.939483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.939531:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.939534:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.939535:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.939537:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.939540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.939542:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.940497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.940542:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.940545:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.940546:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.940548:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.940549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.940550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.940559:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.940562:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.940562:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.940563:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.940564:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.940565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.943221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.943261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.944326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.944369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677469.944676:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.944679:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.944681:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677469.944683:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677469.944683:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.944684:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677469.944685:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96525969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.944687:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.944688:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.944689:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677469.944693:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.944695:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677469.944698:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.944698:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:13.0:1619677469.944699:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:13.0:1619677469.944701:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:13.0:1619677469.944702:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.944702:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.944703:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.945091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.947665:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.947668:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.947672:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.947673:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.947676:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.947678:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677469.947680:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96525951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.947683:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.947684:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.947686:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677469.947691:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.947694:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677469.947698:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.947699:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:12.0:1619677469.947700:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:12.0:1619677469.947702:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:12.0:1619677469.947703:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.947705:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677469.947707:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677469.947708:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.947721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677469.947723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677469.947724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.947725:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.948125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.949193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.949996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.950734:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.950736:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677469.950737:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.950738:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.950740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.950741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677469.950838:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.950840:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.950843:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.950844:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.950845:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.950847:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677469.950848:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96525928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.950849:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.950850:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.950851:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677469.950854:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.950856:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677469.950859:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.950859:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:12.0:1619677469.950860:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:12.0:1619677469.950862:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:12.0:1619677469.950862:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.950863:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.950863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.953097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.953605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.953644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.954641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.954686:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.954688:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.954689:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.954691:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.954692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.954694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.958042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.958693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.959520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.959567:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.959571:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.959572:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.959575:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.959576:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.959579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.963084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.963123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.963256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.963298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.964103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.964148:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.964151:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.964152:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.964154:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.964156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.964157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.965189:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677469.965191:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.965192:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.965193:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.965196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677469.965197:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.967987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.968250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.969172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.969215:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.969218:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.969218:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.969220:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.969222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.969224:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.969233:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677469.969236:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677469.969237:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.969237:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677469.969238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.969239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.972913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.972948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.973301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.973923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.975022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:12.0:1619677469.975662:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.975664:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677469.975667:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677469.975669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677469.975669:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.975671:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677469.975672:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96525788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677469.975674:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677469.975675:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677469.975675:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677469.975679:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677469.975680:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677469.975683:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677469.975684:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:12.0:1619677469.975684:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:12.0:1619677469.975686:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:12.0:1619677469.975686:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677469.975688:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677469.975688:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.977376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.977804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.978566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.978615:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.978618:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.978619:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.978621:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.978622:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.978624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.980296:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677469.980304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677469.980304:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677469.980305:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.980306:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.980309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677469.982301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.982342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.982726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.982763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677469.983448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.983498:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.983501:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.983502:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.983504:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.983506:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.983507:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.986761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.986964:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.986967:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.986968:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.986970:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.986972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.986973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.987705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.988069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.988114:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.988116:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.988117:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.988119:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.988120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.988122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677469.988131:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677469.988134:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677469.988135:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.988136:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677469.988137:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.988138:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677469.989605:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.989608:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677469.989610:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677469.989612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677469.989613:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.989614:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677469.989615:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96525706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677469.989616:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677469.989617:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677469.989618:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677469.989621:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677469.989623:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677469.989626:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677469.989627:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:13.0:1619677469.989627:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:13.0:1619677469.989629:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:13.0:1619677469.989630:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677469.989630:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677469.989631:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677469.991681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.991722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677469.992813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677469.992927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677469.992973:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677469.992975:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677469.992976:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677469.992978:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677469.992980:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677469.992981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.996102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677469.997522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677469.997567:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677469.997569:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677469.997570:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677469.997572:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677469.997574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677469.997575:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677469.997765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.001040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.001081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677470.002419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.002683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.002722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677470.005990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.007251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677470.007302:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.007306:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.007307:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.007310:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.007312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.007313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.007575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.010400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.010440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677470.011918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.011962:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.011965:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.011965:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.011967:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.011969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.011970:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.011980:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.011983:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.011983:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.011984:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.011984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.011985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.012532:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.012535:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.012537:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.012539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.012540:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.012542:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677470.012543:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96525559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.012544:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.012545:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.012546:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677470.012549:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.012551:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677470.012553:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.012554:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:12.0:1619677470.012555:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:12.0:1619677470.012556:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:12.0:1619677470.012557:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.012558:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677470.012560:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677470.012560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:5.2:1619677470.012565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677470.012574:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677470.012576:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677470.012576:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.012577:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677470.015362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.016708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.016752:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.016755:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.016756:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.016758:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.016759:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.016761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.017585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.020227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.020263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677470.021238:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.021241:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.021242:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.021244:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.021246:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.021248:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.021610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.022485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.022526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677470.024683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.026253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.026302:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.026305:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.026306:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.026309:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.026309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.026313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.029244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.029557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.029597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677470.031021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677470.031066:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.031068:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.031069:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.031071:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.031073:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.031074:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.031084:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.031087:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.031088:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.031088:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.031089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.031090:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.032322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:12.0:1619677470.033386:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.033389:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.033389:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.033391:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.033393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.033394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677470.033409:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677470.033412:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677470.033412:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.033413:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677470.033413:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.033414:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.033906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.034238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677470.034530:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.034532:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.034533:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.034534:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.034537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.034538:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.035786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.035831:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.035834:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.035835:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.035837:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.035839:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.035840:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.038769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.038810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677470.039297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.040611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677470.040664:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.040666:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.040667:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.040669:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.040671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.040673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.043149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.044462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.044505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677470.045373:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.045376:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.045377:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.045378:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.045381:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.045382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.045386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.045390:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.045393:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.045394:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.045395:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.045395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.045396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.045430:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.045433:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.045433:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.045435:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.045438:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.045439:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.047912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.047950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677470.049462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.050144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677470.050188:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.050191:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.050192:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.050194:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.050195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.050197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.052666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.054430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.055010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.055056:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.055059:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.055060:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.055062:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.055063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.055065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.057001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.057042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677470.059426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.059865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677470.059914:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.059917:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.059918:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.059921:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.059924:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.059925:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.059926:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.059926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.059927:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.059928:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.059929:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.061854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.064506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.064554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677470.064707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677470.064755:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.064759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.064760:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.064762:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.064765:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.064766:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:24.2:1619677470.065000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2c600] (19)++
00000800:00000200:23.0:1619677470.065056:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91a3aeb2c600] (18)--
00000800:00000200:5.2:1619677470.066195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.066237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677470.066242:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.066245:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.066246:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.066248:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.066249:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.066251:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.069528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.069824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677470.069872:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.069874:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.069875:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.069877:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.069879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.069880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.071182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.074537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.074774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.074822:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.074825:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.074826:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.074828:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.074831:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.074832:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.074844:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.074847:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.074848:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.074848:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.074849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.074850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.076061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.076100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677470.079528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.079716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.079763:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.079767:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.079768:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.079771:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.079771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.079774:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.080733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677470.080778:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.080781:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.080782:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.080784:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.080785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.080786:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.084704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.084750:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.084753:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.084754:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.084756:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.084757:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.084759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.085056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.085099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677470.085536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.085576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677470.089285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.090037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.090260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677470.090305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.090308:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.090309:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.090311:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.090313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.090314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.090322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.090325:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.090326:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.090326:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.090327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.090328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.094181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.094982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.095164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.095201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677470.099050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.099935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.100043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677470.101728:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.101731:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.101732:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.101733:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.101736:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.101737:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.103402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.104056:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.104059:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.104061:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677470.104063:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.104063:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.104065:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677470.104066:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96524960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.104068:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.104069:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.104069:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677470.104073:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.104074:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677470.104076:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.104077:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:13.0:1619677470.104078:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:13.0:1619677470.104079:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:13.0:1619677470.104080:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.104081:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.104081:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677470.104694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.104746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677470.104754:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.104757:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.104758:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.104759:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.104761:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.104763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.104895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.104938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677470.106827:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.106829:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.106830:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.106832:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.106834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.106835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.107449:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.107452:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.107454:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.107456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.107456:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.107458:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677470.107459:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96524935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.107461:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.107462:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.107463:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677470.107467:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.107469:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677470.107471:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.107472:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:12.0:1619677470.107473:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:12.0:1619677470.107474:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:12.0:1619677470.107475:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.107475:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.107476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677470.108271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.109650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.109697:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.109700:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.109701:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.109703:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.109704:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.109706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.109713:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.109716:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.109717:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.109718:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.109719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.109720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.109850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677470.110101:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.110104:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.110105:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.110107:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.110107:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.110109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677470.110411:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.110414:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.110415:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.110417:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.110419:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.110420:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677470.110586:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.110589:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.110590:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.110591:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.110592:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.110594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.112652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.114352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.114392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677470.114399:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.114401:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.114402:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.114404:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.114406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.114407:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.114863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.117472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.119235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.119283:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.119286:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.119287:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.119290:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.119291:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.119304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.119792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.122255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.124100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.124140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677470.124146:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.124149:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.124149:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.124151:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.124153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.124154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.124714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.124756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677470.126563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.127489:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.127493:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.127497:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677470.127499:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.127500:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.127502:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677470.127504:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96524772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.127506:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.127508:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.127509:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677470.127515:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.127518:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677470.127522:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.127523:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:13.0:1619677470.127524:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:13.0:1619677470.127526:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:13.0:1619677470.127527:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.127528:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.127529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677470.128405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677470.128448:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.128450:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.128451:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.128453:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.128455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.128456:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.128465:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.128468:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.128469:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.128470:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.128471:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.128472:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.131336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.132673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677470.133191:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.133193:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.133194:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.133195:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.133197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.133199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677470.133212:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677470.133214:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677470.133215:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.133216:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677470.133216:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.133217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.133284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.133331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677470.133337:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.133340:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.133341:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.133343:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.133345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.133346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.135307:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.135310:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.135313:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.135314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.135316:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.135318:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677470.135319:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96524717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.135320:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.135321:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.135322:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677470.135326:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.135328:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677470.135330:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.135331:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:12.0:1619677470.135332:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:12.0:1619677470.135334:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:12.0:1619677470.135334:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.135336:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677470.135338:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677470.135338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.135351:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677470.135354:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677470.135355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.135355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677470.136121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677470.136892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.136894:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.136897:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677470.136899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.136900:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.136901:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677470.136902:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96524706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.136904:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.136905:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.136906:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677470.136910:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.136911:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677470.136914:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.136915:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:13.0:1619677470.136916:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:13.0:1619677470.136918:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:13.0:1619677470.136918:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.136919:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.136919:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677470.137662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.138089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677470.138133:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.138136:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.138137:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.138139:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.138140:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.138142:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.138940:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.138943:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.138945:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677470.138947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.138948:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.138950:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677470.138951:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96524687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.138953:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.138954:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.138956:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677470.138960:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.138963:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677470.138966:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.138966:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677470.138967:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677470.138969:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677470.138969:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.138970:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.138971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677470.140434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.142550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.142888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.142928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677470.142934:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:17.0:1619677470.142937:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.142937:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.142938:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.142939:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.142941:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.142942:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.143572:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.143575:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.143576:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.143577:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.143580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.143580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.143590:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.143592:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.143593:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.143594:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.143594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.143595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.144286:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:5.2:1619677470.145213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.145219:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.145222:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.145224:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.145227:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.145228:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.145229:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96524638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.145231:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.145232:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.145233:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.145236:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.145239:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.145241:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.145242:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.145243:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.145245:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.145245:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.145246:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.145247:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.147105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.147215:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.147218:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.147219:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.147221:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.147227:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.147228:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.147230:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.147232:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.147233:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.147233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.147261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.147348:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.147350:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.147352:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.147354:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.147357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.147481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.147521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677470.147723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.147732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677470.147767:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.147770:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.147771:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.147773:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.147776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.147777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.148421:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.148423:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.148424:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.148426:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.148427:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.148429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.149990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.149996:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.149999:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.150002:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.150005:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.150007:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.150008:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.150010:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.150011:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.150013:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.150016:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.150018:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.150021:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.150022:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.150023:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.150025:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.150025:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.150026:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.150027:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.150588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677470.150598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677470.152070:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.152072:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.152074:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.152076:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.152079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.152383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677470.153047:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:5.2:1619677470.153496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.153502:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.153505:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.153508:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.153511:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.153512:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.153514:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.153515:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.153516:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.153517:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.153521:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.153522:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.153525:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.153526:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.153527:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.153528:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.153529:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.153530:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.153531:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.153532:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.153537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.153817:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.153820:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.153821:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.153824:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.153826:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.153840:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.156969:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.156972:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.156973:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.156975:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.156979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.157315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.157407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.157421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677470.157468:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.157471:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.157472:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.157474:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.157477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.157478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.158061:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.158065:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.158067:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.158071:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.158076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.158565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.158574:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677470.158610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.158613:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.158614:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.158615:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.158618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.158619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.159699:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.159702:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.159702:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.159704:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.159706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.159707:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.159717:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.159720:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.159721:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.159721:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.159722:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.159723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.161891:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.161894:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.161895:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.161898:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.161901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.161914:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:5.2:1619677470.162297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.162344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.162350:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.162353:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.162355:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.162358:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.162360:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.162361:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.162362:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.162363:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.162364:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.162368:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.162370:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.162373:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.162373:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.162374:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.162376:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.162376:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.162377:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.162378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.162390:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.162392:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.162393:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.162395:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.162397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.162398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.163526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.163536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.163567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.163572:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677470.163572:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:17.0:1619677470.163575:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.163575:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677470.163575:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:17.0:1619677470.163576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677470.163576:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.163578:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.163580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.163581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.163945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.164400:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.164403:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.164404:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.164407:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.164410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.165096:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.165240:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.165243:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.165244:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.165246:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.165249:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.165270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.166811:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.166814:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.166815:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.166818:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.166821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.166833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:5.2:1619677470.167212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.167227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.167233:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.167236:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.167239:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.167242:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.167244:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.167245:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96524483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.167246:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.167247:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.167248:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.167252:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677470.167252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677470.167253:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.167256:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.167257:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.167258:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.167260:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.167260:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.167261:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.167262:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.167274:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.167276:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.167277:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.167279:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.167281:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.167282:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.167483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:5.2:1619677470.168570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.168576:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.168579:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.168582:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.168584:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.168586:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.168587:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.168588:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.168589:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.168590:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.168594:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.168596:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.168599:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.168599:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.168600:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.168602:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.168602:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.168603:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.168603:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.168616:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.168619:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.168620:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.168621:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.168623:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.168624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.168635:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.168637:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.168638:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.168639:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.168639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.168641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.170064:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.170067:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.170068:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.170070:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.170072:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.170073:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:17.0:1619677470.170154:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.170171:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.170173:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.170174:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.170175:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.170176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:13.0:1619677470.170985:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.170988:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.170992:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677470.170993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.170996:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.170998:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677470.171000:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96524453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.171003:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.171004:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.171005:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677470.171012:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.171015:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677470.171018:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.171019:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677470.171019:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:13.0:1619677470.171021:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677470.171021:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.171022:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.171023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677470.171036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.171037:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.171038:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.171039:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.171042:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.171043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.171210:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:5.2:1619677470.172153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.172159:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.172162:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.172165:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.172168:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.172169:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.172170:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.172172:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.172173:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.172174:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.172177:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.172179:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.172182:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.172182:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.172183:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.172185:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.172186:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.172186:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.172187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.172199:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.172202:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.172203:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.172204:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.172207:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.172208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.172237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677470.172850:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:5.2:1619677470.173400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.173406:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.173409:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.173412:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.173415:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.173416:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.173417:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.173419:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.173420:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.173421:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.173424:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.173425:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.173428:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.173429:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.173430:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.173431:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.173432:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.173433:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.173434:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.173435:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.173440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.175009:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.175012:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:17.0:1619677470.175015:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.175017:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.175019:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.175020:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.175022:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.175023:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.175024:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.175027:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.175029:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.175031:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.175032:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.175033:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.175035:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.175035:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.175036:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.175036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677470.176074:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.176369:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.176371:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.176372:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.176375:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.176378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.176561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.176570:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677470.176605:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.176607:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.176608:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.176610:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.176612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.176613:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.177099:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.177102:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.177105:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.177107:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.177110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.177195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.178002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.178012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.178496:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.178498:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:17.0:1619677470.178501:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.178503:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.178504:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677470.178506:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.178507:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.178508:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.178508:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677470.178511:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.178513:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677470.178515:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.178516:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:17.0:1619677470.178516:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:17.0:1619677470.178518:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:17.0:1619677470.178518:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.178519:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.178519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:7.0F:1619677470.179891:0:13229:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.51@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677470.179900:0:13229:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.51@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:7.0:1619677470.180181:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.51@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.180194:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff91a075787000 0x14f5787000(0x14f5787000)
00000800:00000200:7.0:1619677470.180195:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff91a062bd6000 0x14e2bd6000(0x14e2bd6000)
00000800:00000200:7.0:1619677470.180196:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff91a076391000 0x14f6391000(0x14f6391000)
00000800:00000200:7.0:1619677470.180196:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff91a07bbd8000 0x14fbbd8000(0x14fbbd8000)
00000800:00000200:7.0:1619677470.180197:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff91a3e210a000 0x186210a000(0x186210a000)
00000800:00000200:7.0:1619677470.180197:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff91a3aea81000 0x182ea81000(0x182ea81000)
00000800:00000200:7.0:1619677470.180198:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff91a3ae823000 0x182e823000(0x182e823000)
00000800:00000200:7.0:1619677470.180198:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff91a3ae2b6000 0x182e2b6000(0x182e2b6000)
00000800:00000200:7.0:1619677470.180199:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff91a3ae9d7000 0x182e9d7000(0x182e9d7000)
00000800:00000200:7.0:1619677470.180200:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff91a3ae2ab000 0x182e2ab000(0x182e2ab000)
00000800:00000200:7.0:1619677470.180200:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff91a3aea83000 0x182ea83000(0x182ea83000)
00000800:00000200:7.0:1619677470.180201:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff91a3b236e000 0x183236e000(0x183236e000)
00000800:00000200:7.0:1619677470.180201:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff91a3aef4a000 0x182ef4a000(0x182ef4a000)
00000800:00000200:7.0:1619677470.180202:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff91a3aef44000 0x182ef44000(0x182ef44000)
00000800:00000200:7.0:1619677470.180202:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff91a3aef3f000 0x182ef3f000(0x182ef3f000)
00000800:00000200:7.0:1619677470.180203:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff91a3aeeb4000 0x182eeb4000(0x182eeb4000)
00000800:00000200:7.0:1619677470.180203:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff91a3aee9d000 0x182ee9d000(0x182ee9d000)
00000800:00000200:7.0:1619677470.180204:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff91a3aef46000 0x182ef46000(0x182ef46000)
00000800:00000200:17.0:1619677470.181000:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.181014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000800:00000200:7.0:1619677470.181214:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181215:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181216:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181216:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181217:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181217:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181217:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181218:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181218:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181218:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181219:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181219:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181220:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181220:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181221:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181221:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181221:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181222:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181222:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181223:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181223:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181223:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181224:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181224:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181225:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181225:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181225:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181226:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181226:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181226:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181227:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181227:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181227:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181228:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:7.0:1619677470.181228:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (19)++
00000800:00000200:7.0:1619677470.181229:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (20)--
00000800:00000200:5.2:1619677470.181424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.181431:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.181434:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.181437:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.181439:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.181440:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.181442:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96524384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.181443:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.181444:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.181445:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.181448:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.181450:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.181453:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.181454:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.181454:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.181456:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.181456:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.181457:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.181458:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.181471:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.181474:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.181475:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.181476:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.181478:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.181479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.182129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:24.2:1619677470.182350:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:24.2:1619677470.182401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:5.2:1619677470.182589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.182599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.182631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.182637:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677470.182637:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:17.0:1619677470.182639:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.182639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.182640:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.182640:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.182641:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.182642:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.182644:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.182646:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.182656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.182659:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.182660:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.182660:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.182661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.182662:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.185129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.185935:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.185938:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.185939:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.185941:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.185945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.186356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.186365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677470.186412:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.186415:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.186415:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.186417:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.186419:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.186420:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.187029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.187071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677470.187451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.187457:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.187460:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.187463:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.187466:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.187468:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.187469:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96524345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.187471:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.187472:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.187473:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.187475:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.187477:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.187480:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.187481:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.187481:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.187483:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.187484:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.187484:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.187485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.187731:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.187954:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.187957:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:17.0:1619677470.187960:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.187963:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.187965:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.187966:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.187968:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.187969:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.187971:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.187974:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.187976:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.187978:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.187979:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.187980:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.187981:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.187982:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.187982:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.187983:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.187984:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.188815:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.188817:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.188818:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.188821:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.188823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.188935:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.188947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.188949:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:17.0:1619677470.188953:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.188956:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.188958:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677470.188960:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96524334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.188962:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.188964:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.188965:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677470.188970:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.188973:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677470.188976:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.188977:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:17.0:1619677470.188978:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:17.0:1619677470.188980:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:17.0:1619677470.188981:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.188982:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.188983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677470.190846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.190860:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.190862:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.190863:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.190864:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.190864:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.191026:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:24.2:1619677470.191110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2c600] (17)++
00000800:00000200:5.2:1619677470.191361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.191367:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.191370:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.191373:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.191376:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.191378:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.191379:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96524321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.191381:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.191383:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.191384:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.191387:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.191389:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.191393:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.191393:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.191394:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.191396:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.191396:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.191397:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.191398:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.191428:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.191430:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.191431:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.191433:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.191436:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.191438:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.191953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.192428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.192438:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.192468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.192473:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.192475:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.192476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.192477:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.193125:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:5.2:1619677470.193426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe1e00] (20)++
00000800:00000200:5.2:1619677470.193467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe1e00] (21)++
00000800:00000200:17.0:1619677470.194064:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.194066:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.194067:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.194070:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.194073:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.194088:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.195489:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.195491:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.195492:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.195495:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.195497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.196561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.196572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677470.196611:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.196614:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.196615:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.196618:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.196619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.196621:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.196856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.197394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.197400:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.197403:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.197406:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.197408:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.197410:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.197412:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.197413:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.197414:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.197415:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.197418:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.197420:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.197422:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.197423:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.197424:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.197426:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.197426:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.197427:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.197427:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.197439:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.197442:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.197443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.197445:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.197446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.197448:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.197455:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.197458:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.197459:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.197460:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.197461:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.197462:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.200376:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.200378:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.200379:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.200381:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.200383:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.200385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.200508:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.200511:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.200512:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.200515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.200515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.200518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:17.0:1619677470.200719:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:12.0:1619677470.200739:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.200741:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.200742:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.200743:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.200747:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.200751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.200876:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.200879:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.200880:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.200883:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.200883:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.200885:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677470.200885:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)++
00000800:00000200:12.0:1619677470.200886:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (24)--
00000800:00000200:13.0:1619677470.200887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (24)--
00000800:00000200:12.0:1619677470.200888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.200889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.201575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.201581:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.201584:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.201587:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.201590:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.201591:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.201593:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.201594:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.201595:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.201596:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.201600:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.201601:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.201605:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.201605:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.201606:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.201608:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.201608:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.201609:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.201610:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.201622:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.201624:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.201625:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.201627:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.201629:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.201629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.202018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.202379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.202389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.202410:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.202412:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.202412:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.202414:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.202415:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.202415:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.202420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.202426:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.202427:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.202428:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.202429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.202758:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.202761:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.202762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.202764:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.202766:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.202768:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.202769:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.202770:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.202771:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.202772:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.202775:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.202777:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.202779:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.202780:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.202780:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.202782:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.202782:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.202783:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.202783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677470.205245:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.205248:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.205249:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.205249:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.205252:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.205255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.206486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.206492:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.206495:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.206496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.206497:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.206500:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.206502:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.206503:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.206504:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.206505:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.206507:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.206510:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.206512:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.206514:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.206515:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.206516:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.206518:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.206518:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.206518:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.206519:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.206532:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.206534:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.206535:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.206537:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.206539:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.206540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.206549:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.206552:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.206553:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.206554:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.206554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.206556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.207016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.207057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677470.207503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.207509:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.207512:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.207513:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.207515:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.207518:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.207519:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.207521:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.207522:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.207523:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.207524:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.207527:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.207529:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.207532:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.207533:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.207534:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.207535:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.207536:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.207537:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.207537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.207549:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.207552:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.207553:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.207555:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.207556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.207557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.208368:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.208371:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.208371:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677470.208374:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.208377:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.208378:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.208379:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.208380:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.208382:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.208383:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.208385:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.208387:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.208389:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.208390:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.208391:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.208392:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.208393:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.208394:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677470.208395:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677470.208396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.208406:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677470.208408:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677470.208409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.208410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677470.210319:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.210322:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.210323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.210326:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.210328:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.210329:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.210331:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.210332:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.210333:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.210334:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.210337:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.210338:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.210341:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.210342:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:1.0:1619677470.210342:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:1.0:1619677470.210344:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:1.0:1619677470.210344:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.210345:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.210346:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677470.210601:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.210604:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.210605:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.210605:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.210608:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.210611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.211421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.211485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.211491:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.211493:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.211494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.211497:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.211499:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.211500:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.211502:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.211503:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.211504:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.211505:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.211508:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.211510:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.211513:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.211514:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.211515:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.211517:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.211517:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.211518:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.211518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.211531:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.211534:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.211535:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.211536:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.211539:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.211540:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.211824:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.211827:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.211828:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.211828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.211830:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.211833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.212421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.212427:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.212429:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.212430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.212432:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.212436:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.212437:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.212438:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96524164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.212440:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.212440:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.212442:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.212446:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.212448:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.212450:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.212450:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.212451:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.212451:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000400:00000200:17.0:1619677470.212452:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.212452:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000400:00000200:17.0:1619677470.212454:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.212454:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.212456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.212456:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.212457:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.212458:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.212459:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.212459:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.212464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.212473:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.212474:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.212475:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.212477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.212478:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.212480:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.212481:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.212481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.212482:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.212482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.213243:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.213246:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.213247:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.213248:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.213251:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.213253:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677470.213254:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.213256:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.213258:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.213259:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677470.213265:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.213268:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677470.213272:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.213272:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:1.0:1619677470.213273:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:1.0:1619677470.213275:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:1.0:1619677470.213276:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.213276:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.213277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677470.214225:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.214228:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.214229:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.214229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.214231:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.214241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.214246:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.214248:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.214248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.214249:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.214250:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.214251:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.214898:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.214900:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.214901:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.214902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.214904:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.214906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.215842:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.215844:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.215846:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.215846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.215848:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.215850:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.216331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.216337:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.216339:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.216340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.216342:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.216345:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.216346:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.216348:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.216349:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.216350:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.216351:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.216355:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.216356:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.216359:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.216360:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.216361:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.216363:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.216363:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.216364:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.216364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.216378:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.216380:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.216381:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.216383:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.216386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.216387:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.216391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677470.216534:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.216537:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.216538:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.216540:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.216543:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.216544:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.216545:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.216547:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.216548:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.216549:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.216552:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.216554:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.216557:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.216557:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.216558:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.216560:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.216560:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.216561:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.216561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677470.216839:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.216842:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.216843:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000400:00000200:1.0:1619677470.216843:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.216845:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.216848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:13.0:1619677470.217141:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.217144:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.217144:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.217146:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.217148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.217149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.217264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.217269:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.217272:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.217273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.217275:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.217278:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.217279:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.217280:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.217281:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.217282:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.217283:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.217286:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.217288:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.217290:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.217291:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.217292:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.217300:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.217300:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.217301:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.217302:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.217956:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.217959:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.217960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.217960:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.217962:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.217965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.218369:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.218372:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677470.218373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677470.218374:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.218377:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.218378:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677470.218380:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.218381:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.218382:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.218383:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677470.218386:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.218388:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677470.218390:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.218391:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:17.0:1619677470.218392:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:17.0:1619677470.218394:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:17.0:1619677470.218395:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.218396:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.218396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677470.219620:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.219623:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.219624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.219624:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.219626:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.219629:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.220020:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.220022:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.220023:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.220023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.220025:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.220028:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.220038:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.220041:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.220042:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.220042:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.220043:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.220044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.221232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.221238:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.221241:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.221242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.221245:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.221247:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.221249:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.221251:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.221252:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.221253:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.221254:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.221257:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.221259:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.221262:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.221262:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.221263:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.221265:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.221266:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.221266:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.221267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.221291:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.221300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.221304:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677470.221306:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.221308:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.221313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.221315:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.221318:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.221318:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.221319:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.221320:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.221321:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.221769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.222277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.222283:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.222286:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.222287:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.222289:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.222292:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.222300:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.222304:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.222305:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.222306:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.222307:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.222310:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.222312:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.222314:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.222315:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.222316:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.222318:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.222318:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.222320:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.222321:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.222322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.222327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.222333:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.222335:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.222336:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.222337:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.222342:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.222345:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.222346:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.222348:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.222349:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.222351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.223066:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.223069:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.223070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.223070:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.223073:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.223076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.224319:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.224322:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.224322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.224324:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.224327:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.224329:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677470.224330:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.224331:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.224332:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.224333:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677470.224337:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.224339:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677470.224342:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.224343:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:17.0:1619677470.224343:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:17.0:1619677470.224345:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:17.0:1619677470.224345:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.224347:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677470.224348:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677470.224348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.224365:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677470.224367:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677470.224368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.224369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677470.225205:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.225208:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.225209:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.225209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.225211:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.225216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:12.0:1619677470.225338:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.225340:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.225341:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.225344:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.225345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.225347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:1.0:1619677470.225552:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.225555:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.225556:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.225558:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.225560:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.225562:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677470.225564:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96524070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.225565:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.225567:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.225568:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677470.225571:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.225573:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677470.225576:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.225577:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:1.0:1619677470.225577:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:1.0:1619677470.225580:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:1.0:1619677470.225580:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.225581:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.225581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677470.225929:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.225932:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.225933:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.225934:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.225936:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.225939:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.226135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.226184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:5.2:1619677470.226477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.226483:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.226486:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.226487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.226488:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.226492:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.226493:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.226494:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.226496:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.226496:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.226498:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.226501:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.226502:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.226505:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.226506:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.226506:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.226508:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.226508:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.226509:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.226509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.227639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.227645:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.227648:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.227648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.227651:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.227653:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.227654:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.227656:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.227657:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.227658:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.227659:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.227662:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.227664:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.227667:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.227667:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.227668:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.227670:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.227670:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.227671:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.227672:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.227683:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.227686:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.227687:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.227689:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.227690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.227692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.230732:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.230735:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.230736:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.230738:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.230741:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.230742:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.230744:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.230745:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.230746:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.230748:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.230751:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.230753:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.230755:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.230756:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.230757:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.230759:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.230759:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.230761:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.230762:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.230762:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.230773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.230775:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.230776:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.230776:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677470.230991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.231886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677470.231892:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677470.231895:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677470.231896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677470.231898:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.231900:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.231902:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677470.231903:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96524032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.231904:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.231906:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.231906:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677470.231910:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.231911:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677470.231914:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e82fe5580] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.231915:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c000] (20)++
00000800:00000200:1.0:1619677470.231916:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c000] (21)++
00000800:00000200:1.0:1619677470.231918:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c000] (22)--
00000800:00000200:1.0:1619677470.231918:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677470.231920:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677470.231921:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677470.231921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:5.2:1619677470.231927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.231933:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677470.231935:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677470.231936:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677470.231937:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:5.2:1619677470.231958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.231963:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.231964:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.231964:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.231965:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.231966:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677470.231967:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677470.231970:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677470.231970:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.231971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.231972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.231974:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.231976:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.231978:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96524029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.231979:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.231981:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.231982:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.231985:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.231987:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.231990:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.231991:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.231992:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.231993:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.231994:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.231995:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.231995:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.232007:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.232010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.232011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.232013:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.232015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.232016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.232866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.232872:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.232875:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.232876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.232877:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.232880:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.232882:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.232883:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96524026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.232884:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.232885:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.232886:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.232889:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.232890:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.232893:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.232894:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.232895:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.232896:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.232897:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.232898:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.232899:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.232899:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.232904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.232909:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.232911:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.232912:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.232912:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.232912:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.232915:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.232916:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.232919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.232920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.232923:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.232936:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.232939:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.232940:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.232941:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.232942:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.232943:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.233026:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.233029:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.233030:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.233032:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.233034:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.233036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:24.2:1619677470.233135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677470.234103:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.234106:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.234107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.234107:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.234109:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.234113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.234121:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.234124:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.234125:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.234125:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.234126:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.234127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.234218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:13.0:1619677470.234994:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.234997:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.234998:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.234999:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.235002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677470.235002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.235012:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677470.235015:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677470.235016:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.235017:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677470.235017:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.235019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:1.0:1619677470.235441:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.235443:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.235444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.235445:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.235447:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.235450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.235458:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.235461:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.235462:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.235462:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.235463:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.235464:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.235749:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.235752:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.235753:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.235753:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.235756:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.235758:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.236715:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.236717:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.236718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.236720:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.236724:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.236725:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677470.236726:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.236727:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.236728:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.236729:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677470.236732:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.236734:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.236736:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.236736:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:17.0:1619677470.236737:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.236739:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.236739:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.236740:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.236740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677470.236752:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.236754:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:1.0:1619677470.236754:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.236755:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.236756:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.236757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.236976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.236982:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.236984:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.236985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.236987:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.236989:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.236990:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.236992:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.236993:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.236994:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.236995:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.236998:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.237000:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.237003:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.237004:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.237004:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.237006:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.237007:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.237008:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.237008:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.237019:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.237022:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.237023:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.237025:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.237026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.237028:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.237721:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.237724:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.237725:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.237725:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.237727:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.237730:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.237731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.237780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.237786:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.237788:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.237789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.237791:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.237794:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.237795:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.237796:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.237798:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.237799:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.237800:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.237803:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.237806:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.237809:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.237810:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.237810:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.237812:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.237813:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.237813:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.237814:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.237826:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.237829:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.237830:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.237831:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.237833:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.237834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.239892:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.239895:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.239896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.239896:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.239898:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.239901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:24.2:1619677470.240100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677470.240641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:17.0:1619677470.240790:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.240793:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.240794:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.240796:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.240799:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.240800:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.240802:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.240803:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.240804:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.240805:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.240809:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.240811:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.240813:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.240814:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.240815:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.240817:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.240817:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.240818:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.240818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677470.241205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.241246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:13.0:1619677470.241381:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.241383:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.241384:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.241386:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.241388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.241389:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.242052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.242059:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.242061:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.242062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.242064:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.242067:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.242069:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.242070:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.242071:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.242072:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.242073:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.242077:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.242078:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.242081:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.242082:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.242083:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.242084:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.242085:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.242085:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.242086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.242143:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.242146:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.242147:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.242147:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.242150:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.242153:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.242202:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.242205:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.242206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.242206:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.242208:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.242210:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.243029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.243036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.243039:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.243039:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.243042:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.243044:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.243045:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.243047:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.243048:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.243049:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.243050:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.243053:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.243055:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.243057:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.243058:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.243059:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.243061:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.243061:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.243063:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.243064:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.243065:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.243070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677470.243074:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:1.0:1619677470.243075:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677470.243077:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:1.0:1619677470.243077:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:12.0:1619677470.243078:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.243078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.243079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677470.243080:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.243081:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.243082:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.243836:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.243839:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.243840:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.243842:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.243843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.243845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.245490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:12.0:1619677470.246015:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.246018:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.246020:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.246022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.246023:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.246024:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677470.246026:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96523933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.246027:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.246029:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.246030:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677470.246033:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.246035:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677470.246037:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.246038:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:12.0:1619677470.246039:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677470.246040:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.246041:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.246042:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677470.246044:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677470.246044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.246054:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.246056:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.246056:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.246058:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.246058:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677470.246059:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677470.246059:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.246061:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.246062:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677470.246062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:17.0:1619677470.246563:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.246568:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.246569:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.246570:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.246572:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677470.246573:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.246574:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (23)++
00000800:00000200:1.0:1619677470.246576:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (24)--
00000800:00000200:1.0:1619677470.246577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.246578:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.246579:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.246860:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (21)++
00000800:00000200:17.0:1619677470.246865:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000800:00000200:1.0:1619677470.246865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (22)--
00000400:00000200:17.0:1619677470.246869:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.246873:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.246877:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677470.246880:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677470.246881:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (22)++
00000400:00000200:17.0:1619677470.246883:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677470.246883:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000400:00000200:17.0:1619677470.246885:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.246887:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677470.246892:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.246894:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677470.246895:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.246897:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677470.246898:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:1.0:1619677470.246900:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.246902:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.246903:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.246905:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677470.246909:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.246911:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000400:00000200:1.0:1619677470.246911:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.246912:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:1.0:1619677470.246914:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677470.246915:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:17.0:1619677470.246917:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (23)++
00000800:00000200:17.0:1619677470.246918:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (24)--
00000800:00000200:1.0:1619677470.246918:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.246920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)--
00000800:00000200:1.0:1619677470.246920:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:17.0:1619677470.246921:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (22)++
00000800:00000200:1.0:1619677470.246921:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:17.0:1619677470.246924:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000800:00000200:1.0:1619677470.246924:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:1.0:1619677470.246925:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (23)++
00000800:00000200:1.0:1619677470.246926:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (24)--
00000800:00000200:1.0:1619677470.246927:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)--
00000400:00000200:17.0:1619677470.246928:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677470.246929:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (22)--
00000400:00000200:17.0:1619677470.246931:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.246934:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677470.246937:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96523926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.246941:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.246944:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.246946:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677470.246949:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (22)++
00000800:00000200:1.0:1619677470.246952:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000400:00000200:17.0:1619677470.246954:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677470.246957:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677470.246958:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:1.0:1619677470.246961:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.246962:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.246964:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000400:00000200:1.0:1619677470.246964:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:17.0:1619677470.246965:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000400:00000200:1.0:1619677470.246966:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677470.246967:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000400:00000200:1.0:1619677470.246969:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.246971:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.246972:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677470.246977:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.246980:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (23)++
00000800:00000200:1.0:1619677470.246981:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.246982:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07dc600] (24)++
00000800:00000200:17.0:1619677470.246984:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (25)--
00000800:00000200:17.0:1619677470.246985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (24)--
00000800:00000200:1.0:1619677470.246985:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.246987:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)++
00000800:00000200:1.0:1619677470.246987:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:1.0:1619677470.246988:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.246990:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000800:00000200:1.0:1619677470.246991:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.246992:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (24)++
00000800:00000200:1.0:1619677470.246993:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (25)--
00000800:00000200:1.0:1619677470.246994:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (24)--
00000400:00000200:17.0:1619677470.246996:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677470.246996:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)++
00000800:00000200:1.0:1619677470.246997:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07dc600] (24)--
00000800:00000200:1.0:1619677470.246999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)--
00000800:00000200:1.0:1619677470.247000:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (22)++
00000400:00000200:17.0:1619677470.247002:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.247003:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000400:00000200:17.0:1619677470.247005:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.247007:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96523923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.247010:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.247012:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.247014:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.247018:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.247020:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677470.247021:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.247023:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.247024:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677470.247026:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96523922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.247028:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.247030:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.247032:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677470.247036:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.247038:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.247039:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000400:00000200:1.0:1619677470.247040:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.247042:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.247043:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (23)++
00000800:00000200:1.0:1619677470.247043:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677470.247045:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (24)--
00000800:00000200:17.0:1619677470.247046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)--
00000800:00000200:1.0:1619677470.247048:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.247049:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.247049:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:1.0:1619677470.247050:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:17.0:1619677470.247051:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.247052:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.247053:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:1.0:1619677470.247054:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (22)++
00000800:00000200:1.0:1619677470.247055:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (23)--
00000400:00000200:17.0:1619677470.247056:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.247056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (22)--
00000800:00000200:1.0:1619677470.247058:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (21)++
00000800:00000200:17.0:1619677470.247059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.247060:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.247061:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000800:00000200:17.0:1619677470.247062:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.247064:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.247065:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.247066:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.247066:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677470.247068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (22)++
00000800:00000200:17.0:1619677470.247070:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000400:00000200:1.0:1619677470.247070:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677470.247073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:1.0:1619677470.247075:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677470.247076:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.247078:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.247081:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.247083:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.247085:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677470.247090:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.247092:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677470.247093:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677470.247096:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96523919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677470.247096:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677470.247099:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.247100:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.247102:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677470.247106:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49600] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.247108:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953a00] (20)++
00000800:00000200:1.0:1619677470.247109:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953a00] (21)++
00000400:00000200:17.0:1619677470.247110:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.247113:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677470.247113:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953a00] (22)--
00000800:00000200:1.0:1619677470.247114:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (23)++
00000800:00000200:1.0:1619677470.247117:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07dc600] (24)++
00000800:00000200:17.0:1619677470.247118:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.247118:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (25)--
00000800:00000200:17.0:1619677470.247119:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:17.0:1619677470.247120:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:1.0:1619677470.247120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (24)--
00000800:00000200:1.0:1619677470.247122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.247123:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:1.0:1619677470.247123:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)++
00000800:00000200:17.0:1619677470.247124:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (24)++
00000800:00000200:17.0:1619677470.247125:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (25)--
00000800:00000200:1.0:1619677470.247126:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000800:00000200:17.0:1619677470.247127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (24)--
00000800:00000200:17.0:1619677470.247128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.247131:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677470.247131:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.247134:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.247135:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.247137:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677470.247139:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.247142:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.247145:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.247146:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677470.247152:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.247153:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677470.247154:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677470.247157:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677470.247158:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.247162:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.247163:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.247165:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.247169:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49600] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.247170:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953a00] (20)++
00000800:00000200:1.0:1619677470.247171:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953a00] (21)++
00000400:00000200:17.0:1619677470.247174:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.247175:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953a00] (22)--
00000800:00000200:1.0:1619677470.247177:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (23)++
00000800:00000200:17.0:1619677470.247178:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.247178:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (24)--
00000800:00000200:1.0:1619677470.247179:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)--
00000800:00000200:1.0:1619677470.247181:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (22)++
00000800:00000200:17.0:1619677470.247183:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.247183:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000800:00000200:17.0:1619677470.247184:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.247186:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000400:00000200:1.0:1619677470.247187:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677470.247189:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.247191:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)++
00000400:00000200:1.0:1619677470.247191:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.247192:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.247194:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.247194:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677470.247196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677470.247196:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96523915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677470.247198:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)++
00000800:00000200:17.0:1619677470.247200:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000400:00000200:1.0:1619677470.247200:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.247201:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.247203:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.247210:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.247210:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677470.247214:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.247214:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.247217:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.247220:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96523914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.247222:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.247224:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.247225:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.247229:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.247231:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.247232:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000400:00000200:17.0:1619677470.247233:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.247235:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.247236:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (24)++
00000800:00000200:17.0:1619677470.247237:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.247238:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (25)--
00000800:00000200:1.0:1619677470.247239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (24)--
00000800:00000200:17.0:1619677470.247240:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.247241:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)++
00000800:00000200:1.0:1619677470.247242:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.247243:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (23)++
00000800:00000200:1.0:1619677470.247244:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.247245:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (24)--
00000800:00000200:17.0:1619677470.247246:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (23)++
00000400:00000200:1.0:1619677470.247246:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.247248:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07dc600] (24)++
00000400:00000200:1.0:1619677470.247249:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.247250:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (25)--
00000800:00000200:17.0:1619677470.247251:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (24)--
00000800:00000200:17.0:1619677470.247252:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)++
00000800:00000200:17.0:1619677470.247254:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07dc600] (24)--
00000800:00000200:1.0:1619677470.247254:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.247255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)--
00000800:00000200:1.0:1619677470.247256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.247258:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (22)++
00000800:00000200:17.0:1619677470.247263:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000800:00000200:1.0:1619677470.247264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)++
00000800:00000200:1.0:1619677470.247267:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000400:00000200:17.0:1619677470.247268:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.247271:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.247272:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.247275:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.247278:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.247281:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.247283:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.247284:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.247286:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.247290:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677470.247292:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677470.247293:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677470.247302:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.247303:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.247304:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.247306:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677470.247309:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.247311:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.247311:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000400:00000200:1.0:1619677470.247312:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.247314:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.247315:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (24)++
00000800:00000200:1.0:1619677470.247315:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677470.247316:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (25)--
00000800:00000200:17.0:1619677470.247316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (24)--
00000800:00000200:17.0:1619677470.247317:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)++
00000800:00000200:17.0:1619677470.247318:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07dc600] (24)--
00000800:00000200:17.0:1619677470.247319:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)--
00000800:00000200:1.0:1619677470.247319:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.247320:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dc600] (22)++
00000800:00000200:1.0:1619677470.247320:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:1.0:1619677470.247321:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:17.0:1619677470.247322:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.34@o2ib3
00000800:00000200:1.0:1619677470.247324:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:1.0:1619677470.247324:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (23)++
00000400:00000200:17.0:1619677470.247325:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.34@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677470.247325:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (24)--
00000800:00000200:1.0:1619677470.247326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (23)--
00000800:00000200:1.0:1619677470.247327:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (22)--
00000400:00000200:17.0:1619677470.247328:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.247328:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000400:00000200:17.0:1619677470.247330:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:12.0:1619677470.247331:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:17.0:1619677470.247332:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96523910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.247332:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.247334:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.247334:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:17.0:1619677470.247335:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677470.247336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:17.0:1619677470.247337:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677470.247338:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.247339:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677470.247339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000400:00000200:13.0:1619677470.247340:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.247340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:17.0:1619677470.247341:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.34@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677470.247342:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.247344:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677470.247345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677470.247346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.247347:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49600] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.247348:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953a00] (20)++
00000800:00000200:17.0:1619677470.247348:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953a00] (21)++
00000800:00000200:13.0:1619677470.247349:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.247349:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.247350:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953a00] (22)--
00000800:00000200:13.0:1619677470.247350:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.247350:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.247351:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dc600] (21)++
00000800:00000200:13.0:1619677470.247351:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.247352:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dc600] (22)--
00000800:00000200:17.0:1619677470.247353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dc600] (21)--
00000800:00000200:12.0:1619677470.247359:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677470.247361:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677470.247362:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.247363:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677470.247363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.247364:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:24.2:1619677470.247602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677470.247649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:12.0:1619677470.248105:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.248107:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.248108:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.248109:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.248111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.248112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.248122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.248128:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.248131:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.248132:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.248134:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.248137:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.248138:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.248139:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.248141:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.248142:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.248143:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.248147:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.248149:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.248153:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.248153:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.248154:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.248156:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.248157:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.248157:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.248158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.248168:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.248171:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.248172:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.248174:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.248175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.248177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.250172:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.250174:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.250175:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.250177:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.250179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.250180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:24.2:1619677470.250972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:13.0:1619677470.251346:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.251348:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.251349:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.251351:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.251353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.251362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.251382:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.251385:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.251386:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.251389:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677470.251389:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.251395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.251405:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.251408:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.251409:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.251410:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.251411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.251411:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.251414:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.251416:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.251417:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.251418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.251420:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.251424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:13.0:1619677470.251498:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.251501:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.251501:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.251504:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.251504:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.251515:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:1.0:1619677470.251560:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.251563:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.251564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.251564:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.251566:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.251569:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.251812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.252235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.252240:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.252243:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.252244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.252246:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.252248:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.252249:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.252251:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.252252:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.252253:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.252254:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.252258:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.252259:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.252262:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.252263:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.252264:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.252265:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.252266:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.252267:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.252267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.252658:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.252660:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.252662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.252662:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.252664:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.252668:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.253455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.253461:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.253464:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.253465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.253466:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.253469:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.253470:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.253471:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.253473:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.253474:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.253475:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.253479:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.253481:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.253483:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.253484:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.253485:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.253486:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.253487:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.253489:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.253490:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.253490:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.253495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.253501:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.253503:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.253503:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.253504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.2:1619677470.253925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677470.254652:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.254656:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.254656:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.254659:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.254661:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.254662:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.254664:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.254665:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.254666:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.254667:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.254670:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.254671:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.254674:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.254675:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.254675:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.254677:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.254678:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.254678:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.254679:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677470.257332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677470.257416:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:5.2:1619677470.257418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.257420:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.257422:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.257423:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.257425:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677470.257426:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.257427:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.257432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.257434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.257435:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677470.257436:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.257439:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677470.257440:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.257440:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.257442:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.257443:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677470.257443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.257444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000400:00000200:17.0:1619677470.257447:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.257450:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.257452:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.257454:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.257467:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.257471:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.257475:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.257476:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.257477:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.257479:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.257480:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.257481:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.257482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.257942:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.257946:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.257948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.257948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.257951:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.257955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.258419:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.258422:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.258423:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.258424:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.258427:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.258429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.258436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.258442:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.258444:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.258445:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.258447:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.258449:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.258450:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.258451:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.258453:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.258454:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.258455:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.258458:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.258460:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.258462:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.258463:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.258464:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.258465:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.258466:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.258467:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.258467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.2:1619677470.259819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:5.2:1619677470.262283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.262331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677470.262763:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.262766:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.262767:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.262767:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.262769:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.262772:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.263041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.263047:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.263049:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.263050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.263053:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.263055:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.263056:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.263058:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.263060:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.263061:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.263062:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.263066:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.263069:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.263072:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.263073:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.263074:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.263075:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.263076:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.263077:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.263077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.263849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.263856:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.263859:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.263859:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.263862:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.263864:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.263865:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.263866:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.263868:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.263869:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.263870:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.263873:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.263875:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.263877:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.263878:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.263879:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.263880:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.263881:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.263882:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.263883:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.263883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.263889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.263893:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.263895:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.263896:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.263897:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.266966:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.266969:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.266970:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.266972:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.266974:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.266976:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.266978:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.266979:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.266980:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.266981:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.266985:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.266986:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.266988:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.266989:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.266990:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.266992:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.266992:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.266993:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.266993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677470.267150:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.267153:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.267154:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.267154:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.267156:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.267159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.267169:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.267171:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.267172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.267173:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.267174:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.267174:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.267205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677470.267522:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.267525:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.267526:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.267526:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.267528:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.267531:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.268042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.268049:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.268051:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.268052:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.268054:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.268056:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.268058:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.268059:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.268060:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.268061:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.268062:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.268066:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.268069:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.268071:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.268072:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.268073:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.268074:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.268075:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.268075:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.268076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.268858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.268865:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.268867:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.268868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.268870:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.268873:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.268874:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.268875:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.268877:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.268878:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.268879:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.268884:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.268888:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.268890:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.268891:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.268892:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.268893:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.268894:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.268894:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.268895:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.271321:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.271323:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.271324:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.271324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.271326:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.271329:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.272108:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.272110:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.272111:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.272112:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.272114:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.272117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.272206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.272946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.272952:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.272955:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.272956:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.272957:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.272960:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.272962:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.272963:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.272965:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.272966:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.272967:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.272970:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.272972:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.272975:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.272976:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.272977:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.272979:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.272979:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.272980:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.272980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.272994:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.272998:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.272999:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.273002:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.273003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.273006:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.273032:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.273034:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.273035:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.273036:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.273038:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.273041:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.273524:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.273527:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.273528:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.273528:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.273530:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.273534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.273776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.273781:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.273784:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.273785:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.273787:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.273789:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.273790:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.273792:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.273793:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.273794:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.273795:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.273798:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.273799:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.273803:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.273803:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.273804:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.273806:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.273806:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.273808:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.273809:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.273809:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.273814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.273820:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677470.273820:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:17.0:1619677470.273822:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.273822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.273823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.273823:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.273823:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.273825:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.273827:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.273828:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.274658:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.274660:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.274662:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677470.274665:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.274666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.274666:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677470.274668:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96523729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.274669:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.274670:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.274671:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677470.274674:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.274675:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677470.274677:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.274678:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:12.0:1619677470.274679:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:12.0:1619677470.274681:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:12.0:1619677470.274682:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.274682:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.274683:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677470.275235:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.275238:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.275238:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677470.275241:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.275244:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.275245:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.275246:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.275248:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.275249:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.275250:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.275253:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.275255:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.275258:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.275258:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.275259:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.275261:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.275261:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.275262:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.275263:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.275264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.275274:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.275276:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.275277:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.275278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:13.0:1619677470.276211:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.276214:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.276216:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677470.276218:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.276218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.276220:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677470.276221:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96523722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.276223:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.276225:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.276226:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677470.276230:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.276232:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677470.276235:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.276235:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:13.0:1619677470.276236:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:13.0:1619677470.276238:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:13.0:1619677470.276239:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.276239:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.276240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677470.277084:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.277087:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.277088:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.277088:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.277090:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.277093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.277101:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.277104:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.277105:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.277105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.277106:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.277107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.277134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677470.277926:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.277929:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.277930:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.277930:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.277932:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.277936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.278021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.278027:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.278030:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.278031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.278033:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.278035:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.278036:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.278037:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.278039:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.278039:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.278040:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.278044:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.278046:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.278048:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.278049:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.278050:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.278052:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.278053:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.278053:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.278054:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.278733:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.278736:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.278737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.278739:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.278742:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.278743:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677470.278745:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.278746:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.278747:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.278748:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677470.278751:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.278753:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677470.278755:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.278756:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:1.0:1619677470.278757:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:1.0:1619677470.278759:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:1.0:1619677470.278759:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.278760:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.278760:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:5.2:1619677470.278986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.278992:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.278994:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.278996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.278998:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.279000:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.279001:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.279002:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.279003:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.279005:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.279005:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.279009:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.279010:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.279013:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.279013:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.279014:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.279016:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.279016:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.279017:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.279018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.279527:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.279530:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.279531:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.279532:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.279534:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.279537:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.279548:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.279550:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.279551:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.279551:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.279552:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.279553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.282027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.282071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:1.0:1619677470.282661:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.282665:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.282665:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677470.282668:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.282671:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.282672:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.282673:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.282675:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.282676:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.282677:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.282680:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.282682:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.282684:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.282685:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.282686:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.282687:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.282688:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.282688:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.282689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677470.283004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.283010:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.283012:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.283013:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.283015:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.283019:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.283020:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.283022:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.283023:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.283024:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.283025:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.283028:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.283030:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.283033:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.283033:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.283034:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.283036:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.283036:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.283037:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.283037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.283298:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.283301:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.283302:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.283303:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.283305:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.283308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.283901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.283907:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.283909:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.283910:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.283912:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.283914:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.283916:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.283917:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.283918:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.283919:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.283920:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.283924:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.283926:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.283929:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.283929:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.283930:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.283932:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.283933:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.283934:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.283935:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.283935:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.283941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677470.283945:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:1.0:1619677470.283946:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677470.283948:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.283948:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.283948:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.283949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677470.283950:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.283950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.283952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.283954:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.284162:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.284165:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.284166:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.284166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.284168:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.284171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:24.2:1619677470.284224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677470.284357:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.284360:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.284361:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.284361:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.284363:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.284366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.285803:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.285807:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.285807:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.285811:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.285813:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.285815:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677470.285816:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.285818:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.285820:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.285821:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677470.285826:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.285829:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677470.285832:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.285832:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:17.0:1619677470.285833:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:17.0:1619677470.285835:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:17.0:1619677470.285836:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.285837:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.285837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.2:1619677470.286213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677470.286247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:13.0:1619677470.286419:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.286422:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.286423:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.286426:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.286426:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.286429:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.286962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677470.287485:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.287488:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.287489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.287490:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.287492:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.287495:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.287859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.287865:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.287868:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.287869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.287871:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.287873:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.287875:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.287877:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.287878:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.287879:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.287880:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.287884:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.287885:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.287888:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.287889:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.287889:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.287891:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.287891:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.287892:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.287893:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.288842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.288848:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.288851:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.288852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.288854:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.288856:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.288858:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.288859:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.288861:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.288862:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.288863:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.288866:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.288868:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.288871:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.288871:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.288872:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.288874:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.288874:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.288875:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.288875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.291957:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:5.2:1619677470.291957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677470.291960:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.291960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.291963:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.291965:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.291967:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.291968:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.291970:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.291971:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.291972:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.291977:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.291979:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.291982:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.291983:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.291984:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.291985:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.291986:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.291987:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.291988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677470.292823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.292828:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.292831:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.292832:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.292834:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.292836:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.292838:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.292839:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.292841:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.292842:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.292843:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.292846:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.292848:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.292850:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.292851:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.292852:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.292854:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.292854:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.292855:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.292855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.293330:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.293332:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.293333:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.293335:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.293337:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.293339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677470.293356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677470.293359:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677470.293360:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.293360:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677470.293361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.293362:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.293894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.293900:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.293903:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.293904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.293906:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.293908:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.293909:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.293910:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.293912:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.293913:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.293914:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.293917:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.293919:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.293922:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.293923:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.293924:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.293925:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.293926:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.293927:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.293928:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.293929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.293934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:12.0:1619677470.293938:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:1.0:1619677470.293939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677470.293941:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:1.0:1619677470.293941:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.293941:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:12.0:1619677470.293942:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.293943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677470.293944:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.293945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.293946:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.293966:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.293969:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.293970:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.293970:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.293971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.293972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.294269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677470.294275:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.294278:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677470.294278:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677470.294281:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677470.294284:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.294285:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677470.294287:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.294288:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.294289:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.294290:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677470.294299:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677470.294301:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677470.294304:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe0d80d00] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.294305:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb719600] (20)++
00000800:00000200:17.0:1619677470.294306:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb719600] (21)++
00000800:00000200:17.0:1619677470.294308:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb719600] (22)--
00000800:00000200:17.0:1619677470.294308:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.294309:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677470.294310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:12.0:1619677470.294538:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.294540:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.294541:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.294543:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.294544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.294545:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677470.295358:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.295361:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.295363:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677470.295366:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.295366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.295367:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677470.295368:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96523597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.295370:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.295371:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.295372:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677470.295375:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.295376:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677470.295379:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.295379:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:13.0:1619677470.295380:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:13.0:1619677470.295382:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:13.0:1619677470.295383:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.295383:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.295384:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677470.295508:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.295510:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.295511:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.295513:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.295514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.295515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.296504:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.296507:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.296509:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677470.296511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.296512:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.296513:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677470.296514:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96523590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.296516:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.296517:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.296518:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677470.296523:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.296526:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677470.296528:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.296529:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:13.0:1619677470.296529:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:13.0:1619677470.296531:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:13.0:1619677470.296531:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.296533:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677470.296534:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677470.296535:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.296549:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677470.296551:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677470.296552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.296553:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677470.296851:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.296854:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.296855:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.296855:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.296858:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.296861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:24.2:1619677470.297395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677470.297460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.297462:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.297463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.297464:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.297466:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.297469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.297480:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.297483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.297483:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.297484:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.297484:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.297485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.297868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.297874:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.297877:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.297878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.297880:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.297882:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.297884:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.297885:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.297886:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.297887:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.297888:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.297892:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.297893:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.297896:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.297897:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.297897:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.297899:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.297899:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.297900:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.297901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.298192:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.298195:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.298196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.298196:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.298198:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.298201:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.298214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.298216:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.298217:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.298217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.298218:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.298219:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.298421:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.298424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.298424:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677470.298426:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.298428:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.298430:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.298432:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.298433:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.298434:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.298435:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.298439:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.298440:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.298443:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.298444:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.298445:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.298447:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.298447:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.298448:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.298448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677470.298462:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.298465:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.298467:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.298469:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.298470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.298473:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.299061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.299067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.299070:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.299071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.299073:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.299076:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.299077:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.299079:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.299080:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.299081:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.299082:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.299085:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.299087:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.299090:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.299090:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.299091:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.299093:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.299093:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.299094:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.299094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.299142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677470.301853:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.301856:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.301857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.301857:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.301859:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.301862:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.301914:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.301917:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.301917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.301918:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.301920:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.301923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.302428:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.302430:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.302432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.302432:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.302434:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.302437:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.303185:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.303188:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.303189:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.303189:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.303191:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.303194:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.303345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.303350:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.303352:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.303353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.303355:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.303357:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.303358:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.303359:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.303361:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.303361:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.303362:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.303365:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.303367:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.303369:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.303370:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.303371:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.303373:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.303373:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.303373:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.303374:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.2:1619677470.304023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:5.2:1619677470.304255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:5.2:1619677470.304298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677470.304559:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.304561:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.304562:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.304562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.304564:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.304567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.304578:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.304581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.304581:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.304581:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.304583:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.304583:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.304879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.304885:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.304888:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.304889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.304891:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.304893:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.304894:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.304895:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.304896:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.304897:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.304898:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.304902:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.304903:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.304906:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.304907:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.304907:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.304909:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.304910:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.304911:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.304912:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.304913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.304918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.304923:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.304925:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.304926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.304927:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.305144:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.305147:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.305148:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.305149:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.305151:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.305154:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.306341:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.306344:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.306345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.306347:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.306349:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.306351:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677470.306352:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.306353:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.306354:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.306355:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677470.306359:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.306361:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677470.306364:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.306365:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:1.0:1619677470.306366:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:1.0:1619677470.306368:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:1.0:1619677470.306368:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.306370:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677470.306371:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677470.306371:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.306383:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:13.0:1619677470.306384:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:17.0:1619677470.306385:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677470.306386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.306387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677470.306387:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.306389:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.306391:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.306392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.306395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:1.0:1619677470.307391:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.307394:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.307395:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.307395:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.307397:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.307401:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.307711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677470.307752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.2:1619677470.308211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.308816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.308823:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.308825:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.308826:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.308828:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.308830:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.308832:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.308833:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.308835:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.308836:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.308837:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.308840:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.308842:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.308845:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.308846:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.308847:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.308848:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.308849:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.308850:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.308850:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.308863:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.308865:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.308866:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.308868:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.308870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.308871:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.309271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.2:1619677470.309364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.2:1619677470.309826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.309832:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.309835:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.309836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.309838:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.309841:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.309842:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.309843:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.309844:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.309845:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.309846:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.309849:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.309851:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.309854:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.309855:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.309856:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.309858:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.309859:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.309859:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.309860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.309872:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.309875:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.309876:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.309877:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.309879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.309880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.309893:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.309896:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.309897:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.309897:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.309898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.309899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.310182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677470.310599:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.310602:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.310603:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677470.310606:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.310606:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.310607:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677470.310608:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96523508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.310610:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.310611:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.310612:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677470.310615:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.310616:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677470.310619:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.310620:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:13.0:1619677470.310620:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:13.0:1619677470.310622:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:13.0:1619677470.310622:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.310623:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.310624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:2.0:1619677470.310715:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.310720:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.310722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.311270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.2:1619677470.311281:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.311291:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.311328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.2:1619677470.311332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.311338:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.311340:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.311341:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.311342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.312001:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.312004:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.312005:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.312005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.312007:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.312011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.312755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.313710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.313716:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.313719:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.313720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.313722:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.313724:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.313725:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.313726:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.313728:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.313729:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.313730:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.313732:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.313734:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.313737:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.313738:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.313738:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.313740:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.313740:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.313741:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.313742:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.313755:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.313757:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.313758:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.313760:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.313762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.313763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.314318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.314329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.2:1619677470.315114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677470.315218:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.315220:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.315222:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.315224:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.315228:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.315243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.315556:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.315558:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.315559:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.315562:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.315565:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.315658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.315668:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.316339:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.316342:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:2.0:1619677470.316344:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.316345:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.316347:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:2.0:1619677470.316347:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:13.0:1619677470.316348:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.316350:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.316352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:2.0:1619677470.316353:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:13.0:1619677470.316355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:2.0:1619677470.316355:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.316357:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.316358:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.316360:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.316360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:12.0:1619677470.316372:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.316375:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.316375:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.316377:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.316379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.316385:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.316425:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.316428:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.316429:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.316429:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.316432:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.316439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.316525:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:13.0:1619677470.316531:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.316533:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.316534:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.316536:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.316537:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.316543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:1.0:1619677470.316594:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)++
00000800:00000200:1.0:1619677470.316596:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (24)--
00000800:00000200:17.0:1619677470.316597:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)++
00000800:00000200:17.0:1619677470.316598:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (24)--
00000400:00000200:1.0:1619677470.316598:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.316599:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.316601:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677470.316601:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.316609:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.316610:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.316611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.316624:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.316628:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)++
00000800:00000200:1.0:1619677470.316628:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.316629:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (23)++
00000800:00000200:17.0:1619677470.316630:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (25)--
00000800:00000200:1.0:1619677470.316630:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (25)--
00000800:00000200:1.0:1619677470.316630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (24)--
00000400:00000200:17.0:1619677470.316631:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.316632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.316633:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.316642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.316790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677470.316838:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.316841:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:2.0:1619677470.316842:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.316844:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.316849:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.316922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:5.2:1619677470.317311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.317317:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.317320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.317320:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677470.317322:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.317325:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.317326:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.317328:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.317329:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.317330:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.317331:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.317334:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.317336:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.317338:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.317339:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.317340:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.317342:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.317342:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.317343:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.317344:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.317344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.317350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.317356:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.317359:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.317359:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.317360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.317736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.317773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677470.318945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.318951:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.318954:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.318955:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.318957:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.318959:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.318961:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.318963:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.318964:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.318965:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.318967:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.318972:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.318975:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.318981:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.318982:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.318983:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.318985:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.318985:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.318986:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.318987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.318999:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:17.0:1619677470.319001:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:12.0:1619677470.319002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:17.0:1619677470.319003:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:12.0:1619677470.319003:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.319004:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.319004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:12.0:1619677470.319005:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.319007:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:17.0:1619677470.319008:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.319009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.319012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:13.0:1619677470.319016:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.319019:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.319020:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.319020:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:1.0:1619677470.319020:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:13.0:1619677470.319021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.319023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.319023:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.319024:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.319024:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.319025:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.319026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.319035:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.319038:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.319039:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.319042:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.319045:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.319383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.319393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:12.0:1619677470.319477:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.319480:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.319481:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.319484:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.319485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.319487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677470.319497:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677470.319500:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677470.319501:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.319502:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677470.319502:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.319503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:20.2:1619677470.320615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.320655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677470.321067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.321073:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.321076:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:2.0:1619677470.321079:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.321083:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.321085:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677470.321086:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.321088:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.321089:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.321090:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677470.321093:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.321095:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677470.321098:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e82fe5580] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.321099:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c000] (20)++
00000800:00000200:2.0:1619677470.321100:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c000] (21)++
00000800:00000200:2.0:1619677470.321102:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c000] (22)--
00000800:00000200:2.0:1619677470.321102:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.321103:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677470.321104:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677470.321400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.321418:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.321421:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.321422:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.321423:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.321424:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.321770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f74a200] (20)++
00000800:00000200:20.2:1619677470.321772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:13.0:1619677470.321780:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f74a200] (21)++
00000800:00000200:2.0:1619677470.321782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677470.321783:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.44@o2ib3
00000400:00000200:13.0:1619677470.321785:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.3.44@o2ib3 : GET - for me
00000800:00000200:12.0:1619677470.321787:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f74a200] (22)--
00000400:00000200:13.0:1619677470.321789:0:19942:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.3.44@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:13.0:1619677470.321791:0:19942:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.3.44@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:13.0:1619677470.321793:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f74a200] (21)++
00000800:00000200:13.0:1619677470.321794:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f74a200] (22)--
00000400:00000200:13.0:1619677470.321796:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.3.44@o2ib3 local destination
00000800:00000200:13.0:1619677470.321797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f74a200] (21)--
00000800:00000200:2.0:1619677470.321818:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.321820:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.321821:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.321823:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.321825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:12.0:1619677470.322049:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.322052:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.322053:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.322054:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.322056:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.322057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.322325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.322331:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.322334:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.322335:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.322337:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.322340:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.322342:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.322343:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.322345:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.322346:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.322347:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.322350:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.322352:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.322354:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.322355:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.322356:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.322358:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.322358:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.322358:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.322359:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.322789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.323287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.323301:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.2:1619677470.323887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.323893:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.323897:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.323898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.323900:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.323903:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.323904:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.323905:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.323907:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.323908:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.323909:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.323911:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.323913:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.323916:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.323917:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.323917:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.323919:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.323920:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.323920:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.323921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.323956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.324184:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.324187:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.324188:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.324188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.324190:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.324193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.324360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.324366:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.324368:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677470.324371:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.324374:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.324376:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.324377:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.324379:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.324379:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.324380:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.324384:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.324385:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.324388:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.324389:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.324390:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.324392:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.324392:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.324394:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.324395:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.324395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.324400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.324802:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.324805:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.324806:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.324808:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.324811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.324950:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.324953:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.324954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.324954:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.324956:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.324959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.325662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.325688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.325699:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.325729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.325735:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.325736:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.325737:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.325740:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.325741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.326246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.326263:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.326266:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.326267:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.326268:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.326269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.327435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.327441:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.327444:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.327445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.327447:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.327449:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.327450:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.327452:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.327453:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.327454:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.327455:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.327458:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.327460:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.327463:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.327464:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.327465:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.327466:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.327467:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.327468:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.327469:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.327470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.327475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.327480:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.327482:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.327482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.327483:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.327834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:13.0:1619677470.327879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.327882:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.327883:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.327885:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.327886:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.327888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.328613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.328704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.328710:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.328713:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.328714:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.328716:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.328718:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.328720:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.328721:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.328722:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.328723:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.328724:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.328728:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.328729:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.328732:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.328733:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.328734:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.328736:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.328736:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.328736:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.328737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.328749:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.328752:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.328753:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.328754:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.328756:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.328757:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.329439:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.329442:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.329443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.329444:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.329446:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:5.2:1619677470.329447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677470.329449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.329457:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.2:1619677470.329578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.329615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.329653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.329659:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.329661:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677470.329665:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.329668:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.329670:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.329671:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.329672:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.329673:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.329675:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.329678:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.329680:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.329682:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.329683:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.329684:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.329686:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.329686:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.329687:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.329687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.330400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.331002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.331005:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.331006:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.331009:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.331012:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.331181:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:20.2:1619677470.331182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677470.331198:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.331201:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.331202:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.331203:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.331204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.331904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.331915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.331948:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.331950:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.331951:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.331953:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.331954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.332149:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.332152:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.332153:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.332153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.332155:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.332158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.332167:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.332169:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.332170:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.332171:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.332172:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.332172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.332311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.332317:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.332319:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.332320:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.332322:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.332325:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.332326:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.332327:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.332329:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.332330:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.332331:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.332334:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.332336:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.332338:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.332339:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.332339:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.332341:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.332341:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.332342:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.332342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.332597:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.332600:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.332601:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.332601:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.332603:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.332607:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.332874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.332885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.332925:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.332927:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.332928:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.332930:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.332932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.333507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.333513:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.333516:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.333517:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.333518:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.333521:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.333522:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.333524:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.333525:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.333526:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.333527:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.333531:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.333532:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.333535:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.333536:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.333537:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.333539:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.333539:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.333540:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.333541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.333554:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.333556:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.333557:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.333559:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.333561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.333562:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.333706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:12.0:1619677470.333751:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.333753:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.333754:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.333755:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.333757:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.333758:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.333771:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.333774:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.333774:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.333775:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.333775:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.333776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.334121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.334177:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.334180:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.334181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.334181:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.334184:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.334186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.334298:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.334301:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.334302:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.334302:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.334304:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.334307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.334510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.334520:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677470.334686:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.334689:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.334690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.334691:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.334694:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.334695:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677470.334697:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.334698:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.334699:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.334700:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677470.334704:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.334706:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677470.334709:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.334710:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:1.0:1619677470.334710:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:1.0:1619677470.334712:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:1.0:1619677470.334713:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.334714:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.334714:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677470.334725:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.334728:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.334729:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.334731:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.334732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.334734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:17.0:1619677470.334862:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.334865:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.334866:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.334866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.334868:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.334871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:2.0:1619677470.335043:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.335046:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:2.0:1619677470.335047:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.335049:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.335052:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.335393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.335412:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.335415:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.335416:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.335417:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.335418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.335460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:20.2:1619677470.335677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.335683:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.335685:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677470.335688:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.335691:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.335693:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.335694:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.335695:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.335697:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.335697:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.335701:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.335702:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.335705:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.335705:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.335706:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.335708:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.335708:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.335709:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.335709:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.335892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:20.2:1619677470.336103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.336148:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.336151:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.336152:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.336152:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.336155:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.336156:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.336611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677470.336733:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.336736:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.336737:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.336737:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.336740:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.336742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.336753:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.336756:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.336757:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.336757:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.336758:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.336758:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.337190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.337196:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.337199:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.337199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.337202:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.337205:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.337206:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.337208:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.337209:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.337210:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.337211:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.337214:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.337216:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.337219:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.337220:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.337220:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.337222:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.337223:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.337224:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.337225:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.337226:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.337230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.337236:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.337238:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.337239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.337240:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.338507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.338513:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.338515:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.338516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.338518:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.338520:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.338522:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.338523:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.338525:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.338526:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.338527:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.338530:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.338532:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.338535:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.338536:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.338537:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.338539:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.338539:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.338539:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.338540:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.338553:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.338556:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.338556:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.338558:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.338560:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.338561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.339140:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.339143:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.339144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.339145:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.339148:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.339150:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677470.339151:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.339152:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.339153:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.339154:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677470.339157:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.339158:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677470.339160:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.339161:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:1.0:1619677470.339161:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:1.0:1619677470.339163:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:1.0:1619677470.339163:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.339165:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677470.339166:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677470.339166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.339177:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:2.0:1619677470.339178:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:17.0:1619677470.339179:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677470.339180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.339180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677470.339180:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:2.0:1619677470.339181:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.339183:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.339186:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.339203:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.339308:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.339311:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.339312:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.339315:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.339318:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.339455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.340022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.340030:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.340035:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.340036:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.340041:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.340046:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.340048:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.340050:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.340053:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.340055:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.340057:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.340062:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.340065:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.340070:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.340071:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.340072:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.340075:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:1.0:1619677470.340075:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:18.0:1619677470.340076:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.340077:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:1.0:1619677470.340077:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:18.0:1619677470.340078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.340079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.340079:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.340081:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.340084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.340095:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.340098:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.340099:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.340099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.340101:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.340101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.340785:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.340787:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.340789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.340789:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.340791:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.340795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.340952:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.340955:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.340956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000400:00000200:1.0:1619677470.340956:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.340959:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.340962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.341064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.341070:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.341073:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.341074:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.341076:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.341079:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.341080:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.341082:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.341083:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.341084:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.341085:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.341089:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.341090:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.341094:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.341094:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:18.0:1619677470.341096:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:18.0:1619677470.341098:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:18.0:1619677470.341098:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.341099:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.341100:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.341300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.341305:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.341308:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.341309:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.341311:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.341314:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.341316:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.341317:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.341318:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.341319:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.341320:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.341323:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.341324:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.341325:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.341326:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.341326:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.341328:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.341328:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.341330:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.341331:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.341331:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.341338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.341343:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.341345:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.341345:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677470.341346:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.341346:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.341346:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.341347:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.341348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.341349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.341349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.341500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.341552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677470.342326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.342332:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.342335:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.342335:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.342337:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.342339:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.342341:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.342342:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.342343:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.342345:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.342346:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.342350:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.342352:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.342355:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.342355:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.342356:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.342358:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.342358:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.342359:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.342360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677470.342385:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.342388:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.342389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.342389:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.342391:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.342395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.343350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.343356:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.343359:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.343360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.343362:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.343364:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.343365:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.343367:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.343368:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.343370:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.343371:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.343374:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.343375:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.343378:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.343379:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.343380:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.343381:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.343382:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.343382:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.343383:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.343396:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.343399:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.343400:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.343401:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.343403:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.343404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.343959:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.343961:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.343962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.343963:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.343965:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.343968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.344241:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.344244:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.344245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.344245:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.344247:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.344251:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.344327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.344367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677470.344398:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.344401:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.344402:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.344402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.344404:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.344409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.344411:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.344413:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.344413:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.344414:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.344415:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.344416:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.345040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.345046:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.345050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.345050:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677470.345052:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.345055:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.345057:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.345058:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.345060:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.345061:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.345062:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.345065:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.345067:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.345070:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.345071:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.345072:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.345074:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.345074:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.345075:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.345076:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.345077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.345082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.345088:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.345090:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.345090:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.345092:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.345449:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.345452:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.345452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.345454:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.345457:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.345459:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677470.345460:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.345462:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.345463:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.345464:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677470.345468:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.345469:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677470.345472:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.345473:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:17.0:1619677470.345474:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:17.0:1619677470.345475:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:17.0:1619677470.345476:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.345477:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.345477:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:2.0:1619677470.345662:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.345665:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.345666:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.345666:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.345668:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.345672:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.346995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677470.347150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.347156:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.347159:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.347160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.347162:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.347165:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.347167:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.347168:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.347170:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.347171:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.347172:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.347175:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.347177:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.347180:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.347180:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.347181:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.347183:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.347183:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.347185:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.347186:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.347186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.347191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.347197:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.347198:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.347199:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.347200:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.348198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.348204:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.348207:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.348208:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.348209:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.348212:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.348213:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.348214:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.348216:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.348217:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.348217:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.348220:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.348222:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.348225:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.348226:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.348227:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.348229:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.348229:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.348232:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.348232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.348244:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.348248:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.348250:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.348252:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.348254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.348255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.348380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.348387:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.348390:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.348391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.348393:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.348395:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.348397:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.348398:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.348400:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.348403:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.348404:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.348408:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.348409:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.348411:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.348412:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.348413:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.348414:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.348414:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.348415:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.348416:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.348428:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.348431:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.348431:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.348432:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.348433:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.348434:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.348835:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.348838:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.348838:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.348839:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.348841:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.348845:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.348857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.348860:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.348861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.348862:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.348864:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.348867:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.349422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.349800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.349803:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.349804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.349805:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.349807:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.349810:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.350255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.350261:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.350263:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.350264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.350266:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.350269:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.350271:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.350272:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.350274:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.350275:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.350276:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.350279:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.350281:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.350285:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.350285:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.350286:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.350289:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.350290:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.350291:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.350291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.350585:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.350588:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.350589:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.350589:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.350591:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.350595:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.350605:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.350607:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.350608:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.350609:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.350610:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.350610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.350817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.350823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.350825:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.350826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.350828:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.350830:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.350832:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.350834:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.350835:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.350836:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.350837:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.350841:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.350843:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.350846:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.350847:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.350847:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.350850:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.350850:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.350851:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.350852:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.350853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.350858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.350864:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.350866:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.350867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.350868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.350986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.350989:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.350990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.350992:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.350995:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.350997:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677470.350998:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.350999:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.351001:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.351001:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677470.351005:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.351007:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677470.351010:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.351011:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:1.0:1619677470.351011:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:1.0:1619677470.351013:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:1.0:1619677470.351014:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.351014:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.351015:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677470.351029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.351035:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.351038:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.351039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.351041:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.351044:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.351046:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.351047:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.351049:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.351050:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.351051:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.351054:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.351056:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.351059:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.351059:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.351060:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.351062:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.351062:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.351063:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.351064:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677470.351551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.351557:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.351560:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.351561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.351563:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.351565:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.351567:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.351568:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.351570:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.351571:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.351572:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.351575:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.351577:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.351579:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.351580:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.351581:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.351582:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.351583:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.351584:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.351584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.352189:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.352192:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.352193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.352195:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.352197:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.352199:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677470.352200:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.352202:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.352203:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.352204:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677470.352208:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.352210:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.352212:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.352212:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:1.0:1619677470.352213:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.352215:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.352215:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.352216:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.352217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677470.352229:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.352231:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.352232:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.352232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.352234:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.352235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.352251:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.352253:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.352254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.352254:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.352255:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.352256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.352441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677470.352921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.352927:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.352930:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.352931:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.352932:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.352934:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.352936:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.352937:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.352938:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.352939:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.352940:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.352943:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.352944:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.352947:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.352948:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.352949:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.352951:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.352952:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.352952:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.352953:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.352965:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.352968:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.352969:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.352970:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.352972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.352973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.352983:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.352986:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.352987:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.352987:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.352988:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.352989:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:18.0:1619677470.353475:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.353478:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.353479:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.353479:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.353481:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.353485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.353784:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.353786:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.353787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.353788:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.353790:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.353793:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.354118:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.354121:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677470.354121:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677470.354124:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.354128:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.354129:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677470.354130:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96523259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.354132:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.354133:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.354135:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677470.354139:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.354141:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.354143:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.354144:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:17.0:1619677470.354144:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.354145:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.354146:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.354147:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.354147:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677470.354160:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.354162:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.354163:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.354164:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.354165:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.354167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.354315:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.354317:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.354318:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.354318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.354321:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.354333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.354481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.354917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.354923:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.354926:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.354927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.354929:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.354932:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.354933:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.354934:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.354936:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.354937:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.354937:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.354940:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.354942:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.354944:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.354945:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.354946:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.354947:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.354948:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.354949:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.354950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.354961:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.354963:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.354963:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.354964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.354965:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.354967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.354981:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.354983:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.354984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.354984:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.354984:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.354985:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.355221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.355226:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.355229:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.355229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.355231:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.355233:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.355235:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.355236:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.355237:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.355238:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.355240:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.355243:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.355245:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.355248:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.355248:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.355249:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.355250:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.355251:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.355251:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.355252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.355818:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.355820:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.355820:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.355821:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.355822:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.355822:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.355823:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000400:00000200:18.0:1619677470.355824:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.355827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.355828:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.355828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.355829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.356510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.356516:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.356519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.356519:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677470.356521:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.356523:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.356525:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.356526:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.356527:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.356528:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.356529:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.356533:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.356534:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.356537:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.356538:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.356539:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.356540:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.356541:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.356543:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.356544:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.356544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.356549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.356555:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.356557:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.356557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.356558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.356564:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.356570:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.356574:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677470.356575:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677470.356577:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.356580:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.356582:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677470.356584:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.356585:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.356586:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.356587:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677470.356590:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.356592:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677470.356596:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.356596:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:2.0:1619677470.356597:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:2.0:1619677470.356599:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:2.0:1619677470.356599:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.356601:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677470.356602:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677470.356603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677470.356608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.356614:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677470.356616:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677470.356617:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677470.356617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:18.0:1619677470.357013:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.357015:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.357016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.357017:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.357019:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.357022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.357179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.357184:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.357187:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.357188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.357189:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.357192:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.357194:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.357195:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.357197:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.357198:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.357199:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.357202:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.357204:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.357207:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.357207:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.357208:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.357210:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.357210:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.357212:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.357213:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.357213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.357218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.357224:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.357226:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.357227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.357228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.357308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677470.357864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.357870:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.357873:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.357874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.357876:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.357879:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.357880:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.357882:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.357884:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.357886:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.357887:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.357891:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.357893:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.357896:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.357897:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.357898:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.357900:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.357900:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.357901:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.357901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.357914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.357917:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.357918:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.357919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.357921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.357922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.359285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.359857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.359864:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.359866:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.359868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.359870:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.359872:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.359874:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.359875:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.359876:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.359877:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.359878:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.359882:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.359883:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.359888:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.359889:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.359890:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.359892:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.359893:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.359894:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.359894:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.360192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.360198:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.360201:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.360202:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.360204:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.360207:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.360209:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.360211:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.360212:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.360213:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.360214:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.360217:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.360218:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.360221:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.360222:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.360223:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.360225:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.360225:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.360226:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.360227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.360246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.360285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.361020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.361026:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.361030:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.361031:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.361032:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.361036:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.361037:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.361038:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.361040:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.361041:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.361042:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.361045:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.361046:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.361048:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.361049:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.361049:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.361051:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.361052:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.361053:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.361053:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.361064:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.361066:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.361067:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.361068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.361069:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.361070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.361177:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.361180:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.361181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.361183:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.361186:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.361188:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.361189:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.361190:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.361191:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.361192:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.361195:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.361197:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.361199:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.361200:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.361200:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.361202:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.361203:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.361203:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.361204:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:5.2:1619677470.361393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.361399:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.361402:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.361403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.361405:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.361408:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.361409:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.361411:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.361412:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.361413:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.361414:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.361417:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.361419:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.361422:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.361423:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.361423:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.361425:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.361425:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.361426:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.361426:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.361556:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.361559:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.361560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.361562:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.361564:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.361565:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.361567:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.361568:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.361569:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.361570:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.361573:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.361574:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.361577:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.361577:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:1.0:1619677470.361578:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:1.0:1619677470.361579:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:1.0:1619677470.361580:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.361581:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677470.361582:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677470.361582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.361593:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677470.361595:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677470.361596:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.361597:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:2.0:1619677470.362398:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.362400:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.362401:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.362402:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.362404:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.362407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.362416:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.362420:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.362421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (24)--
00000800:00000200:18.0:1619677470.362422:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (23)++
00000800:00000200:18.0:1619677470.362423:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.362424:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.362434:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.362437:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.362438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.362438:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.362440:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.362443:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.362521:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc6400] (21)++
00000800:00000200:1.0:1619677470.362524:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.42@o2ib3
00000800:00000200:17.0:1619677470.362525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc6400] (22)--
00000400:00000200:1.0:1619677470.362527:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.42@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.362530:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.362531:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677470.362533:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.362534:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.362535:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.362536:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677470.362539:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.42@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.362541:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677470.362544:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e82fe5580] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.362545:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c000] (20)++
00000800:00000200:1.0:1619677470.362546:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c000] (21)++
00000800:00000200:1.0:1619677470.362548:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c000] (22)--
00000800:00000200:1.0:1619677470.362548:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc6400] (21)++
00000800:00000200:1.0:1619677470.362549:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc6400] (22)--
00000800:00000200:1.0:1619677470.362550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc6400] (21)--
00000800:00000200:20.2:1619677470.362653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.362694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677470.362782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.362787:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.362790:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.362791:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.362793:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.362796:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.362797:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.362799:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.362800:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.362801:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.362802:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.362805:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.362807:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.362810:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.362811:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.362811:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.362813:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.362814:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.362814:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.362815:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.363245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677470.363251:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677470.363254:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677470.363255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677470.363257:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677470.363260:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.363261:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677470.363262:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.363263:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.363264:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.363265:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677470.363268:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677470.363269:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.363270:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.363271:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:1.0:1619677470.363271:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.363272:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.363273:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677470.363274:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677470.363274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:17.0:1619677470.363286:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.363289:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.363289:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.363290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.363291:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677470.363299:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.364619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.364661:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.364664:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.364665:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.364665:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.364667:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.364669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.364679:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.364682:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.364683:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.364683:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.364684:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.364684:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.364785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.364792:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.364795:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.364796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.364797:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.364799:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.364801:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.364803:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.364804:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.364805:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.364806:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.364810:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.364811:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.364814:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.364815:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.364816:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.364817:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.364818:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.364818:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.364819:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677470.364864:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.364867:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677470.364869:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : GET - for me
00000400:00000200:12.0:1619677470.364872:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.103@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:13.0:1619677470.364873:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677470.364874:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.103@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677470.364876:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.364876:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677470.364878:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:12.0:1619677470.364882:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677470.364884:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.364886:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.364887:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.364888:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.364889:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.364890:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.364903:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.364906:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.364907:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.364910:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:12.0:1619677470.364910:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.364913:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.365126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.365132:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.365135:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.365136:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.365138:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.365141:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.365142:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.365144:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.365145:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.365146:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.365147:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.365150:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.365152:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.365155:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.365156:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.365157:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.365159:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.365159:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.365161:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.365162:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.365162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.365167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.365173:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.365175:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.365175:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.365176:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677470.365408:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.365411:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.365412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.365412:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.365414:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.365417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.365642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.365686:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.365688:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.365689:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.365689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.365691:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.365694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.366299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.366305:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.366308:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.366309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.366310:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.366312:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.366314:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.366315:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.366316:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.366317:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.366318:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.366321:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.366323:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.366326:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.366327:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.366327:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.366328:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.366328:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000400:00000200:17.0:1619677470.366329:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677470.366330:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.366331:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:17.0:1619677470.366332:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.366333:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677470.366334:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:1.0:1619677470.366334:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.366334:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.366335:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677470.366335:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.366336:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.366337:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.366338:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.2:1619677470.366340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:17.0:1619677470.366341:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.366343:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677470.366346:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.366346:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.366347:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:1.0:1619677470.366347:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.366348:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:1.0:1619677470.366348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.366349:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.366350:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:17.0:1619677470.366351:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.366352:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.366353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:2.0:1619677470.366555:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.366557:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.366558:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.366558:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.366561:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.366564:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.366800:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.366803:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.366804:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.366804:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.366807:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.366809:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.367077:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.367079:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.367080:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.367081:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.367083:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.367089:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.367090:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.367092:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.367093:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.367094:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.367094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.367095:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.367118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:18.0:1619677470.367165:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.367168:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.367169:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.367169:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.367171:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.367175:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.367685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.367691:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.367694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.367694:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677470.367696:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.367699:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.367701:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.367702:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.367703:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.367704:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.367705:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.367708:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.367710:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.367712:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.367713:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.367714:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.367715:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.367716:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.367717:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.367717:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677470.367846:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.367849:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.367850:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.367850:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.367853:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.367856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.367866:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:20.2:1619677470.367867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.367869:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.367870:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.367871:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.367871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.367872:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.367872:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.367874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.367874:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677470.367877:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.367881:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.367883:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.367883:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.367885:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.367886:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.367887:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.367891:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.367893:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.367896:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.367896:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.367897:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.367899:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.367900:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.367901:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.367901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.368013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.368719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.368726:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.368729:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.368730:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.368732:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.368736:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.368737:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.368739:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.368740:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.368741:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.368742:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.368745:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.368747:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.368749:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.368749:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.368750:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.368751:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.368752:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.368753:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.368753:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.368765:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.368767:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.368767:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.368768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.368769:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.368770:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:12.0:1619677470.369479:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.369482:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.369486:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.369488:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.369489:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.369492:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677470.369494:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96523170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.369497:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.369498:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.369500:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677470.369505:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.369508:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677470.369511:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.369512:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:12.0:1619677470.369513:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:12.0:1619677470.369516:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:12.0:1619677470.369517:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.369518:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.369519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677470.369531:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.369534:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.369535:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.369535:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.369538:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.369541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.369866:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.369869:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.369870:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.369870:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.369873:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.369878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.369879:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.369881:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.369882:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.369883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.369883:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.369884:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.370002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.370009:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.370012:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.370012:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.370014:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.370017:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.370018:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.370019:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.370021:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.370022:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.370023:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.370026:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.370027:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.370031:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.370031:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.370032:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.370034:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.370035:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.370037:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.370037:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.370310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.370355:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.370357:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.370358:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.370358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.370361:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.370363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.370954:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.370957:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.370958:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.370958:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.370960:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.370964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.371174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.371179:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.371182:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.371183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.371185:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.371187:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.371189:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.371190:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.371191:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.371192:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.371193:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.371197:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.371198:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.371201:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.371201:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.371202:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.371204:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.371204:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.371205:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.371205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.371983:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.371986:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.371987:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.371987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.371990:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.371994:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.372009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.372046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677470.372051:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.372054:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.372055:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.372055:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.372058:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.372060:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.372115:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.372118:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.372119:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.372119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.372121:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.372125:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.372156:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.372158:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677470.372159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677470.372162:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.372164:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.372165:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677470.372166:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.372168:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.372169:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.372170:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677470.372173:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.372175:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.372177:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.372178:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:17.0:1619677470.372178:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.372180:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.372180:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.372182:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677470.372183:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677470.372183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.372194:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677470.372196:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677470.372196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.372197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677470.372206:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.372207:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.372208:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.372209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.372210:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.372211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.372226:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.372229:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.372230:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.372230:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.372231:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.372231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.372556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.372562:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.372564:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.372565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.372566:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.372567:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.372568:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.372569:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.372570:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.372571:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.372572:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.372575:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.372576:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.372578:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.372579:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.372579:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.372581:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.372581:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.372582:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.372582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.372658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.372665:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.372668:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.372669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.372671:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.372673:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.372675:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.372677:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.372678:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.372679:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.372680:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.372683:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.372685:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.372688:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.372689:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.372689:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.372691:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.372691:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.372693:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.372694:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.372694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.372699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.372706:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.372708:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.372709:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.372710:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.372881:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.372883:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.372884:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.372884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.372887:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.372890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.373364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677470.375052:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.375055:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.375056:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.375057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.375058:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.375061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.375066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.375072:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.375074:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.375074:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.375075:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.375076:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.375077:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.375078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.375078:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677470.375079:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.375080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.375081:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.375083:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.375084:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.375085:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.375086:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.375087:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.375091:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.375092:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.375094:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.375095:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.375096:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.375097:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.375097:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.375098:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.375098:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.375110:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.375112:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.375113:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.375113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.375114:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.375116:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.375285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.375290:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.375293:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.375294:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.375302:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.375304:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.375306:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.375307:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.375308:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.375308:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.375309:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.375312:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.375314:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.375317:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.375317:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.375318:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.375320:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.375320:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.375321:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.375322:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.375580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.375586:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.375589:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.375590:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.375591:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.375594:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.375595:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.375596:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.375598:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.375598:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.375599:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.375602:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.375604:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.375606:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.375607:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.375608:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.375609:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.375610:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.375611:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.375612:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.375612:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.375617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.375623:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.375625:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.375626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.375627:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.375641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.376354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.376361:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.376365:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.376366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.376368:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.376372:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.376374:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.376375:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.376377:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.376378:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.376379:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.376382:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.376384:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.376388:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.376388:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:18.0:1619677470.376389:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:18.0:1619677470.376391:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:18.0:1619677470.376392:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.376392:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.376393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.376811:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.376814:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.376815:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.376816:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.376818:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.376821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.376952:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.376955:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.376956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.376956:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.376958:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.376961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.377220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.377226:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.377229:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.377230:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.377231:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.377234:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.377236:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.377237:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.377238:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.377239:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.377240:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.377244:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.377246:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.377249:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.377250:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.377251:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.377253:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.377253:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.377254:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.377255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.377269:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.377272:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.377273:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.377276:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.377278:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.377280:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.377467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:18.0:1619677470.377574:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.377577:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.377578:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.377578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.377580:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.377584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.378233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.378266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.378323:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.378325:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.378326:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.378326:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.378328:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.378332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.379114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.379119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.379122:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.379123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.379126:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.379129:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.379130:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.379132:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.379133:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.379134:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.379135:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.379138:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.379139:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:2.0:1619677470.379140:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677470.379141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.379142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.379143:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.379143:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000400:00000200:12.0:1619677470.379144:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.379144:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:13.0:1619677470.379146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.379146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:2.0:1619677470.379146:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.379146:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.379147:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.379148:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.379203:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.379205:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.379206:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.379207:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.379209:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.379212:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.379221:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.379223:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.379224:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.379224:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.379225:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.379226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.379695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.379702:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.379705:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.379705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.379707:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.379710:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.379711:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.379713:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.379714:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.379716:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.379716:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.379720:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.379721:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.379725:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.379725:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.379726:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.379728:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.379728:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.379729:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.379730:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.379771:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.379774:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.379775:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.379775:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.379778:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.379781:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.380208:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.380210:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.380211:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.380212:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.380213:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.380217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.380415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.380420:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.380423:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.380424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.380426:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.380428:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.380430:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.380431:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96523106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.380433:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.380434:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.380435:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.380436:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.380438:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:1.0:1619677470.380438:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677470.380439:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.380439:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.380440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.380442:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.380442:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.380443:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.380444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.380444:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.380445:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.380446:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.380446:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.380447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.380674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.380804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.380810:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.380813:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.380814:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.380816:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.380819:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.380820:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.380822:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.380824:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.380825:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.380826:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.380829:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.380831:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.380834:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.380835:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.380835:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.380837:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.380838:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.380838:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.380839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.2:1619677470.380926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677470.380938:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:18.0:1619677470.381030:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:13.0:1619677470.381032:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:18.0:1619677470.381033:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.381033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.381034:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.381034:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.381035:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.381037:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:13.0:1619677470.381038:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.381039:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.381040:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:18.0:1619677470.381041:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.381130:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.381132:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.381134:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.381134:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.381136:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.381141:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:2.0:1619677470.381141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:12.0:1619677470.381144:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.381145:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.381146:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.381148:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.381149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677470.381159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677470.381161:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677470.381162:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.381163:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677470.381163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.381165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:17.0:1619677470.381658:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.381661:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.381662:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.381662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.381664:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.381667:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.381730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.381735:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.381737:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.381738:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.381740:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.381742:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.381744:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.381745:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.381746:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.381747:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.381748:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.381751:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.381753:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.381756:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.381757:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.381758:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.381759:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.381760:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.381760:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.381761:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.381978:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.381981:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.381982:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.381982:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.381984:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.381987:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.382604:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.382607:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.382608:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.382608:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.382610:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.382615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.382617:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.382619:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.382619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.382620:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.382621:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.382622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:12.0:1619677470.383008:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.383010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.383011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.383012:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.383014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.383015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.383026:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.383029:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.383030:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.383031:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.383031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.383032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.383108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.383109:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.383110:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.383111:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.383112:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.383112:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.383113:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:13.0:1619677470.383114:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.383122:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.383123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.383125:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677470.383125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.383145:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.383147:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.383148:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.383149:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.383151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.383152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:2.0:1619677470.383158:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.383161:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.383162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.383163:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.383166:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.383169:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.383182:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.383185:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.383186:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.383186:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.383187:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.383188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.383218:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.383220:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.383221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.383221:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.383224:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.383235:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:24.2:1619677470.383261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677470.383279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:20.2:1619677470.383569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.383607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677470.383955:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.383958:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.383959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.383959:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.383962:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.383965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.384283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:17.0:1619677470.384337:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.384340:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.384341:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.384341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.384344:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.384349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.384351:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.384353:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.384354:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.384354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.384355:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.384356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.384558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.384565:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.384568:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.384569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.384570:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.384573:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.384575:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.384576:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.384578:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.384579:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.384580:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.384583:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.384585:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.384588:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.384589:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.384590:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.384591:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.384592:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.384594:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.384595:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.384595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.384600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.384606:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.384610:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.384611:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.384612:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.384801:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.384804:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.384805:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.384805:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.384807:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.384809:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.384831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.384837:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.384839:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.384840:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.384842:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.384845:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.384846:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.384848:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.384849:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.384850:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.384851:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.384854:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.384856:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.384859:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.384860:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.384861:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.384863:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.384864:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.384865:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.384866:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.384867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.384872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.384878:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.384880:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.384881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.384881:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.384960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.385001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677470.385305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.385311:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.385314:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.385315:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.385317:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.385319:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.385320:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.385322:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.385323:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.385324:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.385325:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.385328:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.385330:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.385333:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.385333:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.385334:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.385335:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.385336:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.385337:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.385338:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.385339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.385344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.385350:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.385352:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.385352:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.385353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.386087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.386638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.386644:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.386647:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.386648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.386649:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.386652:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.386653:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.386655:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.386656:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.386657:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.386658:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.386662:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.386663:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.386667:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.386668:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.386669:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.386670:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.386671:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.386671:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.386672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.386686:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.386688:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.386689:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.386691:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.386694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.386695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:18.0:1619677470.386762:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.386765:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.386766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.386767:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.386769:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.386773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.387133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.387139:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.387142:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.387143:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.387145:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.387148:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.387150:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.387151:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.387153:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.387154:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.387155:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.387159:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.387160:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.387163:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.387164:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.387164:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.387166:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.387167:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.387168:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.387169:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.387169:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.387175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.387181:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.387183:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.387184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.387185:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.387504:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.387507:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.387508:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.387508:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.387510:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.387514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.388020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677470.388062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:18.0:1619677470.388280:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.388283:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.388283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.388284:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.388286:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.388293:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.388300:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.388303:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.388304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.388305:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.388306:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.388307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.388379:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.388381:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.388382:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.388382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.388385:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.388388:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.388410:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.388413:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.388413:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.388414:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.388415:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.388416:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.388982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677470.389024:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.389027:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.389027:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.389028:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.389030:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.389032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.389232:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.389234:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.389236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.389236:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.389238:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.389241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.389493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.389499:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.389502:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.389502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.389504:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.389506:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.389508:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.389510:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.389511:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.389512:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.389513:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.389517:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.389519:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.389522:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.389523:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.389524:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.389526:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.389526:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.389527:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.389528:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.389669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.389675:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.389678:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.389679:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.389681:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.389683:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.389685:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.389686:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.389687:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.389689:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.389690:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.389693:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.389695:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.389697:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.389698:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.389699:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.389701:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.389701:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.389702:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.389702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.389752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.389757:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.389760:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.389761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.389764:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.389767:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.389768:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.389770:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.389771:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.389772:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.389773:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.389776:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.389778:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.389780:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.389781:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.389781:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.389783:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.389784:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.389785:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.389785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.389796:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.389799:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.389799:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.389800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.389801:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.389803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.390347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677470.390811:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.390814:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.390815:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.390817:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.390818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.390819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.391316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.391322:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.391325:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.391326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.391328:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.391330:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.391331:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.391333:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.391334:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.391335:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.391336:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.391340:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.391341:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.391345:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.391345:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.391346:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.391348:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.391349:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.391349:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.391350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.391362:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.391365:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.391366:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.391368:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.391370:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.391371:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:2.0:1619677470.391548:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.391550:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.391552:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.391552:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.391554:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.391558:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.391934:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.391937:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.391937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.391939:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.391942:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.391944:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677470.391945:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96523009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.391947:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.391948:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.391949:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677470.391953:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.391954:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677470.391958:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.391959:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:18.0:1619677470.391959:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:18.0:1619677470.391961:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:18.0:1619677470.391962:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.391963:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.391964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:13.0:1619677470.392417:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.392420:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.392421:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.392422:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.392424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.392425:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:20.2:1619677470.392429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677470.392435:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677470.392438:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677470.392440:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.392440:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677470.392441:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.392442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:1.0:1619677470.392476:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.392479:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.392480:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.392480:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.392483:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.392486:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.392524:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.392527:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.392528:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.392529:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.392531:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.392537:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.392540:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.392542:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.392543:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.392543:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.392544:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.392544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.393200:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.393203:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.393204:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.393204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.393206:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.393210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.393398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.393405:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.393407:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.393408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.393410:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.393413:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.393414:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.393416:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.393417:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.393418:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.393419:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.393423:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.393424:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.393426:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.393427:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.393428:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.393429:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.393430:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.393431:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.393431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.393442:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.393444:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.393445:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.393445:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.393447:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.393448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.393837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.393843:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.393845:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.393846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.393848:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.393850:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.393852:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.393853:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.393855:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.393856:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.393857:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.393862:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.393865:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.393868:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677470.393869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.393869:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.393870:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.393872:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.393873:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.393874:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.393875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.393876:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.393879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677470.393879:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:2.0:1619677470.393881:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.393884:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.393886:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677470.393887:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96522994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.393888:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.393889:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.393891:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677470.393895:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.393897:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677470.393901:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.393902:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:2.0:1619677470.393903:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:2.0:1619677470.393905:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:2.0:1619677470.393905:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.393906:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677470.393907:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:12.0:1619677470.393919:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.393922:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.393923:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.393925:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.393927:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.393928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.394397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.394403:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.394406:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.394407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.394409:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677470.394411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000400:00000200:18.0:1619677470.394412:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.394413:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.394415:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.394416:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.394417:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.394418:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.394422:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.394424:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.394428:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.394429:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.394430:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.394432:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.394432:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.394433:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.394434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.394457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.394463:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.394466:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.394467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.394469:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.394472:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.394473:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.394474:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.394475:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.394476:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.394477:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.394480:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.394482:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.394485:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.394485:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.394486:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.394487:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.394488:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.394489:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.394490:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.394491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.394496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.394501:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.394503:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.394504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.394504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.395327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.395366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.395532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.395539:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.395541:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.395543:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.395545:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.395548:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.395550:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.395551:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.395552:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.395553:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.395555:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.395557:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.395559:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.395562:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.395563:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.395563:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.395565:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.395566:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.395567:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.395567:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677470.395888:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.395891:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.395892:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.395892:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.395895:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.395898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.395949:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.395952:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.395953:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.395953:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.395955:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.395958:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.395969:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.395972:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.395973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.395974:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.395975:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.395975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.396127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.396132:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.396135:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.396136:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.396137:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.396140:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.396141:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.396143:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.396144:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.396145:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.396146:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.396150:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.396152:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.396155:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.396156:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.396157:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.396158:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.396159:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.396159:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.396160:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.396172:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.396175:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.396176:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.396178:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.396179:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.396181:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.396190:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.396193:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.396194:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.396195:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.396195:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.396196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.398400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.398446:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.398448:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.398449:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.398449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.398452:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.398454:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.398502:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.398505:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.398505:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.398507:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.398508:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.398509:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.398511:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.398511:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:18.0:1619677470.398513:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677470.398514:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.398516:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.398517:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.398518:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677470.398523:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.398525:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677470.398526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.398527:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677470.398528:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.398530:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.398531:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:18.0:1619677470.398532:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:18.0:1619677470.398533:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:18.0:1619677470.398534:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.398536:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677470.398537:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677470.398537:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677470.398549:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677470.398551:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677470.398552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677470.398553:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677470.399017:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.399020:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.399021:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.399021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.399023:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.399027:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.399278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.399284:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.399286:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.399287:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.399289:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.399292:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.399299:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.399300:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.399301:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.399302:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.399303:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.399306:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.399308:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.399310:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.399311:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.399312:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.399313:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.399314:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.399315:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.399316:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.399319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.399325:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.399328:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.399330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.399331:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.399334:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.399336:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.399338:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.399340:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.399341:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.399342:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.399345:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.399346:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.399350:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.399350:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.399351:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.399353:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.399353:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.399354:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.399355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.399776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677470.400071:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.400074:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.400074:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.400075:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.400077:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.400080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.400104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.400109:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.400112:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.400113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.400115:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.400118:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.400120:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.400121:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.400123:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.400124:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.400125:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.400129:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.400130:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.400133:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.400134:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.400134:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.400136:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.400137:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.400139:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.400140:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.400140:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.400145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.400151:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.400153:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.400154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.400155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.400568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.400848:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.400851:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.400852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.400852:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.400854:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.400857:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.400954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.400960:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.400963:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.400963:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.400965:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.400967:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.400968:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.400969:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.400971:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.400972:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.400973:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.400975:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.400977:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.400980:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.400981:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.400982:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.400984:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.400984:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.400984:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.400985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.400998:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.401000:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.401001:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.401003:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.401005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.401006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.401039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:18.0:1619677470.401085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.401088:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.401090:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.401090:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.401092:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.401097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.401099:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.401101:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.401102:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.401102:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.401104:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.401104:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.402034:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.402036:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.402038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.402038:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.402040:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.402043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.402157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.403886:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.403889:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.403890:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.403891:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.403893:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.403896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.404165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.404171:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.404174:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.404175:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.404177:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.404179:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.404181:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.404182:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.404183:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.404184:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.404185:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.404188:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.404190:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.404192:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.404193:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.404194:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.404195:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.404196:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.404197:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.404198:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.404199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.404204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:5.2:1619677470.404208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677470.404209:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.404211:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.404212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.404213:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677470.404214:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.404216:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.404217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.404219:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.404222:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.404223:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.404225:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.404227:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.404228:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.404229:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.404233:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.404234:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.404238:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.404238:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.404239:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.404241:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.404242:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.404243:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.404244:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.404245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.404250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.404256:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.404258:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.404259:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.404260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.404823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.404829:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.404831:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.404833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.404835:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.404838:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.404840:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.404841:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.404842:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.404843:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.404844:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.404848:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.404850:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.404852:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.404853:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.404854:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.404855:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.404856:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.404857:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.404858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.404869:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.404871:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.404872:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.404872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.404874:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.404875:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.404888:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.404891:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.404892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.404892:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.404893:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.404894:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.405124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.405166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677470.405621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.405627:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.405630:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.405630:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.405632:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.405634:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.405636:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.405637:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.405639:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.405640:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.405641:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.405643:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.405645:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.405648:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.405649:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.405649:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.405651:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.405652:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.405653:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.405653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677470.405734:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.405737:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.405738:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.405738:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.405740:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.405745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.405747:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.405749:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.405750:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.405751:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.405752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.405752:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.405760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:20.2:1619677470.405764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.405766:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.405768:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677470.405770:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677470.405772:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.405775:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.405776:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677470.405778:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.405779:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.405781:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.405782:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677470.405786:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.405790:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677470.405793:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.405794:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:2.0:1619677470.405794:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:2.0:1619677470.405796:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:2.0:1619677470.405796:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.405797:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677470.405798:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677470.405809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.405809:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.405812:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.405813:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.405813:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.405815:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.405817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.405825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.405831:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.405834:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.405835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.405836:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.405839:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.405840:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.405842:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.405843:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.405844:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.405845:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.405849:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.405851:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.405854:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.405855:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.405855:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.405857:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.405858:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.405858:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.405859:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.405881:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.405884:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.405885:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.405886:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.405889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.405890:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:18.0:1619677470.406298:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.406301:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.406302:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.406303:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.406305:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.406308:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.406396:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.406399:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.406400:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.406400:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.406402:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.406405:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.406723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.406772:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.406775:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.406776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.406776:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.406778:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.406782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.406862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.406867:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.406870:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.406871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.406873:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.406875:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.406877:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.406878:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.406880:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.406881:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.406882:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.406885:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.406887:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.406889:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.406890:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.406890:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.406892:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.406893:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.406894:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.406894:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.406905:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.406907:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.406908:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.406909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.406910:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.406911:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.407402:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.407405:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.407406:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.407406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.407408:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.407412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.408578:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.408581:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.408582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.408582:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.408584:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.408588:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.408597:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.408599:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.408600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.408601:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.408602:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.408602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.408773:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.408775:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.408776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.408777:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.408779:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.408782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.409004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.409013:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.409016:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.409017:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.409019:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.409021:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.409023:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.409024:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.409025:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.409026:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.409028:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.409031:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.409033:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.409036:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.409037:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.409037:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.409039:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.409039:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.409040:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.409040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.409111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.409117:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.409120:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.409121:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.409123:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.409126:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.409128:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.409129:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.409131:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.409132:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.409133:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.409137:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.409138:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.409141:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.409142:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.409143:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.409145:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.409146:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.409147:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.409147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.409359:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.409362:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.409362:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.409365:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.409367:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.409369:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677470.409370:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.409372:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.409373:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.409374:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677470.409377:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.409378:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677470.409381:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.409382:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:17.0:1619677470.409383:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:17.0:1619677470.409384:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:17.0:1619677470.409385:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.409386:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.409386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677470.409697:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.409700:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.409701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.409701:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.409703:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.409707:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.410487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677470.410663:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.410666:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.410667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.410668:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.410670:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.410673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.411071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.411076:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.411079:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.411080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.411082:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.411084:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.411085:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.411087:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.411088:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.411089:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.411090:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.411093:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.411094:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.411097:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.411098:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.411099:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.411101:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.411101:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.411102:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.411103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.411115:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.411118:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.411119:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.411120:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.411124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.411125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.411194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.411575:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.411577:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.411579:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.411579:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.411581:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.411585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.412503:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.412505:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.412506:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.412507:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.412509:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.412512:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.412524:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.412526:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.412527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.412527:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.412528:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.412529:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.412820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.412827:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.412829:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.412830:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.412833:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.412836:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.412837:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.412839:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.412841:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.412843:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.412844:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.412849:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.412853:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.412854:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.412855:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.412856:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.412858:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.412859:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.412860:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.412860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.412872:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.412874:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.412875:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.412876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.412877:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.412880:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.412906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.412952:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.412954:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.412955:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.412955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.412957:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.412960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.413027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.413033:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.413036:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.413036:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.413039:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.413041:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.413042:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.413044:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.413046:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.413048:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.413049:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.413054:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.413056:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.413059:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.413060:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:18.0:1619677470.413061:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:18.0:1619677470.413063:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:18.0:1619677470.413064:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.413065:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.413065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.413074:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:13.0:1619677470.413076:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:1.0:1619677470.413076:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.413077:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.413078:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.413079:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:1.0:1619677470.413080:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:13.0:1619677470.413081:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677470.413083:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:1.0:1619677470.413083:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000400:00000200:13.0:1619677470.413084:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.413085:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677470.413086:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96522888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.413088:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.413089:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.413090:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677470.413094:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.413096:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677470.413098:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.413099:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:13.0:1619677470.413100:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:13.0:1619677470.413101:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:13.0:1619677470.413102:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.413103:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.413103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677470.413397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.413403:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.413406:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.413407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.413409:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.413411:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.413413:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.413414:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.413415:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.413417:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.413417:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.413420:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.413422:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.413424:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.413425:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.413426:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.413428:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.413428:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.413430:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.413431:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.413431:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.413436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.413441:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.413443:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.413444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.413445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.413790:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.413792:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.413793:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.413794:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.413797:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.413799:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.413809:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.413812:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.413813:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.413813:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.413814:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.413814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.414005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.414012:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.414015:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.414016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.414018:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.414021:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.414023:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.414024:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.414025:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.414027:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.414027:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.414031:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.414033:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.414036:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.414037:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.414037:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.414039:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.414040:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.414041:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.414041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.414250:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.414253:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.414253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.414254:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.414256:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.414261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.415081:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.415084:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.415085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.415088:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.415090:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.415091:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.415093:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.415094:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.415096:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.415097:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.415101:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.415104:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.415107:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.415107:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.415108:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.415110:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.415111:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.415111:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.415112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677470.415563:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.415567:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.415568:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.415568:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.415570:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.415576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.415577:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.415579:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.415580:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.415581:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.415581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.415582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.415872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677470.415873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.415880:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.415882:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.415883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.415885:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.415887:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.415889:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.415891:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.415893:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.415894:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.415896:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.415900:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.415903:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.415906:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.415907:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.415907:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.415909:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.415910:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.415910:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.415911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.415924:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.415927:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.415928:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.415929:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.415931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.415932:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.415942:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.415945:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.415945:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.415946:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.415946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.415948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.416608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.416646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677470.417033:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.417036:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.417037:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.417038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.417039:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.417043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.417053:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.417056:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.417056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.417057:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.417058:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.417058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.417621:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.417624:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.417625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.417625:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.417627:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.417631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.418136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.418142:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.418145:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.418146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.418148:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.418151:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.418152:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.418154:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.418155:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.418156:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.418159:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.418162:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.418164:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.418167:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.418168:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.418168:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.418170:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.418171:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.418171:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.418172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.418718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.418725:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.418727:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.418728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.418730:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.418733:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.418734:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.418736:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.418737:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.418738:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.418739:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.418742:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.418743:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.418745:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.418746:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.418747:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.418748:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.418749:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.418750:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.418752:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.418752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.418757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.418763:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.418765:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.418765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.418766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.418785:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.418786:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.418787:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.418788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.418789:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.418790:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.418904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.418910:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.418912:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.418913:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.418915:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.418917:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.418919:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.418920:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.418921:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.418922:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.418923:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.418926:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.418928:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.418932:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.418933:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.418933:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.418935:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.418936:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.418937:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.418937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.418939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.418982:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.418985:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.418986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.418986:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.418988:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.418991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.419189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.419194:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.419197:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.419198:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.419200:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.419203:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.419204:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.419206:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.419207:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.419208:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.419209:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.419212:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.419214:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.419215:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.419216:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.419216:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.419218:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.419218:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.419220:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.419221:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.419221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.419227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.419232:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.419234:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.419235:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.419236:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.419255:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.419257:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.419258:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.419258:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.419260:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.419262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.419812:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.419815:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677470.419816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677470.419818:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.419820:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.419821:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677470.419823:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.419824:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.419825:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.419826:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677470.419829:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.419831:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677470.419833:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.419834:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:17.0:1619677470.419834:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:17.0:1619677470.419836:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:17.0:1619677470.419837:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.419837:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677470.419838:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:1.0:1619677470.419844:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.419846:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677470.419847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677470.419848:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677470.419850:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000400:00000200:1.0:1619677470.419850:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.419851:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677470.419852:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:1.0:1619677470.419852:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.419853:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.419854:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:1.0:1619677470.419854:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.419855:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677470.419856:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.419856:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.419858:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:17.0:1619677470.419859:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000400:00000200:1.0:1619677470.419861:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.419862:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677470.419863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000400:00000200:17.0:1619677470.419864:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677470.419865:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677470.419867:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.419869:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677470.419869:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000400:00000200:17.0:1619677470.419870:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677470.419870:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.419871:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.419872:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.419873:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.419873:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677470.419874:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)++
00000800:00000200:1.0:1619677470.419876:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (24)++
00000800:00000200:1.0:1619677470.419877:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (25)--
00000400:00000200:17.0:1619677470.419878:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.419880:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677470.419880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (24)--
00000800:00000200:1.0:1619677470.419881:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (23)++
00000800:00000200:1.0:1619677470.419882:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677470.419883:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.419884:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:17.0:1619677470.419885:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000400:00000200:1.0:1619677470.419885:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677470.419888:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:17.0:1619677470.419888:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (24)++
00000400:00000200:1.0:1619677470.419888:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.419889:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (25)--
00000800:00000200:17.0:1619677470.419889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (24)--
00000400:00000200:1.0:1619677470.419890:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677470.419891:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.419892:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.419892:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.419892:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96522847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.419894:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677470.419895:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.419896:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.419897:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.419899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.419900:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (23)++
00000800:00000200:17.0:1619677470.419901:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (24)--
00000800:00000200:17.0:1619677470.419901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:17.0:1619677470.419903:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.419904:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000400:00000200:1.0:1619677470.419904:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.419906:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677470.419907:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.419909:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677470.419910:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677470.419911:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.419911:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.419912:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000400:00000200:17.0:1619677470.419913:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677470.419914:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000400:00000200:17.0:1619677470.419915:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96522844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.419916:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.419918:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.419920:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.419922:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)++
00000800:00000200:1.0:1619677470.419923:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (24)--
00000400:00000200:17.0:1619677470.419924:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.419924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677470.419925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677470.419927:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.419929:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.419930:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (23)++
00000800:00000200:17.0:1619677470.419931:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (24)++
00000800:00000200:17.0:1619677470.419933:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (24)--
00000800:00000200:17.0:1619677470.419934:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.419935:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677470.419938:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677470.419940:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.419941:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677470.419942:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677470.419944:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.419946:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.419948:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677470.419950:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96522843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.419951:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.419952:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.419953:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677470.419956:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.419958:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.419959:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.419960:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:17.0:1619677470.419960:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.419962:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.419962:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.419963:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:17.0:1619677470.419964:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:17.0:1619677470.419965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677470.419979:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.419980:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677470.419980:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.419981:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000400:00000200:1.0:1619677470.419981:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.419982:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677470.419983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.419984:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000400:00000200:1.0:1619677470.419984:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.419986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.420007:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.420011:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677470.420012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677470.420014:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.420016:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.420017:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677470.420019:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.420020:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.420021:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.420022:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677470.420026:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.420028:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677470.420031:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.420032:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:17.0:1619677470.420032:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:17.0:1619677470.420034:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:17.0:1619677470.420035:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.420035:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677470.420036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:18.0:1619677470.420342:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.420345:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.420345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.420346:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.420348:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.420351:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.420365:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.420368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.420368:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.420372:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.420373:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.420373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.420497:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.420500:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.420501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.420501:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.420503:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.420507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.420702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.420708:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.420710:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.420711:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.420714:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.420717:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.420718:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.420720:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.420721:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.420722:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.420723:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.420726:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.420729:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.420732:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.420733:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.420733:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.420735:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.420736:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.420737:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.420737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.420748:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.420751:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.420752:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.420754:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.420755:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.420756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.421240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677470.421390:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.421393:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.421394:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.421395:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.421397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.421398:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:1.0:1619677470.421699:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.421701:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.421702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.421703:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.421705:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.421708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.421719:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.421722:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.421723:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.421723:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.421724:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.421725:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:24.2:1619677470.422003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:18.0:1619677470.422117:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.422120:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.422121:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.422121:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.422123:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.422126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.422175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.422193:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.422196:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.422197:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.422198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.422200:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.422208:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:12.0:1619677470.422503:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.422506:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.422506:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.422508:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.422510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.422511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.422900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.422907:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.422910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.422910:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677470.422913:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.422915:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.422917:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.422918:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.422919:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.422920:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.422921:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.422924:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.422926:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.422929:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.422929:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.422930:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.422932:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.422932:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.422934:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.422935:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.422935:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.422940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.422946:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.422948:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.422948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.422949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.423122:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.423125:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677470.423126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677470.423128:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677470.423130:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.423132:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.423133:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.423134:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.423135:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.423136:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.423139:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677470.423141:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.423143:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.423143:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:1.0:1619677470.423144:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:1.0:1619677470.423146:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:1.0:1619677470.423146:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.423147:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677470.423148:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:18.0:1619677470.423524:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.423527:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.423528:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.423528:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.423530:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.423533:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.423721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.423774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.423780:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.423783:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.423784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.423786:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.423788:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.423790:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.423791:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.423793:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.423794:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.423795:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.423798:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.423800:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.423804:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.423805:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.423805:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.423807:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.423808:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.423809:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.423810:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.423811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.423816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.423822:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.423824:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.423825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.423826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.424157:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.424160:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.424161:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.424161:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.424164:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.424167:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.424178:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.424181:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.424182:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.424182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.424183:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.424184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.424318:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.424321:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.424322:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.424322:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.424324:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.424327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:17.0:1619677470.424914:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.424917:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.424918:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.424918:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.424920:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.424924:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:18.0:1619677470.425208:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.425211:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.425212:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.425212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.425214:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.425217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.425413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.425450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677470.425457:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.425459:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.425460:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.425460:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.425462:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.425465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.425531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.425536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.425539:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.425540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.425542:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.425544:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.425545:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.425546:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.425547:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.425548:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.425550:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.425553:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.425554:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.425558:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.425558:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.425559:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.425561:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.425561:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.425562:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.425562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.425577:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.425581:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.425582:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.425585:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.425585:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.425588:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.425862:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.425865:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.425866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.425866:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.425868:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.425872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.426098:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.426101:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.426102:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.426104:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.426106:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.426108:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.426109:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.426111:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.426112:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.426113:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.426116:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.426118:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.426121:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.426121:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.426122:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.426124:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.426124:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.426125:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.426126:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.426127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.426138:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.426140:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.426141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.426142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677470.426391:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.426394:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.426395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.426395:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.426397:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.426401:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.427393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677470.427400:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.427403:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677470.427403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677470.427407:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677470.427409:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.427411:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677470.427413:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677470.427414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677470.427415:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.427417:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.427418:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677470.427422:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677470.427425:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677470.427429:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.427430:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:17.0:1619677470.427430:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:17.0:1619677470.427432:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:17.0:1619677470.427433:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.427433:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677470.427434:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:5.2:1619677470.427736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.427742:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.427744:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.427745:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.427748:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.427750:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.427752:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.427753:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.427755:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.427756:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.427757:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.427760:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.427762:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.427764:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.427765:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.427765:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.427767:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.427767:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.427768:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.427769:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.428302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.428309:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.428312:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.428313:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.428315:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.428318:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.428319:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.428320:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.428322:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.428323:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.428324:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.428327:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.428329:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.428331:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.428332:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.428332:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.428334:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.428335:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.428336:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.428337:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.428349:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.428351:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.428352:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.428353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.428354:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.428355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.428367:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.428369:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.428370:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.428370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.428371:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.428371:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.428398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.428441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677470.428904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.428910:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.428913:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.428913:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677470.428916:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.428917:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.428918:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.428920:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.428921:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.428922:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.428923:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.428927:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.428928:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.428930:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.428931:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.428932:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.428933:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.428933:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.428934:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.428935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.428946:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.428948:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.428949:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.428949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.428950:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.428952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.429337:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.429340:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.429340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.429341:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.429343:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.429346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.429357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.430166:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.430168:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.430170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.430170:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.430172:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.430176:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.430402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.430408:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.430411:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.430412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.430414:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.430417:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.430418:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.430419:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96522767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.430421:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.430421:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.430423:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.430426:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.430429:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.430433:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.430434:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.430435:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.430435:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.430437:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.430437:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:17.0:1619677470.430438:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.430438:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.430439:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.430440:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.430440:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.430441:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.430443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:12.0:1619677470.430451:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.430453:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.430454:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.430456:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.430458:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.430459:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.430469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.430471:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.430473:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.430473:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.430475:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.430475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.430731:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.430734:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.430735:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.430735:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.430737:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.430740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.430749:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.430752:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.430753:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.430753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.430754:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.430755:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.431180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.431219:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.431222:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.431223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.431223:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.431225:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.431228:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.431260:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.431262:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.431264:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.431264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.431266:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.431268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.431279:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.431282:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.431283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.431283:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.431284:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.431284:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:18.0:1619677470.432020:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.432023:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.432023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.432024:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.432026:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.432030:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.432197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.432203:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.432206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.432206:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677470.432209:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.432211:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.432213:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.432214:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.432215:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.432216:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.432217:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.432221:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.432222:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.432226:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.432227:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.432227:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.432229:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.432230:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.432231:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.432231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.432468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677470.432473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.432474:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.432477:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.432478:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.432480:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.432483:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.432485:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.432486:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.432488:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.432489:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.432490:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.432499:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.432503:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.432506:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.432507:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.432508:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.432509:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.432510:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.432512:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.432513:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.432514:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.432519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677470.432522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677470.432524:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.432526:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.432527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.432528:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677470.433803:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.433806:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.433807:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.433807:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.433809:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.433813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:24.2:1619677470.433881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677470.433898:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.433901:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677470.433902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677470.433904:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677470.433906:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.433908:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677470.433909:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.433911:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.433912:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.433913:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677470.433916:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677470.433918:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677470.433920:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.433921:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:17.0:1619677470.433922:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:17.0:1619677470.433923:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:17.0:1619677470.433924:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.433924:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677470.433925:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:20.2:1619677470.434124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677470.434359:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.434362:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.434364:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677470.434366:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.434366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.434367:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677470.434368:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96522744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.434370:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.434371:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.434372:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677470.434375:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.434376:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677470.434379:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.434380:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:13.0:1619677470.434381:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:13.0:1619677470.434382:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:13.0:1619677470.434383:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.434385:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677470.434386:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677470.434387:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:2.0:1619677470.434398:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:12.0:1619677470.434401:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:2.0:1619677470.434401:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.434402:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.434402:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.434403:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677470.434404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.434405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:2.0:1619677470.434405:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.434412:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.434414:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.434416:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.434416:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.434417:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.434418:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.434419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.434476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.434544:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.434547:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.434548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.434548:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.434550:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.434554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.434793:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.434796:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.434796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.434798:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.434800:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.434801:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677470.434803:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.434804:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.434805:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.434806:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677470.434808:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.434810:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677470.434813:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.434814:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:17.0:1619677470.434814:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:17.0:1619677470.434816:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:17.0:1619677470.434817:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.434817:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.434818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677470.434922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.434928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.434931:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.434932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.434934:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.434936:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.434938:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.434939:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.434941:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.434942:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.434943:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.434946:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.434948:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.434950:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.434951:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.434951:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.434953:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.434953:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.434954:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.434955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.434966:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.434969:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.434969:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.434970:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.434971:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.434972:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.435145:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.435148:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677470.435149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677470.435151:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677470.435153:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.435155:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677470.435156:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.435158:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.435159:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.435160:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677470.435163:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677470.435164:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677470.435167:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.435168:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:1.0:1619677470.435169:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:1.0:1619677470.435171:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:1.0:1619677470.435171:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.435173:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677470.435174:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677470.435174:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677470.435185:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:17.0:1619677470.435187:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:17.0:1619677470.435187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677470.435188:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:5.2:1619677470.435277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.435282:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.435284:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.435285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.435287:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.435290:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.435291:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.435292:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.435298:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.435299:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677470.435300:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000400:00000200:1.0:1619677470.435300:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:18.0:1619677470.435303:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:1.0:1619677470.435303:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677470.435304:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.435304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:1.0:1619677470.435305:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677470.435306:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.435308:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.435309:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.435309:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.435309:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.435311:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.435312:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.435313:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.435313:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.435326:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.435328:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.435329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.435331:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.435333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.435334:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.435624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.435630:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.435633:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.435634:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.435636:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.435638:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.435639:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.435641:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.435642:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.435643:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.435644:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.435647:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.435649:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.435651:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.435652:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:2.0:1619677470.435653:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:2.0:1619677470.435655:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:2.0:1619677470.435656:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.435656:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.435657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.435720:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.435723:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.435724:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.435724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.435727:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.435730:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.436886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.436930:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.436933:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.436934:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.436934:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.436936:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.436938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.436949:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.436951:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.436952:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.436952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.436953:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.436954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.437124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.437130:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.437133:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.437134:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.437137:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.437140:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.437142:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.437143:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.437145:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.437146:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.437147:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.437150:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.437152:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.437155:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.437156:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.437157:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.437158:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.437159:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.437160:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.437160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.437258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.437264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.437266:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.437267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.437269:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.437272:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.437273:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.437275:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.437276:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.437277:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.437278:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.437281:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.437283:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.437286:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.437286:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.437287:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.437289:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.437289:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.437290:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.437290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.437425:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.437428:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.437429:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.437429:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.437431:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.437435:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.437466:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.437468:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677470.437469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677470.437471:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677470.437473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.437475:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.437476:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.437477:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.437478:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.437479:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.437482:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677470.437483:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.437485:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.437486:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:1.0:1619677470.437487:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:1.0:1619677470.437488:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:1.0:1619677470.437489:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.437489:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677470.437490:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:17.0:1619677470.437767:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.437770:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.437771:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.437771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.437773:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.437777:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.438060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.438102:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.438104:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.438105:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.438106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.438108:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.438110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.438713:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.438716:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677470.438716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677470.438718:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677470.438721:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.438722:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677470.438724:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.438725:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.438727:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.438727:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677470.438731:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677470.438732:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677470.438735:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.438736:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:17.0:1619677470.438736:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:17.0:1619677470.438738:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:17.0:1619677470.438739:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.438739:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677470.438740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:1.0:1619677470.438826:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.438829:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.438830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.438830:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.438832:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.438835:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.439127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.439133:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.439136:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.439137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.439139:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.439142:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.439143:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.439145:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.439146:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.439147:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.439148:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.439151:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.439153:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.439155:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.439156:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:2.0:1619677470.439157:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:2.0:1619677470.439158:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:2.0:1619677470.439159:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.439160:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.439160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.439409:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.439411:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.439412:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.439412:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.439414:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.439417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.439567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39c200] (20)++
00000800:00000200:2.0:1619677470.439573:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c200] (21)++
00000800:00000200:2.0:1619677470.439575:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.7@o2ib3
00000800:00000200:18.0:1619677470.439576:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c200] (22)++
00000800:00000200:18.0:1619677470.439577:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.7@o2ib3
00000400:00000200:2.0:1619677470.439578:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.7@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.439579:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.7@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.439581:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.439582:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.439583:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677470.439585:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.439586:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.439588:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.439589:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677470.439592:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677470.439594:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677470.439594:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677470.439595:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.439598:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.439598:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.439600:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.439604:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.439606:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.439606:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.439608:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:2.0:1619677470.439608:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.439609:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.439609:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.439610:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.439611:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.439612:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.439612:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39c200] (23)++
00000800:00000200:18.0:1619677470.439613:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39c200] (24)++
00000800:00000200:2.0:1619677470.439613:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39c200] (24)--
00000800:00000200:2.0:1619677470.439613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (23)--
00000800:00000200:18.0:1619677470.439614:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39c200] (24)--
00000800:00000200:18.0:1619677470.439614:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (22)--
00000800:00000200:2.0:1619677470.439615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (21)--
00000800:00000200:18.0:1619677470.439627:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.439628:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.439629:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.439629:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.439630:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.439630:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.439631:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.439632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.439632:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.439633:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.439633:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:5.2:1619677470.439634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39c200] (20)++
00000800:00000200:2.0:1619677470.439638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.439641:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c200] (21)++
00000800:00000200:18.0:1619677470.439643:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.439643:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.7@o2ib3
00000800:00000200:18.0:1619677470.439644:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.439645:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000400:00000200:2.0:1619677470.439645:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.7@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677470.439646:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.439646:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.439647:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (22)--
00000400:00000200:2.0:1619677470.439647:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.439648:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000400:00000200:2.0:1619677470.439649:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:5.2:1619677470.439650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39c200] (21)++
00000400:00000200:2.0:1619677470.439650:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.439652:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.439653:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.439654:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677470.439657:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c200] (22)++
00000400:00000200:2.0:1619677470.439657:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.439658:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.7@o2ib3
00000800:00000200:2.0:1619677470.439659:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677470.439660:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.7@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.439661:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.439662:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677470.439663:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677470.439663:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000400:00000200:18.0:1619677470.439664:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677470.439664:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000400:00000200:18.0:1619677470.439666:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677470.439666:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000400:00000200:18.0:1619677470.439667:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.439667:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677470.439668:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39c200] (23)++
00000800:00000200:2.0:1619677470.439669:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39c200] (24)--
00000800:00000200:2.0:1619677470.439669:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (23)--
00000800:00000200:2.0:1619677470.439670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (22)--
00000400:00000200:18.0:1619677470.439671:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.439672:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677470.439675:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.439675:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:18.0:1619677470.439676:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (23)++
00000800:00000200:18.0:1619677470.439677:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (24)--
00000800:00000200:5.2:1619677470.439677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39c200] (21)++
00000800:00000200:18.0:1619677470.439678:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39c200] (22)++
00000800:00000200:18.0:1619677470.439680:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39c200] (23)++
00000800:00000200:17.0:1619677470.439680:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)++
00000800:00000200:18.0:1619677470.439682:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39c200] (24)--
00000800:00000200:18.0:1619677470.439682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (23)--
00000800:00000200:17.0:1619677470.439682:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (24)--
00000800:00000200:1.0:1619677470.439683:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)++
00000800:00000200:18.0:1619677470.439684:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c200] (22)++
00000400:00000200:17.0:1619677470.439684:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.439684:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (24)--
00000400:00000200:1.0:1619677470.439685:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.439686:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.439687:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.7@o2ib3
00000800:00000200:17.0:1619677470.439688:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000800:00000200:2.0:1619677470.439688:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c200] (23)++
00000400:00000200:1.0:1619677470.439688:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.439689:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:2.0:1619677470.439689:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39c200] (24)--
00000400:00000200:18.0:1619677470.439690:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.7@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677470.439690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (23)--
00000800:00000200:1.0:1619677470.439690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.439691:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677470.439691:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (22)--
00000800:00000200:17.0:1619677470.439692:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.439692:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.439693:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.439693:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000400:00000200:18.0:1619677470.439694:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.439696:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677470.439697:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.439698:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.439699:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.439700:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677470.439703:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.439704:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677470.439708:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.439708:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:18.0:1619677470.439709:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:18.0:1619677470.439711:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:18.0:1619677470.439711:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39c200] (21)++
00000800:00000200:18.0:1619677470.439712:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39c200] (22)--
00000800:00000200:18.0:1619677470.439713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (21)--
00000800:00000200:13.0:1619677470.439725:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.439727:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.439728:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.439731:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.439732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.439734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.439741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39c200] (20)++
00000800:00000200:2.0:1619677470.439747:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c200] (21)++
00000800:00000200:2.0:1619677470.439749:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.7@o2ib3
00000800:00000200:18.0:1619677470.439750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (22)--
00000400:00000200:2.0:1619677470.439752:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.7@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.439756:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.439758:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677470.439759:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.439761:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.439762:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.439763:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677470.439768:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.439770:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677470.439774:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe0d80d00] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.439775:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb719600] (20)++
00000800:00000200:2.0:1619677470.439775:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb719600] (21)++
00000800:00000200:2.0:1619677470.439778:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb719600] (22)--
00000800:00000200:2.0:1619677470.439778:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39c200] (21)++
00000800:00000200:2.0:1619677470.439779:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39c200] (22)--
00000800:00000200:2.0:1619677470.439780:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (21)--
00000800:00000200:1.0:1619677470.439940:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.439943:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677470.439944:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677470.439946:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677470.439948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677470.439948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.439950:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.439951:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.439952:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.439953:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.439954:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.439958:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.439960:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.439963:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.439964:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:1.0:1619677470.439965:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:1.0:1619677470.439966:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:1.0:1619677470.439967:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.439968:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677470.439968:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:20.2:1619677470.439984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677470.440116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.440121:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.440124:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.440125:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.440127:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.440129:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.440131:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.440132:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.440133:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.440134:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.440135:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.440139:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.440141:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.440144:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.440145:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.440146:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.440147:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.440148:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.440149:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.440149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.440160:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.440163:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.440164:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.440165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.440167:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.440169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.440221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.440227:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.440230:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.440231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.440233:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.440237:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.440238:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677470.440239:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.440241:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.440242:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.440243:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677470.440247:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.440250:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677470.440253:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e82fe5580] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.440253:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.440254:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c000] (20)++
00000800:00000200:2.0:1619677470.440254:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.440255:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c000] (21)++
00000400:00000200:2.0:1619677470.440255:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.440257:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c000] (22)--
00000400:00000200:2.0:1619677470.440257:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.440259:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.440260:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.440261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677470.440261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.440262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.440275:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.440278:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.440279:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.440279:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.440280:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.440280:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.440509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.440515:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.440517:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.440518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.440520:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.440523:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.440523:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.440525:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.440526:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.440527:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.440528:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.440531:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.440532:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.440534:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.440534:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.440535:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.440536:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.440536:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.440537:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.440538:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.440550:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.440552:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.440553:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.440554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.440555:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.440556:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.441064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677470.441066:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.441069:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.441070:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.441071:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.441073:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.441075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677470.441121:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677470.441123:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677470.441124:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.441125:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677470.441125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.441126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:1.0:1619677470.441140:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.441142:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.441143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.441146:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.441148:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.441150:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677470.441151:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.441153:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.441154:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.441155:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677470.441158:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.441160:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677470.441163:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.441164:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:1.0:1619677470.441165:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:1.0:1619677470.441167:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:1.0:1619677470.441167:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.441169:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677470.441170:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677470.441170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.441182:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677470.441184:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677470.441184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.441185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:12.0:1619677470.441596:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.441599:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.441599:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.441601:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.441603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.441604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.442086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:5.2:1619677470.442091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:1.0:1619677470.442092:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.442095:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.442096:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.442098:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677470.442100:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:1.0:1619677470.442101:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.442103:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.442103:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000400:00000200:1.0:1619677470.442103:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:2.0:1619677470.442104:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:1.0:1619677470.442104:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.442105:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.442106:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.442107:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.442107:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.442108:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677470.442108:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.442111:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.442112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:1.0:1619677470.442112:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677470.442113:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:2.0:1619677470.442113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.442114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.442114:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.442115:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.442117:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677470.442117:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.442117:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000400:00000200:18.0:1619677470.442118:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677470.442118:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000400:00000200:18.0:1619677470.442119:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677470.442120:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.442120:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.442121:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.442122:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:18.0:1619677470.442123:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.442123:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677470.442125:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.442128:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677470.442128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677470.442129:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.442130:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.442131:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.442132:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.442133:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677470.442133:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:18.0:1619677470.442134:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.442135:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.442135:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.442136:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.442267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.442273:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.442276:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.442277:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.442279:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.442281:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.442283:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.442284:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.442286:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.442287:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.442289:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.442301:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.442303:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.2:1619677470.442304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.442306:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.442307:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:2.0:1619677470.442307:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:2.0:1619677470.442309:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:2.0:1619677470.442310:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.442311:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.442312:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.442313:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677470.442316:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:20.2:1619677470.442318:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677470.442319:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.442319:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.442322:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.442323:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:18.0:1619677470.442324:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677470.442324:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:18.0:1619677470.442327:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.442328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.442329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.442341:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (24)++
00000800:00000200:1.0:1619677470.442344:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (24)--
00000400:00000200:1.0:1619677470.442345:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.442347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (23)++
00000800:00000200:17.0:1619677470.442347:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (24)--
00000400:00000200:1.0:1619677470.442348:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677470.442349:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.442352:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.442352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.442353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.442354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.442402:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.442405:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.442407:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.442407:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.442409:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.442412:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:13.0:1619677470.442446:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.442448:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.442449:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.442450:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.442453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.442454:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.442490:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.442493:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.442493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.442494:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.442496:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.442499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.442509:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.442512:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.442513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.442513:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.442514:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.442514:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.442540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:17.0:1619677470.442543:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.442546:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677470.442547:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677470.442550:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677470.442550:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000400:00000200:17.0:1619677470.442552:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.442554:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677470.442554:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:17.0:1619677470.442555:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.442555:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.442556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:17.0:1619677470.442557:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.442558:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677470.442559:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.442560:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.442562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000400:00000200:17.0:1619677470.442565:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677470.442568:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677470.442571:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.442572:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:17.0:1619677470.442572:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:17.0:1619677470.442574:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:17.0:1619677470.442574:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.442576:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:17.0:1619677470.442577:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:17.0:1619677470.442577:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677470.442588:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677470.442591:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677470.442592:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677470.442592:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:13.0:1619677470.442604:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.442606:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.442607:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.442608:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.442610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.442612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677470.442806:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.442808:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.442809:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.442810:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.442812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.442813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677470.442844:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677470.442847:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677470.442848:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.442848:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677470.442849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.442850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:18.0:1619677470.442983:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.442985:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.442986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.442987:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.442989:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.442991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.443561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39c200] (20)++
00000800:00000200:2.0:1619677470.443567:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c200] (21)++
00000800:00000200:2.0:1619677470.443570:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.7@o2ib3
00000800:00000200:18.0:1619677470.443571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (22)--
00000400:00000200:2.0:1619677470.443572:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.7@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.443575:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.443577:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.2:1619677470.443578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:2.0:1619677470.443578:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.443581:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.443582:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.443583:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677470.443584:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.443585:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677470.443588:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.443588:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677470.443590:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.443590:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:18.0:1619677470.443592:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.443593:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.443595:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.443595:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.443596:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.443600:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.443600:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.443601:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.443601:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:2.0:1619677470.443601:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:2.0:1619677470.443603:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:18.0:1619677470.443604:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.443604:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39c200] (21)++
00000800:00000200:18.0:1619677470.443605:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.443605:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39c200] (22)--
00000800:00000200:18.0:1619677470.443606:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.443606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c200] (21)--
00000800:00000200:2.0:1619677470.443607:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.443608:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.443608:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.443610:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.443611:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.443611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.443616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677470.443621:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:2.0:1619677470.443622:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:17.0:1619677470.443623:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.443624:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.443624:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.443624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677470.443625:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:2.0:1619677470.443626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:17.0:1619677470.443627:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.443630:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.443855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.443898:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.443901:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.443902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.443902:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.443904:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.443906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.444950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.444957:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.444959:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.444960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.444962:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.444964:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.444966:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.444967:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.444968:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.444969:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.444970:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.444974:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.444975:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.444979:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.444980:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.444980:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.444982:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.444982:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.444983:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.444984:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.444995:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.444998:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.444999:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.445001:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.445003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.445003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.445014:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.445017:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.445018:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.445019:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.445019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.445020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.445424:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.445427:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.445428:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.445428:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.445431:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.445434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.445658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.445843:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.445845:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.445846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.445847:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.445849:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.445852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.446552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.446558:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.446561:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.446562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.446564:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.446567:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.446568:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.446570:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.446571:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.446572:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.446573:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.446576:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.446578:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.2:1619677470.446580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677470.446581:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.446582:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:18.0:1619677470.446582:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:18.0:1619677470.446584:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:18.0:1619677470.446585:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.446586:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677470.446588:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677470.446588:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677470.446593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.446599:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677470.446601:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677470.446602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677470.446603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:1.0:1619677470.446676:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.446678:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.446679:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.446680:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.446682:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.446687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.446696:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.446699:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.446700:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.446700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.446701:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.446701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.446767:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.446770:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677470.446771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677470.446772:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677470.446775:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.446776:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677470.446778:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.446779:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.446780:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.446781:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677470.446784:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677470.446786:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677470.446789:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.446789:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:1.0:1619677470.446790:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:1.0:1619677470.446792:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:1.0:1619677470.446792:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.446793:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677470.446793:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:5.2:1619677470.447074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.447079:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.447082:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.447083:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.447085:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.447088:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.447089:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.447091:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.447092:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.447093:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.447094:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.447098:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.447100:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.447102:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.447103:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.447104:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.447106:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.447106:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.447107:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.447108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.447194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.447200:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.447203:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.447204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.447206:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.447209:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.447211:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.447212:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.447214:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.447215:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.447216:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.447219:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.447221:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.447224:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.447225:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.447225:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.447227:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.447228:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.447229:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.447230:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.447230:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.447235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.447241:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.447243:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.447244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.447245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.447643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.447681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677470.448347:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.448350:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.448351:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.448352:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.448354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.448355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:24.2:1619677470.448660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:20.2:1619677470.449515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.449523:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.449526:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.449526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.449529:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.449532:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.449533:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.449534:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.449536:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.449537:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.449538:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.449541:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.449542:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.449547:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.449548:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.449549:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.449551:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.449551:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.449552:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.449553:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.449564:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.449566:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.449566:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.449567:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.449568:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.449570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.449795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.449798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.449804:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.449807:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.449808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.449810:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.449812:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.449814:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.449816:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.449817:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.449819:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.449820:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.449825:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.449827:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.449831:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.449831:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:18.0:1619677470.449832:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:18.0:1619677470.449834:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:18.0:1619677470.449835:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.449835:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.449836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.449844:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.449847:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.449848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.449848:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.449850:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.449852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.449863:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.449866:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.449867:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.449867:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.449868:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.449869:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:24.2:1619677470.450175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677470.450247:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.450249:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.450250:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.450252:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.450254:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.450256:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677470.450257:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.450259:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.450260:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.450261:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677470.450264:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.450266:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.450268:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.450268:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:1.0:1619677470.450269:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.450271:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.450271:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.450272:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.450272:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677470.450285:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:5.2:1619677470.450286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.450287:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.450287:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.450289:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.450289:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.450290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.450291:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.450294:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.450299:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677470.450300:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.450302:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.450303:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.450304:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.450305:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.450306:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.450307:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.450310:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.450311:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.450314:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.450315:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.450315:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.450317:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.450317:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.450318:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.450318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.450331:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.450334:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.450335:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.450337:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.450339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.450340:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.450562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.450568:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.450571:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677470.450572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677470.450574:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.450577:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.450579:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.450580:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.450582:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.450583:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.450584:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.450587:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.450589:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.450592:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.450593:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.450593:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.450595:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.450595:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.450596:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677470.450597:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:18.0:1619677470.450641:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.450644:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.450645:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.450645:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.450647:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.450651:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.450948:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.450951:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.450952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.450952:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.450953:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.450954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.451212:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.451214:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.451215:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.451216:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000400:00000200:17.0:1619677470.451217:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.451218:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.451219:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.451220:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.451221:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.451222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.451223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.451224:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.451277:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.451280:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.451281:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.451281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.451284:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.451287:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.451502:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.451504:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.451505:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.451506:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.451508:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.451511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.451744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.451849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.451855:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.451858:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.451859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.451861:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.451863:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.451864:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.451866:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.451867:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.451868:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.451869:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.451872:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.451874:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.451876:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.451877:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.451878:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.451879:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.451880:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.451881:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.451882:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.451883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.451888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.451893:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.451896:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.451897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.451898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.451976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.452016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677470.452032:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.452034:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.452037:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.452039:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.452039:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677470.452041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:12.0:1619677470.452041:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677470.452042:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96522610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.452043:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.452044:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.452045:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677470.452047:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:12.0:1619677470.452048:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.452050:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677470.452050:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.452051:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.452052:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677470.452053:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.452054:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:12.0:1619677470.452054:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:12.0:1619677470.452056:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000400:00000200:2.0:1619677470.452056:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.452057:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.452057:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000400:00000200:2.0:1619677470.452057:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:12.0:1619677470.452058:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:2.0:1619677470.452058:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.452060:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.452061:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.452062:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.452065:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.452067:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.452069:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.452069:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.452070:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:1.0:1619677470.452070:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:2.0:1619677470.452072:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.452072:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.452073:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677470.452073:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.452074:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:2.0:1619677470.452074:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:1.0:1619677470.452074:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.452076:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.452080:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.452086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.452089:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.452091:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.452092:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.452092:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:18.0:1619677470.452094:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.452094:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.452096:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.452097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000400:00000200:2.0:1619677470.452097:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677470.452098:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.452099:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.452100:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.452102:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.452103:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.452104:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.452105:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.452108:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.452109:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.452113:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.452113:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.452114:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.452116:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.452116:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.452117:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.452117:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.452597:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.452600:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.452601:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.452601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.452603:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.452607:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.453681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:13.0:1619677470.453725:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.453727:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.453728:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.453730:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.453732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.453733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.454045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.454051:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.454054:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.454055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.454056:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.454060:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.454061:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.454062:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.454064:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.454065:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.454066:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.454070:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.454072:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.454075:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.454076:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.454077:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.454079:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.454080:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.454080:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.454081:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.454281:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.454284:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.454285:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.454285:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.454287:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.454291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.454569:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.454572:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.454573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.454573:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.454575:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.454579:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.454590:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.454593:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.454594:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.454594:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.454596:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.454596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.455168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.455174:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.455177:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.455178:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.455180:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.455182:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.455184:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.455185:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.455186:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.455187:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.455188:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.455192:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.455193:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.455196:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.455197:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.455198:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.455200:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.455201:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.455201:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.455202:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.455214:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.455216:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.455220:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.455221:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677470.455221:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.455224:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.455446:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.455449:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.455450:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.455452:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.455453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.455455:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:20.2:1619677470.455514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.455560:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.455563:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.455564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.455564:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.455566:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.455570:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.455581:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.455583:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.455584:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.455584:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.455586:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.455588:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.455624:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.455627:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.455628:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.455628:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.455629:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.455630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.456074:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.456077:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.456078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.456078:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.456081:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.456085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.456102:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.456105:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.456105:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.456106:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.456107:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.456108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.456626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.456632:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.456635:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.456636:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.456638:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.456640:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.456642:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.456643:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.456644:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.456646:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.456647:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.456650:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.456651:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.456654:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.456654:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.456655:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.456657:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.456658:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.456658:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.456659:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.456810:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.456812:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.456813:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.456815:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.456817:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.456818:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.457014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.457021:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.457024:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.457025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.457026:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.457030:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.457031:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.457033:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.457034:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.457035:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.457036:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.457040:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.457042:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.457045:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.457046:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.457047:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.457049:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.457049:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.457050:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.457051:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.457111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.457117:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.457119:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.457120:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.457122:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.457125:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.457126:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.457128:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.457130:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.457131:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.457132:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.457135:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.457137:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.457138:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.457139:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.457140:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.457141:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.457142:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.457143:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.457144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.457154:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.457157:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.457158:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.457158:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.457159:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.457161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.457398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677470.457458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.457461:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.457462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.457462:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.457464:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.457468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.457477:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.457480:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.457481:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.457481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.457482:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.457483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.457714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.457758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.457912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.457918:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.457921:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.457922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.457924:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.457925:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.457927:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677470.457928:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.457930:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.457931:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.457932:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677470.457935:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.457937:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677470.457943:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe0d80d00] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.457943:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb719600] (20)++
00000800:00000200:18.0:1619677470.457944:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb719600] (21)++
00000800:00000200:18.0:1619677470.457946:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb719600] (22)--
00000800:00000200:18.0:1619677470.457947:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.457947:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.457948:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677470.457978:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677470.457981:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677470.457982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677470.457984:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.457987:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.457988:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.457989:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.457991:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.457992:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.457993:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.457996:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.457997:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.457998:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.457999:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.457999:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.458000:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.458001:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677470.458001:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677470.458002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677470.458013:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.458015:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.458015:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.458016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.458017:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.458019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:24.2:1619677470.459371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677470.459397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:2.0:1619677470.459683:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.459685:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.459686:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.459687:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.459689:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.459692:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.459998:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.460000:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.460001:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.460002:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.460004:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.460007:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.460044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.460049:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.460052:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.460053:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.460054:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.460057:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.460058:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.460059:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.460061:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.460062:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.460063:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.460066:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.460068:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.460071:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.460072:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.460072:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.460074:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.460075:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.460076:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.460076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.460088:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.460091:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.460092:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.460093:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.460095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.460096:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.460104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:12.0:1619677470.460107:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:18.0:1619677470.460110:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:12.0:1619677470.460110:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.460111:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:18.0:1619677470.460112:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:12.0:1619677470.460112:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.460112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:2.0:1619677470.460113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:12.0:1619677470.460114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000400:00000200:18.0:1619677470.460115:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.460117:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.460118:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.460120:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.460121:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.460122:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.460123:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.460126:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.460128:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.460131:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.460132:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.460133:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.460134:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.460135:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.460136:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.460138:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.460138:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.460143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.460149:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.460151:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.460152:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.460153:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.460839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.460846:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.460849:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.460850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.460852:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.460855:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.460856:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.460858:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.460859:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.460860:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.460861:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.460864:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.460866:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.460868:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.460869:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.460870:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.460872:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.460872:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.460873:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.460874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.460885:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.460887:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.460888:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.460888:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.460890:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.460891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.460906:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.460908:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.460909:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.460909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.460910:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.460910:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.461108:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.461110:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.461111:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.461112:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.461114:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.461117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.461794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.461854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.461861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.461864:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.461865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.461867:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677470.461869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677470.461869:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.461871:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.461872:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.461873:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.461874:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677470.461875:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677470.461876:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677470.461878:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.461878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677470.461881:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677470.461882:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677470.461884:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:17.0:1619677470.461885:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.461886:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677470.461887:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.461888:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:17.0:1619677470.461888:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96522554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.461889:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677470.461889:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677470.461890:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.461890:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677470.461890:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.461891:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000400:00000200:18.0:1619677470.461892:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:17.0:1619677470.461892:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.461893:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000400:00000200:17.0:1619677470.461895:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.461895:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.461896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.461896:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:17.0:1619677470.461897:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.461897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.461898:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:17.0:1619677470.461899:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.461900:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.461901:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.461903:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.461903:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.461905:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.461906:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.461907:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.461911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.461917:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.461920:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.461920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.461921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.462770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677470.463325:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.463328:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.463329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.463330:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.463332:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.463335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.463714:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.463718:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.463719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.463721:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.463724:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.463725:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677470.463726:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.463728:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.463729:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.463730:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677470.463733:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.463734:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677470.463737:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.463738:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:18.0:1619677470.463738:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:18.0:1619677470.463740:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:18.0:1619677470.463741:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.463742:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.463742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:17.0:1619677470.463755:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.463757:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.463758:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.463758:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.463760:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.463762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.464786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.464828:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.464831:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.464832:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.464833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.464835:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.464838:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.465453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.465459:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.465462:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.465463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.465465:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.465467:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.465469:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.465470:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.465472:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.465473:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.465474:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.465477:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.465479:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.465482:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.465483:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.465484:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.465486:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.465486:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.465486:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.465487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.465501:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.465505:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.465506:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.465509:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.465510:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.465512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.465765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.465771:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.465774:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.465776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.465778:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.465780:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.465781:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.465782:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.465784:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.465786:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.465786:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.465789:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.465791:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.465793:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.465794:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.465794:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.465796:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.465797:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.465797:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.465798:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.465809:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.465811:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.465812:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.465812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.465813:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.465815:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:24.2:1619677470.466040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677470.466049:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.466052:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.466053:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.466053:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.466055:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.466061:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.466063:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.466065:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.466066:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.466066:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.466067:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.466067:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.466297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.466304:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.466307:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.466308:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.466309:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.466311:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.466313:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.466314:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.466316:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.466317:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.466318:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.466322:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.466324:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.466327:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.466328:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.466329:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.466331:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.466332:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.466333:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.466334:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.466334:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.466340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.466346:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.466348:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.466349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.466349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.466383:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.466385:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.466386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.466387:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.466389:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.466392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.466788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.466793:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.466796:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.466797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.466799:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.466801:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.466802:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000400:00000200:1.0:1619677470.466802:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.2:1619677470.466803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677470.466804:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.466805:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677470.466806:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:1.0:1619677470.466806:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677470.466807:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.466807:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.466807:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677470.466810:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677470.466812:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.466814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.466815:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.466818:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.466819:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.466820:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.466821:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.466822:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.466823:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.466823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.466835:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.466837:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.466838:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.466840:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.466841:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.466843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.467269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.467275:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.467278:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.467279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.467281:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.467284:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.467285:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.467287:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.467288:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.467289:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.467290:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.467299:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.467300:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.467302:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.467303:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.467304:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.467305:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.467306:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.467307:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.467307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.467320:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.467323:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.467323:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.467324:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.467325:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.467326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:24.2:1619677470.467878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:13.0:1619677470.468044:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.468047:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.468049:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677470.468051:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.468051:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.468052:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677470.468054:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96522523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.468055:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.468056:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.468057:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677470.468060:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.468061:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677470.468063:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.468064:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:13.0:1619677470.468065:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:13.0:1619677470.468066:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:13.0:1619677470.468067:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.468067:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.468068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:24.2:1619677470.468097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677470.468179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677470.468195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.2:1619677470.468280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677470.468420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:5.2:1619677470.469369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.469375:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.469378:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.469379:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.469381:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.469383:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.469385:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.469386:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.469387:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.469388:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.469389:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.469392:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.469394:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.469397:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.469398:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.469399:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.469400:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.469401:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.469403:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.469404:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.469405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.469410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.469416:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.469418:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.469418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.469420:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.469578:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.469581:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.469582:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.469582:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.469585:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.469588:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.469601:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.469604:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.469605:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.469605:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.469606:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.469606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.470240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.2:1619677470.470246:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677470.470352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.470358:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.470361:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.470362:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.470364:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.470367:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.470369:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.470370:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.470371:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.470372:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.470373:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.470377:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.470379:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.470382:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.470383:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.470383:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.470385:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.470385:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.470386:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.470386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.470398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.470401:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.470402:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.470404:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.470406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.470407:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.470791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.470798:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.470800:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.470801:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.470803:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.470806:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.470807:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.470809:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.470810:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.470811:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.470812:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.470816:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.470817:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.470820:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.470821:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.470822:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.470824:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.470824:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.470825:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.470826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.470902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.470941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677470.470947:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.470950:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.470951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.470951:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.470954:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.470958:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.470979:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.470982:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.470983:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.470984:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.470986:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.470989:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.471610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.471615:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.471618:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.471619:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.471621:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.471623:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.471624:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.471626:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.471627:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.471628:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.471629:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.471632:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.471634:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.471637:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.471637:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.471638:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.471640:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.471640:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.471642:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.471643:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.471643:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.471649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.471655:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.471657:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.471657:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.471658:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.471896:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.471899:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.471900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.471900:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.471902:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.471906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:18.0:1619677470.471997:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.472000:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.472000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.472001:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.472003:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.472007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.472132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.472178:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.472180:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.472181:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.472182:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.472184:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.472187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.472202:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.472204:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.472206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.472206:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.472208:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.472212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.472216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.472222:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.472225:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.472225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.472227:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.472229:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.472231:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.472232:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.472234:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.472236:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.472237:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.472241:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.472243:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.472246:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.472247:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.472248:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.472250:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.472251:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.472252:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.472252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677470.472566:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.472569:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:2.0:1619677470.472570:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.472571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.472573:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.472579:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.472581:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.472583:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.472583:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.472584:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.472585:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.472585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.472684:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.472687:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.472687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.472688:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.472690:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.472693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.472926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.472931:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.472934:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.472935:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.472937:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.472939:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.472940:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.472942:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.472943:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.472944:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.472945:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.472949:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.472950:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.472953:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.472954:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.472955:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.472957:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.472957:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.472958:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.472959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.472970:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.472972:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.472973:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.472974:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.472975:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.472977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.472986:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.472988:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.472989:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.472989:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.472990:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.472991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.473433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.2:1619677470.473473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:18.0:1619677470.473763:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.473766:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.473767:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.473768:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.473770:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.473773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.474249:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.474251:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.474252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.474254:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.474257:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.474259:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677470.474260:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.474261:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.474262:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.474263:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677470.474266:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.474268:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.474270:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.474271:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:1.0:1619677470.474271:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.474273:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.474273:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.474274:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.474275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677470.474287:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.474289:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.474289:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.474290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.474291:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.474292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.474307:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.474310:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.474310:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.474311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.474311:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.474312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.474456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.474463:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.474466:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.474467:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.474469:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.474471:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.474472:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.474474:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.474475:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.474477:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.474477:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.474480:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.474482:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.474485:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.474486:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.474487:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.474489:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.474490:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.474491:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.474491:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.474913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.474919:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.474922:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.474922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.474924:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.474927:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.474929:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.474930:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.474931:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.474932:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.474933:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.474936:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.474938:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.474940:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.474941:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.474941:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.474943:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.474944:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.474945:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.474946:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.474947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.474952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.474957:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.474958:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.474959:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.474959:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.474959:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.474960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.474961:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.474961:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.474962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.474962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.475607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.475658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:17.0:1619677470.475665:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.475668:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.475669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.475670:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.475672:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.475675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.475799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.475805:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.475808:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.475809:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.475811:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.475812:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.475813:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.475815:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.475816:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.475817:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.475818:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.475822:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.475824:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.475827:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.475827:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.475828:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.475830:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.475830:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.475831:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.475831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.475844:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.475847:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.475848:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.475850:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.475851:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.475852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.475861:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.475863:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.475864:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.475865:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.475865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.475866:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:18.0:1619677470.476046:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.476048:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.476049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.476050:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.476053:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.476056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:17.0:1619677470.476666:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.476669:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.476670:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.476671:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.476673:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.476676:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.476751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.476756:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.476759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.476759:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677470.476761:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.476764:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.476765:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.476766:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.476768:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.476769:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.476770:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.476773:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.476775:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.476778:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.476779:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.476779:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.476781:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.476782:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.476782:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.476783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.476795:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.476798:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.476799:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.476801:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.476803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.476803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.476915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.476919:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.476922:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:2.0:1619677470.476923:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.476924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.476926:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.476929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.477056:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.477059:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.477059:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.477060:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.477062:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.477064:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:17.0:1619677470.477685:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.477688:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.477688:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.477689:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.477691:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.477694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.477880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.477972:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.477975:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.477976:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.477976:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.477979:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.477982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.478116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.478121:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.478124:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.478124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.478126:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.478128:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.478130:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.478131:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.478132:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.478133:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.478134:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.478138:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.478140:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.478143:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.478144:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:18.0:1619677470.478145:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:18.0:1619677470.478147:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:18.0:1619677470.478148:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.478149:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.478149:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.479330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.479335:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.479339:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.479339:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677470.479342:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.479345:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.479346:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.479348:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.479349:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.479350:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.479351:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.479355:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.479356:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.479359:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.479360:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.479361:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.479363:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.479363:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.479364:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.479365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.479459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.479465:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.479467:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.479468:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.479470:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.479473:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.479474:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.479476:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.479477:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.479478:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.479479:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.479481:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.479483:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.479484:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.479485:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.479485:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.479487:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.479487:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.479488:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.479489:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.479500:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.479502:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.479503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.479503:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.479504:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.479506:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.479522:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.479524:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.479525:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.479525:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.479526:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.479526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.480026:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.480029:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.480030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.480031:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.480033:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.480036:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.480045:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.480048:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.480049:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.480049:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.480050:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.480050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.480336:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.480339:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.480340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.480340:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.480342:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.480345:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.480680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.480686:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.480689:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.480690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.480692:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.480695:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.480696:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.480697:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.480699:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.480700:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.480701:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.480704:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.480705:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.480708:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.480709:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.480710:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.480712:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.480712:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.480713:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.480713:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.480727:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.480731:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.480732:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.480735:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.480735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.480738:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.480923:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.480926:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.480927:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.480927:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.480929:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.480932:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.480944:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.480946:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.480947:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.480947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.480949:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.480949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.480987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677470.481725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.481731:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.481734:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.481735:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.481737:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.481740:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.481741:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.481742:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.481744:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.481745:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.481746:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.481749:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.481751:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.481754:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.481755:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.481755:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.481757:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.481757:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.481759:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.481760:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.481760:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.481765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.481771:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.481773:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.481773:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.481774:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.481996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.482011:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.482013:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.482015:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.482015:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.482017:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.482021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.482033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677470.482033:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.482036:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.482037:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.482037:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.482038:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.482039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.482393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.482399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.482402:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.482403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.482405:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.482407:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.482408:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.482410:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.482411:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.482412:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.482413:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.482416:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.482418:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.482421:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.482422:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.482422:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.482424:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.482425:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.482425:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.482426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.482543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.482549:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.482551:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.482552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.482555:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.482557:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.482558:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.482560:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.482562:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.482564:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.482564:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.482576:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.482579:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.482582:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.482583:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:18.0:1619677470.482584:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:18.0:1619677470.482586:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:18.0:1619677470.482586:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.482587:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.482588:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.482917:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.482920:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.482921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.482921:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.482923:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.482927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.483219:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.483221:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.483222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.483223:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.483225:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.483228:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.483418:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.483421:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.483421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.483422:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.483424:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.483427:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.483616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.483636:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3eb000] (21)++
00000800:00000200:18.0:1619677470.483639:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.9@o2ib3
00000800:00000200:2.0:1619677470.483639:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3eb000] (22)--
00000400:00000200:18.0:1619677470.483642:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.3.9@o2ib3 : GET - for me
00000400:00000200:18.0:1619677470.483645:0:19933:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.3.9@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:18.0:1619677470.483648:0:19933:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.3.9@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:18.0:1619677470.483649:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3eb000] (21)++
00000800:00000200:18.0:1619677470.483650:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3eb000] (22)--
00000400:00000200:18.0:1619677470.483652:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.3.9@o2ib3 local destination
00000400:00000200:18.0:1619677470.483656:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.3.9@o2ib3(10.51.3.9@o2ib3:10.51.3.9@o2ib3) : REPLY try# 0
00000800:00000200:18.0:1619677470.483658:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.3.9@o2ib3
00000800:00000200:18.0:1619677470.483661:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91906ffa8d00] -> 10.51.3.9@o2ib3 (2) version: 12
00000800:00000200:18.0:1619677470.483662:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91907a3eb000] (21)++
00000800:00000200:18.0:1619677470.483662:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907a3eb000] (22)++
00000800:00000200:18.0:1619677470.483664:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91907a3eb000] (23)--
00000800:00000200:18.0:1619677470.483665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3eb000] (22)--
00000800:00000200:2.0:1619677470.483675:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3eb000] (22)++
00000800:00000200:2.0:1619677470.483677:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907a3eb000] (23)--
00000400:00000200:2.0:1619677470.483678:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.483679:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3eb000] (22)--
00000400:00000200:2.0:1619677470.483680:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.3.9@o2ib3: REPLY: OK
00000800:00000200:2.0:1619677470.483683:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3eb000] (21)--
00000800:00000200:1.0:1619677470.483781:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.483785:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.483785:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677470.483788:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.483790:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.483791:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677470.483793:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.483794:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.483795:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.483796:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677470.483799:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.483802:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677470.483804:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.483805:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:1.0:1619677470.483805:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:1.0:1619677470.483807:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:1.0:1619677470.483808:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.483808:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.483809:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677470.483822:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.483825:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.483826:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.483826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.483828:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.483830:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.484250:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.484253:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.484254:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.484254:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.484256:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.484260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.484345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.484351:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.484353:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.484354:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.484356:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.484358:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.484360:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.484361:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.484363:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.484364:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.484365:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.484367:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.484369:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.484373:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.484373:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.484374:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.484376:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.484377:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.484378:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.484378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.485081:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.485083:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.485084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.485085:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.485087:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.485090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.485148:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.485151:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.485152:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.485153:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.485155:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.485157:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677470.485158:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.485159:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.485160:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.485161:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677470.485165:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.485167:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677470.485170:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.485170:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:1.0:1619677470.485171:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:1.0:1619677470.485173:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:1.0:1619677470.485173:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.485174:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.485174:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677470.485297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.485342:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.485345:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.485346:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.485346:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.485348:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.485350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.485852:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.485854:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.485855:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.485855:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.485857:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.485861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.486029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.486035:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.486038:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.486039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.486042:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.486045:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.486046:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.486048:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.486050:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.486051:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.486052:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.486055:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.486057:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.486058:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.486059:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.486060:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.486061:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.486062:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.486063:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.486063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677470.486068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677470.486074:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:17.0:1619677470.486074:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.486076:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677470.486077:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.486077:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.486077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677470.486078:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.486080:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677470.486080:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677470.486082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000400:00000200:17.0:1619677470.486083:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.486084:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.486086:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.486087:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.486088:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.486089:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.486092:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.486094:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.486097:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.486097:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.486098:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.486100:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.486100:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.486101:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.486102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.486115:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.486118:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.486120:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.486122:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.486124:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.486126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.486242:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.486245:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.486246:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.486248:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.486251:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.486252:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.486254:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.486255:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.486256:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.486257:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.486259:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.486261:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.486263:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.486264:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.486264:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.486266:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.486266:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.486267:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.486268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.486327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677470.486535:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.486538:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.486539:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.486539:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.486541:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.486545:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.486845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.486852:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.486855:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.486856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.486858:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.486860:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.486862:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.486863:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.486864:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.486865:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.486866:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.486870:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.486872:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.486875:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.486875:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.486876:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.486878:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.486878:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.486879:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.486879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.486891:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.486894:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.486895:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.486896:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.486899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.486900:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.486911:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.486914:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.486914:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.486915:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.486916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.486917:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:18.0:1619677470.486958:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.486961:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:5.2:1619677470.486961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000400:00000200:18.0:1619677470.486962:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.486962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.486964:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.486966:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:18.0:1619677470.486968:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.486969:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677470.486970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677470.486972:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677470.486975:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.486977:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677470.486978:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.486979:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.486980:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.486981:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677470.486985:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677470.486986:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677470.486990:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e82fe5580] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.486990:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c000] (20)++
00000800:00000200:1.0:1619677470.486991:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c000] (21)++
00000800:00000200:1.0:1619677470.486993:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c000] (22)--
00000800:00000200:1.0:1619677470.486993:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677470.486995:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677470.486996:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677470.486996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:5.2:1619677470.487003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.487009:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677470.487011:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677470.487012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677470.487013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:1.0:1619677470.487648:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.487650:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.487651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.487654:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.487657:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.487658:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677470.487660:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.487661:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.487662:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.487663:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677470.487667:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.487668:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677470.487672:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.487672:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:1.0:1619677470.487673:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:1.0:1619677470.487675:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:1.0:1619677470.487675:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.487676:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.487677:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677470.487689:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.487691:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.487692:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.487694:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.487696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.487697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677470.487707:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677470.487710:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677470.487711:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.487712:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677470.487712:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.487713:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:2.0:1619677470.487799:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.487802:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.487803:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.487804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.487805:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.487812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.487813:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.487815:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.487816:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.487816:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.487817:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.487818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.487984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.488284:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.488287:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.488288:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.488288:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.488290:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.488297:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.488344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.488349:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.488352:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.488353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.488354:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.488356:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.488357:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.488359:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.488360:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.488361:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.488362:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.488365:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.488367:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.488369:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.488370:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.488371:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.488372:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.488373:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.488374:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.488375:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.488376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.488381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.488387:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.488389:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.488389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.488390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.488635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.488640:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.488643:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.488644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.488645:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.488649:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.488650:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.488651:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.488653:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.488653:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.488654:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.488657:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.488658:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.488659:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.488660:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.488660:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.488662:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.488663:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.488664:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.488665:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.488665:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.488671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.488676:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.488677:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.488678:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.488678:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.488679:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.488679:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.488680:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.488680:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.488681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.488681:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.2:1619677470.489101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:20.2:1619677470.489129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.489169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677470.489274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.489279:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.489282:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.489283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.489285:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.489287:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.489289:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.489290:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.489291:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.489292:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.489297:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.489300:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.489302:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.489305:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.489306:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.489307:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.489309:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.489309:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.489311:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.489312:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.489312:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.489318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.489324:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.489326:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.489327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.489328:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.490000:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.490003:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.490004:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.490004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.490006:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.490009:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.490019:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.490021:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.490022:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.490022:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.490023:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.490024:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:13.0:1619677470.490071:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.490073:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.490074:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.490076:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.490078:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.490079:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:20.2:1619677470.490916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.490922:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.490925:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.490926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.490928:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.490931:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.490933:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.490934:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.490936:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.490936:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.490937:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.490941:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.490943:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.490946:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.490946:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.490947:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.490949:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.490950:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.490950:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.490951:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.490978:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.490981:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.490981:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.490982:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.490984:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.490986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.491008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.491014:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.491017:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.491018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.491020:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.491022:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.491023:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.491025:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.491026:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.491027:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.491028:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.491031:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.491033:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.491035:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.491036:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.491037:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.491038:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.491039:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.491039:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.491039:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.491177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677470.491713:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.491716:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.491716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.491717:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.491719:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.491722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.491877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.491919:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.491921:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.491922:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.491922:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.491924:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.491926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.492014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.492020:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.492023:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.492023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.492025:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.492027:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.492029:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.492030:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.492031:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.492032:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.492033:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.492037:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.492038:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.492041:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.492041:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.492042:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.492044:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.492044:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.492046:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.492047:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.492047:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.492052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.492058:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.492060:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.492061:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.492062:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677470.492148:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.492151:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.492153:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.492153:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.492155:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.492159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.492169:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.492171:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.492172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.492173:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.492173:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.492174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.492851:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.492854:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.492855:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.492855:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.492857:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.492860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.493232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.493542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.493549:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.493552:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.493553:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.493554:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.493557:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.493558:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.493560:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.493562:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.493562:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.493563:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.493566:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.493568:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.493570:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.493570:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.493571:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.493573:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.493573:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.493574:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.493575:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.493585:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.493587:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.493588:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.493589:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.493590:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.493592:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.493605:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.493608:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.493609:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.493609:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.493609:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.493610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.493641:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.493644:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.493645:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.493645:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.493648:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.493651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.494174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.494180:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.494183:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.494183:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.494186:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.494188:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.494189:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.494191:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.494192:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.494193:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.494194:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.494198:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.494199:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.494203:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.494204:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.494205:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.494206:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.494207:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.494208:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.494208:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.494405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.494633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.494639:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.494642:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.494643:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.494645:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.494648:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.494650:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.494651:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.494653:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.494654:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.494655:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.494659:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.494661:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.494664:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.494665:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.494666:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.494668:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.494668:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.494669:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.494670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.494941:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.494944:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.494945:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.494945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.494947:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.494950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.495894:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.495897:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.495898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.495898:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677470.495900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:1.0:1619677470.495900:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.495904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.495905:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.495906:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.495907:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677470.495909:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.495911:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.495913:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.495914:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.495916:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.495917:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677470.495918:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000400:00000200:1.0:1619677470.495918:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.495920:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.495921:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.495922:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.495922:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.495923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000400:00000200:1.0:1619677470.495923:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.495925:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.495927:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.495928:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.495929:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.495931:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.495931:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.495932:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.495932:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.495958:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677470.495960:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677470.495963:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.111@o2ib7 : GET - for me
00000800:00000200:13.0:1619677470.495965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:12.0:1619677470.495965:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.111@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677470.495967:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.111@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677470.495969:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677470.495969:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:12.0:1619677470.495971:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677470.495975:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677470.495977:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677470.495979:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.495980:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677470.495981:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.495982:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677470.495982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.495994:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.495996:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.495997:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.495999:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: REPLY: OK
00000800:00000200:12.0:1619677470.496000:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.496001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:20.2:1619677470.496072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.496079:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.496082:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.496083:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.496085:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.496088:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.496090:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.496091:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.496092:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.496093:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.496094:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.496097:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.496099:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.496102:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.496103:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.496103:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.496105:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.496106:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.496107:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.496107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677470.496346:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.496348:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.496349:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.496350:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.496351:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.496354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.496364:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.496367:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.496368:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.496368:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.496369:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.496370:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.496509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.496548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677470.496857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.496863:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.496866:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.496866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.496868:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.496870:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.496872:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.496873:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.496875:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.496876:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.496877:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.496880:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.496882:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.496884:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.496885:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.496886:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.496888:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.496888:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.496888:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.496889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.496900:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.496903:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.496904:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.496906:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.496907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.496908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.497000:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.497003:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.497004:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.497004:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.497006:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.497009:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.497655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.497690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.497698:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.497701:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.497702:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.497702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.497704:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.497706:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.498331:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.498334:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.498335:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.498337:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.498339:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.498341:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.498343:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.498344:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.498345:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.498346:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.498349:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.498351:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.498353:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.498354:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.498355:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.498356:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.498357:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.498357:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.498358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.498388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.498394:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.498397:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.498398:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.498400:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.498402:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.498404:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.498405:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.498407:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.498407:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.498409:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.498412:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.498414:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.498417:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.498418:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.498419:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.498421:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.498421:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.498422:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.498423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.498456:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.498459:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.498460:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.498460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.498462:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.498467:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.498473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.498667:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.498670:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.498671:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.498671:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.498674:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.498677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.499475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.499481:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.499485:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.499485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.499487:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.499490:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.499491:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.499493:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.499494:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.499495:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.499496:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.499499:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.499501:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.499503:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.499504:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.499505:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.499506:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.499507:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.499508:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.499508:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.499519:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:5.2:1619677470.499521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.499521:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.499521:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.499523:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.499524:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.499525:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.499525:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.499526:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.499527:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.499528:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.499530:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.499532:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.499533:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.499534:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.499536:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.499537:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.499540:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000400:00000200:18.0:1619677470.499541:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.499541:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.499542:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.499542:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.499543:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.499543:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.499544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.499546:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.499547:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.499548:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.499550:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.499550:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.499551:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.499551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.500212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.500736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.500743:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.500745:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.500746:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.500748:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.500750:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.500751:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.500753:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677470.500753:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000400:00000200:17.0:1619677470.500754:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.500755:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677470.500755:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:17.0:1619677470.500756:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.500757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.500757:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.500760:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677470.500760:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.500762:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.500763:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:17.0:1619677470.500765:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.500766:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.500767:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.500768:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.500769:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.500769:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.500770:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.501327:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.501330:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.501331:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.501332:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.501334:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.501337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.501752:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.501755:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.501755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.501756:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.501758:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.501761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.501813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677470.501890:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.501893:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.501894:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.501894:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677470.501895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:1.0:1619677470.501896:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.501900:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.501901:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.501902:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.501903:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.501905:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.501907:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.501909:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.501910:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.501911:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.501912:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.501913:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.501916:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.501918:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.501920:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.501921:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.501922:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.501923:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.501924:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.501925:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.501926:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.501927:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.501932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.501937:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.501939:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.501940:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.501941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.502733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.502740:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.502744:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.502745:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.502747:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.502750:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.502751:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.502752:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.502754:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.502755:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.502756:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.502758:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.502760:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.502762:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.502763:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.502763:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.502765:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.502765:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.502767:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.502768:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.502769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.502775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.502780:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.502781:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.502782:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.502782:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:2.0:1619677470.502782:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.502783:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.502784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.502784:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.502785:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.502786:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.502929:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.502932:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.502933:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.502933:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.502935:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.502939:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.503094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.503148:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.503151:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.503152:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.503153:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.503155:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.503160:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.503161:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.503163:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.503164:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.503165:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.503165:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.503166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.503402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.503408:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.503411:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.503412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.503413:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.503415:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.503417:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.503418:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.503420:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.503421:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.503422:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.503425:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.503427:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.503429:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.503430:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.503431:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.503433:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.503434:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.503435:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.503436:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.503436:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.503442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.503448:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.503450:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.503450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.503452:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677470.503775:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.503778:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.503782:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677470.503782:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000400:00000200:13.0:1619677470.503785:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.503785:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:2.0:1619677470.503785:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.503786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.503786:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.503787:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677470.503788:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:13.0:1619677470.503789:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96522319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677470.503791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000400:00000200:13.0:1619677470.503792:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.503793:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.503795:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677470.503800:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.503803:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677470.503807:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.503809:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:13.0:1619677470.503809:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:13.0:1619677470.503812:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:13.0:1619677470.503813:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.503814:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.503815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677470.503826:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.503829:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.503829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.503830:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.503832:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.503835:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.503857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.503900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.504198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.2:1619677470.504235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:5.2:1619677470.504382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.504388:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.504391:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.504392:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.504394:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.504397:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.504399:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.504400:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.504402:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.504403:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.504404:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.504407:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.504408:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.504412:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.504413:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.504413:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.504415:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.504416:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.504416:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.504417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.505591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.505598:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.505601:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.505601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.505604:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.505607:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.505608:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.505610:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.505611:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.505612:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.505613:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677470.505615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000400:00000200:17.0:1619677470.505616:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.505618:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.505621:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.505622:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:17.0:1619677470.505622:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.505622:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.505624:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:17.0:1619677470.505624:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.505624:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.505625:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677470.505626:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.505626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677470.505626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.505628:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.505636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.505975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.506211:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.506214:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.506215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.506215:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.506217:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.506220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.506675:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.506677:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.506678:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.506678:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.506680:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.506685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.506686:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.506689:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.506690:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.506690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.506691:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.506691:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.506792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.506798:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.506801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.506801:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677470.506804:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.506806:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.506807:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.506809:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.506810:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.506811:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.506812:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.506816:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.506817:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.506820:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.506820:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.506821:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.506823:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.506823:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.506824:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.506824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.507040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.507046:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.507049:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.507050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.507052:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.507055:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.507056:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.507057:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.507059:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.507060:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.507061:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.507064:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.507066:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.507067:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.507068:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.507069:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.507071:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.507071:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.507072:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.507073:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.507084:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.507086:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.507087:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.507087:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.507089:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.507091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.507103:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.507105:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.507105:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.507106:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.507106:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.507107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.507129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677470.507146:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.507148:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.507149:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.507150:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.507152:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.507155:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:18.0:1619677470.507292:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.507297:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.507299:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.507300:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.507302:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.507305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.507313:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.507316:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.507317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.507317:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.507318:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.507319:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.507356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677470.507361:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:18.0:1619677470.507362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.507365:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:17.0:1619677470.507365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:2.0:1619677470.507365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677470.507365:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:18.0:1619677470.507369:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.507369:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.507371:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.507372:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.507373:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677470.507374:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677470.507374:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.507376:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.507376:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.507378:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.507378:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.507379:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.507379:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677470.507381:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677470.507383:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677470.507385:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.507386:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677470.507388:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677470.507389:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.507390:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.507390:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:18.0:1619677470.507391:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:1.0:1619677470.507391:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:18.0:1619677470.507392:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.507393:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:1.0:1619677470.507393:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.507393:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:18.0:1619677470.507394:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677470.507394:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.507394:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677470.507395:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.507395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.507406:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.507408:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.507409:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.507410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.507411:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.507414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.508021:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.508024:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.508025:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.508025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.508027:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.508030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.508397:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.508400:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.508401:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.508401:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.508404:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.508406:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.508752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.509271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.509278:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.509281:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.509282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.509284:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.509286:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.509288:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.509289:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.509291:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.509292:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.509293:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.509299:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.509301:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.509304:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.509305:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.509306:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.509307:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.509308:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.509310:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.509311:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.509311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.509317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.509323:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.509325:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.509326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.509327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.509665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.509670:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.509673:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.509674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.509676:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.509679:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.509680:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.509681:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.509683:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.509684:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.509685:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.509688:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.509689:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.509692:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.509692:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.509693:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.509694:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.509695:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.509695:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.509696:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.510131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.510176:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.510176:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.510177:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.510178:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.510178:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:17.0:1619677470.510181:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677470.510181:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677470.510184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.510184:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.510185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.510186:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.510186:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.510187:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.510189:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.510190:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.510191:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.510195:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.510196:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.510199:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.510200:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:1.0:1619677470.510200:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:1.0:1619677470.510202:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:1.0:1619677470.510203:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.510204:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677470.510205:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677470.510205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.510218:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677470.510220:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677470.510221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.510222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677470.510231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677470.510276:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.510279:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:2.0:1619677470.510280:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.510281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.510283:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.510286:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.510510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.510516:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.510519:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.510520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.510522:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.510525:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.510526:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.510528:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.510529:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.510530:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.510531:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.510534:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.510535:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.510538:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.510538:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.510539:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.510540:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:1.0:1619677470.510541:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.510541:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677470.510542:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:1.0:1619677470.510542:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.510542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677470.510543:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.510544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.510545:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.510547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.510654:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.510657:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677470.510657:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677470.510659:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.510661:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.510662:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.510663:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.510665:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.510666:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.510666:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.510669:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.510671:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.510673:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.510674:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.510674:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.510676:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.510676:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.510677:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.510678:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677470.510935:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.510938:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.510939:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.510940:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.510942:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.510945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.511142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.511147:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.511150:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.511151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.511153:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.511155:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.511156:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.511158:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.511159:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.511160:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.511161:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.511164:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.511166:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.511169:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.511170:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:2.0:1619677470.511170:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:2.0:1619677470.511172:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:2.0:1619677470.511173:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.511173:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.511174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.2:1619677470.511358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677470.511394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:18.0:1619677470.511857:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:17.0:1619677470.511857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:18.0:1619677470.511860:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:17.0:1619677470.511860:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:2.0:1619677470.511861:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000800:00000200:1.0:1619677470.511861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:18.0:1619677470.511862:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.511862:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.511865:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677470.511866:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.511869:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:18.0:1619677470.511870:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:17.0:1619677470.511871:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.511873:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.511874:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.511875:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.511875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.511876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.511916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.511921:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.511924:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.511925:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.511926:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.511929:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.511930:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.511931:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.511933:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.511934:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.511935:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.511939:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.511941:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.511943:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.511944:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.511945:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.511946:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.511947:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.511948:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.511949:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.511949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.511954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.511960:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.511962:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.511963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.511964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.512473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677470.512544:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.512547:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.512548:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.512549:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.512551:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.512554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.513057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.513094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.513638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.513645:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.513648:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.513648:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.513651:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.513653:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.513654:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.513655:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.513657:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.513658:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.513659:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.513661:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.513663:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.513665:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.513666:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.513666:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.513668:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.513669:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.513670:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.513670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.513681:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.513683:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.513684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.513684:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.513685:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.513687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.513700:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.513703:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.513704:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.513704:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.513704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.513705:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.513916:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.513919:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.513920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.513922:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.513924:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.513926:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.513927:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.513929:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.513930:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.513931:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.513934:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.513935:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.513938:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.513938:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.513939:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.513941:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.513941:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.513942:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.513943:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.513944:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.513955:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.513957:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.513957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.513958:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677470.514101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.514106:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.514109:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.514110:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.514112:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.514114:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.514116:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.514117:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.514119:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.514119:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.514121:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.514124:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.514126:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.514129:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.514130:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.514131:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.514133:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.514133:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.514134:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.514135:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677470.514898:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.514901:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.514902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.514902:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.514905:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.514908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.515253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677470.515259:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.515262:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677470.515263:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677470.515264:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677470.515267:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.515269:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.2:1619677470.515271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677470.515271:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.515272:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.515273:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.515274:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677470.515277:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677470.515279:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677470.515282:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e82fe5580] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.515283:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c000] (20)++
00000800:00000200:17.0:1619677470.515283:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c000] (21)++
00000800:00000200:17.0:1619677470.515285:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c000] (22)--
00000800:00000200:17.0:1619677470.515286:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.515287:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677470.515287:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:1.0:1619677470.515324:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.515326:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.515327:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.515327:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.515329:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.515332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.515341:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.515344:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.515344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.515345:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.515345:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.515346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.515761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.515764:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.515765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.515768:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.515770:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.515772:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677470.515773:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.515775:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.515776:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.515777:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677470.515780:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.515782:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677470.515785:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.515786:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:1.0:1619677470.515786:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:1.0:1619677470.515788:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:1.0:1619677470.515788:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.515789:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.515790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677470.515898:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.515901:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.515902:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.515902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.515904:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.515907:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.515920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.515927:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.515930:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.515931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.515933:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.515936:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.515937:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.515938:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.515940:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.515941:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.515942:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.515946:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.515947:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.515950:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.515951:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.515951:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.515953:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.515954:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.515955:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.515955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.515959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.515965:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.515968:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.515969:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.515971:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.515975:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.515977:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.515978:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.515980:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.515981:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.515982:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.515985:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.515988:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.515990:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.515991:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.515992:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.515994:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.515994:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.515995:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.515995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.516068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.516073:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.516076:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.516077:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.516079:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.516082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.516085:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.516087:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.516089:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.516090:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.516091:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.516093:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.516095:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.516097:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.516097:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.516098:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.516100:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.516100:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.516102:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.516103:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.516103:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.516108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.516114:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.516115:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.516115:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677470.516116:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.516116:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.516116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.516118:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.516118:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.516118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.516119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.516980:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.516982:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.516983:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.516983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.516985:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.516989:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.517028:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.517031:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.517032:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.517032:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.517034:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.517038:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.517299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.517305:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.517308:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.517309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.517311:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.517313:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.517314:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.517316:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.517319:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.517319:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.517321:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.517324:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.517326:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.517328:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.517329:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.517330:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.517331:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.517332:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.517333:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.517333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.517337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.517378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677470.517425:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.517428:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.517429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.517429:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.517431:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.517434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.517811:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.517814:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677470.517815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677470.517817:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.517819:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.517820:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677470.517822:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.517823:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.517824:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.517825:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677470.517829:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.517831:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677470.517833:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.517834:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:17.0:1619677470.517835:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:17.0:1619677470.517837:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:17.0:1619677470.517838:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.517839:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677470.517841:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677470.517841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.517852:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677470.517854:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677470.517855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.517856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677470.517862:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.517865:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.517866:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.517866:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.517869:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.517871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.517885:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.517888:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.517889:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.517889:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.517890:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.517891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.518104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.518425:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.518428:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.518429:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.518429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.518431:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.518435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.518998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.519003:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.519006:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.519007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.519009:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.519011:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.519013:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.519014:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.519015:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.519016:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.519017:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.519020:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.519022:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.519026:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.519026:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.519027:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.519029:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.519030:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.519030:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.519031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.519213:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.519216:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.519217:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.519217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.519219:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.519222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.519673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.519680:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.519682:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.519683:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.519685:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.519688:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.519689:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.519691:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.519692:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.519693:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.519694:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.519698:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.519700:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.519702:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.519703:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.519704:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.519705:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.519706:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.519707:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.519708:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.519709:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.519714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.519720:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.519722:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.519723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.519724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.519919:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.519922:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.519923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.519923:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.519925:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.519927:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.520331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.520337:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.520340:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.520341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.520344:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.520346:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.520348:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.520350:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.520352:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.520353:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.520355:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.520360:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.520363:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.520366:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.520367:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.520368:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.520369:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.520370:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.520371:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.520372:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.520678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.520685:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.520688:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.520688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.520690:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.520693:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.520694:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.520696:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.520697:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.520698:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.520699:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.520703:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.520704:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.520707:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.520707:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.520708:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.520710:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.520711:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.520712:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.520712:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.520723:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.520725:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.520726:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.520726:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.520728:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.520729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.520743:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.520745:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.520746:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.520746:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.520746:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.520747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.521340:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.521342:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.521344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.521344:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.521346:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.521349:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.521546:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.521549:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.521550:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.521550:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.521553:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.521556:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.521850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.521893:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.521895:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.521896:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.521896:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.521899:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.521901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.521909:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.521911:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.521912:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.521912:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.521913:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.521914:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.522032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.522037:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.522039:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.522040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.522042:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.522044:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.522046:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.522047:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.522049:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.522050:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.522051:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.522054:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.522056:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.522059:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.522060:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.522061:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.522062:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.522063:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.522064:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.522065:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.522065:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.522070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.522075:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677470.522076:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:1.0:1619677470.522077:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.522078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.522078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.522079:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.522080:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.522081:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.522083:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.522084:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:18.0:1619677470.522356:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.522358:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.522359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.522360:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.522361:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.522365:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.522679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677470.522943:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.522946:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.522947:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.522947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.522949:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.522952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.522999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.523004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.523007:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.523008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.523010:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.523011:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.523013:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.523014:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.523016:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.523017:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.523018:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.523022:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.523023:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.523026:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.523027:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.523028:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.523030:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.523030:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.523031:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.523032:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.523249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.523287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.523418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.523424:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.523427:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.523428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.523430:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.523432:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.523434:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.523435:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.523436:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.523437:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.523438:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.523442:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.523444:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.523446:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.523447:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.523448:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.523449:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.523450:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.523451:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.523451:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.523663:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.523666:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.523667:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.523667:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.523670:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.523672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:2.0:1619677470.523739:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.523742:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.523744:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.523744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.523746:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.523749:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.523912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.523918:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.523920:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.523921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.523923:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.523924:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.523926:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.523927:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.523929:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.523930:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.523931:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.523934:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.523936:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.523939:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.523940:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.523941:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.523943:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.523943:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.523944:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.523944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:24.2:1619677470.525057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:5.2:1619677470.525123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.525130:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.525132:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.525133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.525135:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.525138:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.525139:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.525141:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.525142:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.525143:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.525144:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.525148:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.525150:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.525152:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.525153:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.525153:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.525155:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.525156:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.525156:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.525157:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.525315:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.525318:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.525319:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.525319:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.525321:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.525324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.525436:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.525439:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.525439:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.525441:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.525443:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.525445:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.525446:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.525447:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.525448:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.525449:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.525452:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.525454:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.525457:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.525458:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.525458:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.525460:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.525460:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.525461:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.525461:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677470.525998:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.526001:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.526002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.526002:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.526005:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.526007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.526316:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.526318:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.526320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.526320:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.526322:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.526325:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.527191:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.527193:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.527195:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.527195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.527197:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.527200:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.527634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.527640:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.527643:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.527644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.527646:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.527649:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.527650:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.527651:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.527653:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.527654:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.527655:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.527659:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.527661:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.527664:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.527665:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:2.0:1619677470.527665:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:2.0:1619677470.527667:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:2.0:1619677470.527668:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.527668:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.527668:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.527678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.527956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.527963:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.527966:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.527967:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.527969:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.527972:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.527973:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.527974:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.527976:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.527977:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.527978:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.527981:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.527982:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.527985:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.527986:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.527987:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.527989:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.527989:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.527990:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.527990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.528003:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.528005:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.528006:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.528008:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.528010:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.528012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.528014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.528615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.528616:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.528620:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.528620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.528621:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.528624:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.528629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.528631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.528634:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.528634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.528635:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.528636:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.528636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.528934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.528940:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.528942:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.528943:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.528946:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.528948:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.528950:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.528951:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.528953:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.528954:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.528955:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.528959:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.528961:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.528963:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.528964:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.528965:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.528967:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.528967:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.528968:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.528969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.528980:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.528982:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.528983:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.528983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.528985:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.528987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.529463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.529469:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.529472:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.529473:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.529475:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.529477:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.529478:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.529480:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.529481:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.529482:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.529483:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.529487:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.529489:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.529492:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.529493:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.529494:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.529496:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.529496:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.529498:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.529500:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.529500:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.529505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.529511:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.529513:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.529514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.529514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.529894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.529900:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.529903:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.529904:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.529906:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.529910:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.529911:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.529912:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.529914:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.529914:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.529916:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.529919:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.529921:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.529923:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.529924:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.529924:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.529926:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.529926:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.529927:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.529928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.531227:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.531230:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.531231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.531232:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.531234:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.531241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.531242:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.531244:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.531245:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.531246:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.531246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.531247:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.531312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.531573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a4637eea00] (20)++
00000800:00000200:12.0:1619677470.531585:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a4637eea00] (21)++
00000800:00000200:12.0:1619677470.531589:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.16@o2ib3
00000800:00000200:13.0:1619677470.531593:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a4637eea00] (22)--
00000400:00000200:12.0:1619677470.531593:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.4.16@o2ib3 : GET - for me
00000400:00000200:12.0:1619677470.531596:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.4.16@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677470.531599:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.4.16@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677470.531601:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a4637eea00] (21)++
00000800:00000200:12.0:1619677470.531602:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a4637eea00] (22)--
00000400:00000200:12.0:1619677470.531605:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.4.16@o2ib3 local destination
00000800:00000200:12.0:1619677470.531607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a4637eea00] (21)--
00000800:00000200:20.2:1619677470.531709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.531716:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.531719:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.531719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.531721:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.531724:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.531726:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.531727:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.531729:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.531730:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.531731:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.531734:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.531735:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.531738:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.531738:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.531739:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.531741:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.531742:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.531743:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.531744:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.531745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.531750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.531755:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.531757:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.531758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.531759:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.531856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.531862:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.531864:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.531865:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.531867:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.531869:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.531871:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.531872:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.531873:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.531874:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.531875:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.531878:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.531879:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.531881:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.531882:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.531882:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.531884:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.531885:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.531886:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.531886:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.532014:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.532016:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.532017:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.532017:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.532020:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.532023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.532035:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.532037:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.532038:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.532038:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.532040:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.532040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:2.0:1619677470.532247:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.532251:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.532252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.532252:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.532254:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.532258:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.532677:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.532679:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677470.532680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677470.532682:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677470.532685:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.532687:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.532688:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.532691:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.532692:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.532693:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.532698:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677470.532700:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.532703:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.532704:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:1.0:1619677470.532704:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:1.0:1619677470.532706:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:1.0:1619677470.532706:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677470.532707:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677470.532708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:5.2:1619677470.532925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.532931:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.532934:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.532934:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.532937:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.532939:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.532941:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.532942:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.532944:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.532945:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.532946:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.532949:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.532951:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.532954:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.532955:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.532955:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.532957:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.532958:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.532959:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.532960:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.532961:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.532966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677470.532971:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:1.0:1619677470.532971:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.532973:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:13.0:1619677470.532974:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:1.0:1619677470.532974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677470.532975:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.532975:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677470.532976:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.532978:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.532979:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.532988:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.532991:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.532992:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.532992:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.532993:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.532995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.533663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.533700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.533881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:24.2:1619677470.534116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:5.2:1619677470.534376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.534383:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.534386:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.534386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.534388:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.534391:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.534392:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.534394:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.534395:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.534396:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.534397:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.534402:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.534404:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.534407:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.534408:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.534409:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.534411:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.534411:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.534412:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.534413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.534538:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.534541:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.534542:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.534542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.534544:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.534548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.534665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.534670:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.534672:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.534673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.534675:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.534677:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.534678:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.534680:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.534681:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.534682:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.534683:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.534686:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.534688:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.534690:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.534691:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.534692:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.534694:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.534695:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.534696:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.534696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.534802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.535103:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.535106:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.535107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.535109:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.535111:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.535112:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.535113:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.535115:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.535116:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.535117:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.535120:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.535122:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.535124:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.535125:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.535126:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.535127:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.535128:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.535128:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.535129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677470.535181:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.535184:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.535185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.535186:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.535189:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.535190:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677470.535192:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.535193:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.535194:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.535195:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677470.535198:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.535200:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677470.535203:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe0d80d00] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.535204:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb719600] (20)++
00000800:00000200:1.0:1619677470.535205:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb719600] (21)++
00000800:00000200:1.0:1619677470.535207:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb719600] (22)--
00000800:00000200:1.0:1619677470.535207:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.535208:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.535209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:20.2:1619677470.535566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.535572:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.535575:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.535576:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.535577:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.535581:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.535583:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.535584:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.535586:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.535587:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.535588:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.535592:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.535594:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.535597:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.535598:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:2.0:1619677470.535599:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:2.0:1619677470.535601:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:2.0:1619677470.535602:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.535604:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.535605:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.535606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.535612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.535618:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.535620:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.535620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.535621:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.536325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.536546:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.536549:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677470.536550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677470.536552:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677470.536554:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.536556:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677470.536558:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.536559:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.536560:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.536561:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677470.536565:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677470.536567:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677470.536569:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.536570:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:17.0:1619677470.536571:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:17.0:1619677470.536573:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:17.0:1619677470.536573:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677470.536574:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677470.536574:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:2.0:1619677470.537274:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.537277:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.537279:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.537279:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.537281:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.537285:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.537294:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.537300:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.537302:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.537303:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.537304:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.537304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.537670:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.537673:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.537674:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.537674:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.537676:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.537679:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.537986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.537991:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.537995:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.537996:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.537997:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.538000:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.538002:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.538004:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.538005:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.538007:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.538008:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.538011:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.538014:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.538016:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.538017:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.538018:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.538020:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.538021:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.538022:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.538022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677470.538256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.538261:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.538265:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.538265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.538267:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.538270:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.538271:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.538272:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.538274:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.538275:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.538276:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.538279:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.538281:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.538283:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.538284:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.538285:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.538287:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.538287:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.538288:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.538288:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.538302:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.538305:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.538306:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.538307:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.538309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.538310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.538379:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.538382:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.538383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.538384:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.538386:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.538389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.538740:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.538743:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.538743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.538745:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.538748:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.538750:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677470.538751:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.538752:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.538753:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.538755:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677470.538758:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.538760:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677470.538763:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.538764:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:18.0:1619677470.538765:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:18.0:1619677470.538767:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:18.0:1619677470.538768:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.538769:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.538769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.2:1619677470.539002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.539217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.539256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677470.539282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.539288:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.539291:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.539292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.539301:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.539304:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.539306:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.539308:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.539310:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.539311:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.539313:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.539321:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.539324:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.539328:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.539329:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.539330:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.539332:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.539333:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.539334:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.539335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.539497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.539504:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.539507:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.539507:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.539510:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.539512:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.539513:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.539515:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.539516:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.539517:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.539518:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.539522:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.539523:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.539526:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.539527:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.539528:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.539529:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.539530:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.539531:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.539531:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.539556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:17.0:1619677470.539583:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.539585:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.539586:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.539586:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:1.0:1619677470.539586:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.539589:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.539589:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.539591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.539591:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.539592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000400:00000200:2.0:1619677470.539593:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.539597:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:12.0:1619677470.539603:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.539606:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.539607:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.539609:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.539611:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.539612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.539638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.539643:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.539647:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.539648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.539649:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.2:1619677470.539652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (20)++
00000400:00000200:1.0:1619677470.539652:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.539653:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.539655:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.539656:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.539657:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.539658:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.539662:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.539664:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.539667:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.539668:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.539669:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.539671:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.539671:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.539672:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.539672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.539684:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.539686:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.539687:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.539689:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.539690:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.539692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.540368:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.540371:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.540372:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.540373:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.540376:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.540377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:17.0:1619677470.540778:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.540780:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.540781:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.540782:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.540784:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.540786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.540804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.540842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677470.540847:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.540849:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.540850:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.540851:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.540853:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.540855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.540864:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.540867:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.540868:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.540868:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.540870:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.540870:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.540959:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.540962:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.540963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.540963:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.540965:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.540968:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.542143:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.542146:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.542147:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.542147:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.542150:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.542152:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.543015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.543180:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.543182:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.543184:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.543184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000800:00000200:5.2:1619677470.543185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677470.543186:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.543189:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.543191:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.543194:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.543195:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.543197:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.543199:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.543201:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.543202:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.543203:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.543205:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.543205:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.543209:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.543210:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.543213:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.543214:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.543214:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.543216:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.543217:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.543218:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.543220:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.543220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.543225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.543230:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:12.0:1619677470.543230:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:17.0:1619677470.543232:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.543233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:12.0:1619677470.543233:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.543234:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.543234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:12.0:1619677470.543236:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.543237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.543238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.543249:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.543252:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.543253:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.543254:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.543254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.543256:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.544074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677470.544188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.544194:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.544197:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.544198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.544200:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.544202:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.544204:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.544205:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.544207:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.544208:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.544209:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.544213:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.544214:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.544218:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.544219:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.544220:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.544222:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.544222:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.544223:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.544224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.544271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.544277:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.544280:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.544281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.544283:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.544286:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.544287:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.544288:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.544290:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.544291:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.544292:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.544299:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.544301:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.544303:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.544304:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.544304:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.544305:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.544306:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.544307:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.544308:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.544320:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.544322:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.544323:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.544324:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.544325:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.544326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.544523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.544530:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.544532:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.544533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.544535:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.544538:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.544539:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.544540:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.544541:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.544543:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.544543:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.544547:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.544548:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.544551:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.544551:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.544552:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.544554:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.544554:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.544555:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.544555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.544958:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.544961:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:18.0:1619677470.544962:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:17.0:1619677470.544962:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.544964:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677470.544965:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.544966:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.544966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:1.0:1619677470.544966:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.544967:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677470.544969:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677470.544969:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.544971:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.544971:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.544973:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677470.544974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.544976:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000400:00000200:1.0:1619677470.544976:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.544979:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.544979:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:1.0:1619677470.544979:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677470.544980:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.544981:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.544981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.544981:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.544982:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.544983:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.544984:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.544985:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.544986:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.544986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677470.545810:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.545812:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.545814:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.545814:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.545816:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.545820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.545963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.545969:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.545972:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.545972:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.545975:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.545976:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.545978:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.545979:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.545981:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.545982:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.545983:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.545985:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.545987:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.545989:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.545989:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.545990:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.545992:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.545992:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.545993:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.545994:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.546005:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.546007:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.546007:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.546009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.546009:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.546010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.546339:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.546342:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.546343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.546344:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.546346:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.546350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.547043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.547086:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.547089:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.547090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.547090:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.547092:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.547095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.547106:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.547109:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.547110:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (23)++
00000800:00000200:1.0:1619677470.547110:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (23)++
00000800:00000200:17.0:1619677470.547111:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (25)--
00000800:00000200:17.0:1619677470.547111:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (24)--
00000800:00000200:1.0:1619677470.547111:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (25)--
00000400:00000200:1.0:1619677470.547112:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.547113:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.547114:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.547116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.547567:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.547570:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.547571:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.547571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.547573:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.547576:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.548122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.548128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.548131:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.548132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.548134:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.548136:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.548138:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.548139:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.548141:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.548142:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.548143:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.548145:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.548147:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.548150:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.548151:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.548151:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.548153:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.548153:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.548154:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.548154:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.548167:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.548169:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.548170:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.548172:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.548174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.548175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.548258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.548303:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.548306:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.548307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.548307:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.548309:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.548311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.548566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.548573:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.548576:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.548577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.548579:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.548581:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.548582:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.548583:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.548585:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.548586:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.548587:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.548590:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.548591:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.548593:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.548594:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.548595:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.548596:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.548597:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.548598:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.548599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.548609:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.548612:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.548612:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.548613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.548614:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.548615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.549103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.549108:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.549111:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.549112:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.549114:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.549116:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.549118:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.549119:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.549121:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.549122:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.549123:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.549127:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.549128:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.549132:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.549133:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.549134:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.549136:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.549137:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.549138:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.549139:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.549140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.549145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.549150:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.549152:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.549153:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.549154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.549306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.549312:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.549315:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.549316:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.549318:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.549321:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.549322:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.549324:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.549325:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.549326:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.549327:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.549330:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.549332:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.549335:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.549336:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.549336:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.549338:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.549339:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.549340:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.549340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677470.549877:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.549880:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.549881:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.549882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.549884:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.549887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.550125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677470.550730:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.550732:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.550733:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.550734:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.550737:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.550740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.550749:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.550751:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.550752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.550752:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.550754:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.550754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.551320:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.551323:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.551324:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.551324:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.551326:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.551330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.552154:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.552157:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.552158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.552158:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.552160:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.552163:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.552867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.552903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.552920:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.552922:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.552923:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.552923:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.552925:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.552928:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.552939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.552942:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.552943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.552943:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.552944:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.552945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.552972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.552978:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.552980:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.552981:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.552983:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.552985:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.552987:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.552988:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.552989:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.552991:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.552992:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.552996:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.552997:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.553000:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.553001:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.553002:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.553004:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.553004:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.553006:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.553007:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.553008:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.553012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.553017:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:13.0:1619677470.553018:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:1.0:1619677470.553019:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.553020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677470.553021:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:1.0:1619677470.553021:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:13.0:1619677470.553022:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.553023:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.553026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.553026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:20.2:1619677470.553279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.553285:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.553289:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.553290:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.553292:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.553300:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.553301:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.553302:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.553304:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.553305:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.553306:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.553309:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.553311:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.553314:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.553314:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.553315:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.553317:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.553318:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.553319:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.553320:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.553332:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.553334:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.553335:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.553335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.553337:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.553338:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.553349:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.553351:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.553352:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.553352:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.553353:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.553353:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.553366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.553371:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.553374:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.553375:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.553376:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.553377:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.553378:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.553379:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.553381:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.553381:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.553382:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.553384:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.553386:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.553387:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.553387:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.553388:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.553389:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.553390:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.553391:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.553392:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.553393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.553398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.553403:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.553405:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.553405:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.553405:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.553406:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.553407:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.553407:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.553408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.553408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.553409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.553527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.553532:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.553535:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.553536:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.553538:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.553540:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.553542:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.553543:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.553545:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.553546:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.553547:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.553550:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.553552:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.553555:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.553556:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.553557:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.553559:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.553560:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.553560:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.553561:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.553977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.554119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.554126:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.554128:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.554129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.554131:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.554134:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.554136:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.554137:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.554138:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.554139:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.554140:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.554144:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.554146:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.554149:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.554149:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.554150:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.554152:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.554152:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.554153:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.554153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677470.554511:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.554514:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.554515:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.554515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.554518:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.554521:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.554838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.554844:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.554847:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677470.554848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677470.554850:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.554852:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.554854:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677470.554855:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.554857:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.554858:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.554859:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677470.554862:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.554864:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677470.554868:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.554869:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:2.0:1619677470.554870:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:2.0:1619677470.554872:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:2.0:1619677470.554873:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.554874:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677470.554875:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677470.554876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677470.554881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.554887:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:12.0:1619677470.554887:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:18.0:1619677470.554888:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677470.554889:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.554890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:12.0:1619677470.554891:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.554892:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.554895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000400:00000200:12.0:1619677470.554895:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.554898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:17.0:1619677470.555089:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.555092:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.555093:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.555093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.555095:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.555098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.555511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677470.555686:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.555689:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.555690:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.555690:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.555692:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.555696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.555964:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.555967:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.555968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.555968:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.555970:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.555973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.556342:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.556345:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.556346:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.556347:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.556349:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.556352:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:17.0:1619677470.557207:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.557209:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.557210:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.557211:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:17.0:1619677470.557212:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.557213:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.557216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.557216:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677470.557217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.557218:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.557219:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.557221:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.557222:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.557223:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.557224:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.557225:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.557229:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.557231:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.557233:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.557234:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.557234:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.557236:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.557236:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.557238:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.557239:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.557239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.2:1619677470.557240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.557246:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.557249:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.557251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677470.557251:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000400:00000200:2.0:1619677470.557252:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677470.557253:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.557254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.557254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:2.0:1619677470.557255:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.557257:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677470.557258:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.557260:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.557261:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.557262:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677470.557266:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.557268:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.557271:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.557272:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:2.0:1619677470.557273:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:2.0:1619677470.557275:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:2.0:1619677470.557276:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.557277:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.557277:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.557658:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.557660:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.557661:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.557663:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.557666:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.557668:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677470.557669:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.557670:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.557671:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.557672:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677470.557675:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.557677:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677470.557680:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.557680:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:1.0:1619677470.557683:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:1.0:1619677470.557685:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:1.0:1619677470.557685:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.557686:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.557686:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677470.558117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.558349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.558355:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.558357:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.558358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.558361:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.558364:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.558365:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.558366:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.558368:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.558369:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.558370:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:5.2:1619677470.558373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:17.0:1619677470.558374:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.558376:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.558378:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.558379:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677470.558379:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.558380:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.558381:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:18.0:1619677470.558382:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677470.558382:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.558383:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.558383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677470.558383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.558385:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.558388:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.558389:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.558391:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96522009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.558393:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.558394:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.558395:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:13.0:1619677470.558396:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000400:00000200:18.0:1619677470.558398:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.558398:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.558400:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.558401:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:13.0:1619677470.558402:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.558402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:18.0:1619677470.558405:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.558405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:18.0:1619677470.558406:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.558406:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.558409:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.558409:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.558410:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.558411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.559338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.559344:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.559347:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.559348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.559350:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.559352:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.559354:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.559355:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96522006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.559356:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.559357:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.559358:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.559362:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.559363:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.559366:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.559367:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.559368:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.559370:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.559370:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.559371:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.559371:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.559384:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.559387:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.559387:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.559389:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.559391:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.559393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.559402:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:13.0:1619677470.559404:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.559405:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.559406:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:13.0:1619677470.559407:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.559408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:2.0:1619677470.559422:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.559425:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.559427:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.559427:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.559429:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.559434:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.559436:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.559438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.559438:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.559440:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.559440:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.559441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.559782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.559826:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.559829:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.559830:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.559830:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.559832:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.559835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.560789:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.560792:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.560793:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.560793:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.560795:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.560798:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.560809:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.560811:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.560812:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.560812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.560813:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.560814:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.560826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.560868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:1.0:1619677470.560945:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.560948:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.560949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.560950:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.560952:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.560955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.560968:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.560971:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.560971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.560972:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.560973:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.560974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.561126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.561134:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.561136:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.561139:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000400:00000200:18.0:1619677470.561140:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677470.561141:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.561142:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.561143:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.561144:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677470.561145:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.561146:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.561148:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.561149:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.561150:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.561153:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.561155:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.561156:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.561158:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.561158:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.561159:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.561159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.561160:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.561161:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.561162:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.561164:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.561165:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.561165:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.561170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.561176:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.561177:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.561177:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.561177:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.561178:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.561179:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.561179:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.561180:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.561181:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.561181:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.561602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.561608:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.561611:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.561612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.561614:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.561616:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.561618:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677470.561619:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.561621:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.561622:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.561622:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677470.561626:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.561627:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.561630:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.561631:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:2.0:1619677470.561632:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:2.0:1619677470.561634:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:2.0:1619677470.561635:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.561635:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.561636:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.562335:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.562338:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.562339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.562339:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.562341:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.562344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.562919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.562957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677470.563062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.563068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.563071:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.563071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.563073:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.563076:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.563077:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.563079:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.563080:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.563081:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.563082:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.563085:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.563086:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.563089:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.563090:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.563090:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.563092:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.563093:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.563094:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.563095:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.563096:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.563101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.563106:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.563108:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.563108:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.563109:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.563233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.563239:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.563242:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.563243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.563245:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.563248:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.563250:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.563251:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.563253:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.563254:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.563255:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.563258:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.563260:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.563263:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.563264:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.563264:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.563266:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.563267:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.563267:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.563268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.563864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.563871:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.563873:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.563874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.563876:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.563879:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.563880:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.563881:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.563882:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.563883:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.563884:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.563887:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.563889:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.563891:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.563892:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.563893:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.563894:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.563895:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.563895:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.563896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677470.564248:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677470.564251:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677470.564252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677470.564254:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.564257:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.564259:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677470.564260:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.564262:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.564263:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.564264:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677470.564267:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.564268:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677470.564272:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.564273:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:2.0:1619677470.564273:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:2.0:1619677470.564275:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:2.0:1619677470.564276:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677470.564277:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677470.564278:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677470.564279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.564290:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677470.564292:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677470.564292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.564300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677470.564740:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.564742:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.564743:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.564744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.564746:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.564750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.565479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.565485:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.565488:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.565489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.565491:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.565494:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.565495:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.565497:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.565499:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.565500:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.565501:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.565504:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.565506:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.565509:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.565510:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:18.0:1619677470.565511:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:18.0:1619677470.565513:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:18.0:1619677470.565513:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.565514:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.565514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.565557:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.565560:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.565561:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.565561:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.565563:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.565567:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.565642:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.565645:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.565646:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.565646:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.565648:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.565652:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.566139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.566283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.566336:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.566339:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.566340:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.566340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.566342:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.566344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.566651:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.566654:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.566655:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.566656:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.566658:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.566660:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677470.566661:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.566663:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.566663:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.566665:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677470.566668:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.566670:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.566671:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18080] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.566673:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918febd25600] (20)++
00000800:00000200:1.0:1619677470.566673:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.566675:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.566675:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.566676:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.566676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677470.566688:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.566690:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.566690:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.566691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.566692:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.566693:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.566760:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.566763:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.566764:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.566766:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.566768:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.566769:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.566771:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.566772:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.566773:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.566774:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.566777:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.566779:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.566782:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.566783:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.566783:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.566785:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.566785:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.566786:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.566786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677470.567369:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.567372:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.567373:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.567373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.567376:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.567381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.567383:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.567385:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.567385:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.567386:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.567387:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.567388:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.567950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.567956:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.567959:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.567959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.567962:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.567964:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.567965:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.567967:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.567968:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.567969:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.567970:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.567973:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.567975:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.567978:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.567978:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.567979:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.567981:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.567981:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.567982:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.567983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.568086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.568118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.568124:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.568127:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.568128:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.568130:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.568133:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.568134:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.568136:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.568138:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.568139:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.568140:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.568143:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.568145:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.568149:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.568149:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.568150:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.568152:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.568152:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.568154:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.568155:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.568156:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.568161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.568166:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.568168:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.568169:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.568170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.568758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.568764:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.568767:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.568768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:5.2:1619677470.568770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677470.568770:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.568773:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.568774:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.568776:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677470.568776:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677470.568777:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.568778:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.568779:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677470.568779:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.568780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.568783:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.568784:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.568786:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:1.0:1619677470.568786:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.568787:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:2.0:1619677470.568788:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.568789:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000400:00000200:1.0:1619677470.568789:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677470.568790:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000400:00000200:1.0:1619677470.568791:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677470.568792:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.568792:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677470.568792:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677470.568793:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:1.0:1619677470.568793:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677470.568794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:1.0:1619677470.568797:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.568799:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.568802:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.568803:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.568803:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:18.0:1619677470.568805:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:1.0:1619677470.568805:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.568806:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.568806:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677470.568807:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:1.0:1619677470.568807:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677470.568808:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.568809:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.568811:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.568812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:12.0:1619677470.568821:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.568824:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.568826:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.568826:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000400:00000200:12.0:1619677470.568828:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.568829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.568829:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.568830:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:13.0:1619677470.568831:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:2.0:1619677470.568831:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.568831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:12.0:1619677470.568833:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.569504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.569855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.569862:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.569865:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.569866:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.569868:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.569870:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.569872:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.569873:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.569875:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.569876:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.569877:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.569880:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.569882:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.569885:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.569886:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:18.0:1619677470.569887:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:18.0:1619677470.569889:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:18.0:1619677470.569889:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.569891:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.569892:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.569892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.569898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.2:1619677470.569901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677470.569903:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.569905:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.569906:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.569907:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.569956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:18.0:1619677470.569959:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.569962:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.569962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.569963:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.569966:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.569969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.569997:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.570000:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.570001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:2.0:1619677470.570001:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.570003:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.570006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.570022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.570028:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.570031:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.570031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.570033:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.570036:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.570037:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677470.570039:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.570041:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.570042:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.570043:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677470.570047:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.570048:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677470.570052:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.570052:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:18.0:1619677470.570053:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:18.0:1619677470.570055:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:18.0:1619677470.570055:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.570056:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.570057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:13.0:1619677470.570069:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.570071:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.570072:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.570074:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.570076:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.570077:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677470.570089:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:12.0:1619677470.570092:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677470.570093:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.570094:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:12.0:1619677470.570094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.570096:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:2.0:1619677470.571115:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.571117:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.571119:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.571119:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.571122:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.571125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.571330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.571337:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.571340:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.571340:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.571342:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.571345:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.571346:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677470.571347:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.571349:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677470.571349:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000400:00000200:18.0:1619677470.571350:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.571351:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677470.571351:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.571352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.571353:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.571355:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677470.571355:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.571357:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677470.571358:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:18.0:1619677470.571360:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.571361:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:18.0:1619677470.571362:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:18.0:1619677470.571363:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:18.0:1619677470.571364:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.571365:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.571366:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:13.0:1619677470.571379:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.571381:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.571382:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.571384:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.571386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677470.571386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:20.2:1619677470.571446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:1.0:1619677470.571648:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.571651:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.571652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.571652:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.571654:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.571657:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.571666:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.571669:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.571669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.571670:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.571671:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.571672:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.572360:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.572362:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.572364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.572364:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.572366:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.572370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.572834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.572840:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.572843:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.572844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.572846:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.572849:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.572850:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.572852:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.572853:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.572854:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.572855:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.572859:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.572860:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.572863:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.572864:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.572866:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.572867:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.572868:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.572869:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.572870:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.572871:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.572876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.572882:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.572883:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.572884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.572885:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.573005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.573011:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.573014:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.573014:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.573016:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.573019:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.573020:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.573022:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.573023:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.573024:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.573025:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.573029:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.573030:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.573034:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.573035:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.573035:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.573037:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.573038:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.573039:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.573039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.573180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.573216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677470.573672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.573678:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.573681:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.2:1619677470.573682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677470.573682:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.573684:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.573687:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.573688:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:1.0:1619677470.573688:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.573689:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.573690:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677470.573691:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:1.0:1619677470.573691:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677470.573692:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677470.573693:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677470.573695:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.573697:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677470.573698:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.573699:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:1.0:1619677470.573699:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677470.573700:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.573702:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677470.573702:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.573702:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000400:00000200:2.0:1619677470.573703:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677470.573703:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000400:00000200:2.0:1619677470.573704:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677470.573705:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.573705:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.573706:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.573707:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677470.573708:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.573709:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.573713:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.573714:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:2.0:1619677470.573714:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:2.0:1619677470.573716:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:2.0:1619677470.573717:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.573717:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677470.573718:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:2.0:1619677470.573718:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677470.573721:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.573722:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.573724:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.573726:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.573727:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.573882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.575164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.575289:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.575292:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.575293:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.575297:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.575300:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.575304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.575367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.575372:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.575375:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.575376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.575378:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.575380:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.575381:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.575382:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.575384:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.575385:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.575386:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.575389:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.575391:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.575394:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.575394:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.575395:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.575397:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.575397:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.575398:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.575398:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.575574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:18.0:1619677470.576513:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.576516:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.576517:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.576517:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.576520:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.576523:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.576532:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.576535:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.576536:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.576536:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.576537:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.576538:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:17.0:1619677470.576739:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.576742:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677470.576743:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677470.576745:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.576748:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.576750:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677470.576751:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.576752:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.576753:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.576754:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677470.576758:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.576759:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677470.576762:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.576763:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:17.0:1619677470.576763:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:17.0:1619677470.576765:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:17.0:1619677470.576766:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.576767:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.576767:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677470.576779:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.576781:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.576783:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.576783:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.576785:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.576787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.576800:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.576803:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.576804:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.576804:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.576806:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.576809:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.576820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677470.577369:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.577372:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.577373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.577375:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.577377:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.577378:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.577380:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.577381:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.577382:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.577383:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.577387:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.577388:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.577391:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.577392:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.577392:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.577394:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.577395:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.577396:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.577396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677470.577448:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.577451:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:2.0:1619677470.577452:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.577453:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.577455:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.577458:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.577542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.577547:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.577550:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.577550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.577552:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.577555:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.577556:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.577558:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.577559:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.577560:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.577561:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.577564:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.577566:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.577569:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.577569:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:18.0:1619677470.577570:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:18.0:1619677470.577572:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:18.0:1619677470.577572:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.577573:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.577573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.577705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.577711:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.577713:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.577714:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.577717:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.577720:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.577721:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.577723:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.577724:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.577725:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.577726:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.577729:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.577731:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.577733:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.577734:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.577735:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.577736:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.577737:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.577738:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.577738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.577815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.577820:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.577824:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.577824:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677470.577827:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.577829:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.577831:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.577833:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.577834:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.577836:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.577837:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.577840:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.577842:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.577846:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.577847:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.577848:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.577849:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.577850:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.577851:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.577851:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.578543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.578548:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.578550:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.578550:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:18.0:1619677470.578551:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677470.578551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677470.578552:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677470.578553:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000400:00000200:2.0:1619677470.578555:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677470.578557:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677470.578558:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.578558:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.578559:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000400:00000200:2.0:1619677470.578559:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:18.0:1619677470.578560:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677470.578560:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:2.0:1619677470.578561:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677470.578561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.578563:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.578563:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.578564:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.578566:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.578566:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677470.578568:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.578569:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.578570:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677470.578571:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.578572:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677470.578572:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:17.0:1619677470.578573:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677470.578574:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.578576:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.578576:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000400:00000200:17.0:1619677470.578577:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.578578:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:17.0:1619677470.578579:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677470.578579:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.578579:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.578580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677470.578582:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.578583:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.578583:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.578585:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.578586:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.578586:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.578587:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677470.578591:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.578593:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.578594:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.578595:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.578596:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.578598:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:13.0:1619677470.578599:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.578601:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.578602:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.578604:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.578606:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.578607:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:2.0:1619677470.578609:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.578611:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.578612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.578613:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.578614:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.578614:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.578875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.578918:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.578920:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.578921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.578922:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.578924:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.578926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:12.0:1619677470.579428:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.579431:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.579433:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677470.579435:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.579437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.579437:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677470.579438:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96521895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.579440:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.579441:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.579441:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677470.579444:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.579446:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677470.579448:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.579449:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:12.0:1619677470.579450:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:12.0:1619677470.579451:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:12.0:1619677470.579452:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.579453:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677470.579455:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677470.579455:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.579471:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677470.579474:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677470.579475:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.579476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677470.579530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.579570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.579577:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.579580:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.579581:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.579581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.579583:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.579585:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.580177:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.580179:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.580180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.580181:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.580183:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.580187:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.580777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.580820:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.580822:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.580823:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.580824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.580825:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.580828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.581203:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.581206:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.581207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.581208:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.581210:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.581213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.581368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.581374:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.581376:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.581377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.581379:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.581380:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.581382:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.581383:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.581384:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.581386:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.581387:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.581390:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.581392:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.581395:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.581395:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.581396:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.581398:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.581398:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.581400:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.581401:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.581401:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.581407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.2:1619677470.581412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.581416:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.581417:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.581418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.581419:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.581419:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.581420:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.581421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.581424:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.581426:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.581429:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.581431:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.581432:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.581433:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.581434:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.581436:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.581438:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.581439:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.581440:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.581440:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.581442:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.581443:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.581443:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.581444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.581456:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.581458:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.581459:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.581459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.581461:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.581463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.581757:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.581760:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.581761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.581761:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.581764:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.581767:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.582145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.582198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:5.2:1619677470.582531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.582537:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.582540:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.582540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.582542:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.582545:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.582546:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.582548:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.582549:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.582550:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.582551:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.582554:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.582556:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.582558:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.582559:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.582560:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.582561:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.582562:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.582563:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.582565:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.582565:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.582570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.582576:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.582578:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.582578:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.582579:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.582686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.582693:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.582695:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.582696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.582698:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.582701:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.582703:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.582704:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.582706:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.582707:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.582708:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.582711:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.582713:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.582717:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.582717:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.582718:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.582720:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.582721:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.582722:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.582722:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.582898:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.582901:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.582902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.582903:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.582905:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.582908:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.582916:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.582919:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.582920:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.582920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.582921:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.582922:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.582986:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.582989:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.582990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.582990:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.582992:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.582995:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.583354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.583360:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.583363:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.583363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.583366:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.583368:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.583370:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.583371:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.583372:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.583373:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.583374:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.583378:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.583380:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.583383:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.583384:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.583385:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.583386:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.583387:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.583387:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.583388:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.583400:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.583402:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.583403:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.583405:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.583406:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.583408:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.583419:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.583422:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.583422:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.583423:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.583424:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.583425:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:13.0:1619677470.583657:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.583660:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.583660:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.583662:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.583663:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.583665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:20.2:1619677470.584257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.585072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.585079:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.585082:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.585083:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.585085:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.585088:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.585089:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.585091:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.585092:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.585093:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.585094:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.585097:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.585099:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.585100:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.585101:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.585102:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.585104:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.585104:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.585105:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.585106:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.585123:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.585125:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.585126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.585126:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.585128:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.2:1619677470.585129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.585129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.585154:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.585156:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.585157:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.585157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.585158:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.585159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.585174:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.585177:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.585178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.585178:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.585181:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.585184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.585194:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.585197:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.585198:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.585198:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.585199:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.585200:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.585339:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.585342:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.585343:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.585345:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.585348:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.585349:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.585351:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.585352:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.585353:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.585354:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.585357:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.585359:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.585361:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.585362:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.585363:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.585364:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.585365:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.585365:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.585366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677470.585434:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.585436:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.585438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.585438:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.585440:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.585443:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.585862:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.585865:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.585866:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.585866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.585869:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.585871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.586009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.586015:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.586018:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.586019:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.586021:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.586023:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.586024:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.586025:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.586027:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.586028:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.586029:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.586031:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.586033:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.586035:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.586036:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.586037:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.586038:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.586039:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.586041:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.586042:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.586042:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.586048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.586053:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.586055:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.586057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.586059:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.586059:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.586060:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.586061:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.586062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.586063:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.586064:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:23.0:1619677470.586133:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff9197843f0000] (18)--
00000800:00000200:1.0:1619677470.586497:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.586500:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.586501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.586501:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.586503:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.586506:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.586660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.586698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.586703:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.586705:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.586707:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.586707:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.586709:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.586711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.587350:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.587352:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.587354:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.587354:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677470.587356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677470.587356:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.587359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.587362:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.587365:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.587367:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.587369:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.587371:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.587373:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.587374:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.587375:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.587377:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.587378:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.587381:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.587383:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.587385:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.587386:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.587387:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.587388:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.587389:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.587390:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.587390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.587509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677470.587598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.587604:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.587607:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.587608:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.587610:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.587611:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.587613:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.587614:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.587616:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.587617:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.587617:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.587621:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.587622:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.587626:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.587627:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.587627:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.587629:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.587630:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.587631:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.587632:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.587632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.587637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.587643:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.587645:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.587646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.587647:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.588198:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.588202:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.588202:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.588203:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.588205:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.588207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.588232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.588238:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.588241:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.588241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.588243:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.588246:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.588247:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.588248:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.588249:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.588250:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.588251:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.588254:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.588256:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.588259:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.588259:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.588260:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.588262:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.588262:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.588263:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.588263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.588287:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.588290:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.588291:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.588292:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.588294:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.588305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:7.0:1619677470.588628:0:13229:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.52@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677470.588635:0:13229:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.52@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:7.0:1619677470.588757:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.52@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.588767:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff9197838c7000 0xc038c7000(0xc038c7000)
00000800:00000200:7.0:1619677470.588768:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff9197838c8000 0xc038c8000(0xc038c8000)
00000800:00000200:7.0:1619677470.588768:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff9197838c9000 0xc038c9000(0xc038c9000)
00000800:00000200:7.0:1619677470.588769:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff9197838ca000 0xc038ca000(0xc038ca000)
00000800:00000200:7.0:1619677470.588769:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff9197838cb000 0xc038cb000(0xc038cb000)
00000800:00000200:7.0:1619677470.588770:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff9197838cc000 0xc038cc000(0xc038cc000)
00000800:00000200:7.0:1619677470.588770:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff9197838cd000 0xc038cd000(0xc038cd000)
00000800:00000200:7.0:1619677470.588771:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff9197838ce000 0xc038ce000(0xc038ce000)
00000800:00000200:7.0:1619677470.588772:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff9197838cf000 0xc038cf000(0xc038cf000)
00000800:00000200:7.0:1619677470.588772:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff919783eb0000 0xc03eb0000(0xc03eb0000)
00000800:00000200:7.0:1619677470.588773:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff919783eb1000 0xc03eb1000(0xc03eb1000)
00000800:00000200:7.0:1619677470.588774:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919783eb2000 0xc03eb2000(0xc03eb2000)
00000800:00000200:7.0:1619677470.588775:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919783eb3000 0xc03eb3000(0xc03eb3000)
00000800:00000200:7.0:1619677470.588775:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919783eb4000 0xc03eb4000(0xc03eb4000)
00000800:00000200:7.0:1619677470.588776:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919783eb5000 0xc03eb5000(0xc03eb5000)
00000800:00000200:7.0:1619677470.588776:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919783eb6000 0xc03eb6000(0xc03eb6000)
00000800:00000200:7.0:1619677470.588777:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919783eb7000 0xc03eb7000(0xc03eb7000)
00000800:00000200:7.0:1619677470.588777:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919783eb8000 0xc03eb8000(0xc03eb8000)
00000800:00000200:20.2:1619677470.589378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.589529:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.589532:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.589533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.589535:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.589538:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.589540:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.589541:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.589542:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.589543:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.589544:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.589547:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.589549:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.589552:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.589553:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:1.0:1619677470.589554:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:1.0:1619677470.589555:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:1.0:1619677470.589556:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.589556:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.589557:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:7.0:1619677470.589676:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589677:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589678:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589678:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589678:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589679:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589679:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589680:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589680:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589680:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589681:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589681:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589681:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589682:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589682:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589682:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589683:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589683:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589684:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589684:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589685:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589685:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589685:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589686:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589686:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589686:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589687:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589687:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589688:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589688:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589689:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589689:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589689:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589690:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:7.0:1619677470.589690:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (19)++
00000800:00000200:7.0:1619677470.589691:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (20)--
00000800:00000200:2.0:1619677470.590065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.590068:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.590069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.590070:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.590072:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.590075:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.590204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.590210:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.590213:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.590214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.590216:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.590219:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.590220:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.590221:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.590223:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.590224:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.590225:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.590228:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.590230:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.590233:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.590234:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.590235:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.590236:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.590237:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.590238:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.590238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.590321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.590327:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.590329:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.590331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.590332:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.590334:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.590336:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.590337:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.590338:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.590339:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.590340:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.590343:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.590344:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.590345:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.590346:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.590346:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.590348:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.590349:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.590350:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.590350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.590362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.590364:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.590365:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.590366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.590367:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.590368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.590385:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.590387:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.590388:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.590388:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.590389:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.590390:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.590601:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.590604:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.590605:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.590606:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.590608:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.590613:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.590615:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.590617:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.590618:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.590618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.590620:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.590621:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.591000:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.591003:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.591004:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.591004:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.591007:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.591010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.591241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.591282:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.591284:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.591285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.591285:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.591287:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.591290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.591551:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.591554:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.591555:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.591555:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.591558:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.591562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.591706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.591713:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.591715:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.591716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.591718:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.591720:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.591721:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.591723:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.591724:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.591725:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.591726:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.591729:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.591731:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.591734:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.591735:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.591736:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.591737:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.591738:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.591739:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.591740:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.591741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.591746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.591752:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.591753:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.591754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.591755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677470.591817:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.591820:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.591820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.591821:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.591823:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.591826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.591984:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.591987:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.591988:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.591988:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.591990:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.591993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.592360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.592366:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.592368:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.592370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.592371:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.592373:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.592374:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.592376:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.592377:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.592378:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.592379:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.592382:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.592384:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.592386:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.592387:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.592388:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.592390:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.592390:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.592391:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.592391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.592403:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.592406:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.592406:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.592407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.592408:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.592409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.592421:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.592424:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.592424:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.592425:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.592425:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.592426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.592486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.592492:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.592494:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.592495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.592496:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.592498:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.592500:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.592500:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.592502:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.592502:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.592503:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.592507:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.592509:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.592512:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.592513:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.592513:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.592515:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.592516:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.592517:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.592517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.592848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.592854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.592857:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.592858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.592860:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.592862:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.592863:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.592865:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.592866:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.592867:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.592868:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.592871:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.592873:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.592875:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.592876:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.592876:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.592878:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.592879:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.592880:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.592880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.592886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.593064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.593107:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.593110:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.593110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.593111:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.593113:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.593115:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.594579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.594616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.595143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.595149:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.595152:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.595153:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.595155:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.595158:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.595160:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.595161:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.595163:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.595164:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.595165:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.595168:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.595169:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.595171:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.595172:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.595172:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.595174:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.595174:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.595175:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.595176:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.595187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.595189:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.595190:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.595190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.595192:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.595193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:13.0:1619677470.595942:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.595944:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.595945:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.595946:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.595947:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:13.0:1619677470.595948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.595949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.595951:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.595952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.595952:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.595955:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.595958:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.595968:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.595971:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.595971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.595972:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.595973:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.595973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.596088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.596094:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.596097:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.596098:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.596100:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.596102:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.596103:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:5.2:1619677470.596105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000400:00000200:2.0:1619677470.596105:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.596106:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.596107:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.596108:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.596111:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000400:00000200:2.0:1619677470.596112:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.596113:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677470.596114:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:2.0:1619677470.596114:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.596117:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677470.596117:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677470.596118:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.596119:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000400:00000200:1.0:1619677470.596119:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.596120:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.596121:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677470.596121:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:2.0:1619677470.596122:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:1.0:1619677470.596122:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677470.596123:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:1.0:1619677470.596123:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677470.596124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:1.0:1619677470.596125:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.596125:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.2:1619677470.596129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677470.596129:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677470.596130:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677470.596133:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.596134:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.596135:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:1.0:1619677470.596135:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.596136:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:1.0:1619677470.596136:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.596137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677470.596137:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:2.0:1619677470.596138:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677470.596138:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677470.596138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:18.0:1619677470.596226:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.596229:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.596230:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.596230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.596232:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.596236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.596519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.596526:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.596528:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.596529:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.596531:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.596534:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.596535:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.596537:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.596538:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.596539:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.596540:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.596543:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.596545:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.596547:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.596548:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.596548:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.596550:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.596550:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.596551:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.596551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.596918:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.596921:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.596922:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.596922:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.596925:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.596928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.596989:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.596991:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.596992:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.596992:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.596994:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.596998:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.597099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.597141:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.597143:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.597144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.597144:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.597147:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.597149:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:13.0:1619677470.597342:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.597344:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.597345:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.597346:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.597348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.597349:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.597392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.597398:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.597402:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.597403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.597405:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.597408:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.597409:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.597410:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.597412:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.597413:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.597414:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.597417:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.597419:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.597423:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.597424:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.597424:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.597426:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.597427:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.597428:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.597428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.597788:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.597791:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.597792:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.597792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.597794:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.597797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.597806:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.597808:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.597809:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.597809:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.597810:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.597811:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.597908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.597913:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.597916:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.597917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.597918:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.597921:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.597922:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.597924:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.597925:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.597926:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.597927:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.597929:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.597931:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.597933:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.597934:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.597935:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.597936:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.597937:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.597938:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.597938:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.598242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.598437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.598443:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.598447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.598447:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677470.598450:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.598452:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.598454:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.598456:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.598457:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.598459:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.598460:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.598465:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.598467:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.598470:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.598470:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.598471:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.598473:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.598474:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.598475:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.598476:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.598477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.598488:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.598490:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.598491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.598492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.598765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.598772:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.598775:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.598776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.598778:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.598781:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.598783:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.598784:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.598786:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.598787:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.598788:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.598792:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.598794:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.598797:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.598798:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:18.0:1619677470.598799:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:18.0:1619677470.598800:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:18.0:1619677470.598801:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.598802:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.598803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677470.598983:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.598986:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.598986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.598987:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.598989:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.598992:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.599561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.599678:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.599680:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.599682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.599682:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.599684:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.599687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.599696:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.599699:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.599700:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.599700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.599701:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.599702:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.599960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.599965:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.599968:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.599969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.599970:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.599973:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.599975:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.599976:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.599978:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.599978:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.599979:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.599982:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.599984:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.599986:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.599986:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.599987:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.599989:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.599989:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.599990:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.599991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.600002:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.600004:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.600004:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.600005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.600006:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.600008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.600507:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.600510:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.600511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.600511:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.600513:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.600516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.600524:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.600527:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.600527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.600528:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.600529:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.600530:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.600687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.600693:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.600696:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.600697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.600699:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.600701:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.600702:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.600703:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.600705:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.600706:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.600707:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.600709:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.600711:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.600713:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.600714:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.600714:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.600715:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.600716:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.600717:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.600717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.600729:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.600731:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.600732:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.600733:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.600734:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.600735:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.601510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.601516:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.601519:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.601520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.601522:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.601524:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.601525:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.601527:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.601528:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.601529:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.601530:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.601533:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.601535:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.601538:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.601539:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.601539:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.601541:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.601541:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.601543:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.601544:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.601544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.601549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.601555:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.601556:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.601557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.601558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.601784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.601790:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.601792:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677470.601794:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677470.601795:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.601797:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.601799:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677470.601800:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.601802:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.601803:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.601804:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677470.601807:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.601809:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677470.601811:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.601812:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:2.0:1619677470.601813:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:2.0:1619677470.601814:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:2.0:1619677470.601815:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.601816:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677470.601817:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677470.601818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677470.601823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:1.0:1619677470.601828:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:18.0:1619677470.601829:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:1.0:1619677470.601830:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:18.0:1619677470.601831:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677470.601832:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:17.0:1619677470.601832:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.601832:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.601833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:1.0:1619677470.601834:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.601837:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.601873:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.601876:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.601877:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.601877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.601879:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.601882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.602228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.602234:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.602237:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.602238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.602240:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.602242:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.602244:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.602245:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.602247:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.602248:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.602249:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.602253:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.602254:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.602258:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.602258:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.602259:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.602261:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.602261:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.602262:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.602263:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.602495:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.602498:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.602498:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.602499:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.602499:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.602500:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.602860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.602867:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.602870:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.602870:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677470.602872:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.602875:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.602876:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.602878:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.602879:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.602880:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.602881:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.602884:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.602886:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.602889:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.602890:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.602891:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.602893:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.602893:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.602894:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.602894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.602907:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.602909:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.602910:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.602912:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.602914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.602915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.603142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.603185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.603199:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.603202:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.603203:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.603204:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.603206:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.603209:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.603415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:12.0:1619677470.603470:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.603472:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.603473:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.603474:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.603477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.603477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.603487:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.603490:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.603491:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.603492:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.603492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.603494:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.603557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.603594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677470.603974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677470.604223:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.604226:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.604228:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677470.604230:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.604230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.604232:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677470.604234:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96521753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.604235:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.604236:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677470.604237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:13.0:1619677470.604237:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677470.604240:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.604242:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677470.604243:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677470.604245:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.604246:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:13.0:1619677470.604246:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:2.0:1619677470.604246:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.604248:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677470.604248:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:13.0:1619677470.604249:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.604249:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000400:00000200:2.0:1619677470.604249:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.604250:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:2.0:1619677470.604253:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.604254:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677470.604256:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.604257:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.604258:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.604259:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677470.604262:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.604262:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:2.0:1619677470.604264:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.604265:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.604266:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:2.0:1619677470.604266:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677470.604266:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.604267:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:2.0:1619677470.604268:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000400:00000200:1.0:1619677470.604268:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.604270:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:2.0:1619677470.604270:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.604271:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.604272:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.604273:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677470.604274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.604276:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:20.2:1619677470.604278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677470.604278:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.604279:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.604279:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.604280:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.604280:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:18.0:1619677470.604284:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.604286:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.604287:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.604288:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.604378:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.604381:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.604382:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.604382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.604384:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.604387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:18.0:1619677470.605392:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.605395:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.605396:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.605396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.605398:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.605402:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.605436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.605463:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.605468:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.605471:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.605472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.605473:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.605476:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.605477:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.605479:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.605480:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.605481:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677470.605481:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000400:00000200:2.0:1619677470.605482:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677470.605484:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.605485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:2.0:1619677470.605485:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677470.605485:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.605488:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677470.605488:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.605491:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.605491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.605492:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.605493:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.605495:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.605495:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.605496:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.605497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.606346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.606352:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.606355:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.606355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.606357:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.606360:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.606362:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.606363:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.606365:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.606366:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.606367:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.606370:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.606372:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.606374:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.606375:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.606376:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.606377:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.606378:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.606379:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.606379:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.606567:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.606570:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.606571:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.606572:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.606574:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.606577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.606770:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.606773:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.606774:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.606774:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.606777:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.606779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.606790:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.606793:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.606794:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.606794:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.606795:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.606796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.607099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.607106:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.607109:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.607110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.607112:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.607115:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.607117:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.607118:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.607120:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.607121:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.607122:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.607125:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.607126:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.607130:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.607130:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.607131:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.607133:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.607133:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.607135:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.607136:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.607136:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.607141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.607147:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.607149:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.607149:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.607150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.607688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.607694:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.607697:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.607698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.607700:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.607703:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.607704:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.607705:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.607706:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.607707:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.607708:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.607711:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.607713:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.607716:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.607717:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.607718:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.607719:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.607719:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.607720:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.607721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.608201:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.608204:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.608205:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.608205:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.608208:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.608213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.608677:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.608680:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.608681:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.608681:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.608683:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.608686:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.608784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.608880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.608925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.608932:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.608935:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.608935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.608938:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.608940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.608942:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677470.608943:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.608945:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.608946:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.608947:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677470.608951:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.608953:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677470.608955:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.608956:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:18.0:1619677470.608957:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:18.0:1619677470.608958:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:18.0:1619677470.608959:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.608960:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.608960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.609435:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.609438:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.609439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.609439:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.609441:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.609444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.609751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.609787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677470.610133:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.610136:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.610139:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.610141:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.610142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.610144:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677470.610145:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96521718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.610146:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.610147:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.610148:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677470.610151:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.610153:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677470.610155:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.610156:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:12.0:1619677470.610157:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:12.0:1619677470.610159:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:12.0:1619677470.610159:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.610160:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.610160:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677470.610173:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.610176:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.610177:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.610177:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.610179:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.610182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.610191:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.610194:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.610195:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.610195:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.610196:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.610197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.610444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.610450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.610453:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.610454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.610455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:2.0:1619677470.610456:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.610459:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.610460:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677470.610460:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:18.0:1619677470.610461:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677470.610462:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.610463:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.610466:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.610467:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.610472:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.610474:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.610475:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.610477:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.610477:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.610478:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000400:00000200:18.0:1619677470.610479:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677470.610479:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000400:00000200:18.0:1619677470.610480:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677470.610480:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.610482:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.610483:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.610483:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677470.610484:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.610485:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.610486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:18.0:1619677470.610487:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.610488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.610489:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677470.610489:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.610490:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.610491:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.610492:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.610493:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000400:00000200:2.0:1619677470.610493:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.610494:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:2.0:1619677470.610494:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.610495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.610496:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:18.0:1619677470.610496:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.610497:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.610498:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.611078:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.611081:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.611082:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.611082:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.611084:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.611086:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.611232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.611237:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.611240:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.611241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.611243:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.611245:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.611246:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.611248:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.611249:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.611250:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.611251:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.611255:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.611258:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.611260:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.611261:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.611262:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.611263:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.611264:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.611265:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.611266:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.611267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.611272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.611277:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.611279:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.611280:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.611281:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.611369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.611412:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.611415:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.611416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.611416:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.611418:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.611421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.611996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.612002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.612005:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.612006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.612008:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.612010:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.612012:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.612013:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.612014:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.612015:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.612017:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.612020:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.612022:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.612025:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.612026:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.612026:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.612029:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.612029:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.612030:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.612030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.612355:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.612358:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.612358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.612361:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.612363:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.612364:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.612366:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.612367:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.612368:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.612369:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.612372:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.612374:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.612377:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.612378:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.612378:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.612380:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.612381:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.612381:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.612382:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677470.612637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.612643:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.612646:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.612647:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.612649:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.612651:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.612653:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.612654:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.612656:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.612657:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.612658:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.612663:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.612665:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.612668:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.612669:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.612670:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.612671:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.612672:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.612673:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.612673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.612685:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.612687:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.612688:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.612689:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.612692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.612693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.613885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.613892:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.613895:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.613896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.613898:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.613902:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.613903:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.613904:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.613906:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.613907:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.613908:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.613911:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.613913:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.613914:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.613915:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.613916:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.613917:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.613918:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.613919:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.613920:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.613921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.613927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.613933:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.613934:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.613934:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.613934:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.613935:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.613935:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.613936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.613937:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:2.0:1619677470.613937:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.613938:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.614238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.614258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.614261:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.614262:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.614263:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.614265:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.614268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:17.0:1619677470.614347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.614349:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.614350:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.614351:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.614353:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.614356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.614835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.615009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.2:1619677470.615186:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.615484:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.615486:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.615487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.615488:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.615490:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.615495:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.615496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.615498:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.615499:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.615500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.615500:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.615501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.615703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:5.2:1619677470.616059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.616065:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.616068:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.616069:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.616071:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.616073:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.616075:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.616075:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.616077:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.616078:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.616079:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.616082:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.616084:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.616087:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.616087:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.616088:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.616090:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.616090:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.616091:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.616091:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.616275:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.616278:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.616279:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.616279:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.616281:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.616284:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.616643:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.616646:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.616647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.616649:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.616651:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.616653:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677470.616654:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.616655:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.616656:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.616657:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677470.616661:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.616663:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677470.616666:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.616667:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:17.0:1619677470.616668:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:17.0:1619677470.616670:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:17.0:1619677470.616670:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.616672:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677470.616673:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677470.616673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.616684:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677470.616687:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677470.616687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.616688:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:5.2:1619677470.616877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.616884:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.616887:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.616889:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.616890:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.616893:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.616895:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.616896:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.616898:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.616899:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.616900:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.616903:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.616904:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.616907:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.616908:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.616909:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.616911:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.616912:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.616913:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.616913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.617245:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.617248:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.617249:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.617249:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.617251:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.617254:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.617471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.617477:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.617480:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.617481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.617483:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.617486:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.617488:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.617490:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.617491:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.617492:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.617493:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.617497:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.617498:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.617501:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.617502:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.617503:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.617505:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.617505:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.617506:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.617506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.617519:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.617522:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.617523:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.617525:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.617527:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.617528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.617819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.617851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.617857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.617860:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:20.2:1619677470.617861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677470.617861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.617862:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.617865:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.617867:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.617868:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96521667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.617869:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677470.617869:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000400:00000200:2.0:1619677470.617870:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677470.617871:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:2.0:1619677470.617872:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677470.617872:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.617873:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.617876:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677470.617877:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677470.617879:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.617880:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.617883:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.617884:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.617884:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.617886:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.617887:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.617888:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.617889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.617969:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.617971:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.617972:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.617972:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.617975:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.617978:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.617988:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.617990:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.617991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.617991:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.617992:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.617993:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.618325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.618331:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.618333:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.618334:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.618336:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.618339:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.618340:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.618341:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.618343:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.618344:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.618345:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.618348:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.618350:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.618352:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.618352:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.618353:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.618354:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.618355:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.618356:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.618356:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.618367:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.618370:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.618370:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.618371:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.618372:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.618373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.618439:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.618441:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.618442:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.618443:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.618445:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.618448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.618993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.2:1619677470.619610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.619841:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.619844:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.619845:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.619845:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.619847:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.619851:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.619910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.619915:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.619918:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.619918:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.619920:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.619922:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.619923:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.619924:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.619925:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.619926:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.619927:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.619930:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.619931:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.619934:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.619935:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.619935:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.619937:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.619938:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.619939:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.619940:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.619940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.619945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.619950:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.619952:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.619953:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.619954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.620188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:17.0:1619677470.620484:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.620487:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.620488:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.620488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.620490:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.620494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.620624:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.620626:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.620627:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.620628:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.620630:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.620633:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.620835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677470.620837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.620841:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.620843:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.620844:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.620847:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.620849:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.620850:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.620852:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.620854:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.620855:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.620857:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.620865:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.620867:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.620870:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.620871:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.620872:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.620874:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.620875:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.620876:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.620877:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.620878:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.620884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.620889:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.620892:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.620892:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.620894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.621391:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.621394:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.621396:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677470.621399:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.621399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.621401:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677470.621402:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96521644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.621403:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.621404:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.621405:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677470.621408:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.621410:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677470.621413:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.621414:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:13.0:1619677470.621415:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:13.0:1619677470.621416:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:13.0:1619677470.621417:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.621418:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.621418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677470.621431:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.621434:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.621435:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.621435:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.621438:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.621442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.621451:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.621453:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.621454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.621454:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.621455:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.621456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.621528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.621534:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.621537:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.621538:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.621539:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.621542:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.621543:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.621545:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.621546:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.621547:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.621548:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.621552:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.621554:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.621557:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.621557:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.621558:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.621560:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.621560:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.621561:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.621562:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.621740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.621745:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.621748:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.621749:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.621751:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.621754:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.621756:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.621757:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.621758:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.621759:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.621760:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.621763:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.621764:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.621767:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.621768:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.621769:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.621771:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.621771:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.621772:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.621773:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.622309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.622315:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.622318:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.622319:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.622321:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.622323:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.622325:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.622326:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.622328:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.622329:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.622330:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.622333:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.622335:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.622338:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.622339:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.622339:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.622341:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.622342:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.622342:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.622343:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.622365:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.622368:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.622369:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.622371:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.622373:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.622374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.622619:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.622622:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.622623:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.622625:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.622628:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.622630:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.622632:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.622633:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.622634:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.622635:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.622638:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.622639:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.622642:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.622643:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.622643:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.622645:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.622645:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.622646:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.622646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.623707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.623751:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.623753:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.623754:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.623754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.623756:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.623758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.623768:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.623770:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.623771:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.623772:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.623772:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.623773:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.623868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.2:1619677470.623905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:18.0:1619677470.624400:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.624404:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.624406:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.624406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.624409:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.624414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.624802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.624835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.625424:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.625427:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.625428:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.625428:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.625430:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.625433:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.625542:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.625545:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.625546:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.625546:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.625548:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.625551:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.625596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.625603:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.625605:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.625607:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.625608:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.625611:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.625613:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677470.625614:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.625615:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.625616:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.625617:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677470.625621:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.625622:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.625625:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.625626:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:2.0:1619677470.625627:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:2.0:1619677470.625629:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:2.0:1619677470.625629:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.625630:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.625630:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.625670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.625676:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.625679:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.625680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.625682:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.625684:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.625685:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.625687:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.625688:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.625689:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.625690:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.625693:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.625694:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.625696:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.625697:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.625698:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.625700:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.625700:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.625701:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.625701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.625867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.625906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:18.0:1619677470.626212:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.626215:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.626216:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.626216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.626218:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.626222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.626659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.626702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.626708:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.626711:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.626712:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.626714:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.626717:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.626718:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.626720:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.626721:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.626722:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.626723:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.626727:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.626729:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.626732:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.626733:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.626734:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.626736:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.626737:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.626738:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.626739:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.626740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.626745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.626750:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.626752:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.626753:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.626754:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.626782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.626787:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.626790:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.626791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.626794:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.626797:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.626799:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.626801:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.626802:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.626803:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.626804:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.626808:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.626810:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.626812:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.626813:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.626813:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.626815:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.626816:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.626817:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.626817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.626828:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.626830:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.626831:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.626832:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.626833:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.626835:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.627262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.627268:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.627271:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.627272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.627274:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.627276:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.627278:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.627279:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.627280:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.627281:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.627282:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.627286:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.627288:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.627291:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.627292:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.627293:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.627299:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.627300:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.627301:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.627301:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.627314:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.627317:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.627318:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.627320:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.627321:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.627322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.627333:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.627336:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.627337:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.627338:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.627338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.627340:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.628317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.628324:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.628326:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.628328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.628330:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.628332:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.628333:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.628335:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.628336:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.628337:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.628338:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.628342:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.628343:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.628345:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.628346:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.628347:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.628348:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.628348:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.628349:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.628350:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.628351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.628357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.628362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.628363:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.628363:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677470.628364:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.628364:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.628365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.628366:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.628366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.628367:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.628367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.629061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:2.0:1619677470.629357:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.629360:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.629361:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.629361:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.629363:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.629367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.629378:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.629381:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.629382:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.629382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.629383:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.629384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.629623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.629634:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677470.629637:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677470.629638:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677470.629640:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.629643:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.629644:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.629646:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.629648:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.629649:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.629651:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.629655:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.629658:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.629660:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.629662:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.629663:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.629664:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.629665:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677470.629666:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677470.629666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677470.629678:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:17.0:1619677470.629678:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:18.0:1619677470.629680:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.629681:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.629681:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:2.0:1619677470.629681:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:17.0:1619677470.629682:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.629682:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:18.0:1619677470.629683:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677470.629684:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.629685:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.629687:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.629695:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.629698:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.629699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.629699:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.629699:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.629700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.630137:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.630139:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.630140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.630141:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.630143:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.630146:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.630465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:5.2:1619677470.630467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.630468:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.630470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.630470:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.630471:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677470.630472:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.630473:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.630476:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677470.630476:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.630477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.630481:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.630482:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.630483:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.630484:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.630485:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.630486:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.630490:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.630492:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.630494:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.630495:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.630495:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.630497:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.630497:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.630499:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.630500:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.630500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.630505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.630512:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.2:1619677470.630513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.630514:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.630515:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.630516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.630629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.630635:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.630638:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.630639:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.630641:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.630644:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.630645:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.630647:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.630648:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.630649:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.630650:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.630654:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.630656:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.630659:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.630660:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.630661:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.630663:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.630664:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.630665:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.630665:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.631246:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.631249:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.631249:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.631250:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.631252:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.631255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.631266:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.631269:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.631270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.631270:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.631271:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.631271:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.631304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677470.631455:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.631458:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.631459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.631461:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.631464:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.631466:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.631467:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.631468:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.631469:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.631470:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.631473:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.631475:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.631478:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.631479:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.631480:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.631482:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.631482:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.631483:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.631483:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677470.631573:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.631576:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.631577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.631577:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.631580:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.631583:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.631622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.631628:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.631630:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.631631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.631634:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.631637:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.631639:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.631641:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.631642:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.631644:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.631645:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.631649:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.631651:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.631654:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.631655:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.631656:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (22)++
00000800:00000200:18.0:1619677470.631658:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (24)--
00000800:00000200:18.0:1619677470.631659:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.631659:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.631660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.632077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.632083:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.632085:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.632086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.632088:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.632090:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.632092:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.632093:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.632094:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.632095:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.632096:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.632100:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.632102:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.632105:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.632106:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.632107:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.632109:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.632109:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.632110:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.632110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.632123:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.632126:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.632126:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.632128:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.632130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.632131:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.633261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.633299:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.633865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.633871:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.633874:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.633875:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.633877:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.633880:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.633881:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.633883:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.633884:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.633886:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.633887:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.633890:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.633892:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.633894:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.633895:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.633896:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.633897:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.633898:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.633899:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.633899:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.634389:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.634392:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.634393:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.634393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.634395:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.634399:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.634663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.634669:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.634671:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.634672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.634674:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.634676:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.634677:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.634678:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.634680:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.634681:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.634682:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.634685:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.634687:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.634690:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.634690:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:2.0:1619677470.634691:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:2.0:1619677470.634692:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:2.0:1619677470.634693:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.634694:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.634694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.634802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.634808:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.634810:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.634811:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.634814:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.634816:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.634817:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.634819:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.634820:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.634821:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.634822:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.634825:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.634827:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.634829:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.634830:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.634831:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.634832:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.634833:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.634834:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.634834:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.634899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.634917:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.634920:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.634920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.634921:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.634923:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.634927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.635242:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.635245:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.635245:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.635248:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.635251:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.635252:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.635254:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.635255:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.635256:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.635257:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.635260:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.635262:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.635264:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.635265:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.635266:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.635268:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.635268:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.635269:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.635269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677470.635495:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.635498:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.635499:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.635499:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.635501:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.635505:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.636078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.636122:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.636125:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.636126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.636126:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.636128:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.636131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.636140:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.636143:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.636144:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.636144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.636145:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.636146:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.636507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.636513:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.636516:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.636517:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.636519:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.636521:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.636523:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.636524:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.636526:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.636527:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.636528:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.636531:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.636533:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.636536:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.636537:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.636538:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.636539:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.636540:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.636541:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.636541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.636778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.636783:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.636786:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.636787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.636789:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.636792:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.636794:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.636795:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677470.636796:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000400:00000200:2.0:1619677470.636796:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.636797:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.636798:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:12.0:1619677470.636799:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.636799:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.636802:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677470.636802:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.636803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.636804:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:2.0:1619677470.636805:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.636807:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.636808:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.636808:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.636810:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.636811:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.636812:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.636813:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:13.0:1619677470.636815:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677470.636818:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677470.636819:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.636819:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677470.636820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.636821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:18.0:1619677470.636824:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.636826:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.636827:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.636827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.636829:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.636831:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.636843:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.636846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.636846:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.636847:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.636847:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.636848:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.636892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.636899:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.636901:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.636902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.636904:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.636906:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.636908:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.636909:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.636911:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.636912:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.636914:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.636917:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.636920:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.636923:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.636924:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.636925:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.636927:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.636927:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.636928:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.636928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.636939:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.636942:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.636943:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.636944:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.636946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.636947:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:18.0:1619677470.637013:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.637016:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.637017:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.637017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.637019:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.637022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.637232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677470.638372:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.638375:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.638376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.638376:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.638379:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.638383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.638594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.638600:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.638602:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.638603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.638605:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.638607:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.638608:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677470.638610:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.638611:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.638612:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.638613:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677470.638617:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.638619:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677470.638621:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.638622:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:18.0:1619677470.638623:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:18.0:1619677470.638625:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:18.0:1619677470.638625:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.638626:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.638626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.639051:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.639054:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.639054:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.639055:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.639057:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.639061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.639308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.639448:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.639451:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.639453:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.639453:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.639455:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.639458:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.639467:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.639469:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.639470:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.639470:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.639471:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.639472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.639599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.639606:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.639609:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.639610:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.639612:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.639614:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.639616:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.639617:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.639620:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.639621:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.639622:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.639625:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.639627:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.639629:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.639630:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.639630:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.639632:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.639632:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.639634:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.639635:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.639635:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.639641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.639647:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.639649:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.639649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.639650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677470.639758:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.639761:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.639762:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.639762:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.639764:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.639768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.639782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.639787:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.639790:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.639791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.639793:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.639795:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.639799:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.639800:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.639802:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.639802:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.639804:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.639807:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.639809:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.639811:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.639812:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.639813:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.639814:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.639815:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.639816:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.639817:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.639818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.639823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.639828:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.639831:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.639832:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.639833:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677470.640250:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.640253:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.640254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.640256:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.640258:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.640260:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.640261:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.640262:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.640263:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.640264:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.640268:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.640270:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.640272:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.640273:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:1.0:1619677470.640274:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:1.0:1619677470.640275:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:1.0:1619677470.640276:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.640277:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.640277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677470.640445:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.640448:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.640448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.640449:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.640451:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.640454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:13.0:1619677470.640510:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.640513:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.640513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.640515:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.640517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.640518:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.640642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.640678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.640683:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.640686:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.640687:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.640687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.640689:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.640692:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.641676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.641682:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.641685:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.641686:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.641687:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.641690:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.641692:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.641693:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.641694:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.641695:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.641696:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.641699:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.641701:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.641704:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.641705:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.641705:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.641707:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.641708:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.641709:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.641709:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.641721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.641724:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.641725:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.641726:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.641728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.641729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.641740:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.641743:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.641743:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.641744:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.641745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.641746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.642094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.642100:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.642103:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.642104:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.642106:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.642109:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.642110:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.642112:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.642113:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.642114:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.642115:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.642118:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.642120:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.642123:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.642124:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.642124:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.642126:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.642127:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.642128:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.642128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.642463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.642516:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.642519:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.642520:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.642520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.642522:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.642525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.642736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.642767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.642774:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.642777:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.642777:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.642779:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.642783:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.642784:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.642785:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.642787:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.642787:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.642788:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.642791:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.642793:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.642795:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.642796:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.642796:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.642798:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.642799:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.642801:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.642802:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.642802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.642807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.642812:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.642814:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.642814:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.642815:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:2.0:1619677470.642815:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.642816:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.642816:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.642817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.642818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.642818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.643355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.643361:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.643364:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.643365:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.643366:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.643368:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.643369:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.643370:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.643372:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.643373:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.643374:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.643377:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.643379:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.643381:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.643382:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.643382:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.643384:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.643384:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.643385:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.643386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.643397:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.643399:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.643400:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.643400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.643402:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.643403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.643414:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.643417:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.643418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.643418:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.643418:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.643419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.643741:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.643744:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.643744:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.643745:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.643747:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.643750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.643794:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.643796:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.643797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.643797:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.643800:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.643803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:2.0:1619677470.644222:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.644225:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:2.0:1619677470.644226:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.644227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.644229:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.644232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:17.0:1619677470.644399:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.644402:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.644402:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.644405:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.644408:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.644409:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.644411:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.644413:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.644414:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.644415:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.644420:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.644423:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.644425:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.644426:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.644427:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.644428:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.644429:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.644430:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.644432:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.644432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.644443:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.644445:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.644446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:18.0:1619677470.644447:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:1.0:1619677470.644447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677470.644450:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.644451:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.644451:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.644453:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.644457:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.644568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.644574:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.644577:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.644577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.644579:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.644582:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.644583:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.644584:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.644585:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.644586:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.644587:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.644590:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.644592:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.644594:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.644595:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.644596:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.644597:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.644597:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.644598:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.644599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.644691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.645258:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.645261:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.645262:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.645263:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.645265:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.645268:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.645305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.645311:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.645313:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.645314:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.645316:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.645318:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.645320:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.645321:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.645322:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.645323:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.645324:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.645328:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.645329:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.645333:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.645333:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:18.0:1619677470.645334:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:18.0:1619677470.645336:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:18.0:1619677470.645336:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.645337:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.645337:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.645468:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.645471:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.645472:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.645473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.645476:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.645479:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.645784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.646051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.646057:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.646060:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.646061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.646064:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.646066:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.646068:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.646069:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.646070:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.646072:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.646073:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.646076:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.646078:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.646081:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.646082:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.646082:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.646084:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.646084:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.646086:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.646086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677470.646238:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.646241:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.646242:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.646242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.646245:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.646248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.646257:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:1.0:1619677470.646260:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.646261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.646261:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.646262:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.646263:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.646976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.646982:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.646985:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.646986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.646988:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.646990:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.646992:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.646994:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.646995:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.646996:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.646998:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.647002:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.647003:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:5.2:1619677470.647004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:17.0:1619677470.647007:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.647008:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.647008:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:18.0:1619677470.647010:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677470.647010:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.647011:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.647012:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.647012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677470.647013:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.647014:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.647016:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.647019:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.647020:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.647022:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.647023:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.647024:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.647025:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:13.0:1619677470.647026:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000400:00000200:18.0:1619677470.647028:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.647029:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:18.0:1619677470.647030:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:13.0:1619677470.647030:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.647033:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.647034:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.647034:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:12.0:1619677470.647034:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:18.0:1619677470.647035:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.647037:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:13.0:1619677470.647037:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:18.0:1619677470.647038:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.647040:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.647041:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.647042:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.647047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.647052:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.647055:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.647055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.647056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:5.2:1619677470.647141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc5600] (20)++
00000800:00000200:1.0:1619677470.647147:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc5600] (21)++
00000800:00000200:1.0:1619677470.647150:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.43@o2ib3
00000800:00000200:17.0:1619677470.647151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc5600] (22)--
00000400:00000200:1.0:1619677470.647153:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.43@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.647156:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.647157:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.647158:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.647160:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.647161:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.647162:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.647165:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.647167:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.647170:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.647171:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.647172:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.647173:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.647174:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc5600] (21)++
00000800:00000200:1.0:1619677470.647175:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cffdc5600] (22)++
00000800:00000200:1.0:1619677470.647177:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc5600] (23)--
00000800:00000200:1.0:1619677470.647177:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc5600] (22)--
00000800:00000200:5.2:1619677470.647182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc5600] (21)++
00000800:00000200:12.0:1619677470.647186:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:17.0:1619677470.647188:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc5600] (22)++
00000800:00000200:12.0:1619677470.647189:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:17.0:1619677470.647190:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cffdc5600] (23)--
00000400:00000200:12.0:1619677470.647190:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.647191:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc5600] (22)--
00000400:00000200:12.0:1619677470.647191:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.647192:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc5600] (21)--
00000800:00000200:13.0:1619677470.647194:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.647195:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.647918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.647924:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.647926:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.647927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.647929:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.647932:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.647934:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.647935:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.647937:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.647938:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.647939:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.647942:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.647944:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.647947:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.647948:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.647948:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.647950:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.647950:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.647951:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.647952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.648098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.648135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:2.0:1619677470.648435:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.648437:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.648439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.648439:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.648441:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.648444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.648542:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.648544:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.648545:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.648546:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.648548:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.648551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.648561:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.648563:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.648564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.648565:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.648566:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.648566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:5.2:1619677470.649423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.649429:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.649432:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.649433:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.649435:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.649438:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.649439:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.649441:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.649442:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.649444:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.649446:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.649450:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.649452:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.649455:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.649456:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.649457:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.649458:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.649459:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.649460:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.649461:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.649461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.649467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.649472:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.649474:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.649475:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.649476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677470.649485:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.649488:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.649488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.649489:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.649491:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.649495:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.649506:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.649509:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.649509:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[2] from 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677470.649511:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.2.43@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.113@o2ib7 : PUT - routed
00000800:00000200:1.0:1619677470.649513:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:2.0:1619677470.649818:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.649821:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.649822:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.649822:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.649824:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.649827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.649837:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.649840:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.649840:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.649841:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.649842:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.649842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.649999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.650004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.650007:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.650008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.650010:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.650013:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.650015:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.650016:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.650017:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.650018:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.650019:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.650022:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.650024:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.650026:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.650026:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.650027:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.650029:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.650029:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.650030:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.650030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.650042:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.650043:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.650044:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.650045:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.650045:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.650047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.650182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.650221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677470.650243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc5600] (20)++
00000800:00000200:17.0:1619677470.650249:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc5600] (21)++
00000800:00000200:17.0:1619677470.650252:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.43@o2ib3
00000800:00000200:1.0:1619677470.650253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc5600] (22)--
00000400:00000200:17.0:1619677470.650255:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.43@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677470.650257:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.650258:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677470.650260:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.650261:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.650262:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.650263:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677470.650266:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677470.650268:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.650270:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.650270:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:17.0:1619677470.650271:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.650272:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.650273:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc5600] (21)++
00000800:00000200:17.0:1619677470.650273:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc5600] (22)--
00000800:00000200:17.0:1619677470.650274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc5600] (21)--
00000800:00000200:1.0:1619677470.650296:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.650298:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:1.0:1619677470.650298:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.650300:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.650300:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677470.650302:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.650410:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.650413:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.650414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.650414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.650416:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.650419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.650809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc5600] (20)++
00000800:00000200:1.0:1619677470.650814:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc5600] (21)++
00000800:00000200:1.0:1619677470.650817:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.43@o2ib3
00000800:00000200:17.0:1619677470.650818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc5600] (22)--
00000400:00000200:1.0:1619677470.650821:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.43@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677470.650822:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.650824:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677470.650825:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.650827:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.650829:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.650830:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677470.650833:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677470.650836:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.650837:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.650838:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:1.0:1619677470.650839:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.650840:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.650841:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc5600] (21)++
00000800:00000200:1.0:1619677470.650842:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc5600] (22)--
00000800:00000200:1.0:1619677470.650843:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc5600] (21)--
00000800:00000200:17.0:1619677470.650855:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.650858:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.650859:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.650859:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.650861:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677470.650863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:2.0:1619677470.651080:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.651083:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.651084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.651084:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.651086:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.651090:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.651161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.651831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.651837:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.651840:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.651841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.651843:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.651845:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.651847:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.651848:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.651849:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.651850:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.651851:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.651854:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.651856:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.651859:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.651860:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.651860:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.651862:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.651863:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.651864:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.651864:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.651880:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.651883:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.651883:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.651885:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.651887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.651888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:5.2:1619677470.651929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.651936:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.651938:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.651939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.651941:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.651944:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.651945:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.651947:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.651948:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.651949:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.651950:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.651954:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.651955:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.651959:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.651960:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.651960:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.651962:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.651963:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.651963:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.651964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.652181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.652186:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.652189:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.652190:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.652191:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.652194:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.652196:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.652197:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.652198:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.652199:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.652201:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.652203:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.652205:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.652207:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.652208:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:2.0:1619677470.652208:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:2.0:1619677470.652210:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:2.0:1619677470.652210:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.652211:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.652211:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.652898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.652908:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.652910:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.652910:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.652912:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.652913:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.652913:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.652915:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:18.0:1619677470.652916:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.652918:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.652919:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.652921:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.652922:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.652923:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.652924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.652925:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.652926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000400:00000200:18.0:1619677470.652927:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.652929:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.652931:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.652932:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.652933:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.652934:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.652935:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.652937:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.652938:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.652938:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.652943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.652949:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.652951:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.652952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.652954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.653427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677470.654440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.654447:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.654450:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.654450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.654453:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.654456:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.654458:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.654459:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.654460:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.654462:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.654463:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.654466:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.654467:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.654470:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.654471:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.654472:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.654473:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.654474:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.654475:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.654475:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677470.654530:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.654532:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.654533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.654534:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.654537:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.654541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.654702:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.654705:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.654706:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.654706:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.654708:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.654711:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.655124:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.655127:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.655128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.655130:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.655133:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.655135:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.655136:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.655137:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.655138:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.655139:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.655142:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.655144:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.655146:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.655147:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.655148:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.655149:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.655150:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.655150:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.655151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677470.655358:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.655361:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.655361:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.655362:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.655364:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.655366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.655377:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.655380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.655380:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677470.655382:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.655384:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.655385:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.655386:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.655388:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.655389:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.655390:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.655393:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.655395:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.655397:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.655398:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:1.0:1619677470.655399:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:1.0:1619677470.655401:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:1.0:1619677470.655401:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.655402:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.655402:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677470.655495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.655609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.655616:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.655618:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.655619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.655621:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.655623:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.655625:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.655626:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.655627:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.655629:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.655630:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.655633:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.655635:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.655637:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.655638:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.655639:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.655640:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.655641:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.655642:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.655642:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677470.655882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.655888:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.655890:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.655892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.655893:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.655896:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.655897:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.655899:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.655900:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.655901:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.655902:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.655905:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.655907:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.655910:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.655910:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.655911:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.655913:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.655913:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.655914:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.655915:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.655916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.655921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.655927:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.655929:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.655930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.655931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.656605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.656828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.656834:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.656837:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.656838:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.656840:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.656843:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.656845:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.656846:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.656847:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.656848:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.656849:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:5.2:1619677470.656851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677470.656853:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.656854:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677470.656857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677470.656858:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.656859:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.656860:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:17.0:1619677470.656860:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.656860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677470.656862:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.656862:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.656863:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677470.656863:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677470.656865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:17.0:1619677470.656865:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.656867:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.656868:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.656869:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.656870:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.656871:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.656874:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.656876:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.656879:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.656880:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.656880:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.656882:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.656882:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.656883:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.656884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677470.656898:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.656902:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.656903:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.656906:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.656907:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.656909:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.656910:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)++
00000800:00000200:13.0:1619677470.656910:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (24)--
00000800:00000200:12.0:1619677470.656910:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (24)--
00000800:00000200:13.0:1619677470.656911:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.656912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:2.0:1619677470.658092:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.658094:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.658096:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.658096:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.658098:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.658103:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.658104:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.658106:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.658107:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.658108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.658108:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.658109:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.658166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.658172:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.658174:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.658175:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.658177:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.658179:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.658180:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.658181:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.658183:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.658184:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.658184:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.658188:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.658189:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.658191:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.658192:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.658193:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.658194:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.658195:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.658195:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.658196:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.658207:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.658209:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.658209:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.658210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.658211:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.658212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.658670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.658676:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.658679:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.658679:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.658681:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.658683:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.658685:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677470.658686:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.658687:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.658688:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.658689:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677470.658692:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.658694:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677470.658697:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.658698:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:18.0:1619677470.658699:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:18.0:1619677470.658700:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:18.0:1619677470.658701:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.658702:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.658702:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.658789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677470.659217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.659223:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.659226:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.659227:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.659229:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.659232:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.659233:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.659235:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677470.659236:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000400:00000200:1.0:1619677470.659236:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.659237:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.659238:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.659239:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.659240:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.659241:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.659241:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677470.659244:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.659244:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.659247:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.659247:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.659248:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.659248:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.659250:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.659251:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.659252:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.659253:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.659254:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.659258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.659264:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.659265:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.659266:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.659267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.659527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:1.0:1619677470.659564:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.659567:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.659568:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.659570:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.659573:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.659574:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677470.659575:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96521422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677470.659576:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)++
00000400:00000200:1.0:1619677470.659577:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677470.659579:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (24)--
00000800:00000200:2.0:1619677470.659579:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (24)++
00000400:00000200:1.0:1619677470.659579:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.659580:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.659580:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (24)--
00000400:00000200:1.0:1619677470.659580:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677470.659581:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.659582:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677470.659583:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677470.659585:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.659587:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.659589:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.659589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.659590:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:1.0:1619677470.659590:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.659590:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (21)++
00000800:00000200:1.0:1619677470.659591:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (23)++
00000800:00000200:18.0:1619677470.659592:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.659593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (24)--
00000800:00000200:1.0:1619677470.659593:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (24)--
00000800:00000200:18.0:1619677470.659594:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (24)++
00000800:00000200:1.0:1619677470.659594:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:18.0:1619677470.659595:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.659595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:1.0:1619677470.659595:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677470.659596:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677470.659596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:2.0:1619677470.659606:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:17.0:1619677470.659607:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:2.0:1619677470.659608:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:17.0:1619677470.659609:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:2.0:1619677470.659609:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.659610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:17.0:1619677470.659610:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.659611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:2.0:1619677470.659611:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.659614:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.660586:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.660588:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.660589:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.660590:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.660592:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.660595:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.660952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.660993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.661564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.661571:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.661574:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.661575:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.661577:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.661580:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.661581:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.661583:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.661584:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.661585:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.661586:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.661589:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.661591:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.661593:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.661595:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.661596:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.661598:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.661598:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.661599:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.661600:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.661730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.661791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.661796:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.661799:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.661800:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.661801:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.661804:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.661806:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.661807:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.661808:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.661809:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.661811:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.661814:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.661815:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.661818:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.661819:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.661820:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.661822:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.661823:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.661823:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.661824:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.662466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.662473:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.662475:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.662476:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.662478:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.662481:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.662483:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.662484:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.662486:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.662487:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.662488:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.662492:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.662494:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.662496:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.662497:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.662497:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.662498:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000400:00000200:2.0:1619677470.662498:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.662499:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000400:00000200:2.0:1619677470.662500:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.662501:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:18.0:1619677470.662502:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.662503:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.662503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.662503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.662504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.662714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.662720:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.662723:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.662724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.662726:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.662728:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.662729:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.662730:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.662732:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.662732:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.662733:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.662736:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.662738:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.662741:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.662742:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.662743:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.662745:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.662745:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.662746:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.662746:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.662760:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.662764:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.662765:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.662768:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.662770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.662772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.664170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:5.2:1619677470.664452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677470.664458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677470.664461:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677470.664462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677470.664464:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.664467:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.664468:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677470.664470:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.664471:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.664472:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.664473:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677470.664476:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.664477:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677470.664480:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e82fe5580] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.664481:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c000] (20)++
00000800:00000200:1.0:1619677470.664482:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c000] (21)++
00000800:00000200:1.0:1619677470.664484:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c000] (22)--
00000800:00000200:1.0:1619677470.664484:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677470.664485:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677470.664486:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:17.0:1619677470.664623:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.664626:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.664627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.664629:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.664632:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.664633:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677470.664635:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.664636:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.664637:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.664638:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677470.664641:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.664642:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677470.664645:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.664646:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:17.0:1619677470.664646:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:17.0:1619677470.664648:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:17.0:1619677470.664648:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.664649:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.664650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677470.665044:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.665047:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.665048:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.665048:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.665050:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.665054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.665143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.665185:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.665187:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.665189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.665189:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.665191:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.665193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.665206:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.665209:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.665209:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.665209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.665211:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.665211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.665907:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.665909:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.665911:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.665911:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.665913:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.665916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.665925:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.665928:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.665929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.665930:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.665931:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.665932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.666269:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.666272:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.666273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.666274:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.666276:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.666279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.666650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.666656:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.666659:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.666660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.666661:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.666665:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.666667:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.666668:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.666669:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.666670:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.666671:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.666674:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.666676:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.666679:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.666680:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.666681:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.666682:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.666683:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.666684:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.666686:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.666686:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.666691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.666696:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.666699:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.666700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.666701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.666849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.666855:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.666858:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.666859:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.666861:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.666864:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.666866:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.666867:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.666868:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.666869:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.666870:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.666874:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.666876:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.666877:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.666878:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.666879:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.666881:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.666881:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.666882:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.666883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.666894:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.666896:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.666896:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.666897:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.666898:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.666900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.667002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.667018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.667024:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.667027:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.667028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.667031:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.667033:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.667035:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.667037:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.667038:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.667039:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.667040:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.667044:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.667046:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.667048:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.667049:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:18.0:1619677470.667050:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:18.0:1619677470.667052:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:18.0:1619677470.667053:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.667054:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.667054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677470.667183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.667189:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.667192:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.667193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.667195:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.667198:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.667200:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.667201:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.667202:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.667203:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.667204:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.667208:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.667210:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.667212:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.667213:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.667214:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.667216:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.667216:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.667217:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.667217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.667271:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.667273:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.667274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.667276:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.667279:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.667280:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.667281:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.667283:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.667284:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.667285:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.667288:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.667289:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.667291:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.667292:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.667292:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.667299:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.667300:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.667300:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.667301:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:12.0:1619677470.667510:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.667514:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:13.0:1619677470.667518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.667518:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677470.667521:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.667524:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677470.667526:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96521376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.667529:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.667530:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.667532:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677470.667538:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.667541:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677470.667545:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.667546:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:12.0:1619677470.667547:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:12.0:1619677470.667549:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:12.0:1619677470.667550:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.667552:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677470.667554:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677470.667555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.667568:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677470.667571:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677470.667571:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.667572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677470.667816:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.667818:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.667819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.667821:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.667824:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.667826:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.667827:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.667828:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.667829:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.667830:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.667833:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.667835:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.667837:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.667838:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.667839:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.667840:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.667840:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.667841:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.667842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:5.2:1619677470.667991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.667997:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.667999:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.668000:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.668002:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.668004:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.668005:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.668007:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.668008:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.668009:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.668010:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.668013:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.668014:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.668017:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.668018:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.668019:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.668021:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.668021:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.668022:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.668022:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.668033:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.668036:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.668037:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.668038:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.668040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.668042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.668376:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.668379:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.668380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.668380:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.668382:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.668386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.668426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.668432:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.668435:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.668437:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.668439:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.668442:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.668443:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.668445:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.668446:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.668447:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.668448:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.668451:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.668453:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.668455:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.668456:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.668456:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.668458:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.668458:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.668460:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.668461:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.668461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.668468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.668474:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.668475:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.668475:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677470.668476:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.668476:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.668476:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.668477:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.668478:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.668478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.668479:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.668800:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.668803:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.668804:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.668804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.668806:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.668809:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.669309:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.669312:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.669313:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.669315:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.669318:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.669319:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677470.669321:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.669322:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.669323:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.669324:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677470.669327:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.669329:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677470.669332:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.669333:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:1.0:1619677470.669334:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:1.0:1619677470.669336:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:1.0:1619677470.669336:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.669338:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677470.669339:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677470.669339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.669351:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677470.669353:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677470.669354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.669355:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677470.669535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.669578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:12.0:1619677470.669834:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.669837:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.669839:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.669842:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.669842:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.669843:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677470.669844:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96521362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.669846:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.669846:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.669847:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677470.669850:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.669852:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677470.669854:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.669855:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:12.0:1619677470.669856:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:12.0:1619677470.669857:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:12.0:1619677470.669858:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.669859:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.669859:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677470.670042:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.670045:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.670046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.670046:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.670048:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.670051:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.670061:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.670063:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.670064:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.670064:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.670066:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.670066:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.670400:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.670403:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.670404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.670404:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.670407:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.670410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.670421:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.670423:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.670424:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.670424:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.670426:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.670426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.670769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.670774:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.670777:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.670778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.670780:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.670782:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.670784:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677470.670785:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.670787:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.670788:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.670789:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677470.670793:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.670794:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677470.670797:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.670798:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:2.0:1619677470.670799:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:20.2:1619677470.670800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.670801:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:2.0:1619677470.670802:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.670803:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.670804:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.670805:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.670811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.670817:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.670819:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.670820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.670821:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.670836:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.670839:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.670840:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.670840:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.670842:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.2:1619677470.670843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677470.670845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:1.0:1619677470.670848:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.670851:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:1.0:1619677470.670852:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.670853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.670854:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.670856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.671536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.671543:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.671546:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.671547:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.671549:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.671551:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.671553:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.671554:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.671555:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.671556:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.671557:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.671561:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.671563:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.671566:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.671567:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.671567:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.671570:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.671570:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.671572:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.671572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.671952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.672137:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.672139:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.672141:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.672141:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.672143:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.672147:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.672216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.672222:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.672225:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.672226:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.672228:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.672231:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.672232:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.672234:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.672235:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.672236:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.672237:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.672240:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.672242:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.672244:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.672245:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.672246:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.672247:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.672248:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.672249:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.672250:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.672251:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.672256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.672263:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.672264:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.672265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.672266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677470.672977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.672984:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.672987:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.672987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.672990:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.672992:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.672993:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.672995:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.672996:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.672997:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.672998:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.673002:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.673004:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.673006:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.673007:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.673007:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.673009:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.673009:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.673010:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.673010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.673367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.673374:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.673377:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.673377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.673380:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.673383:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.673384:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.673386:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.673387:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.673388:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.673389:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.673393:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.673395:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.673398:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.673399:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:18.0:1619677470.673400:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:18.0:1619677470.673402:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:18.0:1619677470.673403:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.673403:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.673404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.673558:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.673560:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.673561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.673562:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.673564:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.673567:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.673577:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.673580:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.673581:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.673581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.673582:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.673583:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.673607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.673613:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.673616:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.673617:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.673620:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.673624:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.673625:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.673626:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.673628:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.673629:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.673630:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.673634:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.673637:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.673639:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.673640:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.673640:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.673642:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.673643:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.673644:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.673644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.673656:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.673658:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.673659:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.673659:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.673661:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.673663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.674873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677470.674932:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.674935:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.674936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.674936:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.674938:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.674942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.675273:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.675276:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.675276:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.675277:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.675279:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.675283:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.675308:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.675311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.675311:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677470.675313:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.675316:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.675317:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677470.675319:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.675320:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.675323:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.675324:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677470.675327:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.675328:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677470.675331:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.675332:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:1.0:1619677470.675333:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:1.0:1619677470.675335:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:1.0:1619677470.675336:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.675336:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.675337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677470.675350:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.675352:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.675353:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.675355:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.675358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.675359:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:20.2:1619677470.675667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.675673:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.675676:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.675676:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.675677:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:2.0:1619677470.675677:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.675679:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677470.675680:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.675682:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.675682:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.675683:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000400:00000200:18.0:1619677470.675684:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:2.0:1619677470.675684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.675685:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.675687:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.675688:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.675689:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.675693:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.675695:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.675697:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.675698:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.675699:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.675701:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.675701:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.675702:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.675703:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677470.676169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.676175:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.676178:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.676179:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.676181:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.676184:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.676185:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.676186:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.676188:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.676188:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.676189:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.676192:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.676193:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.676196:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.676196:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.676197:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.676198:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.676199:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.676199:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.676200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.676313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.676474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.676481:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.676484:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.676485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.676488:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.676490:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.676492:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.676493:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.676495:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.676497:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.676497:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.676501:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.676504:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.676507:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.676508:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.676509:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.676511:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.676511:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.676512:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.676513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.676544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.676550:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.676553:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.676554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.676557:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.676561:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.676562:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.676564:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.676566:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.676568:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.676569:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.676574:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.676577:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.676580:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.676581:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.676582:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.676584:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.676585:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.676586:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.676587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.676600:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.676602:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.676603:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.676604:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.676605:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.676607:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.676617:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.676620:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.676621:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.676621:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.676622:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.676623:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.677046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.677052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.677055:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.677056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.677058:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.677061:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.677062:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.677064:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.677065:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.677067:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.677067:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.677071:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.677073:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.677076:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.677076:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.677077:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:20.2:1619677470.677079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.677079:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.677079:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.677080:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.677080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.677467:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.677469:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.677470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.677470:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.677473:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.677476:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.677847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.677881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677470.678035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.678041:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.678043:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.678043:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.678044:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677470.678045:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.678047:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677470.678048:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677470.678051:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.678051:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.678052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.678053:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.678053:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.678054:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.678056:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.678057:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.678058:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.678061:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.678063:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.678066:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.678066:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.678067:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.678069:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.678070:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.678070:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.678071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.678083:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.678085:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.678087:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.678088:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.678091:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.678092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:17.0:1619677470.678573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.678576:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.678577:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.678577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.678579:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.678582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.679141:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.679144:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.679145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.679147:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.679150:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.679151:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.679152:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.679154:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.679155:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.679156:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.679159:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.679161:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.679163:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.679164:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.679164:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.679166:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.679166:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.679167:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.679168:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677470.679220:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.679223:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.679224:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.679224:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.679227:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.679230:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.679589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.679596:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.679599:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.679600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.679602:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.679604:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.679606:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.679607:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.679608:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.679609:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.679610:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.679614:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.679616:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.679619:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.679619:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.679620:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.679622:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.679622:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.679623:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.679624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.679794:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.679796:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.679797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.679798:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.679800:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.679802:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.679812:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.679814:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.679815:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.679815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.679819:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.679820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.680153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.680162:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.680165:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.680167:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000400:00000200:2.0:1619677470.680167:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677470.680168:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.680170:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.680170:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.680172:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677470.680172:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.680173:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.680175:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.680176:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.680177:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.680181:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.680183:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.680184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.680185:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.680185:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.680186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.680186:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.680187:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.680189:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.680190:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.680190:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.680191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.680202:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.680204:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.680205:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.680206:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.680206:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.680207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.680213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677470.680410:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.680412:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.680413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.680414:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.680416:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.680419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.680934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.680940:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.680943:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.680944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.680946:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.680948:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.680949:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.680951:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.680952:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.680953:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.680954:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.680957:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.680959:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.680962:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.680963:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.680964:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.680965:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.680966:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.680967:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.680967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.681483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.681856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.681863:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.681866:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.681867:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.681869:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.681871:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.681873:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.681875:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.681876:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.681877:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.681878:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.681882:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.681884:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.681886:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.681887:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.681888:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.681889:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.681890:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.681891:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.681892:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.681893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.681898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.681903:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.681905:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.681905:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.681906:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.682598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.682678:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.682681:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.682682:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.682682:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.682684:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.682687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.682697:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.682699:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.682700:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.682700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.682701:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.682702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.682900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.682905:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.682907:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.682908:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.682910:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.682912:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.682914:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.682915:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.682916:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.682917:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.682918:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.682922:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.682923:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.682927:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.682927:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:1.0:1619677470.682928:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:1.0:1619677470.682930:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:1.0:1619677470.682931:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.682931:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.682932:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.682944:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.682947:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.682948:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.682949:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.682951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.682952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.682962:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:12.0:1619677470.682965:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.682966:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.682967:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fca00] (23)--
00000800:00000200:12.0:1619677470.682967:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:12.0:1619677470.682969:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:20.2:1619677470.683305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.683312:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.683315:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.683316:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.683318:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.683322:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.683323:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.683324:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.683326:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.683327:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.683328:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.683331:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.683333:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.683335:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.683336:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.683337:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.683338:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.683339:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.683341:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.683342:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.683342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.683348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.683354:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.683356:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.683357:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.683357:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677470.683526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.683532:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.683535:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.683536:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.683538:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.683541:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.683542:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.683544:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.683545:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.683546:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.683547:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.683551:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.683552:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.683555:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.683556:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:18.0:1619677470.683557:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:18.0:1619677470.683558:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:18.0:1619677470.683559:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.683560:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.683561:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.683561:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.683567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.683572:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.683574:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.683575:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.683576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.683706:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.683709:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.683710:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.683710:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.683712:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.683716:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.683944:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.683947:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.683948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.683948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.683950:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.683953:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.684590:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.684592:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.684594:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.684594:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.684596:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.684601:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.684602:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.684604:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.684605:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.684605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.684606:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.684607:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.684652:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.684654:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.684656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.684656:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.684658:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.684661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.685118:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.685121:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677470.685122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677470.685124:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.685126:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.685128:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677470.685129:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.685131:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.685132:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.685134:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677470.685137:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.685139:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677470.685141:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.685142:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000800:00000200:17.0:1619677470.685143:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000800:00000200:17.0:1619677470.685145:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:17.0:1619677470.685146:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.685147:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.685147:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677470.685159:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.685162:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.685162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.685163:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.685165:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.685168:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.685183:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.685186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.685186:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.685187:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.685188:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.685188:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.685547:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.685550:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.685550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.685551:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.685553:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.2:1619677470.685554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677470.685556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.685604:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.685606:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.685607:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.685608:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.685610:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.685613:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.685857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.685863:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.685866:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.685867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.685869:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.685872:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.685873:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.685875:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.685876:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.685877:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.685878:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.685882:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.685884:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.685887:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.685888:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.685889:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.685890:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.685891:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.685892:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.685893:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.685894:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.685899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.685905:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.685906:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.685907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.685909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677470.686515:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.686517:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.686518:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.686519:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.686521:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.686522:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:24.2:1619677470.686574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:5.2:1619677470.686740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.686746:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.686748:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.686749:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.686751:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.686753:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.686755:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.686756:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.686757:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.686758:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.686759:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.686763:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.686765:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.686767:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.686768:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.686768:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.686770:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.686771:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.686771:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.686772:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.687038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.687087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.687147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.687153:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.687156:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.687157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.687160:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.687163:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.687164:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677470.687165:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.687167:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.687168:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.687169:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677470.687173:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.687175:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677470.687177:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.687178:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:2.0:1619677470.687179:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:2.0:1619677470.687180:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:2.0:1619677470.687181:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.687182:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.687182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.687336:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.687340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.687340:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:1.0:1619677470.687341:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.687343:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.687346:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.687851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.687856:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.687859:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.687860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.687862:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.687864:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.687865:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.687866:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.687868:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.687869:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.687870:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.687873:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.687875:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.687878:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c0c55f80] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.687879:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:17.0:1619677470.687880:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fca00] (21)++
00000800:00000200:17.0:1619677470.687881:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:17.0:1619677470.687882:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.687882:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.687883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677470.687894:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:13.0:1619677470.687897:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:13.0:1619677470.687898:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.687900:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.687901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000800:00000200:13.0:1619677470.687902:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.688373:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.688376:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.688377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000400:00000200:1.0:1619677470.688377:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.688379:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.688382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.688392:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.688394:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.688395:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.688395:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.688396:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.688397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.688550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.688960:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.688963:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.688964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.688964:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.688966:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.688969:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.689064:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.689067:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.689068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.689069:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.689071:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.689075:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.689306:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.689310:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.689311:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.689312:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.689313:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.689316:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.689330:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.689333:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.689334:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.689335:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.689336:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.689337:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.689471:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.689474:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.689475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.689475:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.689477:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.689481:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.689833:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.689835:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.689836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.689837:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.689839:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.689842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.690198:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.690201:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.690202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.690202:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.690204:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.690207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.690373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.690380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.690383:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.690383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.690385:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.690388:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.690389:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.690390:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.690392:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.690393:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677470.690393:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000400:00000200:18.0:1619677470.690394:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.690397:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.690397:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.690398:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.690398:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:18.0:1619677470.690399:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:17.0:1619677470.690400:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.690401:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.690402:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.690402:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:17.0:1619677470.690403:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.690404:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.690405:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.690406:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.690407:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.690419:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.690421:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.690422:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.690423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.690424:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.690426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.690734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.690740:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.690743:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.690744:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.690746:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.690748:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.690750:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.690751:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.690752:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.690753:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.690755:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.690758:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.690759:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.690762:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.690764:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.690765:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.690767:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.690767:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.690768:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.690769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.690882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.690924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677470.690935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.690941:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.690943:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.690944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.690946:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.690949:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.690951:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677470.690952:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.690953:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.690954:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.690955:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677470.690958:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.690960:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677470.690963:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.690963:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:2.0:1619677470.690964:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:2.0:1619677470.690966:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:2.0:1619677470.690966:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.690967:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.690968:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.691162:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.691165:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.691166:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.691166:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.691168:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.691171:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.691180:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.691182:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.691183:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.691183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.691185:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.691185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.691604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.691610:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.691613:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.691614:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.691618:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.691621:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.691622:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.691624:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.691625:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.691626:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.691627:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.691630:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.691632:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.691635:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.691636:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.691637:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.691638:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.691639:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.691640:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.691641:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.691642:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.691647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.691653:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.691655:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.691656:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.691657:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.691996:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.691999:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.692000:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.692002:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.692005:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.692006:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.692008:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.692009:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.692010:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.692011:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.692014:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.692016:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.692018:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.692019:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.692020:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.692022:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.692022:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.692023:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.692024:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.692025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.692036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.692039:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.692039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.692040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:12.0:1619677470.692074:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)++
00000800:00000200:12.0:1619677470.692077:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fca00] (23)--
00000400:00000200:12.0:1619677470.692078:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.692081:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (22)--
00000400:00000200:12.0:1619677470.692081:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.692088:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:1.0:1619677470.692108:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (23)++
00000800:00000200:1.0:1619677470.692110:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (24)--
00000800:00000200:17.0:1619677470.692111:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (24)++
00000400:00000200:1.0:1619677470.692111:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.692112:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (24)--
00000400:00000200:17.0:1619677470.692112:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.692114:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677470.692115:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.692122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.692123:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:1.0:1619677470.692124:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.692156:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.692159:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.692160:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.692160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.692162:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.692168:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:13.0:1619677470.692171:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.692173:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.692175:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.692177:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:2.0:1619677470.692177:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000400:00000200:13.0:1619677470.692178:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.692180:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.692181:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.692181:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.692182:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.692183:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:13.0:1619677470.692185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:17.0:1619677470.692230:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.692232:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.692233:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.692233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.692235:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.692239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:18.0:1619677470.692275:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.692278:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.692279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.692280:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.692282:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.692289:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:2.0:1619677470.692313:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.692316:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.692317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.692317:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.692321:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.692325:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.692331:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.692337:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.692340:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.692340:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.692344:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.692348:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.692350:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677470.692352:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96521207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.692354:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.692356:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.692357:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677470.692363:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.692366:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677470.692370:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c121bf80] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.692371:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:18.0:1619677470.692372:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:18.0:1619677470.692375:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:18.0:1619677470.692376:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.692377:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.692377:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:1.0:1619677470.692387:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:12.0:1619677470.692388:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:1.0:1619677470.692390:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.692391:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:12.0:1619677470.692391:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:1.0:1619677470.692391:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.692392:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.692393:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:12.0:1619677470.692394:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.692395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.692397:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:1.0:1619677470.692403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:13.0:1619677470.692406:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)++
00000800:00000200:13.0:1619677470.692409:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677470.692410:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.692410:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17fd200] (23)--
00000800:00000200:13.0:1619677470.692411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.692412:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:12.0:1619677470.692463:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.692466:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.692467:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.692469:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.692470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.692472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:5.2:1619677470.692756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.692762:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.692765:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.692766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.692768:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.692770:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.692772:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.692773:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.692775:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.692776:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.692777:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.692780:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.692782:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.692785:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.692785:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.692786:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.692788:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.692788:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.692789:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.692790:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677470.692794:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.692797:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.692798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.692798:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.692800:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.692804:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.692827:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:18.0:1619677470.692830:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.692830:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.692831:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.692832:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:18.0:1619677470.692832:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.692904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.693982:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.693985:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.693986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.693986:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.693988:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.693991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.694064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.694071:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.694074:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.694075:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.694076:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.694080:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.694082:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.694083:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.694084:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.694085:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.694086:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.694090:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.694092:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.694096:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.694097:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.694097:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.694099:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.694100:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.694101:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.694101:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.695650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.695656:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.695659:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.695660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.695662:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.695665:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.695667:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.695668:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.695669:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.695670:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.695671:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.695674:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.695676:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.695679:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.695680:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.695680:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.695682:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.695683:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.695684:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.695684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.695707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.695713:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.695716:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.695717:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.695719:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.695722:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.695723:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.695724:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.695726:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.695727:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.695728:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.695731:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.695732:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.695735:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.695736:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.695736:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.695738:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.695738:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.695739:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.695739:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.695827:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.695830:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.695831:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.695831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.695834:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.695838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.696006:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.696008:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.696009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:2.0:1619677470.696010:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.696012:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.696016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.696226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:13.0:1619677470.696470:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.696474:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.696477:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677470.696479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.696481:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.696484:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:5.2:1619677470.696484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:13.0:1619677470.696486:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96521182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.696488:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.696489:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677470.696491:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:13.0:1619677470.696491:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:17.0:1619677470.696494:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.696495:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:13.0:1619677470.696498:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677470.696499:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.696502:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.696502:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677470.696503:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.696504:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.696505:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.696506:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677470.696506:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fc00] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.696507:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986e00] (20)++
00000400:00000200:17.0:1619677470.696508:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:13.0:1619677470.696508:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986e00] (21)++
00000400:00000200:17.0:1619677470.696511:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.696511:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:13.0:1619677470.696512:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.696513:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:17.0:1619677470.696514:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677470.696514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677470.696516:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.696517:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.696518:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.696519:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.696520:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.696521:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.696522:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677470.696526:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.696529:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.696530:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.696530:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.696532:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.696536:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.696657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.696663:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.696666:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.696667:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.696670:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.696672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.696674:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677470.696676:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.696677:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.696679:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.696680:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677470.696685:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.696688:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.696691:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.696692:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:2.0:1619677470.696693:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:2.0:1619677470.696695:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:2.0:1619677470.696696:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.696698:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.696700:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.696701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.696706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.696712:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.696714:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.696715:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.696716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.697564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.697571:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.697574:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.697574:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.697577:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.697580:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.697581:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.697582:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.697583:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677470.697584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677470.697585:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.697586:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.697589:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.697590:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677470.697590:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677470.697593:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.697593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.697593:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:18.0:1619677470.697594:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.697594:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.697596:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000400:00000200:1.0:1619677470.697596:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677470.697597:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.697598:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.697598:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:1.0:1619677470.697598:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.697599:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.697601:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.697602:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.697603:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.697604:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.697607:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.697608:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.697610:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:1.0:1619677470.697611:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.697611:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:2.0:1619677470.697612:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:1.0:1619677470.697612:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000400:00000200:2.0:1619677470.697613:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.697614:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:1.0:1619677470.697614:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.697614:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677470.697615:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.697615:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.697615:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677470.697617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.697978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.698450:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.698453:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.698454:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.698454:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.698456:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.698459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.698920:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.698923:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.698924:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.698924:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.698927:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.698929:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.698946:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.698949:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.698950:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.698950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.698951:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.698952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.699909:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.699912:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.699913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.699913:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.699915:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.699918:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.700001:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.700004:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000400:00000200:17.0:1619677470.700005:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.700005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.700007:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.700010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:18.0:1619677470.700322:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.700325:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.700326:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.700326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.700329:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.700332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.700506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.700548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677470.700554:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.700557:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.700558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.700558:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.700560:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.700563:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.700628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:20.2:1619677470.700630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.700635:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.700638:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.700638:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.700640:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677470.700641:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.700642:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.700643:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.700645:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.700646:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.700648:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.700649:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.700650:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.700652:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.700653:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.700654:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677470.700655:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.700657:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677470.700657:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.700658:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.700658:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.700661:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.700662:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.700665:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.700665:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.700666:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.700666:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:18.0:1619677470.700667:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.700667:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:18.0:1619677470.700669:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.700669:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:18.0:1619677470.700670:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.700670:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.700671:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.700671:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.700672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.700672:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.700673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.700673:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.700682:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.700686:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.700686:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.700687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.700689:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.700690:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:2.0:1619677470.700691:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.700693:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.700693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.700696:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.700698:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.700699:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:18.0:1619677470.700701:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000400:00000200:17.0:1619677470.700701:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.700702:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.700703:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677470.700704:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677470.700704:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677470.700705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.700706:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.700707:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.700707:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000400:00000200:17.0:1619677470.700708:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.700710:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.700712:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.700713:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.700714:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.700715:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.700716:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.700717:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.700717:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677470.701343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.701349:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.701352:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.701353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.701355:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.701357:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.701358:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.701360:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.701361:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.701362:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.701363:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.701366:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.701367:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.701370:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.701371:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.701371:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.701373:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.701373:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.701375:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.701376:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.701376:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.701381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.701387:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.701389:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.701389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.701390:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.702394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.702398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.702405:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.702408:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.702409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.702411:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.702415:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.702416:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.702418:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.702420:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.702421:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.702422:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.702426:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.702428:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.702431:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49500] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.702432:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950800] (20)++
00000800:00000200:2.0:1619677470.702432:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950800] (21)++
00000800:00000200:2.0:1619677470.702434:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950800] (22)--
00000800:00000200:2.0:1619677470.702434:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.702436:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.702437:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.702437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.702443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.702449:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.702451:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.702452:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.702453:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677470.702506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.702512:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.702515:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.702516:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.702518:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.702520:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.702522:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.702523:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.702524:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.702526:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.702526:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.702530:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.702532:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.702534:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.702535:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.702536:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.702537:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.702538:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.702539:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.702539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.703259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.703266:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.703268:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.703269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.703272:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.703274:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.703276:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.703277:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.703279:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.703280:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.703281:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.703284:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.703286:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.703289:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.703290:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.703290:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.703292:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.703299:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.703300:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.703300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.703362:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.703365:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.703366:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.703366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.703368:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.703373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.703374:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.703377:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.703378:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.703378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.703379:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.703379:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.703618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.704398:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.704401:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.704402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.704402:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.704404:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.704407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.704416:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.704419:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.704420:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.704420:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.704421:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.704422:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:20.2:1619677470.705011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.705057:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.705059:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.705060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.705060:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.705063:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.705065:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.705594:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.705596:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.705597:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.705598:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.705600:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.705602:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.705607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.705613:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.705616:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.705617:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.705619:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.705622:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.705623:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.705625:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.705627:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.705629:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.705630:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.705633:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.705635:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.705638:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.705639:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.705640:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.705642:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.705642:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.705643:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.705644:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.705645:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.705650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.705656:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.705658:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.705659:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.705660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677470.705684:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.705687:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.705688:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.705688:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.705690:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.705693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:2.0:1619677470.706207:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677470.706210:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677470.706211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677470.706213:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.706216:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.706218:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677470.706219:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.706221:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.706222:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.706223:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677470.706226:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.706228:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677470.706231:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.706232:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:2.0:1619677470.706233:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:2.0:1619677470.706234:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:2.0:1619677470.706235:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677470.706236:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677470.706237:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:5.2:1619677470.706244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.706250:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.706253:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.706254:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.706256:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.706258:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.706259:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.706261:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.706262:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.706263:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.706264:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.706268:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.706269:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.706272:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.706273:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.706273:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.706275:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.706276:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.706277:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.706277:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.706333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.706339:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.706343:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.706344:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.706346:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.706349:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.706350:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.706351:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.706353:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.706354:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.706355:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.706358:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.706360:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.706361:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.706362:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.706363:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.706364:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.706365:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.706366:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.706366:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.706378:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.706380:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.706381:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.706382:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.706383:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.706385:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.706456:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.706458:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.706459:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.706459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.706462:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.706466:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.706786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.706792:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.706795:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.706796:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.706797:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.706799:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.706801:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677470.706803:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.706805:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.706806:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.706807:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:23.0:1619677470.706808:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff919b86950800] (18)--
00000400:00000200:2.0:1619677470.706810:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.706813:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677470.706816:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.706817:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:2.0:1619677470.706818:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:2.0:1619677470.706819:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:2.0:1619677470.706820:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.706821:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.706822:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.707025:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.707028:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.707029:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.707029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.707032:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.707035:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.707280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.707434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.707440:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.707443:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.707444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.707446:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.707448:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.707450:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.707452:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.707453:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.707454:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.707455:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.707458:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.707460:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.707462:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.707463:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.707464:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.707466:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.707466:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.707467:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.707467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.707773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:2.0:1619677470.708993:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.708996:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.708997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.708997:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.708999:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.709004:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.709380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.709416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.709463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.2:1619677470.709501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:17.0:1619677470.709582:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.709585:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.709585:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.709586:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.709588:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.709591:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.709987:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.709990:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:17.0:1619677470.709991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:1.0:1619677470.709991:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.709993:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.709996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:17.0:1619677470.710005:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (21)++
00000800:00000200:17.0:1619677470.710007:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.710008:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918febd25600] (22)++
00000800:00000200:1.0:1619677470.710008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.710009:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918febd25600] (22)--
00000800:00000200:17.0:1619677470.710010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:12.0:1619677470.710229:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:12.0:1619677470.710232:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:12.0:1619677470.710233:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.710235:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.710237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:12.0:1619677470.710238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:20.2:1619677470.710388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.710395:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.710398:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.710399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.710401:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.710403:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.710405:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.710406:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.710407:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.710408:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.710409:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.710412:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.710413:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.710415:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.710416:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.710417:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.710419:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.710419:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.710421:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.710422:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.710422:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.710428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.710434:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.710436:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.710436:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.710437:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.710437:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.710437:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.710438:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.710439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.710440:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.710440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.710452:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.710455:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.710456:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.710456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.710457:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.710458:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.710545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.710551:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.710554:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.710555:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.710557:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.710560:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.710561:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.710563:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.710564:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.710565:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.710566:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.710569:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.710570:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.710574:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.710575:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.710575:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.710577:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.710578:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.710579:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.710579:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.711093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.711100:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.711102:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:20.2:1619677470.711103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.711103:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.711105:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.711108:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.711110:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.711112:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.711114:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.711115:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.711117:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677470.711117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677470.711123:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.711123:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677470.711126:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.711126:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.711127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.711130:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677470.711131:0:19933:0:(o2iblnd_cb.c:1635:kiblnd_launch_tx()) peer_ni[ffff919b8e780580] -> 10.0.10.104@o2ib7 (1)++
00000400:00000200:1.0:1619677470.711133:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.711135:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677470.711136:0:19933:0:(o2iblnd_cb.c:1440:kiblnd_connect_peer()) peer_ni[ffff919b8e780580] -> 10.0.10.104@o2ib7 (2)++
00000400:00000200:1.0:1619677470.711136:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96521098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.711138:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.711140:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.711141:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.711145:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.711147:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.711150:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.711152:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.711152:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.711154:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.711155:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.711156:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.711157:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.711158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.711163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.711169:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.711171:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.711172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.711173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.711251:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.711254:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.711254:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.711255:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.711257:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.711260:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.711425:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.711428:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.711429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.711429:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.711431:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.711434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:18.0:1619677470.711744:0:19933:0:(o2iblnd_cb.c:1392:kiblnd_resolve_addr()) bound to port 1023
00000800:00000200:18.0:1619677470.711747:0:19933:0:(o2iblnd_cb.c:1642:kiblnd_launch_tx()) peer_ni[ffff919b8e780580] -> 10.0.10.104@o2ib7 (3)--
00000800:00000200:18.0:1619677470.711748:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.711749:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.711750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:7.0:1619677470.711750:0:13229:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.104@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677470.711757:0:13229:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.104@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:2.0:1619677470.711775:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.711777:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.711778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.711778:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.711781:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.711784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:7.0:1619677470.711878:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.104@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.711889:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919b87002000 0x1007002000(0x1007002000)
00000800:00000200:7.0:1619677470.711890:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff919b87003000 0x1007003000(0x1007003000)
00000800:00000200:7.0:1619677470.711891:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff919b87648000 0x1007648000(0x1007648000)
00000800:00000200:7.0:1619677470.711891:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff919b87649000 0x1007649000(0x1007649000)
00000800:00000200:7.0:1619677470.711892:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff919b878ac000 0x10078ac000(0x10078ac000)
00000800:00000200:7.0:1619677470.711892:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff919b878ad000 0x10078ad000(0x10078ad000)
00000800:00000200:7.0:1619677470.711893:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff919b86e5e000 0x1006e5e000(0x1006e5e000)
00000800:00000200:7.0:1619677470.711893:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff919b86e5f000 0x1006e5f000(0x1006e5f000)
00000800:00000200:7.0:1619677470.711894:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff919b8754c000 0x100754c000(0x100754c000)
00000800:00000200:7.0:1619677470.711894:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff919b8754d000 0x100754d000(0x100754d000)
00000800:00000200:7.0:1619677470.711895:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff919b87286000 0x1007286000(0x1007286000)
00000800:00000200:7.0:1619677470.711895:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919b86493000 0x1006493000(0x1006493000)
00000800:00000200:7.0:1619677470.711896:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919b86494000 0x1006494000(0x1006494000)
00000800:00000200:7.0:1619677470.711896:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919b86495000 0x1006495000(0x1006495000)
00000800:00000200:7.0:1619677470.711897:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919b86496000 0x1006496000(0x1006496000)
00000800:00000200:7.0:1619677470.711898:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919b86497000 0x1006497000(0x1006497000)
00000800:00000200:7.0:1619677470.711898:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919b864c0000 0x10064c0000(0x10064c0000)
00000800:00000200:7.0:1619677470.711899:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919b864c1000 0x10064c1000(0x10064c1000)
00000800:00000200:5.2:1619677470.712252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.712258:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.712261:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.712261:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.712263:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.712265:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.712267:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.712268:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.712269:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.712270:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.712271:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.712275:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.712277:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.712279:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.712280:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.712280:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.712282:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.712283:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.712284:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.712284:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:7.0:1619677470.712670:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712671:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712671:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712671:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712673:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712673:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712673:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712674:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712674:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712674:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712675:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712675:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712675:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712676:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712676:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712677:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712677:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712677:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712678:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712678:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712678:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712679:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712679:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712679:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712680:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712680:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712681:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712681:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712681:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712682:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712682:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712682:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712683:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712683:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:7.0:1619677470.712683:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (19)++
00000800:00000200:7.0:1619677470.712684:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (20)--
00000800:00000200:1.0:1619677470.712999:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.713002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.713002:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677470.713005:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.713007:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.713008:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.713010:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.713011:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.713012:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.713013:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.713016:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.713018:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.713021:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.713022:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:1.0:1619677470.713022:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:1.0:1619677470.713024:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:1.0:1619677470.713024:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.713025:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.713026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.713183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf5800] (20)++
00000800:00000200:20.2:1619677470.713220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf5800] (20)++
00000800:00000200:20.2:1619677470.713238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf5800] (21)++
00000800:00000200:18.0:1619677470.713267:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.713270:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.713271:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.713271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000400:00000200:18.0:1619677470.713273:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.713276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.713277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf5800] (21)++
00000800:00000200:2.0:1619677470.713288:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.713292:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.713293:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.713302:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986e00] (21)++
00000800:00000200:2.0:1619677470.713303:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986e00] (22)--
00000800:00000200:2.0:1619677470.713304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.713346:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.713348:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.713349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.713350:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.713352:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.713360:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:13.0:1619677470.713376:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)++
00000800:00000200:13.0:1619677470.713378:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17fd200] (23)--
00000400:00000200:13.0:1619677470.713379:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.713381:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.713383:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (22)--
00000800:00000200:13.0:1619677470.713388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:20.2:1619677470.713454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf5800] (20)++
00000800:00000200:20.2:1619677470.713485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:20.2:1619677470.713495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf5800] (21)++
00000800:00000200:20.2:1619677470.713520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:20.2:1619677470.713526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf5800] (20)++
00000800:00000200:20.2:1619677470.713555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf5800] (21)++
00000800:00000200:20.2:1619677470.713577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf5800] (21)++
00000800:00000200:20.2:1619677470.713615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf5800] (20)++
00000800:00000200:20.2:1619677470.713654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf5800] (21)++
00000800:00000200:20.2:1619677470.713792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.713798:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.713802:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.713803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.713805:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.713806:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.713808:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677470.713809:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.713811:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.713812:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.713813:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677470.713816:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.713818:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677470.713821:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.713822:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:2.0:1619677470.713822:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:2.0:1619677470.713824:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:2.0:1619677470.713825:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.713826:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.713827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.714308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.714597:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.714600:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.714601:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.714601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.714603:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.714606:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.714702:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.714705:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.714705:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.714708:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.714710:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.714712:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677470.714713:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.714715:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.714716:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.714717:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677470.714720:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.714721:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.714723:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18680] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.714724:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0000] (20)++
00000800:00000200:17.0:1619677470.714724:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0000] (21)++
00000800:00000200:17.0:1619677470.714726:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:17.0:1619677470.714726:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.714727:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.714728:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677470.714741:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.714743:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:1.0:1619677470.714743:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.714745:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.714746:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.714748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:17.0:1619677470.714887:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (22)++
00000800:00000200:17.0:1619677470.714890:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918febd25600] (23)--
00000800:00000200:1.0:1619677470.714890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (22)--
00000400:00000200:17.0:1619677470.714891:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.714893:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.714896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (21)--
00000800:00000200:5.2:1619677470.715500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.715506:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.715509:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.715510:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.715512:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.715515:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.715517:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.715518:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.715520:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.715521:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.715522:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.715525:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.715527:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.715531:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919ea1f0c700] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.715532:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f800] (20)++
00000800:00000200:2.0:1619677470.715532:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f800] (21)++
00000800:00000200:2.0:1619677470.715534:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f800] (22)--
00000800:00000200:2.0:1619677470.715535:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.715536:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.715536:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.715575:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:18.0:1619677470.715577:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:2.0:1619677470.715578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:18.0:1619677470.715579:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.715581:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.715584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:20.2:1619677470.715760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.715766:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.715768:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.715769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.715771:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.715774:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.715775:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.715776:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.715778:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.715779:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.715780:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.715783:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.715784:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.715786:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.715787:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.715788:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.715789:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.715790:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.715791:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.715791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.715803:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.715805:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.715806:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.715806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.715808:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.715809:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.715824:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.715826:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.715827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.715827:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.715827:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.715828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.715964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.715971:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.715973:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.715974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.715976:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.715978:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.715979:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.715981:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.715982:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.715983:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.715984:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.715987:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.715988:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.715991:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.715992:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.715992:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.715994:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.715994:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.715995:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.715995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677470.716226:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.716229:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.716230:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.716230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.716232:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.716236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.716405:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (22)++
00000800:00000200:2.0:1619677470.716407:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986e00] (23)--
00000800:00000200:18.0:1619677470.716409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (23)--
00000400:00000200:2.0:1619677470.716409:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.716411:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.716415:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.716507:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.716509:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.716510:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.716511:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.716513:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.716516:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.716545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.716706:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.716708:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.716709:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.716710:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.716712:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.716716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:17.0:1619677470.717060:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:17.0:1619677470.717063:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:17.0:1619677470.717064:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.717064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:17.0:1619677470.717067:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.717070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.717078:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.717081:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.717082:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.717082:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.717083:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.717084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:5.2:1619677470.717091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.717096:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.717099:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.717100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.717102:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.717103:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.717105:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.717106:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.717107:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.717108:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.717109:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.717112:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.717113:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.717116:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.717116:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.717117:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.717119:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.717119:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.717120:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.717120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677470.717384:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.717387:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.717388:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.717389:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.717391:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.717394:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:20.2:1619677470.717424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.717430:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.717433:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.717434:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.717436:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.717438:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.717440:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677470.717441:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.717443:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.717444:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.717445:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677470.717448:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.717450:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677470.717453:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.717454:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:18.0:1619677470.717455:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:18.0:1619677470.717456:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:18.0:1619677470.717457:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.717458:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.717458:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677470.718576:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (20)++
00000800:00000200:12.0:1619677470.718579:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (21)--
00000800:00000200:12.0:1619677470.718580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (20)--
00000800:00000200:12.0:1619677470.718581:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (19)--
00000800:00000200:23.0:1619677470.718614:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91a7c17fca00] (18)--
00000800:00000200:17.0:1619677470.719140:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.719143:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677470.719144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677470.719146:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.719149:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.719150:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677470.719152:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.719153:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.719154:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.719155:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677470.719158:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:23.0:1619677470.719159:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff919e43e7f800] (18)--
00000800:00000200:17.0:1619677470.719160:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677470.719166:0:19930:0:(o2iblnd_cb.c:1635:kiblnd_launch_tx()) peer_ni[ffff9193c410f580] -> 10.0.10.102@o2ib7 (1)++
00000800:00000200:17.0:1619677470.719169:0:19930:0:(o2iblnd_cb.c:1440:kiblnd_connect_peer()) peer_ni[ffff9193c410f580] -> 10.0.10.102@o2ib7 (2)++
00000800:00000200:20.2:1619677470.719312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.719350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.719565:0:19930:0:(o2iblnd_cb.c:1392:kiblnd_resolve_addr()) bound to port 1023
00000800:00000200:17.0:1619677470.719566:0:19930:0:(o2iblnd_cb.c:1642:kiblnd_launch_tx()) peer_ni[ffff9193c410f580] -> 10.0.10.102@o2ib7 (3)--
00000800:00000200:17.0:1619677470.719567:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.719568:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.719569:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:7.0:1619677470.719570:0:13229:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.102@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677470.719575:0:13229:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.102@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:1.0:1619677470.719659:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (20)++
00000800:00000200:1.0:1619677470.719661:0:19931:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (21)--
00000800:00000200:1.0:1619677470.719662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (20)--
00000800:00000200:1.0:1619677470.719663:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (19)--
00000800:00000200:23.0:1619677470.719691:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff918febd25600] (18)--
00000800:00000200:20.2:1619677470.719845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.719852:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.719855:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.719857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.719858:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.719861:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.719863:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.719865:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.719867:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:7.0:1619677470.719868:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.102@o2ib7 Route resolved: 0
00000400:00000200:2.0:1619677470.719868:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.719869:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.719873:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.719875:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.719879:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.719880:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.719880:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:7.0:1619677470.719882:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff9193c4e03000 0x844e03000(0x844e03000)
00000800:00000200:2.0:1619677470.719882:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:7.0:1619677470.719883:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff9193c4c7f000 0x844c7f000(0x844c7f000)
00000800:00000200:2.0:1619677470.719883:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:7.0:1619677470.719884:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff9193c4c87000 0x844c87000(0x844c87000)
00000800:00000200:7.0:1619677470.719884:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff9193c2def000 0x842def000(0x842def000)
00000800:00000200:2.0:1619677470.719884:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.719884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:7.0:1619677470.719885:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff9193c2dd8000 0x842dd8000(0x842dd8000)
00000800:00000200:7.0:1619677470.719885:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff91907fe90000 0x4ffe90000(0x4ffe90000)
00000800:00000200:7.0:1619677470.719886:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff91907fee6000 0x4ffee6000(0x4ffee6000)
00000800:00000200:7.0:1619677470.719886:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff9193c4c80000 0x844c80000(0x844c80000)
00000800:00000200:7.0:1619677470.719887:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff9193c4c8a000 0x844c8a000(0x844c8a000)
00000800:00000200:7.0:1619677470.719887:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff9193c2e60000 0x842e60000(0x842e60000)
00000800:00000200:7.0:1619677470.719888:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff9193c2dae000 0x842dae000(0x842dae000)
00000800:00000200:7.0:1619677470.719889:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff9193c2d46000 0x842d46000(0x842d46000)
00000800:00000200:7.0:1619677470.719889:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff9193c2d43000 0x842d43000(0x842d43000)
00000800:00000200:7.0:1619677470.719890:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff91907fec1000 0x4ffec1000(0x4ffec1000)
00000800:00000200:7.0:1619677470.719890:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff9193c2da6000 0x842da6000(0x842da6000)
00000800:00000200:7.0:1619677470.719891:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff9190667f4000 0x4e67f4000(0x4e67f4000)
00000800:00000200:7.0:1619677470.719891:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919065381000 0x4e5381000(0x4e5381000)
00000800:00000200:7.0:1619677470.719892:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff9193c2d48000 0x842d48000(0x842d48000)
00000800:00000200:18.0:1619677470.719896:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.719898:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.719899:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.719899:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.719901:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.719903:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.719930:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.719933:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.719934:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.719934:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.719936:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.719936:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.720013:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (20)++
00000800:00000200:18.0:1619677470.720015:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (21)--
00000800:00000200:18.0:1619677470.720016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (20)--
00000800:00000200:2.0:1619677470.720017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (19)--
00000800:00000200:18.0:1619677470.720064:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.720066:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.720067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.720068:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.720070:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.720073:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:23.0:1619677470.720247:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff9193c2986e00] (18)--
00000800:00000200:7.0:1619677470.720543:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720543:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720544:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720544:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720544:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720545:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720545:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720546:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720546:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720546:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720547:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720547:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720547:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720548:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720548:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720548:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720549:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720549:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720550:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720550:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720551:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720551:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720551:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720552:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720552:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720553:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720553:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720553:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720554:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720554:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720554:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720555:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720555:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720555:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:7.0:1619677470.720556:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (19)++
00000800:00000200:7.0:1619677470.720556:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (20)--
00000800:00000200:2.0:1619677470.720643:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.720646:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.720647:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.720647:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.720649:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.720652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.720824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.720830:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.720834:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.720835:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.720837:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.720839:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.720840:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.720842:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96521033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.720843:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.720844:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.720845:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.720848:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.720849:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.720852:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.720853:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.720853:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.720855:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.720856:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.720857:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.720858:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.720859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.720863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.720869:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.720871:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.720872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.720873:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.720952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.721360:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.721363:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.721363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.721364:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.721366:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.721369:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.721712:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (23)++
00000800:00000200:18.0:1619677470.721715:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (24)++
00000800:00000200:2.0:1619677470.721715:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (24)--
00000800:00000200:18.0:1619677470.721716:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (24)--
00000400:00000200:2.0:1619677470.721716:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.721717:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.721719:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677470.721719:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.721722:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.721722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.721723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.721922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.721928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.721931:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.721932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.721934:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.721936:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.721937:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.721939:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.721940:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.721941:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.721943:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.721946:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.721947:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.721948:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677470.721949:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.721950:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.721951:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.721953:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.721954:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.721954:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.721954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.721955:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677470.721957:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.721958:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.721959:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.721960:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.721960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.721966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.721971:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.721973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.721973:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.721974:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677470.721974:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.721975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.721975:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.721976:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.721976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.721977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.721987:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.721989:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.721990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.721990:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.721991:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.721991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.722008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.722014:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.722017:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.722018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.722020:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.722022:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.722024:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.722026:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.722027:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.722028:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.722029:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.722033:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.722034:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.722042:0:19930:0:(o2iblnd_cb.c:1635:kiblnd_launch_tx()) peer_ni[ffff91a7c3537400] -> 10.0.10.103@o2ib7 (1)++
00000800:00000200:17.0:1619677470.722045:0:19930:0:(o2iblnd_cb.c:1440:kiblnd_connect_peer()) peer_ni[ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2)++
00000800:00000200:17.0:1619677470.722139:0:19930:0:(o2iblnd_cb.c:1392:kiblnd_resolve_addr()) bound to port 1023
00000800:00000200:17.0:1619677470.722141:0:19930:0:(o2iblnd_cb.c:1642:kiblnd_launch_tx()) peer_ni[ffff91a7c3537400] -> 10.0.10.103@o2ib7 (3)--
00000800:00000200:17.0:1619677470.722141:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.722142:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.722143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:23.0:1619677470.722146:0:13334:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.103@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677470.722152:0:13334:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.103@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:23.0:1619677470.722471:0:13334:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.103@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677470.722491:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff91a7c08a5000 0x1c408a5000(0x1c408a5000)
00000800:00000200:23.0:1619677470.722492:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff91a7c1bd8000 0x1c41bd8000(0x1c41bd8000)
00000800:00000200:23.0:1619677470.722492:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff91a7c1bd9000 0x1c41bd9000(0x1c41bd9000)
00000800:00000200:23.0:1619677470.722493:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff91a7c1b82000 0x1c41b82000(0x1c41b82000)
00000800:00000200:23.0:1619677470.722493:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff91a7c1b83000 0x1c41b83000(0x1c41b83000)
00000800:00000200:23.0:1619677470.722494:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff91a7c1ad4000 0x1c41ad4000(0x1c41ad4000)
00000800:00000200:23.0:1619677470.722494:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff91a7c1ad5000 0x1c41ad5000(0x1c41ad5000)
00000800:00000200:23.0:1619677470.722495:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff91a7c1b08000 0x1c41b08000(0x1c41b08000)
00000800:00000200:23.0:1619677470.722496:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff91a7c1b09000 0x1c41b09000(0x1c41b09000)
00000800:00000200:23.0:1619677470.722496:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff91a7c111c000 0x1c4111c000(0x1c4111c000)
00000800:00000200:23.0:1619677470.722497:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff91a7c111d000 0x1c4111d000(0x1c4111d000)
00000800:00000200:23.0:1619677470.722497:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff91a7c11d4000 0x1c411d4000(0x1c411d4000)
00000800:00000200:23.0:1619677470.722498:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff91a7c11d5000 0x1c411d5000(0x1c411d5000)
00000800:00000200:23.0:1619677470.722498:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff91a7c0c56000 0x1c40c56000(0x1c40c56000)
00000800:00000200:23.0:1619677470.722499:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff91a7c0c57000 0x1c40c57000(0x1c40c57000)
00000800:00000200:23.0:1619677470.722499:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff91a7c0c3b000 0x1c40c3b000(0x1c40c3b000)
00000800:00000200:23.0:1619677470.722500:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff91a7c0c3c000 0x1c40c3c000(0x1c40c3c000)
00000800:00000200:23.0:1619677470.722500:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff91a7c0c3d000 0x1c40c3d000(0x1c40c3d000)
00000800:00000200:18.0:1619677470.722600:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.722604:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.722605:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.722605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.722608:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.722611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:23.0:1619677470.723119:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723120:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723120:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723120:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723121:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723122:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723122:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723122:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723123:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723123:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723124:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723124:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723124:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723125:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723125:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723125:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723126:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723126:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723126:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723127:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723127:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723127:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723128:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723128:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723129:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723129:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723129:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723130:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723130:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723130:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723131:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723131:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723131:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723132:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:23.0:1619677470.723132:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (19)++
00000800:00000200:23.0:1619677470.723133:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (20)--
00000800:00000200:20.2:1619677470.723175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.723181:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.723184:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.723185:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.723187:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.723189:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.723191:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.723192:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.723194:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.723195:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.723196:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.723199:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.723201:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.723205:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe056bd80] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.723206:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb718a00] (20)++
00000800:00000200:2.0:1619677470.723206:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb718a00] (21)++
00000800:00000200:2.0:1619677470.723208:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb718a00] (22)--
00000800:00000200:2.0:1619677470.723209:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.723210:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.723211:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.723211:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.723216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.723222:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.723225:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.723226:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.723226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.723322:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.723325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.723325:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677470.723329:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.723331:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.723332:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.723334:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96521020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.723335:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.723336:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.723337:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.723341:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.723342:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.723346:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.723347:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677470.723348:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:1.0:1619677470.723351:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:1.0:1619677470.723351:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.723352:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.723352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:12.0:1619677470.724427:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677470.724430:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677470.724431:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.724432:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.724436:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677470.724437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677470.724482:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677470.724485:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[1] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677470.724487:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : PUT - routed
00000800:00000200:12.0:1619677470.724489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677470.724490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677470.724702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:7.0:1619677470.724840:0:13229:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.105@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677470.724846:0:13229:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.105@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:7.0:1619677470.725166:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.105@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.725178:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff918feb8ea000 0x46b8ea000(0x46b8ea000)
00000800:00000200:7.0:1619677470.725179:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff918feb8eb000 0x46b8eb000(0x46b8eb000)
00000800:00000200:7.0:1619677470.725180:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff918feb8c8000 0x46b8c8000(0x46b8c8000)
00000800:00000200:7.0:1619677470.725180:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff918feb8c9000 0x46b8c9000(0x46b8c9000)
00000800:00000200:7.0:1619677470.725181:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff918feb888000 0x46b888000(0x46b888000)
00000800:00000200:7.0:1619677470.725181:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff918feb889000 0x46b889000(0x46b889000)
00000800:00000200:7.0:1619677470.725182:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff918feb86e000 0x46b86e000(0x46b86e000)
00000800:00000200:7.0:1619677470.725182:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff918fe9401000 0x469401000(0x469401000)
00000800:00000200:7.0:1619677470.725183:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff918fe9402000 0x469402000(0x469402000)
00000800:00000200:7.0:1619677470.725184:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff918fe9403000 0x469403000(0x469403000)
00000800:00000200:7.0:1619677470.725184:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff918fe9404000 0x469404000(0x469404000)
00000800:00000200:7.0:1619677470.725185:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff918fe9405000 0x469405000(0x469405000)
00000800:00000200:7.0:1619677470.725186:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff918fe9406000 0x469406000(0x469406000)
00000800:00000200:7.0:1619677470.725187:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff918fe9407000 0x469407000(0x469407000)
00000800:00000200:7.0:1619677470.725188:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff918fe9408000 0x469408000(0x469408000)
00000800:00000200:7.0:1619677470.725189:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff918fe9409000 0x469409000(0x469409000)
00000800:00000200:7.0:1619677470.725189:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff918fe940a000 0x46940a000(0x46940a000)
00000800:00000200:7.0:1619677470.725190:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff918fe940b000 0x46940b000(0x46940b000)
00000800:00000200:17.0:1619677470.725334:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.725338:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.725339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.725341:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.725345:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.725346:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.725348:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96521012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.725349:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.725350:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.725351:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.725355:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.725357:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.725360:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.725361:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.725362:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.725363:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.725364:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.725364:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.725365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677470.725599:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.725602:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.725603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.725604:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.725606:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.725609:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:23.0:1619677470.725693:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91abeb718a00] (18)--
00000800:00000200:7.0:1619677470.725830:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725831:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725831:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725832:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725832:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725832:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725833:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725833:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725833:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725834:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725834:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725834:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725835:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725835:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725836:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725836:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725836:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725837:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725837:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725837:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725838:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725838:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725839:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725839:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725839:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725840:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725840:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725840:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725841:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725841:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725841:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725842:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725842:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725843:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:7.0:1619677470.725844:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (19)++
00000800:00000200:7.0:1619677470.725844:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (20)--
00000800:00000200:20.2:1619677470.726457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.726751:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.726754:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.726755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.726755:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.726757:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.726761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:5.2:1619677470.726828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.726835:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.726838:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.726839:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.726841:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.726844:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.726845:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.726847:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96521003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.726848:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.726849:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.726850:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.726853:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.726854:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.726857:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.726858:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.726859:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.726860:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.726861:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.726862:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.726862:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.726982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677470.727023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:17.0:1619677470.727024:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (24)++
00000800:00000200:17.0:1619677470.727027:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (25)--
00000800:00000200:12.0:1619677470.727027:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:1.0:1619677470.727027:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (25)++
00000400:00000200:17.0:1619677470.727028:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.727028:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (25)--
00000400:00000200:1.0:1619677470.727029:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.727030:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.727030:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:1.0:1619677470.727030:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:12.0:1619677470.727031:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.727033:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.727034:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (24)--
00000800:00000200:12.0:1619677470.727035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677470.727035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.727036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:12.0:1619677470.727038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:17.0:1619677470.727046:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.727047:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.727049:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.727049:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677470.727051:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.727090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.727093:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[3] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.727094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.727096:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.105@o2ib7 : PUT - routed
00000800:00000200:1.0:1619677470.727098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.727116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.727123:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.727127:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.727128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.727130:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.727133:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.727137:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.727139:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96521000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.727140:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.727142:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.727143:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.727146:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.727148:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.727151:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.727152:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:18.0:1619677470.727153:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:18.0:1619677470.727155:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:18.0:1619677470.727156:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.727157:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.727157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.728024:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.728027:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.728028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.728028:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.728030:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.728033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.728043:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:18.0:1619677470.728046:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.728046:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.728047:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.728049:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.728050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.728227:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.728229:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.728230:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:2.0:1619677470.728231:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.728233:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.728237:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.729630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.729637:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.729640:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.729641:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.729643:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.729646:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.729647:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677470.729649:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.729651:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.729652:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.729653:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677470.729655:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.729657:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677470.729661:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe0d80d00] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.729662:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb719600] (20)++
00000800:00000200:18.0:1619677470.729663:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb719600] (21)++
00000800:00000200:18.0:1619677470.729665:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb719600] (22)--
00000800:00000200:18.0:1619677470.729666:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.729667:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677470.729668:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677470.729669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677470.729674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.729680:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677470.729682:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677470.729683:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677470.729684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677470.729706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:18.0:1619677470.729773:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.729776:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.729777:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.729777:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.729779:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.729783:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:23.0:1619677470.729844:0:13334:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.101@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677470.729853:0:13334:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.101@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:20.2:1619677470.730046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.730086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.730093:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.730095:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.730096:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.730097:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.730099:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.730101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:23.0:1619677470.730160:0:13334:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.101@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677470.730174:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919e41f87000 0x12c1f87000(0x12c1f87000)
00000800:00000200:23.0:1619677470.730175:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff919e41f88000 0x12c1f88000(0x12c1f88000)
00000800:00000200:23.0:1619677470.730176:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff919e41f89000 0x12c1f89000(0x12c1f89000)
00000800:00000200:23.0:1619677470.730177:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff919e41f8a000 0x12c1f8a000(0x12c1f8a000)
00000800:00000200:23.0:1619677470.730177:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff919e41f8b000 0x12c1f8b000(0x12c1f8b000)
00000800:00000200:23.0:1619677470.730178:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff919e41f8c000 0x12c1f8c000(0x12c1f8c000)
00000800:00000200:23.0:1619677470.730178:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff919e41f8d000 0x12c1f8d000(0x12c1f8d000)
00000800:00000200:23.0:1619677470.730179:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff919e41f8e000 0x12c1f8e000(0x12c1f8e000)
00000800:00000200:23.0:1619677470.730179:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff919e41f8f000 0x12c1f8f000(0x12c1f8f000)
00000800:00000200:23.0:1619677470.730180:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff919e41f90000 0x12c1f90000(0x12c1f90000)
00000800:00000200:23.0:1619677470.730180:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff919e41f91000 0x12c1f91000(0x12c1f91000)
00000800:00000200:23.0:1619677470.730181:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919e41f92000 0x12c1f92000(0x12c1f92000)
00000800:00000200:23.0:1619677470.730181:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919e41f93000 0x12c1f93000(0x12c1f93000)
00000800:00000200:23.0:1619677470.730182:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919e41f94000 0x12c1f94000(0x12c1f94000)
00000800:00000200:23.0:1619677470.730182:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919e41f95000 0x12c1f95000(0x12c1f95000)
00000800:00000200:23.0:1619677470.730183:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919e41f96000 0x12c1f96000(0x12c1f96000)
00000800:00000200:23.0:1619677470.730184:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919e41f97000 0x12c1f97000(0x12c1f97000)
00000800:00000200:23.0:1619677470.730184:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919e41852000 0x12c1852000(0x12c1852000)
00000800:00000200:2.0:1619677470.730301:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.730304:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.730305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.730305:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.730307:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.730312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.730314:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677470.730317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.730317:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.730318:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677470.730319:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.730320:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.730324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:23.0:1619677470.730823:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730824:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730824:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730824:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730825:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730825:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730825:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730826:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730826:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730827:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730827:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730827:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730828:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730828:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730828:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730829:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730829:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730830:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730831:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730831:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730831:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730832:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730832:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730832:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730833:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730833:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730834:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730834:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730834:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730835:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730835:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730835:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730836:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730836:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:23.0:1619677470.730836:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (19)++
00000800:00000200:23.0:1619677470.730837:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (20)--
00000800:00000200:18.0:1619677470.730854:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.730857:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.730858:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.730858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.730860:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.730864:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.730945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.730950:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.730952:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.730953:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677470.730954:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.730955:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677470.730955:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677470.730957:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677470.730959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.730961:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.730961:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.730962:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.730963:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:2.0:1619677470.730963:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000400:00000200:18.0:1619677470.730964:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677470.730965:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (23)--
00000800:00000200:2.0:1619677470.730965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.730966:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677470.730966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.730967:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677470.730967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000400:00000200:18.0:1619677470.730969:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.730972:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.730974:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.730978:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.730979:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:18.0:1619677470.730980:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:18.0:1619677470.730982:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:18.0:1619677470.730983:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.730983:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.730984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.731018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.731024:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.731027:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.731028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.731029:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.731033:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.731034:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.731036:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.731037:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.731038:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.731039:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.731044:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.731046:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.731053:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.731054:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.731054:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.731056:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.731056:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.731057:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.731057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.731069:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (23)++
00000800:00000200:18.0:1619677470.731072:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (24)--
00000800:00000200:2.0:1619677470.731072:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (24)++
00000400:00000200:18.0:1619677470.731073:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.731073:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (24)--
00000400:00000200:2.0:1619677470.731074:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.731076:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677470.731076:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.731079:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.731080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.731081:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.731091:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.731094:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.731095:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.731095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.731097:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.731099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677470.731697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677470.731703:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.731706:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677470.731707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677470.731709:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.731712:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.731713:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677470.731715:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.731716:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.731717:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.731718:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677470.731721:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.731723:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677470.731726:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.731727:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:1.0:1619677470.731728:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:1.0:1619677470.731730:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:1.0:1619677470.731731:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.731731:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.731732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677470.732028:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.732031:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.732032:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.732032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.732034:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.732037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677470.732045:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.732048:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.732050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.732050:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.732052:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.732055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.732374:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.732377:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.732378:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.732378:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.732380:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.732383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:2.0:1619677470.732392:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.732394:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.732395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.732395:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.732396:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.732396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:1.0:1619677470.732433:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.732435:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.732436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.732438:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.732440:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.732442:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677470.732443:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.732445:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.732445:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.732446:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677470.732449:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.732451:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677470.732454:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9ce680] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.732455:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (20)++
00000800:00000200:1.0:1619677470.732455:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2cc00] (21)++
00000800:00000200:1.0:1619677470.732457:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2cc00] (22)--
00000800:00000200:1.0:1619677470.732458:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.732460:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677470.732461:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677470.732461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.732473:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677470.732475:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677470.732476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.732477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:5.2:1619677470.733698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677470.733704:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.733708:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677470.733708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677470.733710:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677470.733713:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.733714:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677470.733716:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.733717:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.733718:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.733719:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677470.733723:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677470.733724:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677470.733727:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49600] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.733727:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953a00] (20)++
00000800:00000200:17.0:1619677470.733728:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953a00] (21)++
00000800:00000200:17.0:1619677470.733730:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953a00] (22)--
00000800:00000200:17.0:1619677470.733730:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.733731:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677470.733732:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677470.733732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:5.2:1619677470.733738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677470.733744:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677470.733746:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677470.733747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677470.733748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:20.2:1619677470.734365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.734372:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.734375:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.734375:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.734378:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.734380:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.734381:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677470.734382:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.734384:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.734385:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.734386:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677470.734389:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.734392:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677470.734394:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.734395:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:18.0:1619677470.734396:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:18.0:1619677470.734397:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:18.0:1619677470.734398:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.734399:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.734399:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.735069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.736063:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.736066:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.736067:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.736069:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.736072:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.736074:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.736075:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.736077:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.736078:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.736079:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.736082:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.736084:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.736086:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.736087:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:17.0:1619677470.736088:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:17.0:1619677470.736090:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000800:00000200:17.0:1619677470.736090:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.736091:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.736092:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.736093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.736103:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.736105:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.736105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.736106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677470.736165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.736171:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.736174:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.736175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.736177:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.736179:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.736180:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.736182:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96520955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.736183:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.736184:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.736185:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.736187:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.736189:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.736192:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91978a4a3780] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.736192:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3e00] (20)++
00000800:00000200:17.0:1619677470.736193:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3e00] (21)++
00000800:00000200:17.0:1619677470.736195:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3e00] (22)--
00000800:00000200:17.0:1619677470.736195:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.736196:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677470.736197:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677470.736198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677470.736202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677470.736208:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677470.736211:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677470.736211:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677470.736212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677470.736308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.736354:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.736356:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677470.736357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.736358:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.736360:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.736362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677470.736649:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:2.0:1619677470.736652:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.736654:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:2.0:1619677470.736654:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.736656:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.736659:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:23.0:1619677470.736732:0:13334:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.108@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677470.736738:0:13334:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.108@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:23.0:1619677470.737060:0:13334:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.108@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677470.737068:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff91a87d37a000 0x1cfd37a000(0x1cfd37a000)
00000800:00000200:23.0:1619677470.737069:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff91a8793a3000 0x1cf93a3000(0x1cf93a3000)
00000800:00000200:23.0:1619677470.737069:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff91a86b3a7000 0x1ceb3a7000(0x1ceb3a7000)
00000800:00000200:23.0:1619677470.737070:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff91a87db47000 0x1cfdb47000(0x1cfdb47000)
00000800:00000200:23.0:1619677470.737070:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff91a8783b6000 0x1cf83b6000(0x1cf83b6000)
00000800:00000200:23.0:1619677470.737071:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff91a87d78c000 0x1cfd78c000(0x1cfd78c000)
00000800:00000200:23.0:1619677470.737072:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff91a873f84000 0x1cf3f84000(0x1cf3f84000)
00000800:00000200:23.0:1619677470.737072:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff91a875389000 0x1cf5389000(0x1cf5389000)
00000800:00000200:23.0:1619677470.737073:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff91a874f9a000 0x1cf4f9a000(0x1cf4f9a000)
00000800:00000200:23.0:1619677470.737073:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff91a87d7a7000 0x1cfd7a7000(0x1cfd7a7000)
00000800:00000200:23.0:1619677470.737074:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff91a8787cb000 0x1cf87cb000(0x1cf87cb000)
00000800:00000200:23.0:1619677470.737074:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff91a874faa000 0x1cf4faa000(0x1cf4faa000)
00000800:00000200:23.0:1619677470.737075:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff91a87d778000 0x1cfd778000(0x1cfd778000)
00000800:00000200:23.0:1619677470.737075:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff91a875392000 0x1cf5392000(0x1cf5392000)
00000800:00000200:23.0:1619677470.737077:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff91a87a3a0000 0x1cfa3a0000(0x1cfa3a0000)
00000800:00000200:23.0:1619677470.737077:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff91a8637ca000 0x1ce37ca000(0x1ce37ca000)
00000800:00000200:23.0:1619677470.737078:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff91a87e35e000 0x1cfe35e000(0x1cfe35e000)
00000800:00000200:23.0:1619677470.737078:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff91a878bf6000 0x1cf8bf6000(0x1cf8bf6000)
00000800:00000200:1.0:1619677470.737328:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.737331:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.737332:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.737333:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.737335:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.737338:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.737348:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677470.737351:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.737351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677470.737352:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677470.737354:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677470.737354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:23.0:1619677470.737701:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737702:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737703:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737703:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737703:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737704:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737704:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737704:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737705:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737705:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737706:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737706:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737706:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737707:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737707:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737707:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737708:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737708:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737708:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737709:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737709:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737710:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737710:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737710:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737711:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737711:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737711:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737712:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737712:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737712:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737713:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737713:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737714:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737715:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:23.0:1619677470.737716:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (19)++
00000800:00000200:23.0:1619677470.737716:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (20)--
00000800:00000200:20.2:1619677470.738491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.738500:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.738503:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.738504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677470.738506:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.738509:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000400:00000200:18.0:1619677470.738510:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677470.738510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.738510:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.738512:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.738513:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677470.738514:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677470.738515:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96520942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677470.738516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000400:00000200:18.0:1619677470.738517:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.738518:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.738519:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677470.738523:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.738524:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677470.738528:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919785a26f80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.738529:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0600] (20)++
00000800:00000200:2.0:1619677470.738529:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.738530:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0600] (21)++
00000800:00000200:2.0:1619677470.738530:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000800:00000200:18.0:1619677470.738532:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0600] (22)--
00000400:00000200:2.0:1619677470.738532:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.738533:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677470.738534:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.738535:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.738536:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.738537:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000800:00000200:18.0:1619677470.738538:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.738539:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:20.2:1619677470.738544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.738550:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.738552:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.738553:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.738554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.738879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.738885:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.738889:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.738889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.738891:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.738894:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.738896:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677470.738898:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96520940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.738899:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.738901:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.738902:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677470.738905:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.738906:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677470.738910:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56700] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.738910:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (20)++
00000800:00000200:18.0:1619677470.738911:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2f800] (21)++
00000800:00000200:18.0:1619677470.738913:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2f800] (22)--
00000800:00000200:18.0:1619677470.738914:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.738915:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.738915:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677470.739102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.739107:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.739111:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.739112:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.739113:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.739116:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.739118:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.739119:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96520939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.739121:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.739122:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.739122:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.739125:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.739126:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.739129:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919071ba1f00] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.739129:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986600] (20)++
00000800:00000200:2.0:1619677470.739130:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986600] (21)++
00000800:00000200:2.0:1619677470.739131:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986600] (22)--
00000800:00000200:2.0:1619677470.739132:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.739133:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.739133:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.739152:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (22)++
00000800:00000200:18.0:1619677470.739154:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986600] (23)--
00000400:00000200:18.0:1619677470.739154:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.739155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (22)--
00000400:00000200:18.0:1619677470.739156:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.739157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (21)--
00000800:00000200:23.0:1619677470.739436:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff9197843f3e00] (18)--
00000800:00000200:20.2:1619677470.739765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.739771:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.739775:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.739776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.739778:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.739780:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.739782:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.739783:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.739785:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.739786:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.739787:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.739790:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.739791:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.739795:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.739796:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677470.739797:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677470.739800:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677470.739801:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.739802:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.739802:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.739921:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (20)++
00000800:00000200:18.0:1619677470.739924:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (21)--
00000800:00000200:18.0:1619677470.739924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (20)--
00000800:00000200:2.0:1619677470.739925:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (19)--
00000800:00000200:23.0:1619677470.739956:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff9193c2986600] (18)--
00000800:00000200:20.2:1619677470.740273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.740602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.740609:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.740612:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.740612:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.740615:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.740618:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.740619:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.740621:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.740623:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.740624:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.740625:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.740628:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.740630:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.740634:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 0
00000800:00000200:18.0:1619677470.740635:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 0
00000800:00000200:18.0:1619677470.740636:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.740637:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.740637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:7.0:1619677470.741356:0:13229:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.110@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677470.741362:0:13229:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.110@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:2.0:1619677470.741433:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.741436:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.741437:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.741437:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.741440:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.741444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677470.741490:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677470.741493:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[2] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.741493:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.741496:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.102@o2ib7 : PUT - routed
00000800:00000200:18.0:1619677470.741497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:7.0:1619677470.741658:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.110@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.741669:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919063b88000 0x4e3b88000(0x4e3b88000)
00000800:00000200:7.0:1619677470.741670:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff9193c4c65000 0x844c65000(0x844c65000)
00000800:00000200:7.0:1619677470.741671:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff9193c2658000 0x842658000(0x842658000)
00000800:00000200:7.0:1619677470.741671:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff9193c2659000 0x842659000(0x842659000)
00000800:00000200:7.0:1619677470.741672:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff9193c265a000 0x84265a000(0x84265a000)
00000800:00000200:7.0:1619677470.741673:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff9193c265b000 0x84265b000(0x84265b000)
00000800:00000200:7.0:1619677470.741673:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff9193c265c000 0x84265c000(0x84265c000)
00000800:00000200:7.0:1619677470.741674:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff9193c265d000 0x84265d000(0x84265d000)
00000800:00000200:7.0:1619677470.741674:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff9193c265e000 0x84265e000(0x84265e000)
00000800:00000200:23.0:1619677470.741675:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff9197843f0600] (18)--
00000800:00000200:7.0:1619677470.741675:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff9193c265f000 0x84265f000(0x84265f000)
00000800:00000200:7.0:1619677470.741676:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff9193c2660000 0x842660000(0x842660000)
00000800:00000200:7.0:1619677470.741676:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff9193c2661000 0x842661000(0x842661000)
00000800:00000200:7.0:1619677470.741677:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff9193c2662000 0x842662000(0x842662000)
00000800:00000200:7.0:1619677470.741678:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff9193c2663000 0x842663000(0x842663000)
00000800:00000200:7.0:1619677470.741679:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff9193c2664000 0x842664000(0x842664000)
00000800:00000200:7.0:1619677470.741679:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff9193c2665000 0x842665000(0x842665000)
00000800:00000200:7.0:1619677470.741680:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff9193c2666000 0x842666000(0x842666000)
00000800:00000200:7.0:1619677470.741680:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff9193c2667000 0x842667000(0x842667000)
00000800:00000200:7.0:1619677470.742315:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742316:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742316:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742317:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742317:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742317:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742318:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742318:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742318:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742319:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742320:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742320:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742320:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742321:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742321:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742322:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742322:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742322:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742323:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742323:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742323:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742324:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742324:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742324:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742325:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742325:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742325:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742326:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742326:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742327:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742327:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742327:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742328:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742328:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:7.0:1619677470.742328:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (19)++
00000800:00000200:7.0:1619677470.742329:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (20)--
00000800:00000200:20.2:1619677470.742579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:23.0:1619677470.742599:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91a3aeb2f800] (18)--
00000800:00000200:20.2:1619677470.742619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.742625:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.742628:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.742629:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.742630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.742631:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.742634:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.745288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.745563:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.745568:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677470.745570:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.745571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.745573:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.745578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677470.745667:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.745671:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.745673:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.745676:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.745679:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.745682:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677470.745685:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.745688:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.745690:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.745691:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677470.745696:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.745700:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 984 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677470.745704:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.745706:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677470.745707:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677470.745710:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677470.745711:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.745713:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677470.745715:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677470.745716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677470.745728:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677470.745732:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677470.745733:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677470.745735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:13.0:1619677470.745941:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (20)++
00000800:00000200:13.0:1619677470.745943:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (21)--
00000800:00000200:13.0:1619677470.745944:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (20)--
00000800:00000200:13.0:1619677470.745945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (19)--
00000800:00000200:23.0:1619677470.746140:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91a7c17fd200] (18)--
00000800:00000200:20.2:1619677470.746623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.746630:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.746633:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.746634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.746636:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.746638:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.746640:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677470.746642:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.746644:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.746645:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.746646:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677470.746649:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.746651:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677470.746657:0:19933:0:(o2iblnd_cb.c:1635:kiblnd_launch_tx()) peer_ni[ffff919788dc9c80] -> 10.0.10.107@o2ib7 (1)++
00000800:00000200:18.0:1619677470.746661:0:19933:0:(o2iblnd_cb.c:1440:kiblnd_connect_peer()) peer_ni[ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2)++
00000800:00000200:5.2:1619677470.746722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.746728:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.746729:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677470.746732:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.746735:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.746737:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.746739:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96520912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.746741:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.746742:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.746743:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.746747:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.746749:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.746753:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.746754:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.746755:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.746759:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.746760:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.746761:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (23)++
00000800:00000200:18.0:1619677470.746762:0:19933:0:(o2iblnd_cb.c:1392:kiblnd_resolve_addr()) bound to port 1023
00000800:00000200:2.0:1619677470.746762:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (24)--
00000800:00000200:18.0:1619677470.746763:0:19933:0:(o2iblnd_cb.c:1642:kiblnd_launch_tx()) peer_ni[ffff919788dc9c80] -> 10.0.10.107@o2ib7 (3)--
00000800:00000200:2.0:1619677470.746763:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.746764:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.746764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.746765:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.746766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:7.0:1619677470.746767:0:13229:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.107@o2ib7 Addr resolved: 0
00000800:00000200:5.2:1619677470.746768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:7.0:1619677470.746772:0:13229:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.107@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:2.0:1619677470.746773:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.746775:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.746776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.746777:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:7.0:1619677470.747095:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.107@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.747104:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919783eb9000 0xc03eb9000(0xc03eb9000)
00000800:00000200:7.0:1619677470.747104:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff919783eba000 0xc03eba000(0xc03eba000)
00000800:00000200:7.0:1619677470.747105:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff919783b4b000 0xc03b4b000(0xc03b4b000)
00000800:00000200:7.0:1619677470.747106:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff919783b4c000 0xc03b4c000(0xc03b4c000)
00000800:00000200:7.0:1619677470.747106:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff919783b4d000 0xc03b4d000(0xc03b4d000)
00000800:00000200:7.0:1619677470.747107:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff919783b4e000 0xc03b4e000(0xc03b4e000)
00000800:00000200:7.0:1619677470.747107:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff919783b4f000 0xc03b4f000(0xc03b4f000)
00000800:00000200:7.0:1619677470.747108:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff919783b50000 0xc03b50000(0xc03b50000)
00000800:00000200:7.0:1619677470.747109:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff919783b51000 0xc03b51000(0xc03b51000)
00000800:00000200:7.0:1619677470.747109:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff919783b52000 0xc03b52000(0xc03b52000)
00000800:00000200:7.0:1619677470.747110:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff919783b53000 0xc03b53000(0xc03b53000)
00000800:00000200:7.0:1619677470.747110:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919783b54000 0xc03b54000(0xc03b54000)
00000800:00000200:7.0:1619677470.747111:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919783b55000 0xc03b55000(0xc03b55000)
00000800:00000200:7.0:1619677470.747111:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919783b56000 0xc03b56000(0xc03b56000)
00000800:00000200:7.0:1619677470.747112:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919783b57000 0xc03b57000(0xc03b57000)
00000800:00000200:7.0:1619677470.747112:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919783b58000 0xc03b58000(0xc03b58000)
00000800:00000200:7.0:1619677470.747113:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919783b59000 0xc03b59000(0xc03b59000)
00000800:00000200:7.0:1619677470.747114:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919783b5a000 0xc03b5a000(0xc03b5a000)
00000800:00000200:18.0:1619677470.747147:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.747149:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.747150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.747152:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.747156:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.747157:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677470.747159:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96520911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.747160:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.747161:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.747162:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677470.747166:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.747168:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677470.747170:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.747171:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677470.747171:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677470.747173:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677470.747173:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.747174:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.747175:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:1.0:1619677470.747675:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.747679:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.747680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000400:00000200:1.0:1619677470.747680:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.747682:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.747685:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:7.0:1619677470.747735:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747736:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747737:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747737:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747737:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747738:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747738:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747738:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747739:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747739:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747739:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747740:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747740:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747741:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747741:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747741:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747742:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747742:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747742:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747743:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747743:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747743:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747744:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747744:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747745:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747745:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747746:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747746:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747746:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747747:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747747:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747747:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747748:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747748:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:7.0:1619677470.747749:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (19)++
00000800:00000200:7.0:1619677470.747749:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (20)--
00000800:00000200:20.2:1619677470.748556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.748601:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.748604:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.748605:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.748606:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.748607:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.748609:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:23.0:1619677470.748760:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff919e42d9c000] (18)--
00000800:00000200:1.0:1619677470.748974:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)++
00000800:00000200:1.0:1619677470.748977:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (24)--
00000800:00000200:17.0:1619677470.748978:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)++
00000800:00000200:17.0:1619677470.748978:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0000] (24)--
00000400:00000200:1.0:1619677470.748978:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.748979:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.748980:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677470.748981:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.748984:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (23)--
00000800:00000200:17.0:1619677470.748985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.748985:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)++
00000800:00000200:1.0:1619677470.748988:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.748989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (22)--
00000800:00000200:1.0:1619677470.748989:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0000] (22)++
00000800:00000200:1.0:1619677470.748989:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0000] (23)--
00000800:00000200:1.0:1619677470.748990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (21)--
00000800:00000200:2.0:1619677470.749087:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.749091:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.749092:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.749092:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.749095:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.749099:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:23.0:1619677470.749627:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91a3aeb2cc00] (18)--
00000800:00000200:5.2:1619677470.750552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677470.750559:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.750563:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677470.750563:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677470.750567:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.750569:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.750570:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.750572:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96520900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.750574:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.750575:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.750576:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.750580:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.750583:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.750590:0:19930:0:(o2iblnd_cb.c:1635:kiblnd_launch_tx()) peer_ni[ffff919786445900] -> 10.0.10.115@o2ib7 (1)++
00000800:00000200:17.0:1619677470.750595:0:19930:0:(o2iblnd_cb.c:1440:kiblnd_connect_peer()) peer_ni[ffff919786445900] -> 10.0.10.115@o2ib7 (2)++
00000800:00000200:1.0:1619677470.750616:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (20)++
00000800:00000200:1.0:1619677470.750617:0:19931:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (21)--
00000800:00000200:1.0:1619677470.750618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (20)--
00000800:00000200:1.0:1619677470.750619:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (19)--
00000800:00000200:23.0:1619677470.750656:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff918fe95d0000] (18)--
00000800:00000200:18.0:1619677470.751362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.751366:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.751368:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.751369:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.751371:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.751373:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677470.751374:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:506, d:32, s:-96520898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.751376:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.751377:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.751378:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:7.0:1619677470.751381:0:13229:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.109@o2ib7 Addr resolved: 0
00000400:00000200:18.0:1619677470.751382:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.751384:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:7.0:1619677470.751387:0:13229:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.109@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:18.0:1619677470.751388:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.751389:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677470.751389:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677470.751391:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677470.751392:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.751393:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.751393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:17.0:1619677470.751457:0:19930:0:(o2iblnd_cb.c:1392:kiblnd_resolve_addr()) bound to port 1023
00000800:00000200:17.0:1619677470.751458:0:19930:0:(o2iblnd_cb.c:1642:kiblnd_launch_tx()) peer_ni[ffff919786445900] -> 10.0.10.115@o2ib7 (3)--
00000800:00000200:17.0:1619677470.751459:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677470.751460:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677470.751461:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:7.0:1619677470.751462:0:13229:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.115@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677470.751468:0:13229:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.115@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:5.2:1619677470.751626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.751632:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.751635:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.751636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.751638:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.751641:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.751643:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.751645:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:506, d:32, s:-96520897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.751646:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.751647:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.751648:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.751652:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.751653:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.751657:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.751657:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.751658:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.751659:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.751660:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.751661:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.751661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:7.0:1619677470.751664:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.109@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.751678:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919e41b51000 0x12c1b51000(0x12c1b51000)
00000800:00000200:7.0:1619677470.751678:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff919e41b52000 0x12c1b52000(0x12c1b52000)
00000800:00000200:7.0:1619677470.751679:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff919e41b53000 0x12c1b53000(0x12c1b53000)
00000800:00000200:7.0:1619677470.751679:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff919e41b54000 0x12c1b54000(0x12c1b54000)
00000800:00000200:7.0:1619677470.751680:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff919e41b55000 0x12c1b55000(0x12c1b55000)
00000800:00000200:7.0:1619677470.751680:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff919e41b56000 0x12c1b56000(0x12c1b56000)
00000800:00000200:7.0:1619677470.751681:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff919e41b57000 0x12c1b57000(0x12c1b57000)
00000800:00000200:7.0:1619677470.751682:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff919e41b58000 0x12c1b58000(0x12c1b58000)
00000800:00000200:7.0:1619677470.751682:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff919e41b59000 0x12c1b59000(0x12c1b59000)
00000800:00000200:7.0:1619677470.751683:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff919e41b5a000 0x12c1b5a000(0x12c1b5a000)
00000800:00000200:7.0:1619677470.751683:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff919e41b5b000 0x12c1b5b000(0x12c1b5b000)
00000800:00000200:7.0:1619677470.751684:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919e41b5c000 0x12c1b5c000(0x12c1b5c000)
00000800:00000200:7.0:1619677470.751684:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919e41b5d000 0x12c1b5d000(0x12c1b5d000)
00000800:00000200:7.0:1619677470.751685:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919e41b5e000 0x12c1b5e000(0x12c1b5e000)
00000800:00000200:7.0:1619677470.751685:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919e41b5f000 0x12c1b5f000(0x12c1b5f000)
00000800:00000200:7.0:1619677470.751686:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919e41b68000 0x12c1b68000(0x12c1b68000)
00000800:00000200:7.0:1619677470.751687:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919e41b69000 0x12c1b69000(0x12c1b69000)
00000800:00000200:7.0:1619677470.751687:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919e41b6a000 0x12c1b6a000(0x12c1b6a000)
00000800:00000200:20.2:1619677470.751761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.751767:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.751770:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.751771:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.751775:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.751778:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.751780:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.751782:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:506, d:32, s:-96520896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.751783:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.751784:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.751786:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.751790:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.751793:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.751796:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.751797:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677470.751798:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677470.751800:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677470.751800:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.751801:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.751802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:23.0:1619677470.752075:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff919b86953a00] (18)--
00000800:00000200:7.0:1619677470.752314:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752315:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752315:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752316:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752316:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752317:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752317:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752317:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752318:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752318:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752318:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752319:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752319:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752319:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752320:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752320:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752321:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752321:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752321:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752322:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752322:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752322:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752323:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752323:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752323:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752324:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752324:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752324:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752325:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752325:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752326:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752326:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752326:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752327:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752327:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (19)++
00000800:00000200:7.0:1619677470.752328:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (20)--
00000800:00000200:7.0:1619677470.752335:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.115@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.752344:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919783b5b000 0xc03b5b000(0xc03b5b000)
00000800:00000200:7.0:1619677470.752344:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff919783b5c000 0xc03b5c000(0xc03b5c000)
00000800:00000200:7.0:1619677470.752345:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff919783b5d000 0xc03b5d000(0xc03b5d000)
00000800:00000200:7.0:1619677470.752345:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff919783b5e000 0xc03b5e000(0xc03b5e000)
00000800:00000200:7.0:1619677470.752346:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff919783b5f000 0xc03b5f000(0xc03b5f000)
00000800:00000200:7.0:1619677470.752347:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff919783b60000 0xc03b60000(0xc03b60000)
00000800:00000200:7.0:1619677470.752347:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff919783b61000 0xc03b61000(0xc03b61000)
00000800:00000200:7.0:1619677470.752348:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff919783b62000 0xc03b62000(0xc03b62000)
00000800:00000200:7.0:1619677470.752348:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff919783b63000 0xc03b63000(0xc03b63000)
00000800:00000200:7.0:1619677470.752349:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff919783b64000 0xc03b64000(0xc03b64000)
00000800:00000200:7.0:1619677470.752349:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff919783b65000 0xc03b65000(0xc03b65000)
00000800:00000200:7.0:1619677470.752350:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919783b66000 0xc03b66000(0xc03b66000)
00000800:00000200:7.0:1619677470.752350:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919783b67000 0xc03b67000(0xc03b67000)
00000800:00000200:7.0:1619677470.752351:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919783b68000 0xc03b68000(0xc03b68000)
00000800:00000200:7.0:1619677470.752351:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919783b69000 0xc03b69000(0xc03b69000)
00000800:00000200:7.0:1619677470.752352:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919783b6a000 0xc03b6a000(0xc03b6a000)
00000800:00000200:7.0:1619677470.752352:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919783b6b000 0xc03b6b000(0xc03b6b000)
00000800:00000200:7.0:1619677470.752353:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919783b6c000 0xc03b6c000(0xc03b6c000)
00000800:00000200:7.0:1619677470.752946:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752946:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752947:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752947:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752947:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752948:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752948:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752949:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752949:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752949:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752950:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752950:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752950:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752951:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752952:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752952:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752952:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752953:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752953:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752953:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752954:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752954:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752955:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752955:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752955:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752956:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752956:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752956:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752957:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752957:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752957:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752958:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752958:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752959:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:7.0:1619677470.752959:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (19)++
00000800:00000200:7.0:1619677470.752959:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (20)--
00000800:00000200:13.0:1619677470.753150:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.753153:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.753155:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677470.753158:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.753158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.753159:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677470.753161:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:503, d:12, s:-96520891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.753163:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.753164:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.753164:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677470.753168:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.753170:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677470.753174:0:19942:0:(o2iblnd_cb.c:1635:kiblnd_launch_tx()) peer_ni[ffff918feb721980] -> 10.0.10.113@o2ib7 (1)++
00000800:00000200:13.0:1619677470.753178:0:19942:0:(o2iblnd_cb.c:1440:kiblnd_connect_peer()) peer_ni[ffff918feb721980] -> 10.0.10.113@o2ib7 (2)++
00000800:00000200:13.0:1619677470.753563:0:19942:0:(o2iblnd_cb.c:1392:kiblnd_resolve_addr()) bound to port 1023
00000800:00000200:13.0:1619677470.753564:0:19942:0:(o2iblnd_cb.c:1642:kiblnd_launch_tx()) peer_ni[ffff918feb721980] -> 10.0.10.113@o2ib7 (3)--
00000800:00000200:13.0:1619677470.753565:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.753566:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.753567:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:23.0:1619677470.753569:0:13334:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.113@o2ib7 Addr resolved: 0
00000800:00000200:23.0:1619677470.753576:0:13334:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.113@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:20.2:1619677470.753894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:7.0:1619677470.753910:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.113@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.753918:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff918febcdb000 0x46bcdb000(0x46bcdb000)
00000800:00000200:7.0:1619677470.753918:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff918fe940c000 0x46940c000(0x46940c000)
00000800:00000200:7.0:1619677470.753919:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff918fe940d000 0x46940d000(0x46940d000)
00000800:00000200:7.0:1619677470.753919:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff918fe940e000 0x46940e000(0x46940e000)
00000800:00000200:7.0:1619677470.753920:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff918fe940f000 0x46940f000(0x46940f000)
00000800:00000200:7.0:1619677470.753920:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff918fe9410000 0x469410000(0x469410000)
00000800:00000200:7.0:1619677470.753921:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff918fe9411000 0x469411000(0x469411000)
00000800:00000200:7.0:1619677470.753921:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff918fe9412000 0x469412000(0x469412000)
00000800:00000200:7.0:1619677470.753922:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff918fe9413000 0x469413000(0x469413000)
00000800:00000200:7.0:1619677470.753922:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff918fe9414000 0x469414000(0x469414000)
00000800:00000200:7.0:1619677470.753923:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff918fe9415000 0x469415000(0x469415000)
00000800:00000200:7.0:1619677470.753924:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff918fe9416000 0x469416000(0x469416000)
00000800:00000200:7.0:1619677470.753924:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff918fe9417000 0x469417000(0x469417000)
00000800:00000200:7.0:1619677470.753925:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff918fe9418000 0x469418000(0x469418000)
00000800:00000200:7.0:1619677470.753926:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff918fe9419000 0x469419000(0x469419000)
00000800:00000200:7.0:1619677470.753926:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff918fe941a000 0x46941a000(0x46941a000)
00000800:00000200:7.0:1619677470.753927:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff918fe941b000 0x46941b000(0x46941b000)
00000800:00000200:7.0:1619677470.753927:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff918fe941c000 0x46941c000(0x46941c000)
00000800:00000200:1.0:1619677470.753941:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.753944:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677470.753945:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.753946:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.753947:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.753950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.754090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.754097:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.754101:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677470.754102:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677470.754104:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.754107:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.754109:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677470.754110:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96520887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.754112:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.754113:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.754114:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677470.754117:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.754119:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677470.754123:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abe0d80d00] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.754124:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb719600] (20)++
00000800:00000200:2.0:1619677470.754125:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb719600] (21)++
00000800:00000200:2.0:1619677470.754127:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb719600] (22)--
00000800:00000200:2.0:1619677470.754128:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.754129:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677470.754129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:7.0:1619677470.754530:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754531:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754531:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754532:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754532:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754533:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754533:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754533:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754534:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754534:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754534:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754535:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754535:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754535:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754536:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754536:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754536:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754537:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754537:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754538:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754538:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754538:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754539:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754539:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754539:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754540:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754540:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754540:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754541:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754541:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754542:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754542:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754543:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754543:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:7.0:1619677470.754543:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (19)++
00000800:00000200:7.0:1619677470.754544:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (20)--
00000800:00000200:5.2:1619677470.756009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.756015:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.756018:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.756019:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.756021:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.756025:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.756026:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.756027:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96520883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.756029:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.756030:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.756032:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.756035:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.756037:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.756040:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.756041:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.756042:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.756043:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.756044:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.756046:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.756047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.756329:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.756332:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.756333:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.756333:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.756335:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.756339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.757075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.757082:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.757084:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.757085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.757087:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.757090:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.757091:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.757093:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96520880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.757094:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.757095:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.757096:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.757099:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.757101:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.757104:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.757105:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677470.757106:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677470.757109:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677470.757109:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.757111:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.757112:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.757112:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.757118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.757123:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.757126:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.757126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.757127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.757708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:12.0:1619677470.757756:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677470.757759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677470.757760:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.757763:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.757764:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677470.757766:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677470.758027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:7.0:1619677470.758361:0:13229:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.114@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677470.758368:0:13229:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.114@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:7.0:1619677470.758695:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.114@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.758714:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff91a3aef69000 0x182ef69000(0x182ef69000)
00000800:00000200:7.0:1619677470.758715:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff91a3ae254000 0x182e254000(0x182e254000)
00000800:00000200:7.0:1619677470.758716:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff91a3ae23b000 0x182e23b000(0x182e23b000)
00000800:00000200:7.0:1619677470.758717:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff91a3ae5fe000 0x182e5fe000(0x182e5fe000)
00000800:00000200:7.0:1619677470.758718:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff91a3aeb37000 0x182eb37000(0x182eb37000)
00000800:00000200:7.0:1619677470.758719:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff91a3aeb3a000 0x182eb3a000(0x182eb3a000)
00000800:00000200:7.0:1619677470.758720:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff91a3ae27e000 0x182e27e000(0x182e27e000)
00000800:00000200:7.0:1619677470.758721:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff91a3aeb79000 0x182eb79000(0x182eb79000)
00000800:00000200:7.0:1619677470.758722:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff91a3aeb24000 0x182eb24000(0x182eb24000)
00000800:00000200:7.0:1619677470.758723:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff91a3aeb09000 0x182eb09000(0x182eb09000)
00000800:00000200:7.0:1619677470.758724:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff91a3ad80e000 0x182d80e000(0x182d80e000)
00000800:00000200:7.0:1619677470.758725:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff91a3ad80f000 0x182d80f000(0x182d80f000)
00000800:00000200:7.0:1619677470.758726:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff91a3ad810000 0x182d810000(0x182d810000)
00000800:00000200:7.0:1619677470.758728:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff91a3ad811000 0x182d811000(0x182d811000)
00000800:00000200:7.0:1619677470.758729:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff91a3ad812000 0x182d812000(0x182d812000)
00000800:00000200:7.0:1619677470.758730:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff91a3ad813000 0x182d813000(0x182d813000)
00000800:00000200:7.0:1619677470.758731:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff91a3ad814000 0x182d814000(0x182d814000)
00000800:00000200:7.0:1619677470.758732:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff91a3ad815000 0x182d815000(0x182d815000)
00000800:00000200:7.0:1619677470.759412:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759413:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759414:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759414:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759414:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759415:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759416:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759416:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759417:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759417:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759417:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759418:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759418:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759418:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759419:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759419:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759419:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759420:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759420:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759421:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759421:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759421:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759422:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759422:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759422:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759423:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759423:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759423:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759424:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759424:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759425:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759425:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759425:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759426:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:7.0:1619677470.759426:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (19)++
00000800:00000200:7.0:1619677470.759426:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (20)--
00000800:00000200:20.2:1619677470.760085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.760121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.760128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.760131:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.760132:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.760133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.760134:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.760137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.760146:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.760149:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.760150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.760150:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.760151:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.760152:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677470.761178:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.761181:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.761182:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.761182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.761184:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.761188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.761198:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677470.761202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.761202:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.761203:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677470.761204:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.761204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677470.761871:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.761874:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.761875:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.761875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.761878:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.761881:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.761994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.762001:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.762004:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.762005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.762007:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.762010:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.762011:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.762013:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96520865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.762015:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.762015:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.762016:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.762019:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.762021:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.762022:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 0
00000800:00000200:18.0:1619677470.762023:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 0
00000800:00000200:18.0:1619677470.762024:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.762025:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.762026:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.762033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.762038:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.762040:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.762041:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.762042:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.762045:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.762046:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.762048:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:507, d:32, s:-96520864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.762049:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.762049:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.762050:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.762054:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.762055:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.762058:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.762059:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.762060:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.762062:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.762062:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.762063:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.762063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.762800:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (24)++
00000800:00000200:18.0:1619677470.762803:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (25)--
00000800:00000200:2.0:1619677470.762803:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (25)++
00000400:00000200:18.0:1619677470.762805:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.762805:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (25)--
00000400:00000200:2.0:1619677470.762805:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.762807:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677470.762807:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.762811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.762811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (24)--
00000800:00000200:2.0:1619677470.762812:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.762820:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.762821:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.762822:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.762823:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.762826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.762849:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.762853:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.762853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.762856:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.110@o2ib7 : PUT - routed
00000800:00000200:18.0:1619677470.762857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.763251:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.763254:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.763255:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.763256:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.110@o2ib7 : PUT - routed
00000800:00000200:2.0:1619677470.763257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:7.0:1619677470.763585:0:13334:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.116@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677470.763592:0:13334:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.116@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:20.2:1619677470.763593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.763599:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.763603:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.763603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.763605:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.763608:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.763610:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.763611:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96520859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.763612:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.763613:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.763615:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.763618:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.763620:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.763623:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.763623:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.763624:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.763626:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.763627:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.763628:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.763628:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.763640:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.763642:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.763643:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.763644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.763644:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.763646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:23.0:1619677470.763762:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff91abeb719600] (18)--
00000800:00000200:23.0:1619677470.763922:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.116@o2ib7 Route resolved: 0
00000800:00000200:23.0:1619677470.763932:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff91a86afeb000 0x1ceafeb000(0x1ceafeb000)
00000800:00000200:23.0:1619677470.763932:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff91a873387000 0x1cf3387000(0x1cf3387000)
00000800:00000200:23.0:1619677470.763933:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff91a8627e9000 0x1ce27e9000(0x1ce27e9000)
00000800:00000200:23.0:1619677470.763933:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff91a87d342000 0x1cfd342000(0x1cfd342000)
00000800:00000200:23.0:1619677470.763934:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff91a870fc7000 0x1cf0fc7000(0x1cf0fc7000)
00000800:00000200:23.0:1619677470.763934:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff91a860bab000 0x1ce0bab000(0x1ce0bab000)
00000800:00000200:23.0:1619677470.763935:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff91a86abdb000 0x1ceabdb000(0x1ceabdb000)
00000800:00000200:23.0:1619677470.763935:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff91a875387000 0x1cf5387000(0x1cf5387000)
00000800:00000200:23.0:1619677470.763936:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff91a87fb42000 0x1cffb42000(0x1cffb42000)
00000800:00000200:23.0:1619677470.763937:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff91a8783dd000 0x1cf83dd000(0x1cf83dd000)
00000800:00000200:23.0:1619677470.763937:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff91a861397000 0x1ce1397000(0x1ce1397000)
00000800:00000200:23.0:1619677470.763938:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff91a87a3e8000 0x1cfa3e8000(0x1cfa3e8000)
00000800:00000200:23.0:1619677470.763938:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff91abd48c7000 0x20548c7000(0x20548c7000)
00000800:00000200:23.0:1619677470.763939:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff91abd48c8000 0x20548c8000(0x20548c8000)
00000800:00000200:23.0:1619677470.763939:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff91abd48c9000 0x20548c9000(0x20548c9000)
00000800:00000200:23.0:1619677470.763940:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff91abd48ca000 0x20548ca000(0x20548ca000)
00000800:00000200:23.0:1619677470.763940:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff91abd48cb000 0x20548cb000(0x20548cb000)
00000800:00000200:23.0:1619677470.763941:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff91abd48cc000 0x20548cc000(0x20548cc000)
00000800:00000200:18.0:1619677470.764492:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.764496:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.764497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.764498:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.764499:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.764503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:23.0:1619677470.764561:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764562:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764563:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764563:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764563:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764564:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764564:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764564:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764565:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764565:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764565:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764566:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764566:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764567:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764567:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764567:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764568:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764568:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764568:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764569:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764569:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764569:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764570:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764570:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764570:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764571:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764571:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764572:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764572:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764572:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764573:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764573:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764573:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764574:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:23.0:1619677470.764574:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (19)++
00000800:00000200:23.0:1619677470.764574:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (20)--
00000800:00000200:20.2:1619677470.765232:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:2.0:1619677470.765454:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.765458:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677470.765459:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.765460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.765462:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.765466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.765801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.765844:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.765847:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.765848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.765848:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.765850:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.765853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:7.0:1619677470.766226:0:13334:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.106@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677470.766232:0:13334:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.106@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:7.0:1619677470.766509:0:13334:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.106@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.766513:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff91a3ad816000 0x182d816000(0x182d816000)
00000800:00000200:7.0:1619677470.766514:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff91a3ad817000 0x182d817000(0x182d817000)
00000800:00000200:7.0:1619677470.766515:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff91a3ad818000 0x182d818000(0x182d818000)
00000800:00000200:7.0:1619677470.766515:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff91a3ad819000 0x182d819000(0x182d819000)
00000800:00000200:7.0:1619677470.766516:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff91a3ad81a000 0x182d81a000(0x182d81a000)
00000800:00000200:7.0:1619677470.766517:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff91a3ad81b000 0x182d81b000(0x182d81b000)
00000800:00000200:7.0:1619677470.766517:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff91a3ad81c000 0x182d81c000(0x182d81c000)
00000800:00000200:7.0:1619677470.766518:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff91a3ad81d000 0x182d81d000(0x182d81d000)
00000800:00000200:7.0:1619677470.766518:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff91a3ad81e000 0x182d81e000(0x182d81e000)
00000800:00000200:7.0:1619677470.766519:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff91a3ad81f000 0x182d81f000(0x182d81f000)
00000800:00000200:7.0:1619677470.766519:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff91a3ad820000 0x182d820000(0x182d820000)
00000800:00000200:7.0:1619677470.766520:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff91a3ad821000 0x182d821000(0x182d821000)
00000800:00000200:7.0:1619677470.766520:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff91a3ad822000 0x182d822000(0x182d822000)
00000800:00000200:7.0:1619677470.766521:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff91a3ad823000 0x182d823000(0x182d823000)
00000800:00000200:7.0:1619677470.766521:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff91a3ad824000 0x182d824000(0x182d824000)
00000800:00000200:7.0:1619677470.766522:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff91a3ad825000 0x182d825000(0x182d825000)
00000800:00000200:7.0:1619677470.766522:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff91a3ad826000 0x182d826000(0x182d826000)
00000800:00000200:7.0:1619677470.766523:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff91a3ad827000 0x182d827000(0x182d827000)
00000800:00000200:18.0:1619677470.766549:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.766552:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.766553:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.766554:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.766555:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.766559:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677470.766942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.766948:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.766951:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.766952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.766953:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.766956:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.766957:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.766959:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96520847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.766960:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.766961:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.766962:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.766965:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.766967:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.766971:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.766971:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.766972:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.766974:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.766974:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.766976:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.766977:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.766977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.766982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.766988:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.766990:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.766991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.766992:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:7.0:1619677470.767149:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767150:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767150:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767150:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767151:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767151:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767151:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767152:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767152:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767153:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767153:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767153:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767154:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767154:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767154:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767155:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767155:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767155:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767156:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767157:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767157:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767157:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767158:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767158:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767158:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767159:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767159:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767160:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767160:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767160:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767161:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767161:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767161:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767162:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:7.0:1619677470.767162:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (19)++
00000800:00000200:7.0:1619677470.767162:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (20)--
00000800:00000200:2.0:1619677470.767425:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.767427:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.767429:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.767429:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.767430:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.767433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.767732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:18.0:1619677470.767780:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.767783:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.767784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.767785:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.767787:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.767790:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.767800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677470.767803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.767803:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.767804:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.767805:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.767806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677470.768089:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.768092:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.768092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.768095:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.768097:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.768099:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.768101:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96520843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.768102:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.768103:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.768105:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.768107:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.768109:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.768113:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.768113:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677470.768114:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677470.768121:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677470.768122:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.768122:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.768123:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677470.770062:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.770066:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.770068:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.770068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.770070:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.770073:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.770087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.770092:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.770096:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.770097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.770099:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.770102:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.770104:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.770105:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96520837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.770108:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.770109:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.770110:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.770112:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.770114:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.770117:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.770118:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677470.770118:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677470.770120:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677470.770121:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.770122:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.770122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.771373:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.771376:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.771377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.771378:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.771381:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.771385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.771621:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.771623:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.771625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.771625:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.771627:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.771630:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.771646:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.771649:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.771650:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.771650:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.771651:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.771652:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.771797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.771829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.771834:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.771837:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.771838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.771839:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.771842:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.771843:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.771845:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96520831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.771848:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.771849:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.771850:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.771853:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.771854:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.771858:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.771859:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.771860:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.771861:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.771862:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.771863:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.771863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.771974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.771980:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.771983:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.771983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.771986:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.771989:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.771990:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.771992:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96520829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.771993:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.771994:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.771995:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.771999:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.772000:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.772004:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 0
00000800:00000200:18.0:1619677470.772005:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 0
00000800:00000200:18.0:1619677470.772006:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.772007:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.772007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677470.772023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.772028:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.772030:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.772032:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.772033:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.772035:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.772036:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.772037:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:32, s:-96520828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.772039:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.772040:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.772040:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.772043:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.772044:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.772046:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.772046:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.772047:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.772049:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.772049:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.772051:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.772052:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.772053:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.772058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.772064:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.772065:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.772065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677470.772066:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.772066:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.772066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.772067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.772068:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.772068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.772069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.772891:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.772895:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.772896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.772898:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.772901:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.772902:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677470.772904:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.772905:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.772906:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.772907:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677470.772910:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.772912:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677470.772916:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.772917:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677470.772918:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677470.772920:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677470.772921:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.772922:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677470.772923:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677470.772923:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.772934:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677470.772936:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:13.0:1619677470.772936:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:17.0:1619677470.772937:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.772938:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677470.772938:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677470.772940:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.772942:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.772943:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677470.772945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:1.0:1619677470.773034:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.773037:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.773038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.773038:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.773040:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.773043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.773208:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677470.773211:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[1] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.773212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.773214:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.113@o2ib7 : PUT - routed
00000800:00000200:17.0:1619677470.773216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.773465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.773468:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.773469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.773469:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.773471:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.773475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.774265:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.774268:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.774269:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.774269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.774271:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.774274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.775262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.775268:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.775272:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.775273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.775275:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.775278:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.775280:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677470.775281:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.775282:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.775283:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.775284:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677470.775287:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.775289:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.775292:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.775293:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677470.775298:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677470.775301:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677470.775301:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.775302:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.775303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.776181:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.776185:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.776186:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.776187:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.776190:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.776191:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677470.776193:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.776194:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.776195:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.776196:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677470.776199:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.776200:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.776202:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.776203:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677470.776204:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677470.776206:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.776207:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.776207:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.776208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677470.776219:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.776221:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.776222:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.776223:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.776223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.776225:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677470.776725:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.776728:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.776729:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.776729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.776732:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.776735:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677470.776914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.776920:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.776922:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.776923:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.776925:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.776928:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.776929:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.776930:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.776932:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.776933:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.776934:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.776938:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.776940:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.776943:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.776944:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.776945:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.776947:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.776947:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.776948:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.776948:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677470.776977:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.776980:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.776981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.776983:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.776985:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.776987:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677470.776988:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.776990:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.776991:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.776992:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677470.776996:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.776999:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677470.777002:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.777003:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677470.777004:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677470.777007:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677470.777007:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.777008:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.777009:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:7.0:1619677470.777117:0:13334:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.111@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677470.777123:0:13334:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.111@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:20.2:1619677470.777134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.777181:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.777184:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.777185:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.777185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.777187:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.777190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:7.0:1619677470.777453:0:13334:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.111@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.777471:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff91a7c0d35000 0x1c40d35000(0x1c40d35000)
00000800:00000200:7.0:1619677470.777472:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff91a7c0d36000 0x1c40d36000(0x1c40d36000)
00000800:00000200:7.0:1619677470.777473:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff91a7c0d37000 0x1c40d37000(0x1c40d37000)
00000800:00000200:7.0:1619677470.777474:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff91a7c0c30000 0x1c40c30000(0x1c40c30000)
00000800:00000200:7.0:1619677470.777475:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff91a7c0c31000 0x1c40c31000(0x1c40c31000)
00000800:00000200:7.0:1619677470.777476:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff91a7c0c32000 0x1c40c32000(0x1c40c32000)
00000800:00000200:7.0:1619677470.777477:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff91a7c0c33000 0x1c40c33000(0x1c40c33000)
00000800:00000200:7.0:1619677470.777478:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff91a7c11f0000 0x1c411f0000(0x1c411f0000)
00000800:00000200:7.0:1619677470.777479:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff91a7c11f1000 0x1c411f1000(0x1c411f1000)
00000800:00000200:7.0:1619677470.777480:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff91a7c11f2000 0x1c411f2000(0x1c411f2000)
00000800:00000200:7.0:1619677470.777481:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff91a7c11f3000 0x1c411f3000(0x1c411f3000)
00000800:00000200:7.0:1619677470.777482:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff91a7c10c4000 0x1c410c4000(0x1c410c4000)
00000800:00000200:7.0:1619677470.777484:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff91a7c10c5000 0x1c410c5000(0x1c410c5000)
00000800:00000200:7.0:1619677470.777485:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff91a7c10c6000 0x1c410c6000(0x1c410c6000)
00000800:00000200:7.0:1619677470.777486:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff91a7c0e52000 0x1c40e52000(0x1c40e52000)
00000800:00000200:7.0:1619677470.777487:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff91a7c0e53000 0x1c40e53000(0x1c40e53000)
00000800:00000200:7.0:1619677470.777488:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff91a7c0e54000 0x1c40e54000(0x1c40e54000)
00000800:00000200:7.0:1619677470.777489:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff91a7c0e55000 0x1c40e55000(0x1c40e55000)
00000800:00000200:1.0:1619677470.777522:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.777525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.777525:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677470.777527:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.777530:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.777532:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.777533:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.777535:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.777536:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.777537:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.777540:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.777541:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.777544:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.777545:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.777545:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.777547:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.777548:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.777548:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.777549:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:7.0:1619677470.778173:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778174:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778175:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778176:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778176:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778177:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778178:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778179:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778179:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778180:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778181:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778181:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778182:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778183:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778183:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778184:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778185:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778185:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778186:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778187:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778187:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778188:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778189:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778189:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778190:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778191:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778191:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778192:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778193:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778194:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778194:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778195:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778196:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778197:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:7.0:1619677470.778197:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (19)++
00000800:00000200:7.0:1619677470.778198:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (20)--
00000800:00000200:20.2:1619677470.778413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.778452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.778458:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.778461:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677470.778462:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.778463:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.778465:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.778467:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.778468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.778475:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677470.778475:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.778478:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677470.778479:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:17.0:1619677470.778479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:2.0:1619677470.778479:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677470.778480:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.778481:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.778481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:18.0:1619677470.778482:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.778485:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.778486:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.778488:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.778489:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.778491:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.778492:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.778495:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.778497:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.778499:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.778500:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.778501:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.778502:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.778503:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.778504:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.778504:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.778516:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.778518:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.778518:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.778519:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.778520:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.778521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.779019:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.779022:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.779023:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.779023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.779025:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.779030:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.779209:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.779213:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.779214:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.779214:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.779216:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.779219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.779229:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.779232:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.779233:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.779233:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.779235:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.779235:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:12.0:1619677470.779325:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677470.779328:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677470.779329:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.779331:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.779333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677470.779334:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677470.779410:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677470.779413:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[1] from 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677470.779416:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.111@o2ib7 : PUT - routed
00000800:00000200:12.0:1619677470.779417:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677470.779418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677470.779932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677470.779980:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.779983:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.779984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.779984:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.779986:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.779989:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.780018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.780024:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.780026:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.780027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.780029:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.780031:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.780033:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.780034:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.780036:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.780037:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.780038:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.780041:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.780043:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.780046:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.780047:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677470.780048:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677470.780050:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677470.780050:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.780052:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.780053:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.780053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.780058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.780064:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.780066:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.780067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.780068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.781545:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.781548:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.781548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.781549:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.781551:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.781555:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677470.781957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.781962:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.781965:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.781966:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.781968:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.781970:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.781971:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.781972:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.781974:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.781974:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.781975:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.781978:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.781980:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.781983:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.781984:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.781985:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.781987:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.781987:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.781988:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.781988:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.782215:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.782218:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.782219:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.782220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.782221:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.782228:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.782229:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677470.782232:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.782232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.782233:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.782234:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.782235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677470.783275:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.783277:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.783279:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.783282:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.783282:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.783283:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677470.783284:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96520787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.783286:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.783287:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.783287:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677470.783290:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.783292:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677470.783302:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.783302:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677470.783303:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677470.783305:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677470.783306:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.783308:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677470.783309:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677470.783310:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:1.0:1619677470.783317:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.783319:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.783320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.783323:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.783324:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:1.0:1619677470.783325:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.783326:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677470.783327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.783327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:1.0:1619677470.783327:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677470.783329:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.783330:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.783332:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.783333:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677470.783337:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.783339:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.783341:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.783342:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677470.783343:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.783344:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.783345:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.783346:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.783347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677470.783358:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.783360:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.783361:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.783362:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.783363:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.783364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677470.783999:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.784002:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.784002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.784004:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.784007:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.784009:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677470.784010:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.784012:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.784013:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.784014:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677470.784017:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.784018:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677470.784022:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.784023:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677470.784023:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677470.784026:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677470.784027:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.784028:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.784028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.2:1619677470.784286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.784337:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.784340:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.784341:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.784341:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.784343:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.784346:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.2:1619677470.784467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:13.0:1619677470.784591:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677470.784594:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677470.784595:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.784596:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.784598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677470.784599:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677470.784609:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677470.784612:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677470.784613:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677470.784614:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677470.784614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677470.784616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:17.0:1619677470.784623:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.784626:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.784627:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.784627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.784630:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.784641:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677470.784697:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.784700:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.784701:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.784701:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.784704:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.784712:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677470.784720:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677470.784723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677470.784723:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.784725:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.784726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677470.784732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677470.784738:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.784741:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.784742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.784742:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.784745:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.784753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677470.784756:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.784759:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.784760:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.784760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.784762:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.784765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677470.784773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.784775:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.784776:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.784777:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.784780:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.784787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677470.784842:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677470.784845:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677470.784845:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.784847:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.784849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677470.784849:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:7.0:1619677470.784974:0:13334:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.112@o2ib7 Addr resolved: 0
00000800:00000200:7.0:1619677470.784980:0:13334:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.112@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:7.0:1619677470.785244:0:13334:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.112@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677470.785254:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919b864d4000 0x10064d4000(0x10064d4000)
00000800:00000200:7.0:1619677470.785254:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff919b864d5000 0x10064d5000(0x10064d5000)
00000800:00000200:7.0:1619677470.785255:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff919b864d6000 0x10064d6000(0x10064d6000)
00000800:00000200:7.0:1619677470.785256:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff919b864d7000 0x10064d7000(0x10064d7000)
00000800:00000200:7.0:1619677470.785256:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff919b864d8000 0x10064d8000(0x10064d8000)
00000800:00000200:7.0:1619677470.785257:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff919b864d9000 0x10064d9000(0x10064d9000)
00000800:00000200:7.0:1619677470.785257:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff919b864da000 0x10064da000(0x10064da000)
00000800:00000200:7.0:1619677470.785258:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff919b864db000 0x10064db000(0x10064db000)
00000800:00000200:7.0:1619677470.785258:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff919b864dc000 0x10064dc000(0x10064dc000)
00000800:00000200:7.0:1619677470.785259:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff919b864dd000 0x10064dd000(0x10064dd000)
00000800:00000200:7.0:1619677470.785259:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff919b864de000 0x10064de000(0x10064de000)
00000800:00000200:7.0:1619677470.785260:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919b864df000 0x10064df000(0x10064df000)
00000800:00000200:7.0:1619677470.785260:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919b86750000 0x1006750000(0x1006750000)
00000800:00000200:7.0:1619677470.785261:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919b86751000 0x1006751000(0x1006751000)
00000800:00000200:7.0:1619677470.785261:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919b86752000 0x1006752000(0x1006752000)
00000800:00000200:7.0:1619677470.785262:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919b86753000 0x1006753000(0x1006753000)
00000800:00000200:7.0:1619677470.785263:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919b86754000 0x1006754000(0x1006754000)
00000800:00000200:7.0:1619677470.785263:0:13334:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919b86755000 0x1006755000(0x1006755000)
00000800:00000200:7.0:1619677470.785885:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785886:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785886:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785887:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785887:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785888:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785888:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785888:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785889:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785890:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785890:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785890:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785891:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785891:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785891:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785892:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785892:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785893:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785893:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785893:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785894:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785894:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785894:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785895:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785895:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785895:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785896:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785896:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785896:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785897:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785897:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785898:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785898:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785898:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:7.0:1619677470.785899:0:13334:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (19)++
00000800:00000200:7.0:1619677470.785899:0:13334:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (20)--
00000800:00000200:20.2:1619677470.786003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.786009:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.786012:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.786014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.786015:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.786018:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.786020:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.786021:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.786022:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.786024:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.786025:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.786028:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.786030:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.786033:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.786034:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677470.786034:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677470.786036:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677470.786037:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.786037:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.786038:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677470.786155:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.786158:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.786160:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677470.786162:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.786162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.786164:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677470.786165:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96520757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.786166:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.786167:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.786168:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677470.786171:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.786173:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677470.786176:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.786177:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677470.786178:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677470.786180:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677470.786180:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.786181:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.786182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677470.786368:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.786371:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.786372:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.786372:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.786374:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.786381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677470.786383:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677470.786385:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.786386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677470.786387:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677470.786388:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677470.786388:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.786757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:1.0:1619677470.786846:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.786850:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.786850:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677470.786853:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.786855:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.786857:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.786858:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.786860:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.786861:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.786862:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.786866:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.786868:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.786870:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.786871:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.786872:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.786873:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.786874:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.786875:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.786875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677470.786883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.786889:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.786893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.786893:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677470.786896:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.786899:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.786900:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.786902:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.786903:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.786904:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.786905:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.786908:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.786910:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.786914:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.786914:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.786915:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.786917:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.786917:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.786919:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.786920:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.786921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.786926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.786932:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.786934:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.786935:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.786935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.787024:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.787027:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.787028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.787029:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.787031:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.787034:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677470.787251:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.787255:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.787256:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.787256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.787259:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.787266:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.787267:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.787270:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.787270:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.787270:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.787272:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.787272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.788038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:2.0:1619677470.788290:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.788295:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.788298:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677470.788300:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.788302:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.788306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.788919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.788926:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.788929:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.788930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.788932:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.788935:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.788936:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.788938:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.788939:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.788940:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.788941:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.788945:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.788946:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.788948:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.788949:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.788950:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.788952:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.788952:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.788953:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.788954:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.788965:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.788967:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.788967:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.788968:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.788969:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.788970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.790499:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.790502:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.790503:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.790503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.790505:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.790509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.790873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.790913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.790919:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677470.790920:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:2.0:1619677470.790922:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.790923:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677470.790924:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:2.0:1619677470.790925:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677470.790925:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.790926:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:2.0:1619677470.790927:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.790929:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677470.790929:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677470.790930:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.790932:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677470.790932:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:2.0:1619677470.790933:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.790934:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677470.790937:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.790938:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.790942:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.790942:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677470.790943:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677470.790944:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:2.0:1619677470.790945:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677470.790946:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.790946:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.790947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.790948:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.790950:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.790950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677470.790952:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677470.790952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677470.791209:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.791212:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.791213:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.791213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.791216:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.791219:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677470.791831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.791837:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.791840:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.791841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.791843:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.791846:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.791847:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.791849:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.791850:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.791851:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.791852:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.791855:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.791857:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.791860:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.791861:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.791862:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.791863:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.791864:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.791865:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.791865:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677470.792259:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.792262:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.792263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.792264:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.792267:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.792268:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677470.792269:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.792270:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.792271:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.792272:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677470.792275:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.792277:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.792279:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.792280:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677470.792280:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.792282:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.792282:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.792283:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.792283:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677470.792295:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.792301:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.792302:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.792303:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.792304:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.792305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:12.0:1619677470.793626:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.793630:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.793633:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.793636:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.793636:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.793639:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677470.793641:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96520730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.793643:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.793645:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.793646:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677470.793651:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.793654:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677470.793659:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.793660:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:12.0:1619677470.793661:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:12.0:1619677470.793664:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:12.0:1619677470.793665:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.793666:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.793667:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677470.793679:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.793682:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.793683:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.793683:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.793685:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.793689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.793698:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677470.793700:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.793701:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.793702:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.793703:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.793703:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677470.794481:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.794484:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.794485:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.794485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.794487:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.794490:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.794554:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.794557:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.794558:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.794558:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.794560:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.794563:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.794572:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.794574:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.794575:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.794575:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.794576:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.794577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677470.795495:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.795498:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.795500:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677470.795502:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.795503:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.795505:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677470.795506:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96520724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.795507:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.795509:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.795509:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677470.795513:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.795514:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677470.795517:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.795518:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677470.795519:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677470.795521:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677470.795521:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.795522:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.795523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677470.795537:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.795540:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.795541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.795541:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.795543:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.795546:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677470.796048:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.796051:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.796053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.796053:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.796055:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.796059:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.796179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.796185:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.796188:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.796188:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.796192:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.796195:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.796198:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.796200:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.796202:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.796204:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.796206:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.796214:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.796217:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.796219:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.796220:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677470.796221:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (22)++
00000800:00000200:18.0:1619677470.796223:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (23)--
00000800:00000200:18.0:1619677470.796223:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.796224:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.796225:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.796231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.796237:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.796240:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.796241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.796244:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.796248:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.796249:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.796250:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.796252:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.796253:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.796254:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.796258:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.796259:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.796262:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.796263:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.796263:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.796265:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.796265:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.796266:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.796267:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.796826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.796871:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.796874:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.796875:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.796875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.796877:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.796879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.796928:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.796931:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.796932:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.796934:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.796936:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.796937:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.796938:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.796939:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.796940:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.796941:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.796944:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.796946:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.796949:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.796950:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.796951:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.796953:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.796953:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.796955:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.796956:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.796956:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.796967:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.796969:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.796970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.796970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.797114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.797120:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.797123:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.797124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.797126:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.797129:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.797130:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.797132:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.797133:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.797134:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.797135:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.797138:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.797140:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.797142:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.797143:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.797143:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.797145:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.797146:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.797147:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.797148:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.797159:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.797161:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.797162:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.797163:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.797165:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.797166:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.799966:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.799968:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.799969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.799970:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.799972:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.799975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:12.0:1619677470.800192:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.800196:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.800200:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.800202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.800204:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.800206:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677470.800209:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96520707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.800211:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.800213:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.800214:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677470.800219:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.800223:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677470.800226:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.800227:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677470.800228:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677470.800231:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677470.800231:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.800234:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677470.800236:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677470.800237:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.800250:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677470.800252:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677470.800253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.800254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677470.800299:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.800302:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.800303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.800303:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.800305:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.800309:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.800597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.800635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677470.800922:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.800925:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.800926:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.800926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.800928:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.800932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.800942:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677470.800944:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.800945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.800946:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.800947:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.800948:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677470.801128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.801133:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.801135:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.801136:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.801138:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.801140:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.801141:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.801143:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.801144:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.801145:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.801146:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.801150:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.801151:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.801155:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.801156:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.801156:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.801158:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.801159:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.801159:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.801160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.802120:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.802122:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.802123:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.802124:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.802125:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.802129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.802139:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.802141:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.802142:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.802142:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.802144:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.802144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.802227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.802233:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.802235:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.802236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.802238:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.802241:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.802242:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.802243:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.802245:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.802246:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.802247:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.802250:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.802251:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.802254:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.802255:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677470.802256:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677470.802258:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677470.802258:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.802259:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.802260:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.802261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.802266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.802272:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.802274:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.802275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.802275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.802581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.802622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.802624:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677470.802625:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.802626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.802627:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.802630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.803008:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.803010:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.803011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.803013:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.803016:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.803017:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677470.803018:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.803020:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.803021:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.803022:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677470.803028:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.803030:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677470.803033:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.803034:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677470.803035:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677470.803036:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677470.803037:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.803038:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.803038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:2.0:1619677470.803063:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.803066:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.803067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.803067:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.803069:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.803072:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.803375:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.803378:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.803379:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.803381:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.803384:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.803385:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.803386:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.803388:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.803389:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.803390:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.803393:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.803395:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.803398:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.803398:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.803399:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.803401:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.803401:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.803402:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.803403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.805205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.805212:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.805215:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.805216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.805219:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.805222:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.805223:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.805224:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.805226:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.805227:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.805228:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.805233:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.805236:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.805238:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.805239:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.805239:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.805241:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.805242:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.805244:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.805245:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.805246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.805252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.805259:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.805260:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.805260:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.805261:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677470.805261:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.805262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.805263:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.805264:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.805265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.805266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677470.806377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.806385:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.806389:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.806390:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.806392:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.806395:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.806397:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.806398:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.806401:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.806402:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.806404:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.806412:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.806415:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.806419:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.806421:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.806422:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.806424:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.806425:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.806426:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.806428:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.806428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.2:1619677470.806432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.806433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.806439:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.806440:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.806441:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.806442:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.806442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.806442:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.806444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.806444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:2.0:1619677470.806444:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.806448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.806714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.806720:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.806723:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.806723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.806726:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.806729:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.806730:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.806732:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.806733:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.806735:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.806736:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.806739:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.806741:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.806744:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.806745:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677470.806745:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677470.806747:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677470.806748:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.806749:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.806749:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.806800:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.806803:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.806804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.806805:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.806807:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.806810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.807582:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.807585:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.807586:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677470.807587:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.807589:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.807592:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.808232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.808239:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.808242:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677470.808243:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677470.808244:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.808248:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.808250:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677470.808251:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.808253:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.808254:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.808255:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677470.808258:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.808260:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677470.808263:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.808264:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677470.808264:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677470.808267:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677470.808268:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.808269:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677470.808270:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677470.808271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677470.808276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677470.808281:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677470.808282:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:17.0:1619677470.808283:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677470.808284:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:17.0:1619677470.808284:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.808285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:1.0:1619677470.808285:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.808286:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.808286:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677470.808289:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.808303:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677470.808306:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.808307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.808307:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.808308:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.808308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.808412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.808455:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.808458:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.808459:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.808459:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.808461:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.808463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.808474:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.808477:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.808478:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.808478:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.808479:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.808480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.809770:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.809773:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.809773:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.809775:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.809778:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.809780:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677470.809781:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.809782:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.809783:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.809784:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677470.809788:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.809790:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677470.809793:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.809794:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677470.809794:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677470.809797:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677470.809798:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.809798:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.809799:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677470.810551:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.810554:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.810555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.810556:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.810559:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.810562:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677470.810563:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.810564:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.810565:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.810566:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677470.810570:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.810573:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677470.810576:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.810576:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677470.810577:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677470.810579:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677470.810580:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.810581:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677470.810583:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677470.810583:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.810594:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:18.0:1619677470.810595:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:17.0:1619677470.810596:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:18.0:1619677470.810597:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:17.0:1619677470.810597:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:18.0:1619677470.810598:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.810598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677470.810599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.810601:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.810603:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.811291:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.811296:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.811298:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.811299:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.811301:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.811304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.811313:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.811315:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:5.2:1619677470.811315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.811317:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.811317:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.811318:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.811318:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.811319:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.811321:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.811322:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.811324:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.811326:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.811327:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.811328:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.811330:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.811331:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.811332:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.811335:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.811337:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.811340:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.811341:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.811341:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.811343:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.811344:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.811344:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.811345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.811363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.811370:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.811372:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.811373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.811375:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.811378:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.811379:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.811380:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.811382:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.811383:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.811384:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.811386:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.811388:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.811389:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.811390:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.811390:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.811392:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.811392:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.811393:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.811393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.811405:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.811407:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.811407:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.811408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.811409:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.811411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.811525:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.811528:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.811529:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.811531:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.811533:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.811535:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.811536:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.811537:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.811538:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.811539:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.811542:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.811543:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.811546:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.811546:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.811547:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.811549:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.811549:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.811550:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.811550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677470.811918:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.811921:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.811922:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.811922:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.811924:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.811927:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:13.0:1619677470.812007:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677470.812009:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677470.812010:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.812012:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.812014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677470.812015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677470.812028:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.812031:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.812032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.812032:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.812034:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.812037:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.812635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.812642:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.812645:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.812646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.812648:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.812651:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.812653:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.812654:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.812656:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.812657:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.812659:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677470.812660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677470.812663:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.812666:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.812669:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.812669:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677470.812670:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677470.812672:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677470.812672:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.812673:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.812674:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.812700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.814661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.814702:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.814705:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.814706:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.814706:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.814709:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.814711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677470.815761:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.815764:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.815766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.815766:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.815768:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.815772:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677470.816305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.816312:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.816316:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.816316:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.816318:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.816320:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.816322:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.816323:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.816325:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.816325:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.816326:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.816329:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.816331:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.816334:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.816335:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.816336:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.816337:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.816338:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.816339:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.816339:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.816478:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.816481:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.816482:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.816483:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.816485:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.816488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.817181:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.817183:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.817184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.817185:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.817187:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.817190:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.817530:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.817532:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.817533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.817534:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.817535:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.817538:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.817548:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677470.817551:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.817552:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.817552:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.817553:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.817553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.2:1619677470.817657:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677470.817669:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:24.2:1619677470.817683:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:12.0:1619677470.817737:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677470.817739:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677470.817740:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.817743:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.817744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677470.817746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:17.0:1619677470.817752:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.817756:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677470.817756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.817757:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.817759:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.817765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.817785:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.817787:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.817788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.817789:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.817791:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.817793:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677470.817860:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.817863:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.817864:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.817864:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.817866:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.817870:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677470.817914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677470.817916:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677470.817917:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.817919:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.817920:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677470.817925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677470.818482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.818516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.818523:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.2:1619677470.818525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.818525:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.818528:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.818528:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677470.818529:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.818530:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677470.818532:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.818534:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.818534:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677470.818536:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.818537:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.818538:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.818539:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.818542:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.818543:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.818543:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677470.818544:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677470.818546:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.818547:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.818548:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.818550:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.818553:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.818555:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.818557:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.818558:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.818559:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.818559:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.818559:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677470.818559:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677470.818561:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.818562:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.818562:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677470.818562:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.818563:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.818563:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.818564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.818565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.818565:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.818575:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.818577:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.818578:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.818579:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.818581:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.818585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.818597:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.818599:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.818600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.818601:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.818601:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.818602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677470.818777:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.818779:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677470.818780:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.818781:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.818783:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.818785:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.818794:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.818796:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.818797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.818797:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.818798:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.818799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.819117:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.819120:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.819120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.819123:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.819125:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.819126:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.819128:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.819129:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.819130:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.819131:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.819135:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.819137:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.819140:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.819141:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677470.819142:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677470.819143:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677470.819144:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.819145:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.819145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677470.819840:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.819843:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.819844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.819844:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.819846:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.819850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.820459:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.820462:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.820463:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.820463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.820465:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.820468:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.820606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.820651:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.820654:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.820655:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.820655:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.820657:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.820660:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677470.821318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.821324:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.821327:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.821328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.821329:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.821332:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.821334:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.821335:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.821337:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.821337:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.821338:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.821341:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.821343:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.821346:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.821347:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.821348:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.821349:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.821350:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.821351:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.821351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.821532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e2feb400] (20)++
00000800:00000200:20.2:1619677470.821561:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e2feb400] (21)++
00000800:00000200:18.0:1619677470.821579:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.821582:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.821582:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.821583:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.821585:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.821590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.821608:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.821611:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.821612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.821612:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.821614:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.821616:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.821635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e2feb400] (20)++
00000800:00000200:12.0:1619677470.822875:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677470.822877:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677470.822878:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.822880:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.822882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677470.822883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677470.822891:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677470.822893:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677470.822894:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677470.822895:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677470.822896:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677470.822897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677470.822909:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.822912:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.822913:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.822913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.822915:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.822918:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.824331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.824371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677470.825334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.825342:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.825347:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.825348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.825352:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.825356:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.825359:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.825361:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.825364:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.825366:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.825368:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.825374:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.825377:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.825380:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.825382:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677470.825383:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677470.825385:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677470.825386:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.825388:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.825390:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.825391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.825396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.825404:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.825407:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.825408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.825410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.825647:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.825651:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.825652:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:18.0:1619677470.825653:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.825655:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677470.825656:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.825657:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (23)++
00000800:00000200:2.0:1619677470.825658:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (24)--
00000800:00000200:2.0:1619677470.825660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.825661:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.825662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677470.825904:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.825907:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.825909:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.825909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.825913:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.825918:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.825921:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.825924:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.825924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.825925:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.825927:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.825928:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.826165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.826172:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.826176:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677470.826177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677470.826180:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.826184:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.826186:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677470.826188:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.826191:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.826193:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.826194:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677470.826199:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.826202:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677470.826206:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.826207:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677470.826208:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677470.826212:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677470.826212:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.826214:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677470.826214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:5.2:1619677470.826230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.826238:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.826242:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.826244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.826248:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.826252:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.826254:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.826256:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.826259:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.826260:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.826263:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.826269:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.826273:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.826277:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.826278:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.826279:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.826281:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.826282:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.826284:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.826286:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.826287:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:20.2:1619677470.826288:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.2:1619677470.826292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.826302:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.826304:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.826305:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.826306:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.826307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.826310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.826311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000400:00000200:2.0:1619677470.826311:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.826317:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.826320:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.826322:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.826324:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.826326:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.826328:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.826333:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.826336:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.826339:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.826340:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.826341:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.826342:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.826343:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.826344:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.826345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.826357:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.826359:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.826360:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.826360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.826362:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.826363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.826976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.827020:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.827023:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.827024:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.827024:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.827026:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.827029:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.828376:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.828378:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.828379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.828380:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.828382:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.828384:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.828465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:1.0:1619677470.829503:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.829506:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.829507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.829509:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.829511:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.829513:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.829514:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.829516:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.829517:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.829518:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.829521:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.829523:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.829525:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.829526:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.829526:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.829528:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.829529:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.829530:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.829531:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.829532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.829543:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.829545:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.829546:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.829547:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.830091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.830157:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.830160:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.830161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.830161:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.830164:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.830167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677470.830178:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.830181:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.830182:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.830182:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.830185:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.830187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.830404:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.830406:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.830407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.830408:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.830410:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.830413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.830887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.830893:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.830895:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.830896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.830898:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.830900:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.830902:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.830903:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.830904:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.830905:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.830906:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.830909:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.830911:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.830914:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.830915:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677470.830916:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677470.830918:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677470.830918:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.830919:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.830919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.831105:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.831108:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.831109:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.831109:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.831111:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.831114:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.831126:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.831129:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.831130:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.831130:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.831131:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.831132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677470.831192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.831198:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.831202:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.831202:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.831205:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.831207:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.831208:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.831210:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.831211:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.831212:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.831213:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.831217:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.831218:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.831222:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.831223:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.831223:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.831225:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.831226:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.831226:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.831227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.831582:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.831585:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.831586:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.831586:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.831588:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.831591:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677470.831727:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.831729:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.831731:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.831731:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.831733:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.831737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.831889:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.831892:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.831893:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.831894:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.831897:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.831899:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.831900:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.831902:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.831903:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.831904:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.831907:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.831909:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.831911:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.831912:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.831912:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.831914:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.831915:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.831916:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677470.831917:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677470.831918:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677470.831928:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.831929:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.831930:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000400:00000200:2.0:1619677470.831930:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.831931:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677470.831931:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.831932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.831932:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.831933:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677470.831934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.831943:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.831946:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.831946:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.831946:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.831947:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.831948:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.832171:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.832174:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.832175:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.832175:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.832177:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.832180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.832566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.832572:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.832574:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.832575:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.832577:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.832579:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.832580:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.832582:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.832583:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.832584:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.832585:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.832588:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.832590:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.832592:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.832593:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.832594:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.832595:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.832596:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.832596:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.832598:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.832598:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.832604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.832610:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.832611:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.832611:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677470.832612:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.832612:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.832612:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.832613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.832614:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.832614:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.832615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.832805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.832844:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.832847:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.832848:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.832848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.832849:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.832852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677470.833405:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.833407:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.833409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.833409:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.833410:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.833413:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.834190:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.834192:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.834193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677470.834194:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.834196:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.834199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677470.834548:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677470.834550:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677470.834551:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.834552:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.834555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677470.834556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:18.0:1619677470.835195:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.835198:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.835199:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.835200:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.835202:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.835205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.835405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.835411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.835414:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.835415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.835416:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.835418:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.835420:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677470.835421:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.835422:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.835423:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.835424:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677470.835428:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.835430:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.835432:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.835433:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677470.835434:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677470.835436:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677470.835437:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.835437:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.835438:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.835861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.835899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677470.836087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.836093:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.836096:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.836096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.836099:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.836101:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.836103:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.836104:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.836106:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.836107:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.836108:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.836112:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.836114:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.836117:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.836118:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.836119:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.836121:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.836121:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.836122:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.836123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.837508:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.837510:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.837512:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.837512:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.837514:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.837518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.839027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.839072:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.839074:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.839075:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.839075:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.839078:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.839080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.839090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.839093:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.839093:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.839094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.839094:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.839095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677470.839393:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.839396:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.839397:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.839397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.839399:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.839404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.839406:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.839408:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.839409:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.839409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.839410:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.839411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677470.839869:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.839872:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.839872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.839873:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.839875:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.839878:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.839887:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677470.839890:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.839890:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.839891:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.839892:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.839892:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677470.840009:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.840012:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.840013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.840013:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.840016:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.840020:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677470.840052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.840054:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.840055:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.840056:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.840058:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.840061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.840491:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.840494:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.840495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.840497:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.840499:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.840500:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.840502:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.840503:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.840504:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.840505:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.840508:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.840510:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.840512:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.840513:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.840513:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.840515:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.840516:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.840517:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.840518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677470.840844:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677470.840847:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677470.840848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677470.840850:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.840852:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.840854:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:17.0:1619677470.840855:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.840857:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.840858:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.840859:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:17.0:1619677470.840863:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.840864:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:17.0:1619677470.840867:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.840868:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:17.0:1619677470.840869:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:17.0:1619677470.840871:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:17.0:1619677470.840872:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677470.840872:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677470.840873:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:24.2:1619677470.840878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:1.0:1619677470.840883:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677470.840886:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:17.0:1619677470.840887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:1.0:1619677470.840888:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.840890:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.840892:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.840894:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.840895:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.840896:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.840897:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.840901:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.840903:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.840905:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.840906:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677470.840906:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677470.840908:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677470.840909:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677470.840909:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677470.840910:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:17.0:1619677470.840969:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677470.840972:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677470.840972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677470.840974:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.840977:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.840978:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677470.840980:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.840981:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.840982:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.840983:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677470.840986:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.840988:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677470.840991:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.840992:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:17.0:1619677470.840992:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677470.840994:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:17.0:1619677470.840995:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677470.840996:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:17.0:1619677470.840998:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:17.0:1619677470.840998:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677470.841008:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:1.0:1619677470.841010:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:1.0:1619677470.841011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677470.841012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:17.0:1619677470.841023:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677470.841026:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677470.841026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677470.841029:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.841031:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.841033:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677470.841034:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.841036:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.841037:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.841037:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677470.841040:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.841041:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677470.841044:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.841044:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677470.841045:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677470.841047:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677470.841047:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677470.841048:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677470.841049:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:1.0:1619677470.841069:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677470.841072:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:17.0:1619677470.841073:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:1.0:1619677470.841075:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.841077:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.841078:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677470.841080:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.841081:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.841082:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.841083:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677470.841086:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.841087:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677470.841089:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.841090:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677470.841090:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677470.841092:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677470.841092:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677470.841093:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677470.841093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:17.0:1619677470.841262:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677470.841265:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677470.841265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677470.841268:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.841270:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.841271:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.841273:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.841274:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.841275:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.841276:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.841279:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.841280:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.841283:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.841284:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677470.841284:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677470.841286:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677470.841287:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677470.841288:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677470.841288:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:1.0:1619677470.841382:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677470.841386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677470.841386:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000400:00000200:1.0:1619677470.841388:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.841391:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.841393:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677470.841394:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.841395:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.841396:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.841397:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677470.841400:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.841403:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677470.841406:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.841407:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677470.841407:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677470.841409:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677470.841410:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677470.841411:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:1.0:1619677470.841412:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:1.0:1619677470.841413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677470.841423:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:17.0:1619677470.841425:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:17.0:1619677470.841426:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677470.841427:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:1.0:1619677470.841454:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677470.841459:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:17.0:1619677470.841460:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:1.0:1619677470.841461:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.841464:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.841465:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677470.841467:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.841468:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.841469:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.841470:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677470.841473:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.841475:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677470.841478:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.841479:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:1.0:1619677470.841479:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677470.841482:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677470.841483:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677470.841484:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677470.841484:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:12.0:1619677470.841496:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677470.841498:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677470.841499:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.841501:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.841503:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677470.841504:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677470.841560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.841632:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677470.841635:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677470.841635:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677470.841637:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.841640:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.841642:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677470.841643:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.841644:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.841645:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.841646:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677470.841650:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.841652:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677470.841655:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.841656:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:17.0:1619677470.841656:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677470.841658:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:17.0:1619677470.841659:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677470.841659:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677470.841660:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:20.2:1619677470.841756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677470.841758:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677470.841761:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:17.0:1619677470.841762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:18.0:1619677470.841763:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677470.841763:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677470.841766:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.841766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:1.0:1619677470.841766:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.841768:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677470.841769:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.841770:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.841771:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.841772:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.841772:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.841773:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677470.841774:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.841775:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.841776:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677470.841777:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677470.841778:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677470.841779:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.841780:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:18.0:1619677470.841781:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677470.841781:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:18.0:1619677470.841782:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677470.841782:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.841783:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677470.841783:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677470.841784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000400:00000200:18.0:1619677470.841785:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.841787:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.841789:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.841790:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.841790:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.841792:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.841793:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.841794:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.841794:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677470.841795:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.841797:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.841798:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.841799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.841800:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.841802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677470.841806:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.841808:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.841809:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.841810:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.841811:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.841812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.842273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.842279:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.842283:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.842283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.842285:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.842287:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.842289:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.842290:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.842292:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.842293:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.842300:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.842303:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.842305:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.842307:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.842308:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677470.842309:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677470.842311:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677470.842312:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.842313:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.842313:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677470.842410:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677470.842412:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677470.842413:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.842414:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.842417:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677470.842417:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677470.842431:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677470.842433:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.842434:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677470.842435:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677470.842435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677470.842437:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:5.2:1619677470.842943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.842949:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.842952:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.842953:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.842955:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.842958:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.842960:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.842961:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.842963:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.842964:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.842965:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.842968:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.842971:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.842974:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.842975:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.842976:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.842977:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.842978:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.842979:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.842980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677470.843524:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.843527:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.843528:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.843528:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.843531:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.843534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677470.843609:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.843613:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.843614:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.843614:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.843616:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.843619:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677470.844277:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.844279:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.844280:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.844281:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.844283:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.844286:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.845103:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.845106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.845106:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677470.845108:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.845110:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.845112:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.845113:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.845114:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.845115:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.845116:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.845119:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.845121:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.845124:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.845124:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677470.845125:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677470.845127:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677470.845127:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.845129:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677470.845130:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677470.845130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.845143:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677470.845146:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677470.845147:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.845148:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677470.845279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.845329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.845331:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.845333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677470.845333:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.845335:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.845337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.846032:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.846035:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.846035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.846036:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.846038:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.846041:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.846056:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677470.846058:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.846059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.846059:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.846061:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.846061:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.847007:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.847010:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677470.847011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677470.847013:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.847015:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.847017:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.847018:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.847019:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.847020:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.847021:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.847025:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.847026:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.847029:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.847030:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:17.0:1619677470.847031:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677470.847033:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:17.0:1619677470.847034:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.847035:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.847035:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677470.847154:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (18)++
00000800:00000200:12.0:1619677470.847156:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (19)--
00000800:00000200:12.0:1619677470.847157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (18)--
00000800:00000200:12.0:1619677470.847158:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (17)++
00000800:00000200:12.0:1619677470.847158:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (18)--
00000800:00000200:12.0:1619677470.847159:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (17)--
00000800:00000200:12.0:1619677470.847159:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (16)++
00000800:00000200:12.0:1619677470.847160:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (17)--
00000800:00000200:12.0:1619677470.847161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (16)--
00000800:00000200:12.0:1619677470.847162:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (15)++
00000800:00000200:12.0:1619677470.847164:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (16)--
00000800:00000200:12.0:1619677470.847164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (15)--
00000800:00000200:12.0:1619677470.847165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (14)++
00000800:00000200:12.0:1619677470.847165:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (15)--
00000800:00000200:12.0:1619677470.847166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (14)--
00000800:00000200:12.0:1619677470.847166:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (13)++
00000800:00000200:12.0:1619677470.847167:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (14)--
00000800:00000200:12.0:1619677470.847167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (13)--
00000800:00000200:12.0:1619677470.847168:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (12)++
00000800:00000200:12.0:1619677470.847168:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (13)--
00000800:00000200:12.0:1619677470.847169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (12)--
00000800:00000200:13.0:1619677470.847170:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (11)++
00000800:00000200:13.0:1619677470.847172:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (12)--
00000800:00000200:13.0:1619677470.847172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (11)--
00000800:00000200:13.0:1619677470.847173:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (10)++
00000800:00000200:13.0:1619677470.847174:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (11)--
00000800:00000200:13.0:1619677470.847174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (10)--
00000800:00000200:13.0:1619677470.847175:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (9)++
00000800:00000200:13.0:1619677470.847176:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (10)--
00000800:00000200:13.0:1619677470.847176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (9)--
00000800:00000200:12.0:1619677470.847177:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (8)++
00000800:00000200:12.0:1619677470.847180:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (9)--
00000800:00000200:12.0:1619677470.847180:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (8)--
00000800:00000200:12.0:1619677470.847181:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (7)++
00000800:00000200:12.0:1619677470.847181:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (8)--
00000800:00000200:12.0:1619677470.847182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (7)--
00000800:00000200:12.0:1619677470.847182:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (6)++
00000800:00000200:12.0:1619677470.847183:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (7)--
00000800:00000200:12.0:1619677470.847183:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (6)--
00000800:00000200:12.0:1619677470.847185:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (5)++
00000800:00000200:12.0:1619677470.847187:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (6)--
00000800:00000200:12.0:1619677470.847187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (5)--
00000800:00000200:12.0:1619677470.847187:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (4)++
00000800:00000200:12.0:1619677470.847188:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (5)--
00000800:00000200:12.0:1619677470.847188:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (4)--
00000800:00000200:12.0:1619677470.847189:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fca00] (3)++
00000800:00000200:12.0:1619677470.847190:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fca00] (4)--
00000800:00000200:12.0:1619677470.847190:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (3)--
00000800:00000200:12.0:1619677470.847191:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fca00] (2)--
00000800:00000200:20.2:1619677470.847366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.847374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.847377:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.847379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.847381:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.847383:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.847385:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.847386:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.847388:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.847389:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.847390:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.847395:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.847398:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.847401:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.847402:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677470.847403:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677470.847405:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677470.847406:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.847407:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.847408:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.847409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.847414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.2:1619677470.847419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.847420:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.847422:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.847423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.847423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.847454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677470.847563:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.847565:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.847566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.847569:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.847572:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.847573:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.847575:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.847576:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.847577:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.847578:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.847582:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.847583:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.847586:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.847587:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677470.847588:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677470.847589:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677470.847590:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.847591:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.847591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:5.2:1619677470.847707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677470.847713:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677470.847716:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677470.847717:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677470.847719:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677470.847722:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.847723:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677470.847725:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.847726:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.847727:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.847728:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677470.847730:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677470.847732:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677470.847734:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.847735:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:1.0:1619677470.847736:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677470.847738:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677470.847739:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677470.847740:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677470.847740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:2.0:1619677470.847752:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.847754:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.847755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.847756:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.847758:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.847760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677470.847984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.847990:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.847993:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.847993:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.847995:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.847998:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.847999:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.848000:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.848002:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.848003:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.848004:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.848008:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.848010:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.848013:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.848014:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.848014:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.848016:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.848017:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.848018:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.848019:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.848019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.848024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.848030:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.848031:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.848032:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.848034:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.848140:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (18)++
00000800:00000200:17.0:1619677470.848142:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (19)--
00000800:00000200:17.0:1619677470.848143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (18)--
00000800:00000200:17.0:1619677470.848144:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (17)++
00000800:00000200:17.0:1619677470.848145:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (18)--
00000800:00000200:17.0:1619677470.848145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (17)--
00000800:00000200:17.0:1619677470.848146:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (16)++
00000800:00000200:17.0:1619677470.848148:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (17)--
00000800:00000200:17.0:1619677470.848149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (16)--
00000800:00000200:17.0:1619677470.848149:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (15)++
00000800:00000200:17.0:1619677470.848150:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (16)--
00000800:00000200:17.0:1619677470.848151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (15)--
00000800:00000200:17.0:1619677470.848152:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (14)++
00000800:00000200:17.0:1619677470.848153:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (15)--
00000800:00000200:17.0:1619677470.848153:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (14)--
00000800:00000200:17.0:1619677470.848154:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (13)++
00000800:00000200:17.0:1619677470.848155:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (14)--
00000800:00000200:17.0:1619677470.848155:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (13)--
00000800:00000200:17.0:1619677470.848156:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (12)++
00000800:00000200:17.0:1619677470.848156:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (13)--
00000800:00000200:17.0:1619677470.848157:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (12)--
00000800:00000200:17.0:1619677470.848158:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (11)++
00000800:00000200:17.0:1619677470.848158:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (12)--
00000800:00000200:17.0:1619677470.848159:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (11)--
00000800:00000200:17.0:1619677470.848159:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (10)++
00000800:00000200:17.0:1619677470.848160:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (11)--
00000800:00000200:17.0:1619677470.848161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (10)--
00000800:00000200:17.0:1619677470.848161:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (9)++
00000800:00000200:17.0:1619677470.848162:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (10)--
00000800:00000200:17.0:1619677470.848163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (9)--
00000800:00000200:17.0:1619677470.848163:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (8)++
00000800:00000200:17.0:1619677470.848164:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (9)--
00000800:00000200:17.0:1619677470.848164:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (8)--
00000800:00000200:17.0:1619677470.848166:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (7)++
00000800:00000200:17.0:1619677470.848166:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (8)--
00000800:00000200:17.0:1619677470.848167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (7)--
00000800:00000200:17.0:1619677470.848167:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (6)++
00000800:00000200:17.0:1619677470.848168:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (7)--
00000800:00000200:17.0:1619677470.848169:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (6)--
00000800:00000200:17.0:1619677470.848169:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (5)++
00000800:00000200:17.0:1619677470.848170:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (6)--
00000800:00000200:17.0:1619677470.848170:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (5)--
00000800:00000200:17.0:1619677470.848171:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (4)++
00000800:00000200:17.0:1619677470.848172:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (5)--
00000800:00000200:17.0:1619677470.848172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (4)--
00000800:00000200:17.0:1619677470.848173:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918febd25600] (3)++
00000800:00000200:17.0:1619677470.848174:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918febd25600] (4)--
00000800:00000200:17.0:1619677470.848174:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (3)--
00000800:00000200:17.0:1619677470.848175:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918febd25600] (2)--
00000800:00000200:18.0:1619677470.848457:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (18)++
00000800:00000200:1.0:1619677470.848457:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677470.848459:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (19)--
00000800:00000200:18.0:1619677470.848460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (18)--
00000800:00000200:1.0:1619677470.848460:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677470.848461:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (17)++
00000800:00000200:17.0:1619677470.848461:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.848461:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.848463:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (18)--
00000400:00000200:1.0:1619677470.848463:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.848464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (17)--
00000800:00000200:18.0:1619677470.848465:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (16)++
00000800:00000200:18.0:1619677470.848467:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (17)--
00000800:00000200:1.0:1619677470.848467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677470.848468:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (16)--
00000800:00000200:18.0:1619677470.848469:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (15)++
00000800:00000200:18.0:1619677470.848470:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (16)--
00000800:00000200:18.0:1619677470.848471:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (15)--
00000800:00000200:18.0:1619677470.848472:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (14)++
00000800:00000200:18.0:1619677470.848473:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (15)--
00000800:00000200:18.0:1619677470.848473:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (14)--
00000800:00000200:18.0:1619677470.848474:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (13)++
00000800:00000200:18.0:1619677470.848475:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (14)--
00000800:00000200:18.0:1619677470.848475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (13)--
00000800:00000200:18.0:1619677470.848476:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (12)++
00000800:00000200:18.0:1619677470.848477:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (13)--
00000800:00000200:18.0:1619677470.848477:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (12)--
00000800:00000200:18.0:1619677470.848478:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (11)++
00000800:00000200:18.0:1619677470.848480:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (12)--
00000800:00000200:18.0:1619677470.848480:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (11)--
00000800:00000200:18.0:1619677470.848481:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (10)++
00000800:00000200:18.0:1619677470.848482:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (11)--
00000800:00000200:18.0:1619677470.848482:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (10)--
00000800:00000200:18.0:1619677470.848483:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (9)++
00000800:00000200:18.0:1619677470.848484:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (10)--
00000800:00000200:18.0:1619677470.848484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (9)--
00000800:00000200:18.0:1619677470.848485:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (8)++
00000800:00000200:18.0:1619677470.848486:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (9)--
00000800:00000200:18.0:1619677470.848486:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (8)--
00000800:00000200:18.0:1619677470.848487:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (7)++
00000800:00000200:18.0:1619677470.848487:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (8)--
00000800:00000200:18.0:1619677470.848488:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (7)--
00000800:00000200:18.0:1619677470.848489:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986e00] (6)++
00000800:00000200:18.0:1619677470.848490:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986e00] (7)--
00000800:00000200:18.0:1619677470.848491:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (6)--
00000800:00000200:18.0:1619677470.848492:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986e00] (5)--
00000800:00000200:2.0:1619677470.849633:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.849635:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.849637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.849637:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.849639:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.849644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.849646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.849648:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.849649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.849649:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.849650:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.849651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.850285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.850291:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.850295:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.850298:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677470.850300:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.850303:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.850304:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.850305:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.850307:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.850308:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.850309:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.850311:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.850313:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.850315:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.850316:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.850316:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.850318:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.850319:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.850320:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.850320:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.850332:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.850334:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.850335:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.850336:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.850337:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.850338:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.850556:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.850558:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.850559:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.850559:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.850561:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.850564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677470.850582:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.850585:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.850586:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.850588:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.850590:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.850591:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.850593:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.850594:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.850595:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.850596:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.850599:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.850601:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.850604:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.850604:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677470.850605:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677470.850607:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677470.850607:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.850608:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.850609:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677470.851307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f74be00] (20)++
00000800:00000200:12.0:1619677470.851315:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f74be00] (21)++
00000800:00000200:12.0:1619677470.851318:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.40@o2ib3
00000400:00000200:12.0:1619677470.851320:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.40@o2ib3 : GET - for me
00000800:00000200:13.0:1619677470.851322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f74be00] (22)--
00000400:00000200:12.0:1619677470.851323:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.40@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677470.851325:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.40@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677470.851326:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f74be00] (21)++
00000800:00000200:12.0:1619677470.851327:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f74be00] (22)--
00000400:00000200:12.0:1619677470.851328:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.40@o2ib3 local destination
00000800:00000200:12.0:1619677470.851329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f74be00] (21)--
00000800:00000200:20.2:1619677470.851753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.851797:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.851800:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.851801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.851801:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.851803:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.851805:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.851990:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.851993:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.851993:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.851995:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.851997:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.851999:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677470.852000:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.852002:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.852003:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.852004:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677470.852008:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.852010:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677470.852013:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.852014:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677470.852014:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677470.852016:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677470.852017:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.852017:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.852018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677470.852029:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677470.852032:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677470.852033:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.852035:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.852035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677470.852038:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677470.852066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.852072:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.852074:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.852076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.852078:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.852080:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.852082:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677470.852083:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.852085:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.852086:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.852087:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677470.852087:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.852090:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.852091:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:2.0:1619677470.852091:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677470.852091:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.852092:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677470.852093:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.852095:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.852096:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677470.852096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677470.852097:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677470.852099:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677470.852100:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.852100:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.852101:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.853163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.854067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.854073:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:17.0:1619677470.854074:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677470.854077:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:17.0:1619677470.854077:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677470.854077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:17.0:1619677470.854078:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.854078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:18.0:1619677470.854080:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.854081:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677470.854083:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.854084:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:18.0:1619677470.854085:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.854086:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.854087:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.854088:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.854090:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.854093:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.854095:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.854098:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.854099:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.854100:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.854101:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.854102:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.854103:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.854104:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677470.855334:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.855338:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.855338:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677470.855340:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.855343:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.855344:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.855346:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.855348:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.855349:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.855350:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.855354:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.855357:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.855359:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.855360:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.855361:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.855363:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.855363:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.855364:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.855365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677470.855393:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.855396:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.855397:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.855397:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.855399:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.855403:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.856574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677470.856597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:20.2:1619677470.856614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:18.0:1619677470.856620:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.856623:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.856624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.856625:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.856628:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.856635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.856637:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.856639:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.856639:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.856640:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.856642:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.856642:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.856657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:17.0:1619677470.856665:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.856668:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.856669:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.856669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.856671:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.856675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.856684:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677470.856687:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.856688:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.856688:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.856689:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.856690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.857514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.857559:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.857561:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.857562:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.857562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.857564:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.857567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.857577:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.857580:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.857581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.857581:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.857582:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.857583:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677470.857939:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.857942:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.857943:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.857944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.857945:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.857949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.858936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:5.2:1619677470.858963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.858969:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.2:1619677470.858972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677470.858972:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.858973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.858975:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.858978:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.858980:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.858981:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.858983:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.858984:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.858985:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.858988:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.858990:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.858994:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.858995:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.858996:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.858997:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.858998:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.858999:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.859000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.859600:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.859603:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.859604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.859604:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.859607:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.859611:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677470.859620:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677470.859623:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.859624:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677470.859624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677470.859626:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677470.859626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.859727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.859733:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.859736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.859736:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677470.859739:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.859742:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.859743:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.859745:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.859746:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.859747:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.859748:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.859751:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.859753:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.859755:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.859756:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.859757:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.859758:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.859759:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.859759:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.859760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.859771:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.859773:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.859774:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.859775:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.859776:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.859777:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.859853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.859858:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.859860:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.859862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.859862:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.859864:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.859865:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677470.859866:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.859868:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.859868:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.859869:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677470.859873:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.859874:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677470.859877:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.859878:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677470.859878:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677470.859880:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677470.859880:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.859881:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.859882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.860237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.860242:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.860245:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.860246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.860248:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.860251:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.860252:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.860254:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.860255:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.860256:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.860257:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.860261:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.860263:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.860265:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.860266:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677470.860266:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677470.860268:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677470.860269:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.860269:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.860270:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677470.861047:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.861050:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.861051:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.861051:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.861053:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.861056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677470.862605:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.862610:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.862611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.862615:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.862618:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.862621:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.862623:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.862625:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.862627:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.862628:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.862634:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.862637:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.862640:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.862642:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677470.862643:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677470.862645:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677470.862645:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.862648:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.862649:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.862650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.862663:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.862666:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.862667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.862669:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.863499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.863549:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.863552:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.863553:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.863554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.863555:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.863555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677470.863557:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:17.0:1619677470.863558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.863559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:1.0:1619677470.863560:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : GET - for me
00000400:00000200:1.0:1619677470.863564:0:19931:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.29@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1619677470.863566:0:19931:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.29@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:1.0:1619677470.863568:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:1.0:1619677470.863569:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:1.0:1619677470.863571:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (23)--
00000400:00000200:1.0:1619677470.863573:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.29@o2ib3 local destination
00000400:00000200:1.0:1619677470.863577:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.29@o2ib3(10.51.5.29@o2ib3:10.51.5.29@o2ib3) : REPLY try# 0
00000800:00000200:1.0:1619677470.863580:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.29@o2ib3
00000800:00000200:1.0:1619677470.863582:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918ce0fb8000] -> 10.51.5.29@o2ib3 (2) version: 12
00000800:00000200:17.0:1619677470.863583:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (23)++
00000800:00000200:17.0:1619677470.863583:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1200] (24)--
00000800:00000200:1.0:1619677470.863583:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918cfdbb1200] (23)++
00000800:00000200:1.0:1619677470.863583:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1200] (25)++
00000800:00000200:17.0:1619677470.863584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (25)--
00000800:00000200:1.0:1619677470.863585:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918cfdbb1200] (26)--
00000800:00000200:1.0:1619677470.863585:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:17.0:1619677470.863586:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:1.0:1619677470.863595:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:1.0:1619677470.863597:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1200] (23)--
00000400:00000200:1.0:1619677470.863598:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.863599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:1.0:1619677470.863599:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.29@o2ib3: REPLY: OK
00000800:00000200:1.0:1619677470.863601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:5.2:1619677470.863855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.863861:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.863864:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.863864:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.863867:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.863870:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.863871:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.863872:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.863874:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.863875:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.863876:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.863880:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.863881:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.863885:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.863886:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.863886:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.863888:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.863889:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.863889:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.863890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.863967:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.863969:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.863970:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.863971:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.863973:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.863976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.863985:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.863988:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.863988:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.863989:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.863990:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.863991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.864235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.864240:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.864243:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.864244:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.864246:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.864249:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.864251:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677470.864252:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.864253:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.864254:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.864255:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677470.864258:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.864260:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.864263:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.864264:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677470.864265:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677470.864267:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677470.864267:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.864268:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.864268:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.864430:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.864433:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.864433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.864434:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.864436:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.864440:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.864760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.864807:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.864810:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.864811:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.864811:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.864813:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.864815:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.866154:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.866157:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.866158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.866160:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.866162:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.866163:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677470.866165:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.866166:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.866167:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.866168:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677470.866172:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.866173:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677470.866176:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.866177:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677470.866178:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677470.866181:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677470.866181:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.866182:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.866182:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677470.866719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.866725:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.866728:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.866729:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.866730:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.866733:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.866734:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.866736:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.866737:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.866738:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.866739:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.866742:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.866744:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.866746:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.866747:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.866747:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.866749:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.866750:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.866751:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.866752:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.866752:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.866758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.866763:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.866765:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.866765:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677470.866766:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.866766:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.866766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.866767:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.866768:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.866768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.866769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.866841:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.866844:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.866845:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.866845:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.866847:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.866850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.867116:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.867118:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.867119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.867120:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.867122:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.867125:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.867292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.2:1619677470.867339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:2.0:1619677470.867747:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (18)++
00000800:00000200:2.0:1619677470.867749:0:19932:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (19)--
00000800:00000200:2.0:1619677470.867750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (18)--
00000800:00000200:2.0:1619677470.867751:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (17)++
00000800:00000200:2.0:1619677470.867752:0:19932:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (18)--
00000800:00000200:2.0:1619677470.867753:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (17)--
00000800:00000200:2.0:1619677470.867754:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (16)++
00000800:00000200:18.0:1619677470.867755:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (17)++
00000800:00000200:2.0:1619677470.867755:0:19932:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (17)--
00000800:00000200:2.0:1619677470.867755:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (17)--
00000800:00000200:18.0:1619677470.867756:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (16)--
00000800:00000200:2.0:1619677470.867756:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (16)++
00000800:00000200:18.0:1619677470.867757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (16)--
00000800:00000200:18.0:1619677470.867757:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (15)++
00000800:00000200:2.0:1619677470.867757:0:19932:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (15)--
00000800:00000200:18.0:1619677470.867758:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (14)--
00000800:00000200:2.0:1619677470.867758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (15)--
00000800:00000200:2.0:1619677470.867758:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (14)++
00000800:00000200:18.0:1619677470.867759:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (14)--
00000800:00000200:2.0:1619677470.867759:0:19932:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (13)--
00000800:00000200:18.0:1619677470.867760:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (13)++
00000800:00000200:2.0:1619677470.867760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (13)--
00000800:00000200:18.0:1619677470.867761:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (12)--
00000800:00000200:18.0:1619677470.867761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (12)--
00000800:00000200:2.0:1619677470.867761:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (12)++
00000800:00000200:18.0:1619677470.867762:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (11)++
00000800:00000200:2.0:1619677470.867762:0:19932:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (11)--
00000800:00000200:18.0:1619677470.867763:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (10)--
00000800:00000200:2.0:1619677470.867763:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (11)--
00000800:00000200:2.0:1619677470.867763:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (10)++
00000800:00000200:18.0:1619677470.867764:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (10)--
00000800:00000200:2.0:1619677470.867764:0:19932:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (9)--
00000800:00000200:18.0:1619677470.867765:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (9)++
00000800:00000200:2.0:1619677470.867765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (9)--
00000800:00000200:18.0:1619677470.867766:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (8)--
00000800:00000200:18.0:1619677470.867766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (8)--
00000800:00000200:2.0:1619677470.867766:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (8)++
00000800:00000200:18.0:1619677470.867767:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986600] (7)++
00000800:00000200:2.0:1619677470.867767:0:19932:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (7)--
00000800:00000200:18.0:1619677470.867768:0:19933:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff9193c2986600] (6)--
00000800:00000200:18.0:1619677470.867768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (5)--
00000800:00000200:2.0:1619677470.867768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (7)--
00000800:00000200:2.0:1619677470.867769:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986600] (4)--
00000800:00000200:20.2:1619677470.868487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.868494:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.868497:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.868498:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.868500:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.868502:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.868504:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.868505:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.868507:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.868507:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.868508:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.868512:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.868513:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.868516:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.868517:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677470.868518:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677470.868520:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677470.868520:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.868522:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.868523:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.868523:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.868529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.868534:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.868536:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.868537:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.868538:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.868760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.868766:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.868769:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.868769:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.868771:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.868774:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.868775:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.868777:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.868778:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.868779:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.868780:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.868783:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.868785:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.868788:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.868789:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.868789:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.868791:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.868791:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.868792:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.868793:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.868793:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.868798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.868804:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.868806:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.868806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.868807:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.869197:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.869200:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.869201:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.869201:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.869203:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.869205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.869214:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.869216:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.869217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.869218:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.869219:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.869220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.869691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.870453:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.870456:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.870457:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.870457:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.870459:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.870462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.870948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.870955:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.870962:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.870965:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.870965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.870969:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : GET - for me
00000400:00000200:18.0:1619677470.870972:0:19933:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.14.9@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:18.0:1619677470.870975:0:19933:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.14.9@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:18.0:1619677470.870977:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.870978:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.870980:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.14.10@o2ib3 local destination
00000400:00000200:18.0:1619677470.870984:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.14.10@o2ib3(10.51.14.10@o2ib3:10.51.14.10@o2ib3) : REPLY try# 0
00000800:00000200:18.0:1619677470.870987:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.870989:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c610f100] -> 10.51.14.10@o2ib3 (2) version: 12
00000800:00000200:20.2:1619677470.870990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677470.870990:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.870990:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.870991:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.870992:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.870997:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.871002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.871004:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677470.871005:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.871006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.871007:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.14.10@o2ib3: REPLY: OK
00000800:00000200:2.0:1619677470.871009:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677470.872550:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.872553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.872553:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677470.872556:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.872558:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.872560:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.872561:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.872562:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.872563:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.872565:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.872568:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.872571:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.872573:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.872574:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.872575:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.872577:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.872577:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.872578:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.872578:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677470.872643:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.872646:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.872646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.872648:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.872651:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.872653:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677470.872654:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.872656:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.872657:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.872658:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677470.872661:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.872663:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677470.872667:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.872667:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677470.872668:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677470.872670:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677470.872671:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.872671:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.872672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677470.873214:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.873216:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.873217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.873218:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.873220:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.873223:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.873438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.873444:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.873447:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.873448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.873450:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.873453:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.873454:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.873455:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.873456:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.873457:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.873458:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.873461:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.873462:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.873464:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.873465:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.873466:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.873467:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.873468:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.873469:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.873469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.873480:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.873483:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.873483:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.873484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.873485:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.873486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:12.0:1619677470.873552:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (18)++
00000800:00000200:12.0:1619677470.873554:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (19)--
00000800:00000200:12.0:1619677470.873554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (18)--
00000800:00000200:12.0:1619677470.873555:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (17)++
00000800:00000200:12.0:1619677470.873556:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (18)--
00000800:00000200:12.0:1619677470.873556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (17)--
00000800:00000200:12.0:1619677470.873556:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (16)++
00000800:00000200:12.0:1619677470.873557:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (17)--
00000800:00000200:12.0:1619677470.873557:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (16)--
00000800:00000200:12.0:1619677470.873558:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (15)++
00000800:00000200:12.0:1619677470.873559:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (16)--
00000800:00000200:12.0:1619677470.873559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (15)--
00000800:00000200:13.0:1619677470.873560:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (14)++
00000800:00000200:13.0:1619677470.873562:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (15)--
00000800:00000200:13.0:1619677470.873563:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (14)--
00000800:00000200:13.0:1619677470.873563:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (13)++
00000800:00000200:13.0:1619677470.873564:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (14)--
00000800:00000200:13.0:1619677470.873565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (13)--
00000800:00000200:13.0:1619677470.873565:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (12)++
00000800:00000200:13.0:1619677470.873566:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (13)--
00000800:00000200:12.0:1619677470.873567:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (13)++
00000800:00000200:13.0:1619677470.873568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (13)--
00000800:00000200:13.0:1619677470.873569:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (12)++
00000800:00000200:12.0:1619677470.873569:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (12)--
00000800:00000200:12.0:1619677470.873570:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (12)--
00000800:00000200:13.0:1619677470.873571:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (11)--
00000800:00000200:12.0:1619677470.873571:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (11)++
00000800:00000200:13.0:1619677470.873572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (11)--
00000800:00000200:12.0:1619677470.873573:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (10)--
00000800:00000200:13.0:1619677470.873574:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (10)++
00000800:00000200:12.0:1619677470.873574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (10)--
00000800:00000200:13.0:1619677470.873575:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (9)--
00000800:00000200:12.0:1619677470.873576:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (9)++
00000800:00000200:13.0:1619677470.873577:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (9)--
00000800:00000200:13.0:1619677470.873578:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (8)++
00000800:00000200:12.0:1619677470.873578:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (8)--
00000800:00000200:12.0:1619677470.873579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (8)--
00000800:00000200:13.0:1619677470.873580:0:19942:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (7)--
00000800:00000200:12.0:1619677470.873580:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17fd200] (7)++
00000800:00000200:13.0:1619677470.873581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (7)--
00000800:00000200:13.0:1619677470.873582:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (6)--
00000800:00000200:12.0:1619677470.873582:0:19943:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff91a7c17fd200] (6)--
00000800:00000200:12.0:1619677470.873583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17fd200] (4)--
00000800:00000200:5.2:1619677470.873709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.873715:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.873718:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.873718:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.873720:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.873721:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.873723:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.873724:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.873726:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.873726:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.873728:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.873731:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.873733:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.873736:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.873737:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.873738:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.873740:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.873741:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.873742:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.873742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.873938:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.873940:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.873941:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.873942:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.873944:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.873947:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.874736:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.874738:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.874739:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.874740:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.874742:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.874745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.875837:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.875839:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.875840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.875841:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.875843:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.875845:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.875854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677470.875857:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.875858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.875858:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.875859:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.875859:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677470.876271:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.876274:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.876275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.876275:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.876278:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.876280:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.876384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.876390:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.876393:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.876393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.876395:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.876398:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.876399:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.876400:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.876402:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.876402:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.876404:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.876407:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.876409:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.876411:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.876412:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677470.876413:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677470.876415:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677470.876415:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.876416:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.876417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.876866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.877995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.878001:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.878004:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677470.878005:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677470.878007:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.878010:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.878012:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677470.878013:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677470.878015:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (18)++
00000400:00000200:2.0:1619677470.878015:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.878016:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677470.878017:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (19)--
00000800:00000200:17.0:1619677470.878018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (18)--
00000400:00000200:2.0:1619677470.878018:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677470.878018:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (17)++
00000800:00000200:1.0:1619677470.878020:0:19931:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (18)--
00000800:00000200:1.0:1619677470.878020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (17)--
00000400:00000200:2.0:1619677470.878021:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.878021:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (16)++
00000800:00000200:17.0:1619677470.878022:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (17)++
00000800:00000200:1.0:1619677470.878022:0:19931:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (17)--
00000800:00000200:1.0:1619677470.878022:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (17)--
00000800:00000200:17.0:1619677470.878023:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (16)--
00000800:00000200:1.0:1619677470.878023:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (16)++
00000800:00000200:17.0:1619677470.878024:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (16)--
00000800:00000200:2.0:1619677470.878024:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677470.878024:0:19931:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (15)--
00000800:00000200:17.0:1619677470.878025:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (15)++
00000800:00000200:1.0:1619677470.878025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (15)--
00000800:00000200:17.0:1619677470.878026:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (14)--
00000800:00000200:17.0:1619677470.878026:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (14)--
00000800:00000200:1.0:1619677470.878026:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (14)++
00000800:00000200:17.0:1619677470.878027:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (13)++
00000800:00000200:2.0:1619677470.878027:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.878027:0:19931:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (13)--
00000800:00000200:17.0:1619677470.878028:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (12)--
00000800:00000200:2.0:1619677470.878028:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:1.0:1619677470.878028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (13)--
00000800:00000200:1.0:1619677470.878028:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (12)++
00000800:00000200:17.0:1619677470.878029:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (12)--
00000800:00000200:17.0:1619677470.878029:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (11)++
00000800:00000200:2.0:1619677470.878029:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677470.878029:0:19931:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (11)--
00000800:00000200:17.0:1619677470.878030:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (10)--
00000800:00000200:1.0:1619677470.878030:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (11)--
00000800:00000200:1.0:1619677470.878030:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (10)++
00000800:00000200:17.0:1619677470.878031:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (10)--
00000800:00000200:1.0:1619677470.878031:0:19931:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (9)--
00000800:00000200:17.0:1619677470.878032:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (9)++
00000800:00000200:2.0:1619677470.878032:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677470.878032:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:1.0:1619677470.878032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (9)--
00000800:00000200:17.0:1619677470.878033:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (8)--
00000800:00000200:17.0:1619677470.878033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (8)--
00000800:00000200:2.0:1619677470.878033:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:1.0:1619677470.878033:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (8)++
00000800:00000200:17.0:1619677470.878034:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0000] (7)++
00000800:00000200:2.0:1619677470.878034:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:1.0:1619677470.878034:0:19931:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (7)--
00000800:00000200:17.0:1619677470.878035:0:19930:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff918fe95d0000] (6)--
00000800:00000200:1.0:1619677470.878035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (7)--
00000800:00000200:17.0:1619677470.878036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (5)--
00000800:00000200:1.0:1619677470.878037:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0000] (4)--
00000800:00000200:13.0:1619677470.878045:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677470.878048:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677470.878049:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.878050:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.878052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677470.878053:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677470.878256:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.878259:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.878260:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.878262:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.878264:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.878266:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677470.878267:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.878268:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.878269:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.878270:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677470.878273:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.878275:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677470.878278:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.878279:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677470.878279:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677470.878281:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677470.878282:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.878282:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.878283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:5.2:1619677470.878644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.878650:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.878653:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.878654:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.878656:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.878659:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.878661:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.878662:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.878663:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.878665:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.878666:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.878670:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.878672:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.878675:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.878676:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.878676:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.878678:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.878679:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.878680:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.878680:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.879055:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.879058:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.879059:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.879060:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.879062:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.879066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.879078:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.879080:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.879080:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.879081:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.879081:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.879082:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.879082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.879083:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.879084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:2.0:1619677470.879084:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.879085:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.879088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.879305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.879348:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.879350:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677470.879351:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.879352:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.879354:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.879356:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.880627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.880634:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.880637:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.880637:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.880639:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.880641:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.880643:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.880644:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.880645:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.880646:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.880647:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.880650:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.880652:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.880654:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.880655:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677470.880657:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677470.880658:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677470.880659:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.880660:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.880660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.880831:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.880834:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.880834:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.880837:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.880839:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.880841:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677470.880842:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.880844:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.880845:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.880846:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677470.880849:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.880850:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677470.880854:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.880854:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677470.880855:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677470.880857:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677470.880857:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.880859:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677470.880860:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677470.880860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.881002:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.881004:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677470.881006:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.881007:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.881007:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677470.881008:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:1.0:1619677470.881008:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.881009:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.881010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677470.881011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.882007:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.882010:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.882010:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.882012:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.882015:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.882016:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.882017:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.882019:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.882020:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.882021:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.882024:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.882026:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.882029:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.882030:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677470.882030:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677470.882032:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677470.882033:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.882033:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.882034:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.882230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.2:1619677470.882267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.2:1619677470.882741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.882748:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.882750:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.882752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.882754:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.882757:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.882758:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.882759:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.882761:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.882762:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.882762:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.882765:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.882767:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.882770:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.882771:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.882771:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.882773:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.882774:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.882774:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.882775:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.882787:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.882789:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.882790:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.882790:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.882792:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.882794:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677470.883003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.883008:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.883011:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.883012:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.883014:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.883016:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677470.883018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677470.883018:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.883019:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.883021:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.883022:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.883023:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.883028:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.883030:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.883034:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.883035:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.883035:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.883037:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.883038:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.883038:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.883039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.883941:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.883944:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.883945:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.883945:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.883947:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.883951:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.885244:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.885246:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.885248:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.885248:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.885250:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.885253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.885353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.885391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677470.885398:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.885400:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677470.885401:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.885402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.885403:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.885406:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.885419:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677470.885421:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.885422:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.885422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677470.885424:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677470.885424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.885467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.885474:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.885476:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.885477:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.885479:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.885482:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.885483:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.885485:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.885486:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.885487:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.885488:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.885491:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.885493:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.885496:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.885496:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677470.885497:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677470.885499:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677470.885500:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.885501:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.885501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.885658:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.885661:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.885662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.885662:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.885664:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.885667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677470.886156:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.886159:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.886160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.886160:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.886162:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.886165:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677470.886947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.886950:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.886950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.886951:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.886953:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.886955:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677470.887417:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.887420:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.887420:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.887422:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.887425:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.887426:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.887428:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.887429:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.887430:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.887431:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.887435:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.887436:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.887439:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.887440:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677470.887440:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677470.887442:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677470.887443:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.887444:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677470.887444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:5.2:1619677470.887908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.887914:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.887917:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.887918:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.887920:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.887923:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.887925:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.887926:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.887927:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.887928:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.887929:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.887932:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.887934:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.887937:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.887938:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.887938:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.887940:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.887941:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.887942:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.887943:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.887944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.887949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.887954:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.887956:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.887957:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.887958:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.888866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677470.888890:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.888893:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.888893:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.888894:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.888896:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.888900:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.888909:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677470.888912:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.888913:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.888913:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.888914:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.888915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677470.889892:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.889895:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.889896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.889896:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.889899:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.889901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.891057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.891064:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.891067:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.891067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.891070:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.891073:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.891074:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.891076:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.891077:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.891078:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.891079:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.891082:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.891084:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.891086:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.891087:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.891087:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.891089:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.891090:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.891091:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.891091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.891102:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.891104:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.891105:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.891106:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.891107:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.891108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.891126:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.891129:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.891130:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.891130:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.891131:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.891131:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.891465:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.891467:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.891469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.891469:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.891471:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.891475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.891871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.891913:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.891916:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.891917:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.891917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.891919:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.891921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.892133:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.892135:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.892136:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.892138:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.892140:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.892142:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.892143:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.892144:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.892145:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.892146:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.892149:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.892150:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.892153:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.892154:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.892154:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.892156:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.892157:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.892157:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.892158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.892801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.892808:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.892811:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.892812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.892814:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.892816:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.892818:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.892819:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96520293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.892821:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.892822:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677470.892823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:18.0:1619677470.892824:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.892825:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.892826:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.892828:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677470.892829:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.892832:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.892832:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.892833:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.892835:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.892837:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677470.892837:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677470.892839:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677470.892840:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.892841:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.892842:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.892843:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.892848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.892854:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.892856:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.892857:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.892858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.892891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.892897:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.892900:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.892901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.892903:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.892906:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.892908:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.892909:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.892911:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.892911:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.892913:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.892916:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.892917:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.892920:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.892921:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.892921:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.892923:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.892924:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.892925:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.892925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.893168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.893174:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.893177:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677470.893178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677470.893179:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.893183:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.893184:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677470.893185:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.893187:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.893188:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.893189:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677470.893191:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.893193:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677470.893197:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.893198:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677470.893198:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677470.893201:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677470.893201:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.893203:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677470.893204:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677470.893204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677470.893210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.893215:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677470.893218:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677470.893218:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677470.893219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:18.0:1619677470.894103:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.894106:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.894106:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.894107:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.894109:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.894113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.894271:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.894274:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.894275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.894275:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.894277:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.894281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.894562:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.894564:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.894565:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.894566:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.894568:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.894572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.894843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.896374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.896377:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.896379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.896379:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.896381:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.896385:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.897529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.897536:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.897540:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.897541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.897544:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.897546:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.897548:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.897550:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.897551:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.897552:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.897554:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.897557:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.897559:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.897562:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.897563:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677470.897564:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677470.897566:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677470.897567:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.897568:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.897568:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.897818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.897863:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.897869:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.897872:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.897873:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.897873:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.897875:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.897878:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677470.897893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.897898:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.897901:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.897902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.897904:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.897907:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.897909:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.897910:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.897912:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.897913:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.897915:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.897918:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.897920:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.2:1619677470.897921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.897925:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.897926:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.897927:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.897928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.897929:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.897929:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.897930:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.897931:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.897932:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677470.897932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.897933:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.897934:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.897936:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.897937:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.897938:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.897939:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.897940:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.897943:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.897944:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.897946:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.897946:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.897947:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.897948:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.897949:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.897950:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.897951:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.897952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.897957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.897962:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.897964:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.897964:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.897965:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677470.897965:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.897966:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.897967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.897967:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.897968:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.897969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.897983:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.897985:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.897986:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.897986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.897987:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.897987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.898089:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.898092:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.898093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.898093:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.898095:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.898097:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.898108:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677470.898111:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.898112:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.898112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.898113:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.898114:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.898474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:12.0:1619677470.898516:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677470.898519:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677470.898520:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.898522:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.898524:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677470.898524:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677470.899587:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677470.899590:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677470.899590:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.899592:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.899594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677470.899595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677470.899606:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677470.899608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:13.0:1619677470.899609:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677470.899610:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677470.899610:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677470.899611:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677470.899612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677470.899655:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677470.899657:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677470.899658:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.899660:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.899661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677470.899663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677470.900351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.2:1619677470.900355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677470.900360:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.900363:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677470.900364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677470.900367:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.900370:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.900372:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677470.900374:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96520259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.900375:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.900377:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.900378:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677470.900382:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.900384:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677470.900388:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.900388:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677470.900389:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677470.900391:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677470.900392:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677470.900392:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677470.900393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677470.901117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.901955:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.901958:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.901959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.901961:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.901963:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.901965:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.901966:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.901968:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.901969:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.901970:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.901973:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.901975:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.901977:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.901978:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.901979:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.901980:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.901981:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.901982:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.901983:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.901984:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.901995:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.901997:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.901998:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.901999:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677470.902242:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.902244:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.902245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.902246:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.902248:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.902251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677470.902838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.902845:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.902848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.902848:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677470.902851:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.902853:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.902855:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.902856:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.902858:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.902859:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.902860:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.902863:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.902864:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.902868:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.902868:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.902869:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.902871:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.902872:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.902872:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.902873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.902913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.902920:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.902922:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.902923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.902925:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.902928:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.902929:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.902930:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.902931:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.902933:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.902934:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.902937:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.902939:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.902941:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.902942:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677470.902943:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677470.902945:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677470.902945:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.902946:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.902946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677470.903858:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.903860:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.903862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.903862:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.903864:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.903869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.903871:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677470.903873:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.903874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.903874:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.903875:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.903875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677470.904067:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.904070:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.904071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.904071:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.904073:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.904077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677470.904823:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.904826:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.904827:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.904827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.904829:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.904832:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677470.904917:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.904920:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.904921:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.904921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.904923:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.904926:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677470.905026:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.905030:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.905034:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.905036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.905037:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.905039:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677470.905041:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96520242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.905044:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.905045:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.905047:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677470.905052:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.905055:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677470.905059:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.905061:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677470.905062:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677470.905065:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677470.905066:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.905067:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:20.2:1619677470.905068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677470.905068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677470.905078:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.905081:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.905082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.905082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.905085:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.905089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677470.905114:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.905116:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.905117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.905117:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.905119:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.905122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.905131:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677470.905134:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.905135:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.905135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677470.905136:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677470.905137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.906816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677470.907025:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.907028:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.907030:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677470.907032:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.907034:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.907036:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677470.907037:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96520233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.907038:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.907039:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.907040:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677470.907044:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.907046:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677470.907049:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.907050:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677470.907050:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677470.907052:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677470.907053:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.907053:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.907054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677470.907492:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.907494:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.907495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.907496:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.907498:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.907501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677470.907790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.907797:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.907800:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.907800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.907802:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.907805:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.907806:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.907808:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.907810:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.907812:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.907813:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.907818:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.907821:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.907825:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.907826:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.907826:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.907828:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.907829:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.907830:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.907831:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.907832:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.907837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.907842:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.907845:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.907846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.907847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.907980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.907986:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.907989:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.907989:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.907992:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.907995:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.907996:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.907997:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.907999:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.908000:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.908001:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.908004:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.908006:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.908008:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.908009:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.908010:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.908012:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.908013:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.908014:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.908014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.908025:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.908027:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.908027:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.908028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.908030:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.908031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677470.908369:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.908373:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.908373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.908376:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.908378:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.908380:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.908381:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.908383:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.908384:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.908385:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.908388:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.908390:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.908392:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.908393:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677470.908394:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677470.908396:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677470.908396:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.908397:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.908398:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677470.908401:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.908404:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.908405:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.908405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.908407:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.908410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.908422:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.908425:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.908426:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.908427:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.908427:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.908428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.908606:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.908609:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.908610:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.908610:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.908613:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.908616:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.909034:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.909037:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.909038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.909038:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.909040:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.909043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.909371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.909378:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.909381:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.909382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.909384:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.909386:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.909388:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.909390:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.909391:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.909392:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.909393:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.909396:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.909398:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.909401:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.909401:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677470.909402:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677470.909404:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677470.909404:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.909405:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.909406:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.910906:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.910909:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.910910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.910910:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.910913:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.910916:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.911099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.911136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.911142:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.911144:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.911145:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.911146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.911148:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.911150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.912506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.912721:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.912724:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.912725:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.912727:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.912729:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.912731:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677470.912732:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.912733:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.912735:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.912735:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677470.912739:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.912741:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677470.912744:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.912744:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677470.912745:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677470.912747:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677470.912747:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.912748:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.912748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677470.912763:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.912767:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677470.912769:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.912770:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.912772:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.912776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677470.912809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.912816:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.912821:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.912821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.912826:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.912829:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.912831:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.912833:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.912836:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.912838:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.912839:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.912845:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.912848:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.912852:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.912853:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.912854:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.912856:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.912857:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.912859:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.912860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.913353:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.913357:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.913359:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.913359:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.913363:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.913370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.913372:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677470.913375:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.913376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.913377:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.913379:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.913380:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677470.913776:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.913779:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.913781:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677470.913784:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677470.913785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.913786:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677470.913787:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96520203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.913789:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.913790:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.913791:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677470.913794:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.913796:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677470.913798:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.913799:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:12.0:1619677470.913800:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:12.0:1619677470.913802:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:12.0:1619677470.913803:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.913804:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.913804:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677470.915530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.915538:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.915542:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.915542:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.915545:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.915548:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.915550:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.915551:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.915553:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.915554:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.915555:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.915559:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.915561:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.915564:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.915565:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677470.915566:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677470.915568:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677470.915569:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.915570:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.915572:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.915572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.915577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.915584:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.915586:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.915587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.915588:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.915738:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.915740:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.915741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.915742:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.915744:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.915747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677470.916421:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.916425:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.916427:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.916427:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.916430:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.916436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.916438:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677470.916441:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.916442:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.916442:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.916444:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.916445:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.917186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.917228:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.917230:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.917231:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.917231:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.917233:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.917236:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677470.917806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.917812:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.917815:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.917816:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.917818:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.917821:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.917823:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.917824:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.917825:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.917826:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.917827:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.917830:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.917832:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.917836:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.917836:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.917837:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.917839:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.917840:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.917841:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.917841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.918072:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.918075:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.918075:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.918076:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.918078:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.918081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.918383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.918424:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.918427:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.918428:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.918428:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.918430:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.918432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.918441:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.918444:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.918446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.918446:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.918447:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.918447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.918937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.918943:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.918947:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.918948:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.918950:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.918953:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.918954:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.918956:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.918957:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.918958:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.918959:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.918962:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.918964:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.918966:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.918967:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.918967:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.918969:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.918970:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.918971:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.918971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.918987:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.918990:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.918990:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.918991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.918992:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.918993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.919037:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.919040:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.919041:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.919041:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.919044:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.919047:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:13.0:1619677470.919060:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677470.919063:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677470.919063:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.919065:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.919067:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677470.919068:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677470.919239:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677470.919241:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677470.919242:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.919244:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.919245:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677470.919252:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677470.919314:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677470.919316:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677470.919317:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.919319:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.919321:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677470.919322:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677470.919325:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677470.919326:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677470.919327:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677470.919328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677470.919329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677470.920416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.920423:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.920426:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.920427:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.920429:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.920431:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.920433:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.920434:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.920436:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.920437:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.920438:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.920442:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.920444:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.920447:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.920448:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677470.920448:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677470.920450:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677470.920451:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.920452:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.920452:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.920692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677470.920698:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.920702:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677470.920703:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677470.920705:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677470.920707:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.920709:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677470.920710:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.920712:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.920713:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.920714:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677470.920718:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677470.920719:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677470.920722:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.920723:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677470.920723:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677470.920725:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677470.920726:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.920727:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677470.920727:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:1.0:1619677470.921579:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.921582:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.921583:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.921583:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.921586:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.921588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.922243:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.922246:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.922246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.922247:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.922249:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.922252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.922574:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.922577:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.922578:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.922580:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.922582:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.922584:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.922585:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.922586:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.922587:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.922588:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.922592:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.922594:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.922596:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.922597:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.922598:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.922599:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.922600:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.922600:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.922601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677470.922645:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.922647:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677470.922648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.922649:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.922651:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.922654:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677470.922707:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.922710:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.922711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.922711:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.922713:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.922717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.922807:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.922810:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.922811:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.922811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.922814:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.922817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.922833:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677470.922835:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.922836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.922837:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.922839:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.922839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.922952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.922991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677470.922996:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.922999:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.923000:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.923000:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.923002:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.923005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.923559:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.923562:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.923562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.923564:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.923567:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.923568:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677470.923570:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.923571:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.923572:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.923574:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677470.923577:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.923578:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.923580:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.923581:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677470.923582:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677470.923583:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.923583:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.923584:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.923584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677470.923596:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.923599:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677470.923599:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.923600:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.923601:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.923602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.924163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.925775:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.925778:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677470.925779:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677470.925780:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.925782:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.925784:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677470.925785:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.925787:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.925787:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.925789:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677470.925792:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.925794:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677470.925796:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.925797:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677470.925797:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677470.925799:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677470.925800:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.925800:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.925801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677470.925814:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.925817:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.925817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.925818:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.925820:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.925822:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677470.925845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.925851:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.925853:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.925854:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.925856:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.925858:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.925860:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.925861:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.925862:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.925863:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.925864:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.925868:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.925870:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.925873:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.925874:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.925875:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.925876:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.925877:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.925878:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.925878:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.925992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.925998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.926000:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.926001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.926003:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.926006:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.926007:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.926008:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.926010:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.926011:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.926012:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.926015:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.926016:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.926019:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.926020:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677470.926021:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677470.926023:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677470.926023:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.926024:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.926025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.926479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.926485:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.926488:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.926489:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.926491:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.926494:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.926495:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.926497:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.926498:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.926499:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.926500:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.926503:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.926505:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.926507:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.926508:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.926508:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.926510:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.926511:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.926511:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.926512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.926524:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.926526:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.926527:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.926527:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.926528:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.926530:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.926543:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.926545:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.926546:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.926546:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.926547:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.926547:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.927575:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.927578:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.927578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.927579:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.927581:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.927584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677470.928078:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.928080:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.928082:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.928082:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.928084:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.928087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.928248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677470.928289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:2.0:1619677470.928293:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.928298:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.928301:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.928302:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.928304:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.928307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677470.928387:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.928390:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.928391:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.928391:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.928393:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.928396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.928404:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677470.928407:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.928408:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.928408:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.928409:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.928410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677470.928697:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.928699:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.928700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.928701:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.928703:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.928706:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.928716:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.2:1619677470.928717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677470.928718:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.928719:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.928720:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.928721:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.928721:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677470.928774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.928777:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.928778:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.928778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.928780:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.928782:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.928792:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.928795:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.928796:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.928796:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.928797:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677470.928797:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.929245:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.929248:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.929249:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.929249:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.929252:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.929255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.929926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.929969:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.929972:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.929973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.929973:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.929975:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.929977:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:13.0:1619677470.930859:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.930861:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:5.2:1619677470.930862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:13.0:1619677470.930863:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677470.930865:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.930866:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.930867:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:18.0:1619677470.930868:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:13.0:1619677470.930868:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96520122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.930869:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.930870:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677470.930871:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:13.0:1619677470.930871:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.930872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.930875:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.930879:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.930881:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:13.0:1619677470.930882:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677470.930883:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96520120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.930885:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677470.930885:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.930886:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.930888:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:13.0:1619677470.930888:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.930889:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:13.0:1619677470.930889:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:18.0:1619677470.930892:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.930892:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:13.0:1619677470.930892:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.930893:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:18.0:1619677470.930894:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:13.0:1619677470.930895:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677470.930895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:18.0:1619677470.930897:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.930897:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.930898:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.930900:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.930901:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.930903:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.930904:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.930904:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.930909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:12.0:1619677470.930912:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:2.0:1619677470.930914:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:12.0:1619677470.930915:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677470.930915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.930916:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677470.930916:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.930918:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.930919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.931607:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.931610:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.931610:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.931613:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.931615:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.931616:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.931617:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.931619:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.931620:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.931621:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.931623:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.931625:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.931628:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.931628:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677470.931629:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677470.931631:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677470.931632:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.931633:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.931633:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677470.932299:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.932302:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.932303:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.932303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.932305:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.932309:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.933028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.933034:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.933037:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.933038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.933040:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.933043:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.933045:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.933046:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.933048:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.933049:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.933049:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.933053:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.933054:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.933057:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.933058:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677470.933058:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677470.933060:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677470.933061:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.933062:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.933062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677470.934006:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.934009:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.934010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677470.934011:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.934013:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.934016:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.934695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.934701:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.934704:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.934705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.934707:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.934711:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.934712:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.934713:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.934715:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.934716:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.934717:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.934720:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.934722:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.934724:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.934724:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.934725:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.934727:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.934728:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.934729:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.934730:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677470.934731:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.934736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.934742:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.934743:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.934743:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.934744:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677470.934744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.934745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.934746:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.934746:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.934747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.934747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.934783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.934822:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677470.935082:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.935085:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.935089:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.935092:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.935092:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.935095:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677470.935097:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96520108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.935099:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.935100:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.935102:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677470.935107:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.935111:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677470.935114:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.935116:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:12.0:1619677470.935117:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:12.0:1619677470.935120:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:12.0:1619677470.935121:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.935122:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.935123:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677470.935134:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.935137:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.935138:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.935138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.935140:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.935143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677470.935830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.935836:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.935839:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.935841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.935842:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.935845:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.935846:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.935847:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.935849:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.935850:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.935851:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.935854:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.935856:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.935859:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.935860:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.935861:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.935863:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.935863:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.935864:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.935864:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.935962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.935962:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.935965:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677470.935966:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677470.935967:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.935969:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.935972:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677470.935973:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.935975:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.935976:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.935977:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677470.935982:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.935984:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677470.935987:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.935988:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677470.935989:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677470.935991:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677470.935992:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.935992:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.935993:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677470.937062:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.937065:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.937066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.937067:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.937069:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.937073:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.937390:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.937393:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.937394:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.937394:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.937396:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.937399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677470.937823:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.937826:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677470.937827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677470.937829:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.937832:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.937834:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677470.937835:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.937836:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.937837:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.937838:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677470.937841:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.937843:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677470.937846:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.937847:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677470.937847:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677470.937849:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677470.937850:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.937852:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677470.937853:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677470.937853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.937864:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677470.937865:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677470.937866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.937867:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677470.937873:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677470.937875:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677470.937876:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.937878:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.937879:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677470.937881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677470.938319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.938326:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.938329:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.938329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.938332:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.938334:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.938336:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.938337:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.938339:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.938340:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.938341:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.938344:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.938346:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.938349:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.938350:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677470.938350:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677470.938352:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677470.938353:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.938354:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.938355:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.938356:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.938362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.938368:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.938370:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.938371:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.938372:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.940033:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.940036:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.940037:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.940037:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.940039:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.940041:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.940052:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677470.940055:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.940056:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.940057:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.940058:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.940058:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.940482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:5.2:1619677470.940750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.940757:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.940760:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.940760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.940763:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.940766:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.940768:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.940770:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.940771:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.940772:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.940774:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.940777:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.940778:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.940781:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.940782:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.940783:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.940785:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.940785:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.940786:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.940787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.940880:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.940883:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.940884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.940884:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.940886:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.940890:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677470.940901:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677470.940904:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.940905:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.940905:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677470.940906:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677470.940907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.941027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.941069:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.941072:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.941073:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.941073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.941075:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.941077:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.941253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.941259:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.941262:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.941263:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.941265:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.941268:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.941269:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.941271:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.941272:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.941273:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.941275:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.941278:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.941280:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.941282:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.941283:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.941283:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.941284:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.941285:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.941286:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.941286:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.941300:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.941302:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.941302:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.941303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.941304:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.941305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.941824:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.941827:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.941828:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.941828:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.941830:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.941834:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677470.942512:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.942515:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.942515:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677470.942517:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.942520:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.942522:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.942523:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.942524:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.942525:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.942526:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.942529:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.942531:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.942533:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.942534:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.942535:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.942537:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.942537:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.942539:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.942540:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.942540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.942551:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.942553:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.942553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.942554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.943402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.943409:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.943412:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.943413:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.943415:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.943418:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.943419:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.943420:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.943422:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.943423:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.943424:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.943427:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.943429:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.943432:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.943433:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677470.943433:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677470.943435:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677470.943436:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.943437:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.943437:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.945599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.945606:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.945611:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.945612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.945616:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.945619:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.945622:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.945624:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.945626:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.945628:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.945630:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.945638:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.945641:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.945644:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.945645:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.945646:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.945648:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.945649:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.945650:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.945651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.945974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.946328:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.946332:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.946334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.946334:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.946337:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.946341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.946536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.946540:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.946542:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.946543:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.946546:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.946550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677470.946647:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.946650:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.946651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.946652:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.946654:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.946658:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.946821:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.946823:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.946824:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.946825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.946826:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.946829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.947382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.947389:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.947392:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.947393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.947395:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.947397:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.947400:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.947401:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.947403:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.947405:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.947406:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677470.947409:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.947411:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677470.947414:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.947415:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677470.947416:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677470.947418:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677470.947419:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.947420:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.947420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677470.948734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.948740:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.948743:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.948744:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.948747:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.948751:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.948752:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.948754:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.948755:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.948756:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.948757:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.948760:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.948762:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.948764:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.948765:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.948765:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.948767:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.948768:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.948769:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.948769:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.948781:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.948783:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.948784:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.948785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.948786:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.948788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.948800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.948803:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.948803:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.948804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.948804:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.948805:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.2:1619677470.948827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:18.0:1619677470.949057:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.949060:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.949061:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.949061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.949063:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.949066:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.949970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.949976:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.949979:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.949980:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.949982:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.949985:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.949986:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.949987:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.949989:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.949990:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.949991:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.949995:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.949996:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.949999:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.950000:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677470.950000:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677470.950002:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677470.950002:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.950003:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.950004:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.950115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.950122:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.950125:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.950125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.950127:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.950130:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.950131:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677470.950133:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.950134:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.950135:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.950136:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677470.950140:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.950142:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677470.950145:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.950146:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677470.950146:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677470.950149:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677470.950149:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.950150:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677470.950151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:5.2:1619677470.950585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.950591:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.950594:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.950595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.950597:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.950600:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.950601:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.950603:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.950604:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.950605:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.950606:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.950609:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.950610:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.950616:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.950617:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.950617:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.950619:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.950620:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.950622:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.950623:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.950623:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.950628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.950634:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.950636:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.950637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.950637:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.950811:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.950813:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.950814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.950815:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.950817:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.950821:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.950905:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677470.950907:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677470.950908:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:2.0:1619677470.950910:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.950912:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.950914:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677470.950915:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.950916:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.950917:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.950918:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677470.950921:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.950923:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677470.950927:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.950928:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677470.950929:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677470.950931:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677470.950931:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677470.950932:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677470.950933:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:12.0:1619677470.950944:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677470.950947:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677470.950947:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.950949:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677470.950952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677470.950953:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677470.951233:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.951236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.951236:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677470.951239:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.951241:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.951243:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677470.951244:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.951247:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.951247:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.951249:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677470.951252:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.951254:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677470.951256:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.951257:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677470.951258:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677470.951259:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677470.951260:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677470.951261:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.951261:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677470.951600:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.951603:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.951603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.951604:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.951606:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.951611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677470.951613:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677470.951615:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.951616:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677470.951617:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.951618:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677470.951619:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.951653:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.951655:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.951656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.951656:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.951658:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677470.951661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.951894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.951936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677470.951937:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.951940:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.951941:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.951941:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.951943:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.951946:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.952397:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.952399:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.952400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.952401:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.952403:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.952406:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.952415:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677470.952418:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.952419:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.952419:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.952420:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.952421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.955471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.955477:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.955480:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.955481:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.955483:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.955486:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.955488:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677470.955489:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677470.955490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000400:00000200:18.0:1619677470.955491:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.955493:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.955494:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.955496:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000400:00000200:18.0:1619677470.955498:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.955498:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:2.0:1619677470.955500:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677470.955501:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:2.0:1619677470.955502:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.955504:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:18.0:1619677470.955505:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.955506:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:2.0:1619677470.955506:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677470.955507:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:2.0:1619677470.955507:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.955508:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677470.955509:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:2.0:1619677470.955509:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.955510:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.955512:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.955512:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.955513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.955513:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.955515:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.955518:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.955519:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.955519:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.955521:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.955522:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.955522:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.955523:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677470.955751:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.955754:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677470.955755:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677470.955757:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.955760:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.955762:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677470.955763:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.955765:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.955766:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.955767:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677470.955769:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.955771:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677470.955773:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.955774:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677470.955774:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677470.955776:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677470.955776:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677470.955778:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677470.955779:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677470.955779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677470.955790:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677470.955792:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677470.955792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677470.955793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.2:1619677470.955818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.955824:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.955827:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.955828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.955829:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.955831:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.955833:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.955834:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96520020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.955836:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.955837:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.955838:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.955841:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.955842:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.955844:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.955844:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.955845:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.955846:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.955847:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.955848:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.955848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.955859:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.955861:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.955862:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.955863:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.955864:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.955865:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.955945:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.955948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.955948:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677470.955951:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.955953:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.955954:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.955956:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96520018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.955958:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.955959:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.955960:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.955964:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.955965:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.955968:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.955969:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.955970:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.955971:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.955972:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.955972:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.955973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.956131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.956178:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.956181:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.956182:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.956182:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.956185:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.956187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.957787:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.957790:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.957791:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.957791:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.957793:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.957796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.957994:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.957997:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.957998:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.957998:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.958000:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.958002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.959136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.960365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.960372:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.960376:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.960376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.960378:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.960381:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.960382:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677470.960383:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.960385:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.960386:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.960387:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677470.960391:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.960393:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677470.960396:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.960397:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677470.960398:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677470.960400:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677470.960401:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.960402:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.960403:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:5.2:1619677470.960403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.960406:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.960407:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:20.2:1619677470.960409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.960409:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677470.960411:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677470.960411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.960413:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.960413:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.960414:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.960414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.960415:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:2.0:1619677470.960415:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:18.0:1619677470.960416:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.960418:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.960419:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.960420:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.960424:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.960426:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.960429:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.960430:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.960430:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.960432:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.960433:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.960433:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.960434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.960681:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.960683:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.960684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.960685:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.960687:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.960690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677470.960778:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.960781:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:13.0:1619677470.960781:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:18.0:1619677470.960782:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.960782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677470.960784:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:18.0:1619677470.960785:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:13.0:1619677470.960787:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677470.960788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:18.0:1619677470.960789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:13.0:1619677470.960790:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.960791:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:13.0:1619677470.960792:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96519993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.960794:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.960795:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.960796:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677470.960798:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:13.0:1619677470.960799:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.960801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677470.960801:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677470.960801:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.960802:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.960803:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.960803:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677470.960804:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.960804:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:13.0:1619677470.960805:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:13.0:1619677470.960807:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:13.0:1619677470.960808:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.960809:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.960809:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677470.961260:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.961263:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.961264:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.961264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.961266:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.961269:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.962816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.962823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.962826:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.962827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.962829:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.962831:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.962833:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.962834:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.962836:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.962837:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.962838:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.962841:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.962842:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.962844:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.962845:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.962846:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.962847:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.962849:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.962850:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.962851:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.962852:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.962857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.962862:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.962863:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677470.962864:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.962864:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.962864:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.962865:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.962866:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.962867:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.962867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.962868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.963009:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.963013:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.963015:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.963015:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.963019:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.963022:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.963680:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.963685:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.963687:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.963687:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.963690:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677470.963691:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.963692:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:1.0:1619677470.963694:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (24)--
00000800:00000200:1.0:1619677470.963695:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:20.2:1619677470.963696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.963696:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.963697:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.963741:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.963744:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.963745:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.963745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.963748:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.963750:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.963898:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.963901:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.963902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.963902:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.963905:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.963908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.963917:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.963920:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677470.963920:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)++
00000800:00000200:17.0:1619677470.963921:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (24)++
00000800:00000200:1.0:1619677470.963921:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (24)--
00000800:00000200:17.0:1619677470.963922:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (24)--
00000400:00000200:1.0:1619677470.963922:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677470.963923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.963924:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.963925:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.963928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677470.964619:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.964621:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.964622:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.964623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.964624:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.964627:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.965218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.965255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677470.965314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.965321:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.965324:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.965324:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.965326:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.965329:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.965330:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.965332:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.965333:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.965335:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.965336:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.965339:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.965340:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.965344:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.965345:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.965345:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.965347:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.965348:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.965349:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.965349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:17.0:1619677470.965460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.965463:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.965463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.965466:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.965468:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.965469:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.965471:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.965473:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.965473:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.965475:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.965477:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.965479:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.965481:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.965482:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677470.965483:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677470.965485:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677470.965485:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.965486:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.965486:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677470.965651:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.965654:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677470.965655:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.965656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.965658:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.965661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.966899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.966906:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.966909:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.966910:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.966912:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677470.966915:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.966916:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.966918:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.966919:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.966920:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.966921:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.966924:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677470.966926:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677470.966928:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.966929:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677470.966930:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677470.966931:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677470.966932:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.966933:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.966933:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.2:1619677470.968822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:2.0:1619677470.969249:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.969252:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.969253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.969253:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.969256:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.969259:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677470.969438:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.969441:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.969442:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.969442:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.969444:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.969446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.969455:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.969458:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.969458:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.969459:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.969460:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.969461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.969634:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.969637:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.969638:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.969638:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.969640:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.969643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.969818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.969864:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.969867:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.969868:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.969868:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.969870:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.969872:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677470.970219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.970225:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.970229:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.970229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.970232:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.970234:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.970236:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.970237:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.970239:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.970240:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.970241:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.970245:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.970246:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.970250:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.970251:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.970251:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.970253:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.970254:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.970255:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.970256:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.970257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.970262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.970267:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.970269:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.970270:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.970271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.970411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.970415:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.970416:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677470.970416:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.970419:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.970423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.970425:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677470.970426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.970427:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.970428:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677470.970429:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.970429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677470.970526:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.970529:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.970530:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.970530:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.970532:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.970535:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.971449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.971563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.971570:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.971573:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.971574:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.971576:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.971578:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.971580:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.971581:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.971583:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.971584:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.971585:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.971588:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.971590:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.971591:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.971592:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.971593:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.971595:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.971595:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.971596:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.971597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.971607:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.971610:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.971610:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.971611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.971612:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.971613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677470.971693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677470.971699:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.971702:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677470.971703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677470.971704:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.971706:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.971708:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677470.971709:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.971710:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.971711:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.971711:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677470.971714:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.971715:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677470.971718:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.971718:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677470.971719:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677470.971720:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677470.971721:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677470.971722:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677470.971723:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677470.971723:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677470.971728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677470.971734:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677470.971734:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677470.971736:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677470.971736:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:17.0:1619677470.971737:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677470.971737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:17.0:1619677470.971738:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.971738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.971740:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.971742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.971751:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677470.971754:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.971755:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.971755:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.971756:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.971757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.2:1619677470.971923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677470.972002:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.972005:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.972006:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.972006:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.972009:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.972012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.2:1619677470.972185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677470.972207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:20.2:1619677470.972243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:1.0:1619677470.972337:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.972340:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677470.972341:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677470.972343:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.972346:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.972347:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677470.972349:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.972350:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.972351:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.972352:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677470.972355:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.972357:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677470.972360:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.972361:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677470.972361:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677470.972363:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677470.972364:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677470.972365:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677470.972365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677470.972378:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.972381:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.972381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.972382:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.972384:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.972387:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.973725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.973731:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.973734:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.973736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.973737:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.973740:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.973741:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677470.973743:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.973744:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.973745:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.973746:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677470.973750:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.973751:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.973754:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.973756:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677470.973756:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677470.973758:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677470.973759:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.973759:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.973760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677470.974898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677470.974904:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.974907:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677470.974908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677470.974910:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677470.974913:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.974914:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677470.974916:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.974918:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.974919:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.974920:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677470.974924:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677470.974925:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677470.974928:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.974928:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:17.0:1619677470.974929:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677470.974931:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677470.974931:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677470.974932:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677470.974932:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:18.0:1619677470.974946:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.974948:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.974949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.974950:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.974952:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677470.974955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.974993:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.974995:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.974996:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.974997:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.974999:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.975002:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677470.975143:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.975146:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.975147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.975147:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.975149:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.975154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677470.975171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.975177:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.975180:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.975180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.975184:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.975187:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.975188:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.975190:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.975191:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.975192:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.975193:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.975197:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.975198:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.975202:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.975203:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.975203:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.975205:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.975206:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.975207:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.975207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.975728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3a3000] (20)++
00000800:00000200:2.0:1619677470.975734:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3a3000] (21)++
00000800:00000200:2.0:1619677470.975737:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.22@o2ib3
00000800:00000200:18.0:1619677470.975738:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)--
00000400:00000200:2.0:1619677470.975741:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.22@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.975744:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.975746:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677470.975747:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.975749:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.975751:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.975752:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677470.975755:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.975757:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 968 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677470.975760:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.975761:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677470.975761:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677470.975763:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677470.975764:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3a3000] (21)++
00000800:00000200:2.0:1619677470.975765:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3a3000] (22)--
00000800:00000200:2.0:1619677470.975765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (21)--
00000800:00000200:20.2:1619677470.975982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.976029:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677470.976031:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677470.976032:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.976032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.976034:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.976037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677470.976048:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677470.976050:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677470.976051:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677470.976051:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.976052:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677470.976052:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677470.976385:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.976387:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.976388:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.976388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.976390:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.976393:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.976462:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.976464:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.976466:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.976466:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.976468:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.976470:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.977215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.977254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677470.977855:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.977858:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.977859:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.977859:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.977861:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.977864:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677470.978283:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.978287:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.978291:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.978293:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.978304:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.978306:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677470.978309:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96519909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677470.978311:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.978313:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.978314:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677470.978319:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.978322:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677470.978325:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.978327:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677470.978327:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677470.978329:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677470.978330:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.978332:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677470.978333:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677470.978669:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.978672:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.978673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.978675:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.978677:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.978679:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.978680:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.978682:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.978683:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.978684:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.978687:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.978689:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.978692:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.978693:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677470.978693:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677470.978695:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677470.978695:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.978696:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.978696:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.979425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.979431:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.979434:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.979435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.979436:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.979439:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.979441:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.979442:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.979444:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.979445:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.979446:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.979449:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.979451:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.979454:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.979455:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677470.979455:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677470.979457:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677470.979458:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.979458:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.979459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677470.979975:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.979977:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.979978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.979979:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.979981:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.979985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.979993:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677470.979996:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.979996:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677470.979997:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.979998:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.979999:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677470.980211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.980217:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.980219:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.980220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.980223:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.980225:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.980227:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.980228:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.980229:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.980230:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.980231:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.980235:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.980237:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.980240:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.980241:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.980241:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.980243:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.980244:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.980244:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.980244:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677470.980952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.980958:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.980961:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.980962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.980964:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.980967:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.980969:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.980970:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.980972:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.980973:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.980974:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.980978:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.980979:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.980981:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.980982:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.980983:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.980984:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.980985:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.980986:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677470.980986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677470.980997:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.980999:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677470.981000:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.981000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677470.981002:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.981003:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.981975:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.981978:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.981979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.981979:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.981981:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.981984:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.981994:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.981997:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.981998:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.981999:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.982000:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.982000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.982194:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.982196:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.982198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.982198:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.982200:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.982203:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.982212:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677470.982215:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.982216:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.982216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.982217:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677470.982217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.982598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.983843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.983849:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.983852:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.983853:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.983855:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.983857:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.983858:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677470.983860:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.983861:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.983862:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.983863:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677470.983866:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.983868:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677470.983871:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.983871:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677470.983872:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677470.983874:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677470.983875:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.983876:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677470.983877:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677470.983877:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677470.983883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.983889:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677470.983891:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677470.983892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.983892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677470.984509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677470.984552:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.984554:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.984555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.984555:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.984557:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.984560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677470.984752:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.984756:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.984756:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.984757:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.984759:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.984763:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.984798:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.984800:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.984801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.984802:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.984804:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.984808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:13.0:1619677470.985023:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677470.985025:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677470.985026:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677470.985028:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.985030:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677470.985032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:18.0:1619677470.985080:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.985083:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.985084:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.985084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.985087:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.985098:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.985115:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.985117:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.985117:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.985118:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677470.985118:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.985119:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.985121:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677470.985122:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.985125:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.985125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.985126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.985127:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677470.985131:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.985132:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.985132:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677470.985133:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:5.2:1619677470.985133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.985136:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677470.985137:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.985139:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.985139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.985141:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.985143:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.985144:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.985146:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.985148:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.985149:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.985150:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.985154:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.985155:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.985159:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.985160:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.985161:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.985163:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.985163:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.985164:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677470.985164:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677470.986464:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.986466:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677470.986468:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677470.986468:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.986470:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.986474:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677470.986820:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.986823:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.986823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.986826:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.986828:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.986830:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.986831:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.986832:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.986833:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.986834:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.986837:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.986838:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.986841:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.986842:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677470.986842:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677470.986846:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677470.986846:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.986848:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677470.986849:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677470.986849:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.986860:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677470.986862:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677470.986863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.986864:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677470.987317:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.987319:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677470.987320:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.987321:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.987323:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.987325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677470.987342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677470.987349:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.987352:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677470.987353:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.987354:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.987357:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.987359:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677470.987360:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.987362:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.987363:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.987364:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677470.987367:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.987369:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677470.987371:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.987372:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677470.987372:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.987374:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.987375:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.987376:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677470.987376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677470.987387:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.987390:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677470.987390:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.987391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.987393:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.987394:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677470.987830:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.987832:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.987833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.987834:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.987836:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.987839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.988510:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.988513:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.988514:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.988516:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.988518:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.988519:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677470.988521:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.988522:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.988523:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.988524:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677470.988528:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.988529:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677470.988532:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.988533:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677470.988534:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677470.988535:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677470.988536:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.988537:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677470.988538:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677470.988539:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.988552:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677470.988554:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677470.988555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677470.988556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677470.988633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677470.989311:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.989313:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677470.989314:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677470.989316:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.989319:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.989321:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677470.989322:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.989323:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.989324:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.989325:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677470.989328:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.989330:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.989332:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.989333:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:17.0:1619677470.989334:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677470.989335:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:17.0:1619677470.989336:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677470.989336:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677470.989337:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677470.989547:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.989549:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.989550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.989551:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.989553:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.989556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.990457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.990494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677470.990501:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677470.990504:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677470.990505:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677470.990505:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.990507:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.990509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677470.990847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677470.990853:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.990856:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677470.990857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677470.990858:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677470.990861:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.990862:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677470.990864:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.990866:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.990868:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.990869:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677470.990874:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677470.990875:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677470.990878:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.990879:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677470.990880:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677470.990882:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677470.990883:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677470.990883:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677470.990884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.991083:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.991086:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677470.991086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677470.991090:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.991092:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.991093:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677470.991095:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.991096:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.991097:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.991099:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677470.991102:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.991104:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677470.991107:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.991108:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677470.991108:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677470.991110:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677470.991111:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677470.991111:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677470.991112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677470.991687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677470.991693:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.991696:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677470.991697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677470.991699:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.991702:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.991704:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677470.991705:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.991707:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.991707:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.991709:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677470.991712:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.991714:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677470.991717:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677470.991718:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677470.991719:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677470.991720:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677470.991721:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677470.991722:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677470.991723:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677470.991724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677470.991729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.991734:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677470.991736:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677470.991737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.991738:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677470.993042:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.993044:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.993045:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.993046:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.993048:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.993050:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.993784:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.993787:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.993788:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.993788:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.993790:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.993793:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677470.993802:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677470.993805:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677470.993806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.993806:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.993807:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677470.993808:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.994304:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.994307:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677470.994308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677470.994310:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677470.994313:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677470.994315:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677470.994316:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677470.994318:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677470.994319:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677470.994320:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677470.994323:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677470.994325:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677470.994328:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.994329:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677470.994330:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677470.994331:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677470.994332:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677470.994332:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677470.994333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677470.994337:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.994340:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.994341:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.994341:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.994343:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.994347:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677470.994733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677470.994738:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.994741:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677470.994742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677470.994744:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.994747:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.994748:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677470.994749:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.994751:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.994752:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.994753:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677470.994756:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.994758:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.994761:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.994762:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677470.994762:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677470.994763:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.994764:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677470.994765:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677470.994766:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.994767:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677470.994772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677470.994778:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.994779:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677470.994779:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677470.994780:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.994780:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677470.994781:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677470.994782:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.994782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677470.994783:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677470.994783:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:12.0:1619677470.994906:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.994909:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677470.994911:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677470.994913:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677470.994914:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677470.994915:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677470.994917:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96519826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677470.994918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:12.0:1619677470.994918:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677470.994919:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677470.994920:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677470.994923:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677470.994924:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677470.994928:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677470.994928:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677470.994929:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677470.994931:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677470.994932:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677470.994933:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677470.994935:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677470.994935:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:2.0:1619677470.994945:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.994947:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.994949:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677470.994949:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:2.0:1619677470.994949:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.994951:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:13.0:1619677470.994952:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677470.994953:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.994953:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677470.994955:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.994965:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677470.994969:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677470.994969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (24)--
00000800:00000200:18.0:1619677470.994970:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677470.994971:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.994972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677470.994982:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.994984:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.994985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.994985:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.994987:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.994990:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677470.995141:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.995144:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677470.995145:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677470.995145:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.995148:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.995151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.995160:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677470.995162:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677470.995164:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.995164:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677470.995165:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677470.995165:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677470.995792:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.995795:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677470.995796:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677470.995799:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677470.995802:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.995803:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677470.995805:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677470.995806:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677470.995807:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677470.995808:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677470.995812:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677470.995813:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677470.995816:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677470.995817:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677470.995818:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677470.995819:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677470.995820:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677470.995821:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677470.995821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677470.996337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677470.996353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677470.996360:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.996363:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677470.996364:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677470.996366:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677470.996368:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.996370:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677470.996371:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677470.996373:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677470.996374:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677470.996375:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677470.996378:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677470.996380:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677470.996380:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:18.0:1619677470.996382:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677470.996382:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:18.0:1619677470.996383:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:17.0:1619677470.996383:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677470.996384:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677470.996384:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677470.996385:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.996386:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677470.996386:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677470.996387:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677470.996388:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677470.996388:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677470.996609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677470.996615:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.996618:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677470.996619:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677470.996621:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677470.996623:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.996625:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677470.996627:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677470.996628:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677470.996629:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677470.996630:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677470.996633:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677470.996635:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677470.996638:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677470.996639:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677470.996640:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677470.996641:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677470.996642:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677470.996643:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677470.996643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677470.996756:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.996758:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677470.996761:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677470.996763:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677470.996763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677470.996765:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677470.996766:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96519814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677470.996768:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677470.996769:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677470.996770:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677470.996773:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677470.996776:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677470.996778:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677470.996779:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677470.996780:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677470.996782:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677470.996782:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677470.996783:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677470.996784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677470.996839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:1.0:1619677470.997862:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677470.997865:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677470.997866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677470.997866:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677470.997868:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677470.997871:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677470.998264:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677470.998267:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677470.998268:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677470.998268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677470.998270:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677470.998273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677470.999050:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677470.999052:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677470.999053:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677470.999054:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677470.999056:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677470.999060:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677470.999806:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677470.999809:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677470.999810:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677470.999810:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677470.999812:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677470.999814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.000037:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.000040:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.000041:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.000043:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.113@o2ib7 : GET - for me
00000400:00000200:1.0:1619677471.000046:0:19931:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.113@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1619677471.000047:0:19931:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.113@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:1.0:1619677471.000049:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.000050:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.000052:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677471.000055:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : REPLY try# 0
00000800:00000200:1.0:1619677471.000058:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.000060:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.000060:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.000061:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.000063:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.000063:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.000075:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.000077:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.000078:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.000079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.000080:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: REPLY: OK
00000800:00000200:17.0:1619677471.000081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.000731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.000776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:1.0:1619677471.000776:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.000779:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.000780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.000780:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.000782:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:1.0:1619677471.000782:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.000784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.000785:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.000785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.000793:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677471.000794:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:18.0:1619677471.000796:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.000798:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677471.000798:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.000798:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:18.0:1619677471.000799:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677471.000799:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.000800:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:18.0:1619677471.000801:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677471.000801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:18.0:1619677471.000802:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.000803:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.000805:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.000808:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.000810:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.000811:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.000812:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.000813:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.000814:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.000815:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.000815:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.000816:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.000816:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.000817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:2.0:1619677471.000818:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.000819:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.000821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677471.001489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.001495:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.001498:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.001499:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.001501:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.001504:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.001505:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.001507:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.001508:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.001509:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.001510:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.001513:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.001515:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.001518:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.001519:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.001520:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.001522:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.001523:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.001524:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.001524:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677471.002888:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.002891:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.002892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.002892:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.002895:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.002898:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.003812:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.003815:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (24)--
00000400:00000200:18.0:1619677471.003816:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.003816:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.003818:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.003823:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.003825:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.003827:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.003827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.003828:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.003830:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.003831:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.003845:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.003849:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.003849:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.003850:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.003851:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.003851:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.003999:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.004002:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.004003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.004003:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.004005:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.004008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.004242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.004249:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.004252:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.004252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.004255:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.004257:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.004258:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.004260:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.004262:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.004263:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.004264:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.004267:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.004269:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.004271:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.004272:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.004273:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.004275:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.004276:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.004277:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.004278:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.004288:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.004290:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.004291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.004291:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.004292:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.004299:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.004312:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.004315:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.004315:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.004316:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.004316:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.004317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.004419:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.004422:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.004422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.004424:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.004426:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.004428:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.004429:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.004430:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.004431:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.004432:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.004435:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.004437:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.004440:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.004441:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.004442:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.004443:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.004444:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.004445:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.004445:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.006033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.006067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677471.006406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677471.006411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.006414:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677471.006415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677471.006417:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.006420:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.006421:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677471.006423:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.006424:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.006425:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.006426:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677471.006430:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.006431:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677471.006435:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.006436:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677471.006436:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677471.006438:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677471.006439:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.006439:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677471.006440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677471.006906:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.006909:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.006910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.006912:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.006914:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.006915:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677471.006917:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.006918:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.006919:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.006920:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677471.006923:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.006925:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.006927:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.006928:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.006929:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.006931:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.006931:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.006932:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.006932:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677471.006944:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.006946:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.006946:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.006947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.006948:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.006949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.007412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.007456:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.007459:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.007460:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.007460:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.007462:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.007464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.007947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.007954:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.007957:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.007957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.007960:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.007962:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.007964:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.007965:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.007967:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.007968:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.007969:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.007973:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.007975:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.007977:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.007978:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.007979:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.007980:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.007981:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.007982:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.007984:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.007985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.007989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.007995:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.007998:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.007999:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.008000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.008591:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.008593:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.008594:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.008594:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.008597:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.008602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.009620:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.009622:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.009623:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.009624:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.009625:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.009628:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.010113:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.010116:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.010117:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.010117:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.010119:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.010123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:12.0:1619677471.010605:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.010608:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.010610:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.010612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.010612:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.010614:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677471.010615:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96519754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.010617:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.010618:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.010619:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677471.010622:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.010624:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677471.010626:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.010627:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677471.010627:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677471.010629:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677471.010629:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.010630:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.010631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677471.010703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.010709:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.010712:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.010713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.010715:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.010718:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.010719:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.010721:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.010722:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.010723:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.010724:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.010727:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.010729:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.010730:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.010731:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.010732:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.010734:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.010735:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.010736:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.010736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.010747:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.010749:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.010750:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.010751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.010752:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.010753:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.010897:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.010901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.010901:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.010903:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.010905:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.010906:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.010908:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.010909:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.010910:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.010911:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.010914:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.010915:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.010918:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.010919:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.010919:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.010921:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.010921:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.010923:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.010924:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.010924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.010936:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.010938:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.010938:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.010939:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677471.011360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.011367:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.011370:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.011371:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.011373:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.011375:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.011377:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.011378:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.011380:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.011381:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.011383:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.011388:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.011390:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.011394:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.011395:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.011396:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.011398:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.011398:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.011400:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677471.011401:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677471.011402:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677471.011407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.011412:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677471.011414:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677471.011415:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677471.011416:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677471.011475:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.011478:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.011479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.011479:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.011480:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:1.0:1619677471.011482:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.011483:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.011484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.011485:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.011485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:18.0:1619677471.011487:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.011491:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.011508:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.011510:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.011511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.011511:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.011513:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.011515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.011525:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.011529:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.011529:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.011530:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.011531:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.011531:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.011662:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.011665:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.011666:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.011666:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.011668:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.011672:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.012360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.013292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.013300:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.013303:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.013304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.013306:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.013309:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.013311:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.013312:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.013314:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.013315:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.013316:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.013319:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.013321:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.013323:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.013324:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.013326:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.013328:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.013329:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.013330:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.013331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.013343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.013349:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.013352:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.013353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.013354:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.013357:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.013359:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677471.013360:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.013362:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.013363:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.013364:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677471.013367:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.013368:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677471.013370:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.013371:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677471.013372:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677471.013373:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677471.013374:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.013374:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.013375:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677471.013815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.013855:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.013857:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.013858:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.013858:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.013860:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.013863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.013873:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.013876:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.013877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.013877:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.013878:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.013878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.015096:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.015099:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.015099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.015100:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.015102:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.015105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.015637:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.015640:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.015641:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.015643:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.015645:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.015647:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.015648:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.015649:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.015650:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.015652:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.015655:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.015656:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.015659:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.015659:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.015660:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.015662:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.015662:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.015663:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.015664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.015852:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.015854:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.015855:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.015855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.015857:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.015860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.015870:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.015873:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.015874:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.015874:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.015875:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.015878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.015883:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.015886:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677471.015886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.015889:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.015891:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.015893:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677471.015894:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.015896:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.015897:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.015898:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677471.015901:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.015903:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677471.015907:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.015908:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677471.015908:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677471.015910:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677471.015911:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.015912:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.015912:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:5.2:1619677471.016298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677471.016304:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.016307:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677471.016308:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677471.016312:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.016315:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.016317:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677471.016318:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.016321:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.016322:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.016324:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677471.016328:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.016331:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677471.016334:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.016335:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677471.016336:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677471.016338:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677471.016339:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.016340:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677471.016340:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677471.017136:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.017139:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.017140:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.017140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.017142:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.017145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.017685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.017691:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.017694:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.017695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.017697:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.017700:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.017701:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.017702:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.017704:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.017705:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.017706:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.017710:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.017711:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.017714:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.017715:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.017715:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.017717:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.017718:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.017718:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.017719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.018066:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.018069:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.018070:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.018071:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.018073:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.018077:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.018138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.018175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.018181:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.018183:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.018185:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.018185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.018186:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.018189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.019034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.019040:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.019044:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.019045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.019046:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.019049:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.019051:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.019052:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.019054:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.019055:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.019056:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.019059:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.019061:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.019063:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.019064:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.019065:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.019066:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.019067:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.019067:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.019068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.019079:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.019082:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.019082:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.019083:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.019085:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.019086:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.019814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.019855:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.019858:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.019859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.019859:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.019861:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.019863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.020029:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.020032:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.020033:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.020036:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.020038:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.020040:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.020041:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.020043:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.020044:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.020045:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.020048:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.020050:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.020052:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.020053:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.020053:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.020055:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.020056:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.020056:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.020057:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.020672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.020678:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.020681:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.020682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.020684:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.020686:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.020688:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677471.020689:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.020690:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.020691:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.020692:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677471.020696:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.020698:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677471.020702:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.020703:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677471.020704:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677471.020706:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677471.020706:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.020707:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.020707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:5.2:1619677471.021286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.021292:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.021296:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.021300:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677471.021302:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.021305:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.021307:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.021308:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.021310:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.021311:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.021312:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.021315:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.021317:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.021321:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.021321:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.021322:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.021324:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.021325:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.021325:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.021326:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677471.021529:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.021531:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.021533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.021533:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.021535:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.021538:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.021866:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.021869:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.021870:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.021871:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.021873:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.021875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.022875:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.022878:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.022879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.022881:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.022883:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.022885:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.022886:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.022887:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.022888:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.022889:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.022893:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.022894:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.022897:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.022898:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.022898:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677471.022900:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.022901:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.022901:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.022902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677471.023980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.023986:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.023988:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.023990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.023992:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.023995:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.023996:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.023998:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.023999:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.024001:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.024001:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.024005:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.024007:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.024010:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.024010:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.024011:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677471.024013:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:2.0:1619677471.024013:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.024013:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.024014:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.024015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.024016:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.024017:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.024020:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.024022:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.024024:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.024025:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.024027:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.024028:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.024029:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.024033:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.024037:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.024040:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.024040:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.024041:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.024043:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.024043:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.024044:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.024045:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.024177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.025918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.025959:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.025962:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.025963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.025963:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.025965:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.025968:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.025978:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.025981:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.025981:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.025982:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.025983:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.025983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.026202:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.026205:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.026206:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.026206:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.026208:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.026212:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.026220:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.026223:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.026224:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.026224:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.026225:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.026226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.026229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.026234:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.026237:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.026238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.026239:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.026242:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.026244:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.026245:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.026246:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.026247:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.026248:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.026251:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.026253:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.026256:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.026257:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.026258:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.026259:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.026260:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.026261:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.026261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677471.027400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.027406:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.027409:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.027410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.027412:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.027415:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.027417:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.027418:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.027419:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.027420:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.027421:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.027424:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.027427:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.027429:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.027430:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.027431:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.027432:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.027433:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.027434:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.027435:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677471.027703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.027710:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.027713:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.027713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.027715:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.027718:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.027718:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.027720:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.027722:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.027722:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.027723:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.027726:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.027728:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.027730:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.027730:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.027731:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.027732:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.027733:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.027734:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.027735:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.027736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.027742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.027748:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.027749:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.027749:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.027749:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.027750:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.027750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.027751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.027751:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.027752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.027753:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.027980:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.027983:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.027984:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.027984:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.027986:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.027989:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.028134:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.028137:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.028138:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.028138:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.028142:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.028146:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.029908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.029946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.029952:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.2:1619677471.029953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677471.029955:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.029956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.029958:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.029961:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.029962:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.029964:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96519649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.029966:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.029968:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.029969:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.029972:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.029974:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.029976:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.029977:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.029978:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.029980:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.029981:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.029982:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.029983:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.029984:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.029990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.029996:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.029999:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.030000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.030000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.030007:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.030011:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.030011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.030013:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.030015:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.030017:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.030019:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.030020:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.030021:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.030022:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.030025:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.030027:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.030029:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.030030:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.030031:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.030033:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.030033:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.030034:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.030035:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.030036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.030047:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.030049:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.030050:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.030051:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:24.2:1619677471.030393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:18.0:1619677471.030459:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.030462:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.030463:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.030463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.030466:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.030470:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.031090:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.031092:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.031093:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.031093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.031095:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.031098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677471.031206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677471.031211:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.031214:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677471.031215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677471.031217:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.031219:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.031220:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677471.031222:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.031223:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.031224:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.031225:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677471.031228:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.031230:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677471.031233:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.031234:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677471.031234:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677471.031236:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677471.031237:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.031238:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677471.031239:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677471.031239:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677471.031244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.031250:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677471.031252:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677471.031253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677471.031254:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677471.032329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.032373:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.032376:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.032377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.032377:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.032379:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.032381:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.032849:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.032852:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.032853:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.032853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.032855:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.032858:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.034035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:17.0:1619677471.034358:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.034361:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.034362:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.034362:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.034365:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.034367:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.034381:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.034384:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.034385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.034385:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.034386:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.034387:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.034582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.034588:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.034591:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.034592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.034593:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.034596:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.034598:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.034599:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.034600:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.034602:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.034603:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.034607:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.034608:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.034611:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.034612:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.034613:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.034615:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.034615:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.034616:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.034617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.034825:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.034828:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677471.034828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.034830:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.034833:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.034835:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.034836:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.034837:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.034838:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.034839:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.034843:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.034844:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.034846:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.034847:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.034847:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.034849:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.034849:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.034850:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.034851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677471.034872:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.034874:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.034875:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.034875:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.034876:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.034877:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.034894:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.034896:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.034896:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.034897:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.034897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.034898:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.035548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.035659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.035665:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.035668:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.035669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.035670:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.035672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.035674:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.035675:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.035676:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.035677:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.035678:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.035682:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.035683:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.035685:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.035686:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.035686:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.035688:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.035689:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.035690:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.035690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.035702:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.035704:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.035705:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.035705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.035707:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.035708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.035723:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.035727:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.035727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.035730:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.035733:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.035734:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.035735:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.035737:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.035738:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.035739:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.035742:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.035744:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.035747:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.035748:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.035749:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.035751:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.035752:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.035752:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.035753:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677471.036119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677471.036125:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.036128:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677471.036129:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677471.036131:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.036133:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.036135:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677471.036136:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.036138:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.036139:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.036140:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677471.036143:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.036145:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677471.036149:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.036149:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677471.036150:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677471.036152:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677471.036153:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.036154:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677471.036155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677471.037581:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.037584:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.037585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.037586:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.037588:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.037592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.037601:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.037604:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.037605:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.037605:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.037606:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.037607:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.037799:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.037801:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.037803:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.037803:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.037805:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.037808:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.039700:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.039703:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.039704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.039705:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.039707:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.039710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.039811:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.039814:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.039815:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.039815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.039817:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.039820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.040072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.040588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.040594:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.040597:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.040598:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.040601:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.040605:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.040607:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.040609:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.040611:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.040612:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.040613:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.040617:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.040619:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.040621:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.040622:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.040623:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.040624:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.040625:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.040626:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.040627:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677471.041045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.041051:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.041054:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.041054:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.041057:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.041060:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.041061:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.041063:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.041064:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.041065:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.041066:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.041068:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.041070:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.041073:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.041074:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.041075:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.041077:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.041077:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.041078:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.041079:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677471.042105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.042111:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.042114:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.042115:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.042116:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.042120:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.042121:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.042122:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.042124:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.042125:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.042126:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.042128:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.042130:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.042132:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.042132:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.042133:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.042135:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.042135:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.042137:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.042137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.042149:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.042151:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.042151:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.042152:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.042153:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.042155:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.042826:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.042828:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.042830:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.042830:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.042832:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.042836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.043843:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.043846:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.043847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.043849:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.102@o2ib7 : GET - for me
00000400:00000200:18.0:1619677471.043851:0:19933:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.102@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:18.0:1619677471.043853:0:19933:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.102@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:18.0:1619677471.043854:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.043856:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.043857:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677471.043861:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : REPLY try# 0
00000800:00000200:18.0:1619677471.043863:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.043864:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.043865:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.043866:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.043867:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.043868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.043880:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.043882:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.043883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.043883:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.043885:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: REPLY: OK
00000800:00000200:2.0:1619677471.043886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.044192:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.044194:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.044195:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.044195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.044198:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.044200:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.044274:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.044277:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.044278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.044279:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.044281:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.044284:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.045317:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.045320:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677471.045321:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677471.045322:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.045325:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.045326:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677471.045328:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.045329:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.045331:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.045332:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.045336:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.045338:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.045341:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.045342:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677471.045343:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677471.045345:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677471.045345:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.045346:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.045347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:17.0:1619677471.045457:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.045459:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.045460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.045461:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.045463:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.045468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.045469:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.045471:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.045472:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.045473:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.045473:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.045474:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.045855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.045889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:5.2:1619677471.045982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.045987:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.045990:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.045991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.045993:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.045996:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.045998:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.045999:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.046002:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.046003:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.046004:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.046007:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.046009:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.046013:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.046013:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.046014:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.046016:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.046016:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.046017:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.046018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677471.046652:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.046654:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.046655:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.046656:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.046659:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.046660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677471.046672:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677471.046675:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677471.046676:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.046676:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677471.046677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.046679:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677471.046863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5e00] (20)++
00000800:00000200:24.2:1619677471.046903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:20.2:1619677471.046906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5e00] (21)++
00000800:00000200:20.2:1619677471.046918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:12.0:1619677471.046918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.046921:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:20.2:1619677471.046922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5e00] (21)++
00000400:00000200:12.0:1619677471.046922:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.046924:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.046925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:2.0:1619677471.046925:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677471.046928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:2.0:1619677471.046928:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.046929:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.046932:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.046937:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.046939:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.046940:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.046942:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.046943:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.046945:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.046951:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.046954:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.046957:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.046958:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.046959:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.046960:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.046961:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.046962:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.046962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677471.046978:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.046980:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.046981:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.046983:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.046984:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.046988:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677471.047001:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5e00] (20)++
00000800:00000200:20.2:1619677471.047024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5e00] (21)++
00000800:00000200:20.2:1619677471.047049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5e00] (21)++
00000800:00000200:20.2:1619677471.047074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5e00] (21)++
00000800:00000200:20.2:1619677471.047120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5e00] (20)++
00000800:00000200:20.2:1619677471.047268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5e00] (22)++
00000800:00000200:18.0:1619677471.047272:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.047275:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.047276:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.047276:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.047278:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.047282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:20.2:1619677471.047297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5e00] (20)++
00000800:00000200:2.0:1619677471.047298:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.047301:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.047302:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.047302:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.047304:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.047306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.047498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.047501:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.047501:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.047502:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.047504:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.047511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.047513:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.047515:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.047516:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.047516:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.047517:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.047517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677471.048995:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.048998:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.048999:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.049002:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.049003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.049005:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677471.049577:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.049581:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.049583:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.049584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.049587:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.049592:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.049615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.049621:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.049624:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.049625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.049627:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.049630:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.049631:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677471.049633:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.049635:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.049636:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.049638:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.049641:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.049642:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.049646:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.049647:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677471.049648:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677471.049649:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677471.049650:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.049652:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677471.049653:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677471.049653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677471.049658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.049664:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677471.049666:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677471.049667:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.049668:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677471.050541:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.050543:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.050544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.050545:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.050547:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.050550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.050854:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.050858:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.050858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.050859:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.050861:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.050865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.050928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677471.050934:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.050936:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677471.050937:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677471.050939:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.050942:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.050943:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677471.050944:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.050945:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.050946:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.050947:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677471.050951:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.050952:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677471.050956:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.050956:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677471.050957:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677471.050959:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677471.050959:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.050961:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677471.050963:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677471.050964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677471.050969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.050975:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677471.050977:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677471.050978:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677471.050978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677471.051266:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.051269:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.051270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.051270:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.051272:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.051275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.051525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.051531:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.051534:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.051535:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.051538:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.051541:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.051542:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.051544:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.051546:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.051546:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.051548:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.051551:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.051553:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.051555:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.051556:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.051557:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.051559:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.051560:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.051561:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.051562:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.051563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.051568:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.051573:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.051576:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.051577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.051578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.051583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.051656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.051662:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.051665:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.051665:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.051668:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.051671:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.051672:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.051673:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.051675:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.051677:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.051678:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.051681:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.051683:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.051684:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.051685:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.051686:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.051687:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.051688:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.051689:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.051689:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.051712:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.051714:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.051715:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.051716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.051717:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.051718:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.051736:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.051739:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.051739:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.051739:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.051740:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.051741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.052176:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.052179:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.052180:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.052180:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.052182:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.052186:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.052733:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.052736:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.052736:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.052738:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.052741:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.052742:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677471.052744:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.052746:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.052747:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.052748:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677471.052751:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.052753:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677471.052756:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.052757:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677471.052758:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677471.052760:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677471.052760:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.052761:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.052762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677471.052774:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.052777:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.052778:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.052779:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.052781:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.052782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677471.055567:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.055570:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.055571:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.055574:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.055577:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.055578:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677471.055580:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.055581:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.055582:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.055583:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677471.055586:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.055588:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677471.055590:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.055591:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677471.055592:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677471.055594:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.055595:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.055596:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677471.055597:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677471.055598:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677471.055609:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:17.0:1619677471.055609:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677471.055611:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677471.055611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677471.055612:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:17.0:1619677471.055612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:18.0:1619677471.055613:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.055613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.055615:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.055617:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.055736:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.055739:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.055740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.055740:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.055743:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677471.055746:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677471.056034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.056040:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.056043:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.056044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.056046:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.056048:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.056050:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.056051:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.056053:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.056054:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.056058:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.056061:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.056063:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.056067:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.056068:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.056068:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.056070:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.056071:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.056072:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.056072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677471.056280:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.056283:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.056284:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.056284:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.056286:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.056289:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.056315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.056361:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.056364:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.056365:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.056365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.056367:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.056369:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.056379:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.056382:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.056383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.056383:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.056384:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.056385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.056584:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.056588:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:1.0:1619677471.056588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000400:00000200:17.0:1619677471.056590:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.056592:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.056594:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677471.056595:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.056596:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.056597:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.056598:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677471.056602:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.056604:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677471.056607:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.056608:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677471.056609:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677471.056612:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677471.056613:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.056614:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b4200] (22)++
00000800:00000200:17.0:1619677471.056615:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (23)--
00000800:00000200:17.0:1619677471.056615:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000800:00000200:1.0:1619677471.056626:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)++
00000800:00000200:1.0:1619677471.056628:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b4200] (23)--
00000800:00000200:1.0:1619677471.056628:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000800:00000200:1.0:1619677471.056629:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000800:00000200:20.2:1619677471.056712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677471.056716:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.056719:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:2.0:1619677471.056719:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677471.056720:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000400:00000200:17.0:1619677471.056722:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677471.056722:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.056724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677471.056724:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.056726:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677471.056726:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677471.056727:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.056728:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.056729:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.056729:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.056730:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.056731:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.056733:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.056734:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677471.056734:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677471.056736:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 600 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.056736:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.056737:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677471.056739:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.056740:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677471.056741:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:2.0:1619677471.056741:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677471.056743:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677471.056743:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.056743:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (22)--
00000800:00000200:2.0:1619677471.056743:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677471.056744:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000800:00000200:2.0:1619677471.056746:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.056747:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.056748:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.056750:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.056750:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.056751:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.056752:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.056833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000800:00000200:1.0:1619677471.056836:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:17.0:1619677471.056837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000400:00000200:1.0:1619677471.056839:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.056842:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.056844:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677471.056845:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.056846:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.056847:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.056848:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677471.056851:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.056853:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 616 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677471.056856:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.056857:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677471.056858:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677471.056860:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677471.056861:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000800:00000200:1.0:1619677471.056861:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (22)--
00000800:00000200:1.0:1619677471.056862:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000800:00000200:12.0:1619677471.056874:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:17.0:1619677471.056876:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:12.0:1619677471.056876:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677471.056877:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.056879:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:12.0:1619677471.056879:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.056879:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:13.0:1619677471.056881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:17.0:1619677471.056882:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677471.056883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:17.0:1619677471.056885:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.056886:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.056888:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.056889:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.056890:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.056891:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.056894:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.056895:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.056898:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.056898:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.056899:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.056901:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.056901:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.056902:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.056902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.056960:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000800:00000200:1.0:1619677471.056963:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:17.0:1619677471.056964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000400:00000200:1.0:1619677471.056966:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.056968:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.056969:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677471.056971:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.056972:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.056973:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.056974:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677471.056977:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.056979:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 600 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677471.056981:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.056982:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677471.056982:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677471.056984:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677471.056985:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000800:00000200:1.0:1619677471.056985:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (22)--
00000800:00000200:1.0:1619677471.056986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000800:00000200:13.0:1619677471.056997:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.057000:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.057001:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.057002:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.057004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.057005:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677471.057015:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677471.057018:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677471.057019:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.057020:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677471.057020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.057022:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:17.0:1619677471.057153:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.057156:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:1.0:1619677471.057157:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000400:00000200:17.0:1619677471.057159:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.057162:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.057163:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677471.057164:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.057166:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.057167:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.057168:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677471.057171:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.057173:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 600 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677471.057176:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.057177:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677471.057177:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677471.057179:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677471.057179:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.057181:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b4200] (22)++
00000800:00000200:17.0:1619677471.057182:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (23)--
00000800:00000200:17.0:1619677471.057182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000800:00000200:1.0:1619677471.057192:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)++
00000800:00000200:1.0:1619677471.057194:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b4200] (23)--
00000800:00000200:1.0:1619677471.057195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000800:00000200:1.0:1619677471.057196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000800:00000200:17.0:1619677471.057231:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.057234:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:1.0:1619677471.057235:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000400:00000200:17.0:1619677471.057236:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.057239:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.057241:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677471.057242:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.057243:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.057244:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.057245:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677471.057248:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.057249:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 520 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.057251:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.057251:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.057252:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.057253:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.057254:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.057255:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (22)--
00000800:00000200:17.0:1619677471.057255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000800:00000200:1.0:1619677471.057267:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.057269:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677471.057270:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.057271:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.057272:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.057273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.057288:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.057291:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.057291:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.057292:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.057292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.057293:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.057389:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677471.057390:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000800:00000200:18.0:1619677471.057392:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.057393:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.057393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.057393:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:17.0:1619677471.057394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000400:00000200:18.0:1619677471.057396:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677471.057396:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.057398:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.057399:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.057401:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.057402:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677471.057403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:1.0:1619677471.057403:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677471.057404:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000400:00000200:1.0:1619677471.057404:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677471.057407:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.057407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.057408:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000400:00000200:1.0:1619677471.057408:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.057409:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.057410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.057411:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.057413:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.057414:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.057415:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.057416:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.057417:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000800:00000200:1.0:1619677471.057418:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (22)--
00000800:00000200:1.0:1619677471.057418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000800:00000200:17.0:1619677471.057560:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.057563:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:1.0:1619677471.057564:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000400:00000200:17.0:1619677471.057566:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.057568:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.057570:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677471.057571:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.057572:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.057573:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.057574:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677471.057577:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.057579:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677471.057581:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.057582:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677471.057583:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677471.057585:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677471.057585:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.057586:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (22)--
00000800:00000200:17.0:1619677471.057586:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000800:00000200:1.0:1619677471.057747:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000800:00000200:1.0:1619677471.057750:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:17.0:1619677471.057751:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000400:00000200:1.0:1619677471.057752:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.057755:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.057756:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677471.057758:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.057759:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.057760:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.057761:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677471.057765:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.057767:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 552 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677471.057769:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.057770:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677471.057770:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677471.057771:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677471.057772:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000800:00000200:1.0:1619677471.057773:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b4200] (22)++
00000800:00000200:1.0:1619677471.057774:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (23)--
00000800:00000200:1.0:1619677471.057775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000800:00000200:17.0:1619677471.057786:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)++
00000800:00000200:17.0:1619677471.057787:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b4200] (23)--
00000800:00000200:17.0:1619677471.057788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000800:00000200:17.0:1619677471.057789:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000800:00000200:1.0:1619677471.057818:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.057821:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.057822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.057822:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.057824:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.057828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.057867:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.057869:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:1.0:1619677471.057870:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000400:00000200:17.0:1619677471.057872:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.057874:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.057876:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677471.057878:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.057880:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.057881:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.057882:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677471.057885:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.057887:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 512 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677471.057889:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.057890:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677471.057891:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677471.057893:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677471.057893:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.057894:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (22)--
00000800:00000200:17.0:1619677471.057895:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000800:00000200:13.0:1619677471.057907:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.057909:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.057910:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.057912:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.057914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.057915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677471.058005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677471.058052:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.058054:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.058056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.058056:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.058058:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.058061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.058083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000800:00000200:1.0:1619677471.058086:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:17.0:1619677471.058087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000400:00000200:1.0:1619677471.058089:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.058092:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.058093:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.058094:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.058096:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.058096:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.058097:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.058100:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.058102:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 632 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.058105:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.058105:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.058106:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677471.058108:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.058108:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000800:00000200:1.0:1619677471.058109:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (22)--
00000800:00000200:1.0:1619677471.058110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000800:00000200:17.0:1619677471.058209:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.058212:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:1.0:1619677471.058213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000400:00000200:17.0:1619677471.058215:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.058217:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.058218:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677471.058220:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.058221:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.058222:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.058223:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677471.058226:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.058228:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 552 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677471.058230:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.058231:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677471.058232:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677471.058233:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677471.058234:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000800:00000200:17.0:1619677471.058235:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (22)--
00000800:00000200:17.0:1619677471.058235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000800:00000200:12.0:1619677471.058246:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.058248:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.058249:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.058251:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.058252:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.058254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677471.058264:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677471.058267:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677471.058267:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.058268:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677471.058269:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.058270:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677471.058812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.059326:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.059329:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.059330:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.059330:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.059332:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.059336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.061001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.061008:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.061011:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.061012:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.061014:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.061017:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.061018:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.061020:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.061021:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.061022:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.061023:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.061027:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.061028:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.061031:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.061032:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.061033:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.061035:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.061035:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.061037:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.061038:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.061039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.061044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.061049:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.061051:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.061051:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.061051:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.061052:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.061052:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.061053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.061053:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.061054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.061055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677471.061289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.061295:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.061300:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.061302:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000400:00000200:18.0:1619677471.061305:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.061307:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.061309:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.061310:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.061312:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.061313:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.061314:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.061317:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.061319:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.061322:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.061323:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.061323:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.061325:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.061326:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.061327:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.061327:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:2.0:1619677471.061881:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.061884:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.061885:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.061885:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.061887:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.061890:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.061903:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.061907:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.061907:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.061908:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.061909:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.061909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.062045:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.062049:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.062049:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.062052:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.062055:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.062056:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.062058:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.062060:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.062061:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.062063:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.062068:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.062071:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.062076:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.062078:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.062078:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.062080:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.062081:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.062082:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.062082:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677471.062097:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677471.062098:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677471.062100:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.062101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.062101:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.062101:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.062102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.062102:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.062105:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677471.062105:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.062109:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.062109:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.063079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.063114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677471.063840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.063847:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.063850:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.063851:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.063853:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.063856:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.063857:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.063859:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.063860:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.063861:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.063862:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.063865:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.063867:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.063870:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.063870:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.063871:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.063873:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.063874:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.063875:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.063875:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.064352:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.064354:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.064356:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.064356:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.064358:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677471.064361:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.064390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:18.0:1619677471.064395:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677471.064398:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:2.0:1619677471.064399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:18.0:1619677471.064401:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.064403:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.064405:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677471.064406:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.064407:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.064408:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.064410:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677471.064413:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.064415:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677471.064418:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.064418:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677471.064419:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677471.064421:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677471.064422:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677471.064423:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:18.0:1619677471.064423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:20.2:1619677471.064621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.065293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:12.0:1619677471.065344:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:17.0:1619677471.065345:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:12.0:1619677471.065347:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:17.0:1619677471.065348:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.065349:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.065349:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.065349:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:12.0:1619677471.065351:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:17.0:1619677471.065352:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.065352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.065355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:17.0:1619677471.065357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.065359:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.065361:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.065362:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.065362:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.065363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.065363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.066084:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.066087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.066087:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.066090:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.066092:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.066093:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.066095:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.066096:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.066097:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.066098:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.066101:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.066103:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.066105:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.066106:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.066107:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.066109:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.066109:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.066110:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.066112:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.066112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.066123:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.066125:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.066126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.066127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677471.066517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677471.066523:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.066526:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677471.066527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677471.066529:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.066532:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.066533:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677471.066535:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.066537:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.066538:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.066539:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677471.066543:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.066544:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677471.066548:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.066549:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677471.066549:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677471.066551:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677471.066552:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.066553:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677471.066553:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:18.0:1619677471.066808:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.066810:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.066811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.066812:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.066814:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.066817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.067158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.067163:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.067166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.067166:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677471.067169:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.067171:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.067172:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.067174:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.067175:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.067176:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.067177:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.067180:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.067182:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.067184:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.067185:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.067185:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.067187:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.067188:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.067188:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.067189:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.067201:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.067203:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.067203:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.067204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.067205:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.067206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.067347:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.067349:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.067350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.067351:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.067353:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.067356:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.067525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.2:1619677471.067561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.2:1619677471.068062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.068068:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.068070:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.068071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.068073:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.068075:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.068077:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.068078:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.068080:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.068081:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.068082:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.068085:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.068087:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.068089:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.068090:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.068091:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.068092:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.068093:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.068094:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.068096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.068955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.070153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.070568:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.070571:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.070572:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.070572:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.070574:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.070578:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.070588:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.070591:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.070591:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.070593:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.070595:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.070597:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.070598:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.070599:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.070600:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.070601:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.070604:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.070606:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.070609:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.070610:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.070610:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.070612:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.070612:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.070613:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.070614:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677471.070940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.070946:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.070949:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.070950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.070953:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.070955:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.070957:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.070958:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.070960:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.070961:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.070962:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.070965:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.070968:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.070971:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.070972:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.070972:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.070974:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.070975:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.070976:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677471.070977:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677471.070978:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677471.070983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.070988:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677471.070990:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677471.070991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677471.070992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677471.071184:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.071187:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.071187:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.071189:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.071192:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.071194:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:18.0:1619677471.071515:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.071517:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.071518:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.071518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.071520:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.071524:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.073005:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.073008:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.073009:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.073009:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.073011:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.073014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.073261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.073268:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.073271:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.073272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.073273:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.073276:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.073278:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.073279:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.073280:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.073281:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.073282:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.073286:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.073287:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.073290:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.073291:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.073292:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.073298:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.073298:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.073300:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.073301:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.073301:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.073307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.073314:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.073315:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.073316:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.073318:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.073737:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.073740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.073740:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677471.073742:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.073744:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.073746:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677471.073747:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.073749:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.073750:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.073751:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677471.073754:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.073756:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.073758:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.073758:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.073759:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.073760:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.073761:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.073762:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.073762:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677471.073774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.073777:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.073777:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.073778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.073779:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.073780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.073970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.073976:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.073979:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.073980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.073982:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.073985:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.073986:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.073987:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.073989:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.073990:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.073991:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.073994:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.073996:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.073998:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.073999:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.073999:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.074001:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.074002:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.074003:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.074003:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.074015:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.074017:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.074017:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.074018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.074019:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.074021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.074979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.075027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677471.075465:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.075468:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.075469:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.075469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.075471:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.075475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.075483:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.075486:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.075487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.075487:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.075488:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.075489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677471.075815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.075822:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.075825:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.075825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.075827:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.075830:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.075831:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.075832:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.075834:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.075835:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.075836:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.075840:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.075841:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.075845:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.075845:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.075846:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.075848:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.075848:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.075849:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.075850:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677471.075990:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.075992:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.075993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.075995:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.075997:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.075999:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.076000:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.076001:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.076002:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.076004:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.076007:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.076009:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.076011:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.076012:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.076012:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677471.076014:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.076014:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.076016:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.076017:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.076018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.076029:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.076031:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.076032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.076033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677471.076199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677471.076235:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.076238:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.076239:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.076240:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.076242:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.076245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.076254:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.076257:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.076258:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.076258:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.076260:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.076260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.076427:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.076430:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.076431:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.076431:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.076434:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.076439:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.076440:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.076442:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.076443:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.076444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.076444:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.076445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.076640:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.076643:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.076644:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.076646:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.076648:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.076650:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.076650:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.076652:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.076653:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.076654:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.076657:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.076659:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.076661:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.076662:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.076662:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.076664:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.076664:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.076665:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.076665:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677471.077358:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.077361:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.077363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.077363:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.077365:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677471.077368:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.078708:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.078711:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.078712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.078712:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.078714:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.078717:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.080259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.080266:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.080269:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.080270:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.080272:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677471.080273:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:18.0:1619677471.080274:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.080276:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677471.080276:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.080277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:18.0:1619677471.080278:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.080279:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.080280:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.080281:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.080282:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677471.080282:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.080284:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.080286:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677471.080286:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.080287:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.080288:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677471.080289:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677471.080290:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.080292:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.080292:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.080296:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.080298:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:17.0:1619677471.080298:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.080299:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.080299:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.080300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.080300:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677471.080303:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.080304:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677471.080305:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677471.080306:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677471.080307:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.080307:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.080308:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:5.2:1619677471.080756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677471.080762:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.080765:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677471.080766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677471.080768:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.080771:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.080772:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677471.080774:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.080775:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.080776:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.080778:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677471.080782:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.080784:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677471.080788:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.080788:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677471.080789:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677471.080791:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677471.080792:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.080793:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677471.080793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677471.082167:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.082170:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.082171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.082173:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.082176:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.082177:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.082179:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.082180:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.082181:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.082182:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.082186:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.082187:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.082190:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.082190:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.082191:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.082193:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.082193:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.082194:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.082195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.082406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.082412:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.082415:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.082416:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.082419:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.082421:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.082423:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.082424:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.082426:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.082427:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.082428:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.082431:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.082433:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.082435:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.082436:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.082437:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.082439:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.082439:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.082440:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.082441:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.082452:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.082454:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.082454:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.082455:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.082456:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.082457:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.082951:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677471.082954:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677471.082955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677471.082957:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.082960:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.082961:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677471.082963:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.082964:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.082965:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.082966:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677471.082969:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.082971:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677471.082973:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.082974:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677471.082975:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677471.082976:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677471.082976:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677471.082977:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677471.082977:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:12.0:1619677471.083546:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.083549:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.083551:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677471.083553:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.083554:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.083555:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677471.083556:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96519376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.083557:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.083558:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.083559:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677471.083562:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.083563:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677471.083566:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.083567:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:12.0:1619677471.083567:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:12.0:1619677471.083569:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:12.0:1619677471.083570:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.083571:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.083571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677471.083585:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.083587:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.083589:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.083589:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.083591:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.083594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.083692:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.083695:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.083696:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.083696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.083698:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.083701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.084825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.084832:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.084835:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.084836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.084838:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.084840:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.084841:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.084843:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.084844:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.084845:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.084846:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.084849:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.084851:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.084853:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.084854:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.084855:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.084857:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.084858:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.084859:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.084859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677471.084983:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.084986:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.084988:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677471.084990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.084991:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.084993:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677471.084994:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96519370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.084995:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.084996:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.084997:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677471.085007:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.085009:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677471.085016:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.085016:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677471.085017:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677471.085018:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677471.085019:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.085020:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677471.085022:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677471.085022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.085036:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677471.085038:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677471.085038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.085040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677471.085163:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.085166:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677471.085166:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.085168:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.085170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.085170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:2.0:1619677471.085248:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.085251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.085251:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.085252:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.085254:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.085258:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.085277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:13.0:1619677471.085327:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.085330:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.085330:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.085332:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.085334:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.085335:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677471.085344:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677471.085346:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677471.085347:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.085348:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677471.085348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.085349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:5.2:1619677471.085652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.085659:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.085662:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.085662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.085665:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.085667:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.085669:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.085671:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.085673:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.085674:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.085675:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.085679:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.085680:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.085684:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.085685:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.085685:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.085687:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.085688:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.085689:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.085689:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:13.0:1619677471.086070:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.086073:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.086073:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.086075:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.086077:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.086078:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:1.0:1619677471.087979:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.087982:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.087983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.087985:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.087988:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.087989:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.087990:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.087992:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.087993:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.087994:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.087997:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.087999:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.088002:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.088002:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.088003:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.088005:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.088005:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.088007:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.088008:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.088008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.088019:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.088021:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.088021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.088022:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.088726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.088733:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.088736:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.088737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.088739:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.088742:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.088743:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.088745:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.088746:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.088747:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.088748:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.088751:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.088753:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.088755:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.088756:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.088756:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.088757:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.088758:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.088760:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.088761:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.088762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.088767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.088772:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.088774:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.088774:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.088774:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.088775:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677471.088776:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.088776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.088777:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.088777:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.088778:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677471.089086:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.089089:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.089090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.089090:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.089092:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.089095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.089106:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.089108:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.089109:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.089109:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.089111:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.089111:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.089345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.089353:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.089357:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.089358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.089361:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.089364:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.089366:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.089368:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.089371:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.089373:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.089374:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.089379:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.089382:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.089387:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.089388:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.089389:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.089392:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.089393:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.089394:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.089395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.090206:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.090209:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.090210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.090210:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.090213:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.090217:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.090226:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.090228:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.090229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.090230:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.090231:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.090231:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.090248:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.090250:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.090252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.090252:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.090254:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.090257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.090590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.090596:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.090599:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.090600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.090602:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.090604:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.090606:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.090607:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.090609:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.090610:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.090611:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.090614:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.090616:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.090619:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.090620:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.090621:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.090622:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.090623:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.090624:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677471.090625:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677471.090625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677471.090631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.090636:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677471.090638:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677471.090639:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677471.090641:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677471.091009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.091054:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.091057:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.091058:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.091058:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.091060:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.091063:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.092115:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.092117:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.092118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.092119:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.092121:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.092124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677471.092315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677471.092321:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677471.092324:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677471.092325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677471.092327:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677471.092329:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.092330:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677471.092332:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.092333:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.092334:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.092335:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677471.092338:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677471.092340:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677471.092342:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.092343:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677471.092344:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677471.092346:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677471.092346:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677471.092347:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677471.092348:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677471.092349:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:5.2:1619677471.092355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.092361:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677471.092363:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677471.092364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677471.092365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:18.0:1619677471.092582:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.092585:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677471.092585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.092588:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.092591:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.092593:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677471.092594:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.092596:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.092597:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.092598:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677471.092601:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.092603:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677471.092606:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.092606:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677471.092607:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677471.092608:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677471.092609:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.092610:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677471.092611:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677471.092612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.092622:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677471.092624:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677471.092625:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.092626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:1.0:1619677471.093192:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.093195:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.093196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.093198:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.093201:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.093203:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.093204:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.093206:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.093207:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.093208:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.093211:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.093213:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.093215:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.093216:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.093217:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.093218:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.093219:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.093219:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.093219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.094585:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.094588:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.094589:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.094589:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.094591:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.094594:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.094732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677471.094776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:18.0:1619677471.094781:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.094783:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.094784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.094785:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.094787:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.094791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.094987:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.094990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.094990:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.094991:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.094993:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.094996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.095005:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.095008:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.095008:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.095009:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.095010:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.095010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.095745:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.095748:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.095749:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.095749:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.095752:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.095754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.096093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.096098:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.096101:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.096101:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.096104:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.096106:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.096108:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.096109:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.096111:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.096112:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.096112:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.096116:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.096117:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.096120:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.096121:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.096122:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.096124:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.096124:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.096125:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.096125:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:1.0:1619677471.096761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.096764:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.096765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.096766:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.096769:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.096771:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.096771:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.096773:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.096774:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.096775:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.096777:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.096779:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.096782:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.096783:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (23)++
00000800:00000200:1.0:1619677471.096784:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:1.0:1619677471.096786:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677471.096786:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.096787:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.096787:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:12.0:1619677471.096829:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.096833:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:13.0:1619677471.096837:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.096837:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677471.096839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:12.0:1619677471.096840:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.096843:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677471.096845:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96519319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.096845:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:12.0:1619677471.096847:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677471.096848:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.096849:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:12.0:1619677471.096849:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.096851:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677471.096852:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677471.096856:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.096859:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677471.096861:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.096862:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:12.0:1619677471.096864:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.096865:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:2.0:1619677471.096865:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677471.096866:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:2.0:1619677471.096866:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.096867:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677471.096868:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.096868:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:12.0:1619677471.096869:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.096871:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.096872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:2.0:1619677471.096872:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.096873:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.096877:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.096878:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.096879:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.096881:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.096882:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.096883:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.096883:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.096884:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.096884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:18.0:1619677471.096885:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.096885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.096887:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.096891:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.096892:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.096894:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.096894:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.096895:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.096896:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.096897:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.097014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.097019:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.097022:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.097023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.097025:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.097027:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.097028:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.097030:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.097032:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.097033:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.097034:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.097037:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.097039:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.097042:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.097043:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.097043:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.097045:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.097046:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.097047:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.097048:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.097049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.097054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.097060:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.097062:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.097063:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.097063:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.097437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.097494:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.097496:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.097497:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.097497:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.097499:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.097502:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.097514:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.097517:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.097518:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.097518:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.097519:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.097519:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.098661:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.098664:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.098665:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.098665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.098667:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.098670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.098854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.098857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.098857:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.098860:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.098862:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.098863:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.098864:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.098866:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.098867:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.098868:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.098871:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.098873:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.098875:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.098876:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.098876:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.098878:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.098878:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.098879:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.098880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677471.099062:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.099066:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.099068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.099068:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.099071:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.099076:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.099150:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.099153:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.099154:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.099156:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.099158:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.099160:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677471.099161:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.099162:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.099163:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.099165:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677471.099168:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.099170:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.099171:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.099172:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.099173:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.099174:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.099175:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.099175:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.099176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677471.099187:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.099190:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.099190:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.099191:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.099192:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.099193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.099923:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.099925:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.099926:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.099926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.099928:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.099931:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677471.101092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.101099:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.101102:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.101102:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.101105:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.101107:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.101109:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.101110:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.101112:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.101113:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.101114:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.101117:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.101119:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.101122:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.101123:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.101123:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.101125:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.101126:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.101127:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.101127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677471.101452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.101457:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.101460:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.101461:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.101463:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.101466:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.101468:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.101470:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.101471:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.101472:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.101473:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.101476:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.101477:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.101479:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.101480:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.101481:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.101483:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.101483:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.101484:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.101485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.102056:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.102059:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.102060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.102062:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.102064:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.102066:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677471.102067:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.102068:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.102069:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.102070:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677471.102073:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.102075:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677471.102078:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.102079:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677471.102079:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677471.102081:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677471.102082:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.102082:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.102083:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677471.103849:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.103852:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.103853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.103855:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.103858:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.103859:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.103860:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.103862:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.103863:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.103864:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.103867:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.103869:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.103872:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.103873:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.103874:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.103876:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.103876:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.103877:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.103877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.103962:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.103965:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.103965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.103966:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.103968:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.103972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.104721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.104728:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.104731:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.104732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.104734:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.104737:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.104738:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.104740:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.104742:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.104743:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.104745:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.104749:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.104753:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.104755:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.104756:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.104756:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.104758:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.104758:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.104759:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.104760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.104771:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.104773:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.104774:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.104774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.104776:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.104778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.104899:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.104902:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.104903:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.104903:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.104905:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.104909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.104918:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.104921:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.104921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.104923:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.104924:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.104924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:13.0:1619677471.105868:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.105871:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.105871:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.105873:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.105875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.105876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677471.105974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.105980:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.105983:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.105984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.105986:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.105988:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.105990:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.105991:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.105993:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.105994:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.105995:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.105998:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.106000:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.106003:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.106004:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.106004:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.106006:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.106007:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.106008:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.106008:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677471.106060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:18.0:1619677471.106066:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.106069:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:2.0:1619677471.106070:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:18.0:1619677471.106072:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.106074:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.106076:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677471.106077:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.106078:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.106080:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.106081:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677471.106084:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.106086:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677471.106089:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.106089:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677471.106090:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677471.106092:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.106092:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.106093:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (22)--
00000800:00000200:18.0:1619677471.106093:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:12.0:1619677471.106746:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.106748:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.106749:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.106750:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.106753:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.106754:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677471.107263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.107302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.107780:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.107783:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.107784:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.107785:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.107787:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.107790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.107889:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.107891:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.107893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.107893:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.107894:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.107897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.108026:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.108029:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.108030:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.108030:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.108032:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.108034:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.108046:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.108049:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.108050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.108050:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.108051:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.108052:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.108512:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.108515:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.108515:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.108517:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.108519:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.108521:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.108522:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.108523:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.108525:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.108525:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.108529:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.108531:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.108533:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.108534:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.108535:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.108537:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.108537:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.108539:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.108540:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.108540:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.108551:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.108552:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.108553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.108554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.109679:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.109682:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.109683:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.109684:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.109686:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.109690:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.111009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (20)++
00000800:00000200:2.0:1619677471.111015:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.111019:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.43@o2ib3
00000800:00000200:18.0:1619677471.111020:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000400:00000200:2.0:1619677471.111022:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.43@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.111024:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.111026:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677471.111027:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.111029:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.111030:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.111031:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677471.111035:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.43@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.111037:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677471.111040:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.111041:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677471.111042:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677471.111044:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677471.111045:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dfbd200] (21)++
00000800:00000200:2.0:1619677471.111047:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dfbd200] (22)++
00000800:00000200:2.0:1619677471.111048:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dfbd200] (23)--
00000800:00000200:2.0:1619677471.111048:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:5.2:1619677471.111053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dfbd200] (21)++
00000800:00000200:18.0:1619677471.111059:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)++
00000800:00000200:18.0:1619677471.111061:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dfbd200] (23)--
00000800:00000200:18.0:1619677471.111062:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (22)--
00000800:00000200:2.0:1619677471.111063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dfbd200] (21)--
00000800:00000200:20.2:1619677471.111150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.111156:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.111159:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.111160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.111162:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.111165:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.111166:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.111168:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.111169:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.111171:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.111172:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.111175:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.111177:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.111178:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.111179:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.111179:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.111181:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.111182:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.111183:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.111183:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.111194:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.111196:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.111197:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.111198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.111199:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.111200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.111217:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.111219:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.111220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.111221:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.111221:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.111222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.111917:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.111920:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.111921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.111922:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.111924:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.111927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.112765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677471.112810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.112817:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.112820:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.112820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.112823:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.112826:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.112828:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.112830:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.112832:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.112833:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.112834:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.112838:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.112840:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.112844:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.112845:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.112846:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.112848:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.112849:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.112850:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.112851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677471.112986:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.112989:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.112990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.112990:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.112992:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.112996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.113082:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.113344:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.113347:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.113348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.113350:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.113352:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.113353:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.113355:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.113356:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.113357:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.113358:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.113362:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.113363:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.113366:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.113367:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.113367:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.113369:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.113369:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.113370:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.113370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.113434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.113441:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.113444:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.113445:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.113446:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.113450:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.113451:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.113453:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.113454:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.113456:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.113457:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.113461:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.113463:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.113466:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.113467:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.113468:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.113470:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.113472:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.113473:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.113473:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.114210:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.114212:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.114214:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.114214:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.114216:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.114219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.114312:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.114314:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.114315:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.114318:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.114320:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.114322:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677471.114323:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.114324:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.114325:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.114326:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677471.114330:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.114332:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677471.114335:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.114336:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677471.114336:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677471.114338:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677471.114339:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.114340:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.114341:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.114341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.114354:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.114356:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.114357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.114358:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677471.114694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.114697:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.114698:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.114699:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.114701:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.114705:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.116568:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.116571:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.116572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.116572:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.116574:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.116577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.117300:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.117303:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.117304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.117305:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.117307:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.117309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.117786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.117793:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.117797:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.117798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.117800:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.117803:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.117805:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.117806:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.117808:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.117809:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.117810:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.117814:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.117816:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.117820:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.117821:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.117821:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.117823:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.117824:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.117826:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.117828:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.117828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.117833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.117840:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.117842:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.117843:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.117843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.117992:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.117995:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.117996:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.117998:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.118001:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.118002:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.118003:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.118006:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.118007:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.118008:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.118011:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.118013:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.118015:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.118016:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.118017:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.118018:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.118019:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.118020:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.118020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.118892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.118900:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.118903:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.118904:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.118908:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.118911:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.118912:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.118914:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.118916:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.118917:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.118918:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.118921:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.118923:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.118925:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.118926:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.118926:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.118928:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.118929:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.118931:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.118932:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.118932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.118937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.118943:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.118945:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.118945:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.118946:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677471.118946:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.118947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.118948:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.118949:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.118950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.118950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.118963:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.118966:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.118966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.118968:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.118969:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.118969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.120604:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.120607:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.120608:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.120609:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.120612:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.120619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.120620:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.120623:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.120624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.120624:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.120625:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.120626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.122630:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.122633:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.122634:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.122634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.122637:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.122639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.122652:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.122655:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.122655:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.122656:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.122657:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.122657:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.122658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.122665:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.122668:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.122668:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.122670:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.122673:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.122675:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.122676:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.122678:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.122679:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.122680:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.122684:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.122686:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.122688:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.122689:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.122690:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.122692:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.122693:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.122694:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.122694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.123194:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.123197:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.123198:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.123200:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.123202:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.123204:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.123205:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.123207:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.123208:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.123209:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.123212:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.123214:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.123216:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.123217:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.123218:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.123220:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.123220:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.123221:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.123221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677471.123930:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.123933:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.123935:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.123935:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.123937:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.123941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.126113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.126121:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.126123:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.126124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.126126:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.126129:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.126130:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.126131:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.126133:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.126134:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.126136:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.126139:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.126141:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.126143:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.126144:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.126145:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.126148:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.126148:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.126149:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.126150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.126161:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.126163:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.126164:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.126164:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.126166:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.126168:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.126478:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:18.0:1619677471.126481:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:2.0:1619677471.126482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:18.0:1619677471.126484:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.126486:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.126488:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677471.126489:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.126491:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.126492:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.126493:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677471.126497:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.126499:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677471.126502:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.126502:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677471.126503:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677471.126505:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677471.126506:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:18.0:1619677471.126507:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:18.0:1619677471.126507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:20.2:1619677471.126955:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.2:1619677471.126993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:2.0:1619677471.128046:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.128049:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.128050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.128050:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.128053:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.128056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.128328:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.128331:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.128332:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.128332:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.128335:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.128338:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.128466:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.128468:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.128469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.128470:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.128472:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.128475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.128974:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.128977:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.128977:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.128979:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.128981:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.128982:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.128984:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.128985:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.128986:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.128987:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.128989:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.128992:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.128994:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.128995:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.128996:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.128997:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.128998:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.128999:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.129000:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.129001:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.129012:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.129013:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.129014:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.129015:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677471.129312:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.129315:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.129316:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.129316:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.129318:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.129322:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.129982:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.129985:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.129986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.129987:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.129989:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.129993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.130005:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.130008:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.130009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.130009:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.130011:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.130012:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.130105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.2:1619677471.130110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.130114:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.130116:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.130117:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:18.0:1619677471.130119:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677471.130119:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:2.0:1619677471.130121:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.130122:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.130124:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.130125:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.130127:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.130128:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.130130:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.130132:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.130134:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677471.130134:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.130136:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677471.130137:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.130138:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.130139:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.130140:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.130144:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.130145:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:2.0:1619677471.130145:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.130146:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.130147:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.130149:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.130150:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.130150:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.130151:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.130151:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.130152:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.130152:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.130153:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.130154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.130154:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.130155:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.130156:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.130156:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677471.132448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.132455:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.132458:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.132459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.132462:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.132465:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.132466:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.132468:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.132469:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.132470:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.132472:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.132476:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.132479:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.132481:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.132482:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.132483:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.132485:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.132486:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.132487:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.132487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.132498:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.132500:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.132501:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.132502:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.132503:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.132505:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.132612:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.132615:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.132616:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.132618:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.132621:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.132623:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677471.132624:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.132627:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.132628:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.132629:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677471.132633:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.132635:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677471.132638:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.132639:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:17.0:1619677471.132640:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677471.132642:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:17.0:1619677471.132642:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.132643:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.132643:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677471.133716:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.133719:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.133720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.133720:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.133723:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.133726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.133878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.133922:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.133924:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.133925:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.133925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.133927:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.133930:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.133938:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677471.133942:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.2.45@o2ib3
00000800:00000200:2.0:1619677471.133942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:18.0:1619677471.133945:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.133947:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.133949:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:18.0:1619677471.133951:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.133953:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.133954:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.133956:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000800:00000200:2.0:1619677471.133956:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)++
00000800:00000200:2.0:1619677471.133958:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000400:00000200:18.0:1619677471.133961:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677471.133961:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.133963:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.133964:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000400:00000200:2.0:1619677471.133965:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677471.133966:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.133970:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.133971:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.133972:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677471.133976:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.133978:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.133980:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.133981:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.133981:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677471.133982:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:18.0:1619677471.133982:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:2.0:1619677471.133982:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.133984:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.133985:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (23)++
00000800:00000200:18.0:1619677471.133986:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:18.0:1619677471.133987:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (24)++
00000800:00000200:18.0:1619677471.133987:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (24)--
00000800:00000200:2.0:1619677471.133987:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (24)--
00000800:00000200:2.0:1619677471.133987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (23)--
00000800:00000200:18.0:1619677471.133988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000800:00000200:2.0:1619677471.133989:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:24.2:1619677471.133992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:18.0:1619677471.133998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.133999:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.134000:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.134001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.134002:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.134004:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.134014:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677471.134017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000800:00000200:2.0:1619677471.134017:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000400:00000200:2.0:1619677471.134020:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.134023:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.134025:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677471.134026:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.134028:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.134030:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.134030:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.134034:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.134036:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.134039:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.134040:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677471.134040:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677471.134043:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677471.134044:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677471.134045:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:2.0:1619677471.134045:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:18.0:1619677471.134093:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677471.134096:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:2.0:1619677471.134097:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:18.0:1619677471.134099:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.134103:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.134105:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677471.134107:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.134107:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:18.0:1619677471.134109:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677471.134109:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.134110:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.134111:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.134112:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677471.134113:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.134116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.134117:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.134118:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.134119:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677471.134120:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.134121:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.134122:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.134122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.134123:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.134124:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677471.134124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.134125:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677471.134128:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677471.134129:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677471.134130:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907a3e8400] (22)++
00000800:00000200:18.0:1619677471.134131:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (23)--
00000800:00000200:18.0:1619677471.134132:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000800:00000200:12.0:1619677471.134142:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677471.134143:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)++
00000800:00000200:12.0:1619677471.134145:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:2.0:1619677471.134145:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907a3e8400] (23)--
00000800:00000200:2.0:1619677471.134146:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:12.0:1619677471.134147:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.134147:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000400:00000200:12.0:1619677471.134148:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.134150:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:18.0:1619677471.134151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000800:00000200:12.0:1619677471.134151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.134152:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:2.0:1619677471.134153:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.134156:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.134158:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677471.134160:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.134161:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.134162:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.134163:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677471.134164:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677471.134167:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677471.134167:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.134168:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.134169:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677471.134169:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677471.134169:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677471.134170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677471.134172:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.134173:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677471.134174:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677471.134176:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677471.134176:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677471.134177:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:2.0:1619677471.134178:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:1.0:1619677471.134189:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.134192:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.134193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.134193:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.134195:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.134197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.134422:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677471.134425:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:2.0:1619677471.134426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:18.0:1619677471.134428:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.134430:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.134432:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677471.134433:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.134435:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.134436:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.134437:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677471.134440:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.134442:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.134443:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.134444:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677471.134444:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.134446:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.134447:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677471.134447:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:18.0:1619677471.134448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:2.0:1619677471.134460:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.134462:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.134462:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.134463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.134464:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.134466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.134508:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677471.134511:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:2.0:1619677471.134512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:18.0:1619677471.134514:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.134515:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.134517:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677471.134518:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.134519:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.134520:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.134521:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677471.134524:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.134526:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677471.134528:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.134529:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:18.0:1619677471.134529:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:18.0:1619677471.134532:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:18.0:1619677471.134532:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677471.134533:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:18.0:1619677471.134533:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:17.0:1619677471.134547:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.134549:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.134550:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.134550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.134552:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.134554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.134568:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:20.2:1619677471.134569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.134570:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:18.0:1619677471.134573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:2.0:1619677471.134573:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677471.134575:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.134576:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677471.134576:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.134577:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677471.134579:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.134581:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.134583:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.134584:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677471.134587:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.134588:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677471.134590:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.134590:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677471.134591:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677471.134592:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677471.134593:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.134593:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677471.134593:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:18.0:1619677471.134595:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.134596:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677471.134596:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:18.0:1619677471.134597:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.134598:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677471.134599:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907a3e8400] (22)++
00000800:00000200:2.0:1619677471.134600:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (23)--
00000800:00000200:2.0:1619677471.134601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:18.0:1619677471.134602:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.134602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.134604:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.134607:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.134608:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677471.134609:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.134609:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:1.0:1619677471.134611:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677471.134612:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:17.0:1619677471.134612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677471.134612:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)++
00000400:00000200:1.0:1619677471.134612:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.134613:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.134613:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907a3e8400] (23)--
00000800:00000200:18.0:1619677471.134614:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.134614:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:1.0:1619677471.134614:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.134615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.134615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:1.0:1619677471.134616:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.134628:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.134629:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.134630:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.134632:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.134632:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.134633:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.134634:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.134635:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:1.0:1619677471.134635:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677471.134636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.134637:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.134640:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.134641:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.134642:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.134643:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.134644:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.134648:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.134650:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.134652:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.134653:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.134653:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.134655:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.134655:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.134656:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.134657:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.135150:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.135153:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.135153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.135154:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.135156:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.135159:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.135819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.135861:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.135864:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.135865:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.135865:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.135867:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.135869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.135882:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.135885:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.135885:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.135886:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.135887:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.135887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.136055:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.136058:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.136059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.136059:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.136061:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.136064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.136258:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.136260:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.136261:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.136263:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.136265:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.136267:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677471.136268:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.136269:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.136270:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.136272:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677471.136275:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.136277:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.136278:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.136279:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.136280:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.136281:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.136282:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.136282:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.136283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677471.136294:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.136301:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.136304:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.136304:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.136306:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.136307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.139366:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.139369:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.139371:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.139371:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.139374:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.139378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677471.139576:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.139578:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.139580:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.139583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.139583:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.139584:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677471.139585:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96519118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.139587:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.139587:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.139588:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677471.139591:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.139593:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677471.139595:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.139596:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:12.0:1619677471.139596:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:12.0:1619677471.139598:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:12.0:1619677471.139598:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:20.2:1619677471.139599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:12.0:1619677471.139599:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.139600:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677471.139605:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.139608:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.139609:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.139611:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.139614:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.139616:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.139618:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.139619:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.139621:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.139622:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.139625:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.139628:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.139631:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.139632:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.139632:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.139634:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.139636:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.139637:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.139639:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.139639:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.139644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.139651:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.139653:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.139654:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.139654:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.140187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.140190:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.140191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.140192:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.140194:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.140198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.140423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.2:1619677471.140529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.140563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.141262:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.141265:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.141266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.141268:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.141270:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.141271:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.141272:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.141274:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.141275:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.141276:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.141279:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.141281:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.141283:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.141283:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.141284:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.141286:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.141286:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.141287:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.141287:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677471.141700:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.141703:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.141704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.141704:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.141706:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.2:1619677471.141707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677471.141710:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.141819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.141825:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.141828:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.141829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.141831:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.141834:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.141835:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.141837:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.141839:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.141840:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.141842:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.141845:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.141847:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.141850:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.141851:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.141852:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.141853:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.141854:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.141855:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.141856:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.141867:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.141869:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.141870:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.141871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.141872:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.141874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.141884:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.141887:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.141888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.141889:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.141890:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.141890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.142779:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.142782:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.142783:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.142783:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.142785:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.142788:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.142799:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.142803:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.142803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.142804:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.142805:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.142806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.143744:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.143746:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.143747:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.143748:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.143750:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.143752:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.144095:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (23)++
00000800:00000200:18.0:1619677471.144099:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (23)++
00000800:00000200:2.0:1619677471.144099:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (24)--
00000400:00000200:2.0:1619677471.144100:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.144101:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (24)--
00000400:00000200:18.0:1619677471.144102:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.144102:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677471.144104:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.144107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.144107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.144108:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.144110:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.144112:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.144112:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.144113:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.144114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.145851:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.145854:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.145855:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.145855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.145857:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.145860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.146132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.146139:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.146143:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.146144:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.146146:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.146148:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.146150:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.146151:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.146153:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.146155:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.146156:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.146159:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.146161:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.146164:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.146165:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.146166:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.146168:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.146169:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.146170:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.146171:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.146466:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.146469:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.146470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.146472:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.146474:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.146475:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.146477:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.146479:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.146480:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.146481:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.146484:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.146485:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.146488:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.146489:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.146489:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.146491:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.146491:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.146492:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.146493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.146606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.146646:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.146648:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.146649:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.146649:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.146651:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.146654:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.147554:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.147556:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.147558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.147558:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.147560:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.147563:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.148756:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.148759:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.148759:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.148760:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.148762:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.148765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.148836:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.148839:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.148840:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.148841:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.148843:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.148847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.149483:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.149487:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.149489:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.149489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.149493:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.149497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.150074:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.150077:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.150078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.150078:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.150081:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.150084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.150093:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.150096:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.150097:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.150100:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.150102:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.150103:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677471.150105:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.150106:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.150107:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.150108:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677471.150111:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.150113:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677471.150117:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.150118:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677471.150119:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677471.150121:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677471.150122:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.150123:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677471.150124:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677471.150125:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.150135:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:13.0:1619677471.150135:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:17.0:1619677471.150137:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677471.150138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:13.0:1619677471.150138:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:17.0:1619677471.150139:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:13.0:1619677471.150139:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677471.150141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:13.0:1619677471.150141:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.150144:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.150146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:18.0:1619677471.150148:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.150151:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.150151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.150154:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.150156:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.150158:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.150159:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.150161:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.150162:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.150164:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.150167:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.150169:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.150171:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.150172:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.150172:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.150174:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.150175:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.150177:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.150178:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.150178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.150183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.150189:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.150191:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.150191:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.150191:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.150192:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.150192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.150193:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.150194:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.150195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.150195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.150982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.150989:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.150992:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.150993:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.150995:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.150997:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.150999:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.151000:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.151002:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.151003:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.151004:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.151008:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.151010:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.151012:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.151013:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.151014:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.151016:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.151017:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.151018:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.151018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.151390:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.151392:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.151393:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.151394:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.151396:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.151399:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.151408:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.151410:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.151411:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.151411:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.151412:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.151412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.152382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.152426:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.152429:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.152430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.152430:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.152432:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.152434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.153042:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.153045:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.153045:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.153048:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.153051:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.153052:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.153054:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.153055:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.153056:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.153057:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.153061:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.153062:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.153065:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.153065:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.153066:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.153068:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.153068:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.153069:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.153070:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.153071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.153082:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.153084:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.153084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.153085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677471.153184:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.153187:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.153189:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.153189:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.153191:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.153195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.153540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677471.153546:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.153549:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677471.153550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677471.153552:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677471.153554:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.153555:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677471.153557:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.153558:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.153559:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.153560:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677471.153562:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677471.153564:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677471.153566:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:17.0:1619677471.153566:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.153566:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.153567:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:18.0:1619677471.153569:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:17.0:1619677471.153569:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.153569:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000400:00000200:18.0:1619677471.153570:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.153570:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677471.153570:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:2.0:1619677471.153570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.153572:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.153576:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.153593:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.153595:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.153596:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.153596:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.153598:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677471.153599:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.153615:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.153618:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.153618:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.153619:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.153619:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.153620:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.154762:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.154765:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.154766:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.154766:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.154768:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.154771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.155396:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.155399:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.155399:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.155400:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.155402:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.155404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.155504:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.155507:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.155509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.155509:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.155512:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.155516:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.155525:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.155527:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.155528:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.155529:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.155530:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.155531:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.156778:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.156781:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.156782:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.156782:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.156784:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.156788:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.157037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.157043:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.157046:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.157047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.157049:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.157052:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.157053:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.157055:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.157057:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.157058:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.157059:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.157063:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.157065:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.157067:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.157068:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.157069:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.157072:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.157072:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.157073:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.157074:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.158221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.158227:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.158230:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.158231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.158234:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.158237:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.158238:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.158240:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96519032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.158241:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.158243:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.158244:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.158247:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.158249:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.158252:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.158253:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.158254:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.158255:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.158256:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.158257:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.158258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.158269:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.158271:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.158272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.158272:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.158274:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.158277:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.158305:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.158308:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.158309:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.158310:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.158312:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.158317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.158319:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.158321:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.158322:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.158323:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.158324:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.158324:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.158551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.158588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.158595:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.158597:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.158598:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.158598:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.158600:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.158602:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:12.0:1619677471.159425:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.159429:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.159431:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677471.159433:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.159434:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.159435:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677471.159436:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96519025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.159438:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.159439:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.159440:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677471.159443:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.159445:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677471.159447:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.159448:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677471.159449:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677471.159450:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677471.159451:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.159452:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.159453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677471.159754:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.159757:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.159757:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677471.159758:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.159760:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.159763:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.159772:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.159774:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.159775:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.159775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.159776:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.159776:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.160292:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.160296:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.160298:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.160300:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.160301:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.160304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.160401:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.160404:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.160405:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.160405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.160407:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.160410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.160531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.160572:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.160575:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.160576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.160576:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.160578:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.160580:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.160623:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.160627:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.160627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.160630:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.160631:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.160632:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.160633:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.160635:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.160636:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.160637:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.160639:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.160641:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.160643:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.160644:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.160645:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.160647:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.160647:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.160648:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.160648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:13.0:1619677471.160991:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.160994:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.160996:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677471.160998:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677471.160998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.160999:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:1.0:1619677471.161000:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:13.0:1619677471.161001:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96519014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.161002:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.161003:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677471.161003:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.161004:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:13.0:1619677471.161004:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677471.161004:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.161007:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:13.0:1619677471.161009:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.161011:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677471.161011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:13.0:1619677471.161013:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.161014:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677471.161014:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677471.161016:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677471.161017:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.161018:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677471.161020:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677471.161020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.161034:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677471.161036:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677471.161037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.161038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677471.161246:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.161250:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.161251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.161251:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.161254:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.161258:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.161633:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.161636:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.161636:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.161637:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.161639:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.161642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.162473:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.162475:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.162477:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.162477:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.162479:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.162482:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.162682:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.162687:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.162690:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.162691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.162693:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.162696:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.162698:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.162699:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.162701:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.162703:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.162704:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.162707:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.162710:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.162713:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.162714:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.162715:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.162717:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.162717:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.162719:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.162720:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.162720:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.162725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.162731:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.162734:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.162735:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.162736:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.162807:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.162810:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.162811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.162812:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.162814:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.162818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.163667:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.163670:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.163671:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.163672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.163674:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.163678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.164145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.164151:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.164154:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.164154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.164157:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.164160:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.164161:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677471.164163:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96519003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.164165:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.164166:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.164167:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677471.164171:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.164173:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677471.164176:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.164177:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677471.164178:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677471.164180:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677471.164181:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.164182:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.164183:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:1.0:1619677471.164195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.164198:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.164199:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.164199:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.164201:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.164204:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.164214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.164216:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.164217:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.164217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.164218:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.164219:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.164537:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.164540:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.164542:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.164542:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.164544:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.164548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.164557:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.164560:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.164561:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.164561:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.164562:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.164563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.165038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.165405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.165411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.165414:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.165415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.165417:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.165420:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.165421:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.165423:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.165425:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.165426:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.165427:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.165431:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.165433:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.165435:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.165436:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.165437:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.165439:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.165440:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.165441:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.165441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.165452:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.165454:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.165455:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.165456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.165457:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.165458:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.165766:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.165769:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.165770:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.165772:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.165775:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.165776:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677471.165777:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.165779:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.165780:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.165781:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677471.165784:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.165786:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677471.165788:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.165789:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677471.165790:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677471.165792:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.165792:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.165793:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.165794:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677471.165805:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.165808:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.165809:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.165809:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.165811:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.165814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.166411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.166451:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.166454:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.166455:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.166455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.166460:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.166466:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.166467:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.166469:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.166470:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.166470:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.166471:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.166471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.166768:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.166770:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.166771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.166773:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.166775:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.166776:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.166777:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.166778:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.166779:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.166780:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.166783:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.166785:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.166788:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.166788:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.166789:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677471.166791:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.166791:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.166791:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.166792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677471.167045:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.167048:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.167049:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.167049:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.167052:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.167055:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.167202:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.167205:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.167206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.167208:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.167211:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.167212:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.167213:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.167214:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.167216:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.167216:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.167220:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.167222:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.167224:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.167225:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.167226:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.167227:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.167228:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.167229:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.167229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.167895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.167902:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.167905:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.167906:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.167908:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.167910:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.167912:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.167913:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.167915:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.167916:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.167917:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.167921:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.167923:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.167926:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.167927:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.167928:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.167931:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.167932:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.167933:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.167934:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.168465:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.168468:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.168470:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.168470:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.168472:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.168477:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.168486:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.168489:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.168490:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.168490:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.168491:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.168492:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.169625:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.169628:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.169629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.169629:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.169631:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.169635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.170922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.170958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677471.171221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.171228:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.171231:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.171231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.171234:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.171236:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.171238:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.171239:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.171241:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.171242:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.171244:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.171247:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.171249:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.171251:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.171252:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.171253:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.171255:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.171256:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.171259:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.171260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.171268:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.171270:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.171271:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.171272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.171273:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.171274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.172141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:17.0:1619677471.172150:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.172153:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.172154:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.172154:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.172158:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.172161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.172653:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.172656:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.172656:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.172658:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.172661:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.172662:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.172664:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.172665:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.172666:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.172667:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.172671:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.172672:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.172675:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.172676:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.172676:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.172678:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.172678:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.172679:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.172679:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.172805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.172843:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.172846:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.172847:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.172847:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.172849:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.172851:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.173245:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.173248:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.173249:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.173250:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.173252:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.173260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.173262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.173264:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.173265:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.173265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.173267:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.173268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.173278:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.173281:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.173282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.173282:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.173284:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.173287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.174501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.174508:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.174511:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.174512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.174514:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.174516:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.174517:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.174519:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.174521:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.174522:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.174523:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.174527:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.174529:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.174531:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.174532:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.174533:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.174535:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.174536:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.174537:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.174538:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.176057:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.176060:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.176061:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.176062:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.176064:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.176067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.176631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.177141:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.177144:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.177145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.177145:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.177147:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.177150:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.178111:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.178113:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.178115:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.178115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.178118:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.178121:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.178331:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.178334:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.178335:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.178335:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.178338:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.178340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.178349:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.178351:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.178353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.178353:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.178354:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.178354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.178562:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.178566:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.178566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.178567:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.178569:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.178572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.178720:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.178723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.178723:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.178725:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.178727:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.178729:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.178730:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.178731:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.178732:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.178734:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.178737:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.178739:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.178741:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.178742:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.178743:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.178744:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.178745:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.178746:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.178747:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.178748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.178758:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.178760:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.178761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.178762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.178810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.178816:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.178819:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.178820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.178822:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.178824:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.178826:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.178827:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.178829:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.178831:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.178832:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.178836:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.178838:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.178841:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.178843:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.178844:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.178846:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.178847:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.178848:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.178849:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.178881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.178924:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.178927:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.178928:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.178928:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.178930:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.178932:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.179209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.179215:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.179219:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.179219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.179222:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.179225:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.179226:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.179228:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.179230:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.179231:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.179233:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.179237:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.179240:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.179242:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.179243:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.179244:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.179246:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.179247:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.179249:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.179251:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.179251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.179256:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.179261:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.179263:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.179263:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.179264:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677471.179264:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.179265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.179266:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.179267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.179268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.179269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:12.0:1619677471.181341:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.181343:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.181344:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.181345:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.181348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.181349:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677471.181879:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.181882:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.181884:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677471.181886:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.181887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.181887:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:12.0:1619677471.181888:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96518925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.181890:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.181891:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.181892:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:12.0:1619677471.181895:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.181896:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677471.181899:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.181900:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:12.0:1619677471.181900:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:12.0:1619677471.181902:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:12.0:1619677471.181903:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.181904:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.181904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677471.181944:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.181948:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.181950:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.181950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.181954:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.181959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.182420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677471.182442:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.182445:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.182446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.182446:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.182449:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.182456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.2:1619677471.182458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677471.182458:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.182460:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.182461:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.182461:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.182463:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.182463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.182926:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.182929:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.182930:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.182930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.182933:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.182936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.183300:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.183302:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.183303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.183304:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.183306:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.183309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.183495:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.183498:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.183499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.183503:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.183505:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.183507:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.183508:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.183510:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.183512:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.183513:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.183518:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.183521:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.183524:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.183525:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.183527:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.183528:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.183529:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.183530:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.183531:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.184814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.184820:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.184823:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.184825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.184826:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.184829:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.184831:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.184833:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.184835:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.184836:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.184837:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.184841:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.184843:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.184847:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.184848:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.184849:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.184851:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.184852:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.184854:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.184855:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.184856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.184862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.184868:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.184870:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.184871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.184872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.185274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.185322:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.185324:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.185325:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.185325:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.2:1619677471.185326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:17.0:1619677471.185327:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.185330:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.185332:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.185335:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.185336:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.185338:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.185341:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.185343:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.185344:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.185346:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.185347:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.185348:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.185352:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.185354:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.185356:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.185357:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.185358:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.185360:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.185361:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.185362:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.185363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.185374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.185376:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.185377:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.185378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.185379:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.185380:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.186268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.186274:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.186278:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.186279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.186281:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.186283:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.186285:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677471.186286:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.186289:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.186290:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.186291:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677471.186301:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.186304:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677471.186307:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.186308:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677471.186309:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677471.186311:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677471.186312:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.186313:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677471.186314:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677471.186315:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677471.186320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.186326:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677471.186329:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677471.186330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677471.186330:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:1.0:1619677471.186388:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.186391:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.186392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.186394:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.186396:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.186397:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677471.186399:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.186400:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.186401:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.186402:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677471.186404:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.186406:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.186408:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.186408:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.186409:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.186411:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.186411:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.186412:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.186413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677471.186425:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.186427:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.186428:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.186428:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.186429:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.186431:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.186446:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.186448:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.186449:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.186449:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.186449:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.186450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.186610:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.186612:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.186614:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.186614:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.186617:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.186622:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.186624:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.186626:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.186627:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.186628:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.186628:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.186629:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.187782:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.187785:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.187786:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.187787:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.187789:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.187792:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.187924:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.187926:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.187927:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.187929:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.187931:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.187933:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677471.187934:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.187936:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.187937:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.187938:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677471.187942:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.187944:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677471.187947:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.187948:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677471.187948:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677471.187950:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677471.187951:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.187952:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.187952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677471.188119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.188376:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.188379:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.188380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.188382:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.188385:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.188387:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.188388:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.188389:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.188390:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.188391:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.188395:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.188396:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.188399:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.188400:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.188400:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.188402:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.188402:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.188403:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.188404:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.188826:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.188829:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.188830:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.188830:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.188832:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.188835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.189057:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.189060:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.189061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.189061:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.189063:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.189066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.190634:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.190637:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.190638:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.190638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.190641:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.190645:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.191233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.191274:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.191276:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.191277:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.191278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.191279:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.191282:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.191631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.191638:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.191641:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.191642:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.191648:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.191650:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.191652:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.191653:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.191655:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.191656:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.191657:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.191661:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.191663:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.191667:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.191667:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.191668:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.191669:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.191669:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:18.0:1619677471.191670:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.191672:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.191672:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.191674:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.191675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.191675:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.191676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.191677:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.191766:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.191769:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.191770:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.191772:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.191774:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.191775:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677471.191776:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.191778:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.191778:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.191779:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677471.191782:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.191784:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677471.191786:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.191787:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677471.191788:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677471.191789:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677471.191790:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.191791:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.191791:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:20.2:1619677471.191798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.191804:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.191805:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.191806:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.191807:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677471.191807:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.191809:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677471.191810:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.191813:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.191813:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.191815:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677471.191815:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:18.0:1619677471.191816:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.191816:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.191818:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.191819:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.191820:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.191824:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.191825:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.191827:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.191828:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.191829:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.191831:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.191831:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.191832:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.191833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.191843:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.191845:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.191846:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.191847:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.191848:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.191849:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.191863:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.191866:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.191867:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.191867:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.191867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.191868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.192303:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.192306:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.192307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.192307:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.192310:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.192313:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.192572:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.192575:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.192576:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.192576:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.192578:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.192582:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.192591:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.192593:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.192594:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.192595:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.192596:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.192597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.192691:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.192694:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.192694:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.192696:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.192698:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.192699:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.192701:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.192702:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.192703:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.192704:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.192707:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.192709:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.192712:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.192713:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.192713:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.192715:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.192716:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.192716:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.192717:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.193339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.193378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.193746:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.193749:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.193750:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.193751:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.193753:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.193756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.193765:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.193768:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.193769:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.193769:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.193770:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.193770:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.195141:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.195144:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.195145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.195145:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.195148:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.195151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.195846:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.195849:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.195850:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.195850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.195853:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.195856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.196228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.196234:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.196237:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.196238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.196240:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.196242:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.196244:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.196245:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.196248:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.196249:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.196251:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.196255:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.196257:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.196259:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.196260:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.196261:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.196263:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.196264:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.196265:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.196266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.196397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.196404:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.196407:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.196408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.196410:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.196412:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.196414:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677471.196415:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.196417:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.196418:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.196419:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677471.196422:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.196424:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677471.196428:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.196429:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677471.196430:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677471.196432:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677471.196432:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.196433:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.196434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:13.0:1619677471.196448:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.196451:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.196453:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.196455:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.196456:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.196459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677471.197261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.197306:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.197308:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.197309:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.197310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.197312:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.197314:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.197676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.197682:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.197685:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.197686:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.197689:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.197693:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.197694:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.197695:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.197697:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.197698:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.197699:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677471.197703:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:2.0:1619677471.197703:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.197705:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677471.197706:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.197707:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:2.0:1619677471.197708:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.197709:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:17.0:1619677471.197710:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677471.197710:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:17.0:1619677471.197712:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.197712:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:17.0:1619677471.197713:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677471.197713:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.197714:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677471.197715:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.197715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:17.0:1619677471.197716:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.197717:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.197718:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.197721:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.197723:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677471.197726:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:17.0:1619677471.197726:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.197727:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.197727:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677471.197728:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.197729:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.197729:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:2.0:1619677471.197729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677471.197730:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.197731:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000400:00000200:18.0:1619677471.197732:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.197732:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.197733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:18.0:1619677471.197734:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.197744:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.197746:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.197747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.197748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677471.197864:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.197867:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.197868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.197868:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.197871:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.197874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677471.198486:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.198489:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.198490:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.198491:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.198493:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.198495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677471.198892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a464ba8a00] (21)++
00000800:00000200:13.0:1619677471.198895:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.17@o2ib3
00000400:00000200:13.0:1619677471.198898:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.15.17@o2ib3 : GET - for me
00000800:00000200:12.0:1619677471.198898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a464ba8a00] (22)--
00000400:00000200:13.0:1619677471.198902:0:19942:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.15.17@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:13.0:1619677471.198904:0:19942:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.15.17@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:13.0:1619677471.198905:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a464ba8a00] (21)++
00000800:00000200:13.0:1619677471.198907:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a464ba8a00] (22)--
00000400:00000200:13.0:1619677471.198909:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.15.17@o2ib3 local destination
00000800:00000200:13.0:1619677471.198912:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a4773b0000] -> 10.51.15.17@o2ib3 (2) version: 12
00000800:00000200:13.0:1619677471.198913:0:19942:0:(o2iblnd.c:1251:kiblnd_query()) peer_ni 10.51.15.17@o2ib3 ffff91a4773b0000, alive 0 secs ago
00000800:00000200:13.0:1619677471.198914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a464ba8a00] (21)--
00000800:00000200:18.0:1619677471.199158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.199161:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.199162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.199163:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.199165:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.199169:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.199341:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.199344:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.199345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.199345:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.199348:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.199351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.199577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677471.199768:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.199770:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.199771:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.199772:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.199774:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.199779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.199780:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.199783:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.199783:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.199784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.199785:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.199785:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.200591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.200597:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.200600:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.200601:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.200603:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.200606:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.200608:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677471.200610:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.200612:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.200613:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.200614:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677471.200618:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.200620:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677471.200624:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.200625:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677471.200627:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677471.200629:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677471.200630:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.200631:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.200631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677471.200700:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.200702:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.200704:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.200704:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.200706:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.200709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.201738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.201744:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.201747:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.201747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.201750:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.201753:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.201754:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.201756:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.201757:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.201758:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.201760:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.201763:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.201765:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.201768:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.201770:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.201771:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.201773:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.201773:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.201774:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.201775:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677471.202517:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.202519:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.202520:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.202521:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.202523:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.202525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677471.202547:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677471.202549:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677471.202550:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.202551:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677471.202551:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.202552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677471.202659:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.202661:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.202663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.202663:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.202665:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.202669:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.202976:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.202978:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.202979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.202981:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.202984:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.202985:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677471.202986:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.202988:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.202989:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.202990:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677471.202993:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.202995:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677471.202997:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.202998:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677471.202999:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677471.203000:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.203002:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.203003:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677471.203004:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677471.203004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677471.203014:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:17.0:1619677471.203015:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:18.0:1619677471.203017:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:17.0:1619677471.203017:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:18.0:1619677471.203018:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.203018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:2.0:1619677471.203018:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677471.203019:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:18.0:1619677471.203020:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.203022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.203341:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.203344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.203344:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.203347:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.203349:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.203350:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.203352:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.203353:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.203354:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.203355:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.203358:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.203360:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.203362:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.203363:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.203364:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.203366:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.203366:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.203367:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.203367:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.203580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.203624:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.203626:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.203627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.203628:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.203630:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.203633:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.203642:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.203645:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.203646:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.203646:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.203647:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.203648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.205224:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.205227:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.205228:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.205228:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.205230:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.205233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.205242:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.205245:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.205246:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.205246:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.205247:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.205248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.207005:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.207008:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.207010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.207010:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.207012:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.207019:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.207021:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.207024:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.207025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.207025:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.207026:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.207027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.207069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.207076:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.207079:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.207079:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.207082:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.207085:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.207086:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.207087:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.207089:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.207090:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.207091:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.207095:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.207096:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.207099:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.207099:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.207100:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.207102:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.207103:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.207104:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.207105:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.207106:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.207111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.207117:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.207118:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.207118:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.207119:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677471.207119:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.207120:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.207121:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.207121:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.207122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.207123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677471.207618:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.207621:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.207622:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.207622:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.207625:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.207627:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.208122:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.208125:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.208126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.208126:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.208129:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.208132:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.208293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.208299:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.208302:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.208303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.208305:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.208308:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.208309:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.208311:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.208313:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.208314:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.208315:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.208318:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.208320:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.208324:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.208325:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.208326:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.208328:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.208328:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.208330:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.208331:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.208332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.208337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.208342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.208345:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.208346:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.208347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.209264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.209267:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.209268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.209270:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.209272:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.209273:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.209275:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.209276:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.209277:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.209278:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.209281:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.209283:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.209285:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.209286:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.209287:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.209289:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.209289:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.209290:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.209290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.209662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.209698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.209703:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.209706:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.209707:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.209708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.209709:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.209712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.209722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.209729:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.209733:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.209734:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.209737:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.209739:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.209741:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677471.209743:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.209746:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.209747:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.209749:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677471.209754:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.209757:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677471.209760:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.209762:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677471.209762:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677471.209765:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677471.209766:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.209767:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.209768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:5.2:1619677471.210349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (20)++
00000800:00000200:1.0:1619677471.210356:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.210359:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.15@o2ib3
00000800:00000200:17.0:1619677471.210360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000400:00000200:1.0:1619677471.210362:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.15@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.210364:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.210366:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:1.0:1619677471.210368:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.210369:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.210370:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.210371:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:1.0:1619677471.210375:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.210377:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 520 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:1.0:1619677471.210380:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.210381:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:1.0:1619677471.210382:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677471.210384:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:1.0:1619677471.210384:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.210385:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:1.0:1619677471.210385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)--
00000800:00000200:24.2:1619677471.210390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:2.0:1619677471.211401:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.211404:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677471.211406:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677471.211407:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.211411:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.211413:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677471.211414:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.211416:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.211417:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.211419:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.211423:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.211425:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.211429:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.211430:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677471.211431:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677471.211434:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677471.211434:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.211435:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.211436:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:5.2:1619677471.211758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (20)++
00000800:00000200:17.0:1619677471.211764:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:17.0:1619677471.211768:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.15@o2ib3
00000800:00000200:1.0:1619677471.211768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000400:00000200:17.0:1619677471.211771:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.15@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.211774:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.211775:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677471.211777:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.211778:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.211779:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.211780:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677471.211783:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.211785:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677471.211788:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.211789:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677471.211789:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677471.211791:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677471.211792:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:17.0:1619677471.211793:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cffdc7c00] (22)++
00000800:00000200:17.0:1619677471.211794:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc7c00] (23)--
00000800:00000200:17.0:1619677471.211795:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:5.2:1619677471.211800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.211805:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)++
00000800:00000200:1.0:1619677471.211807:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cffdc7c00] (23)--
00000800:00000200:1.0:1619677471.211808:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:1.0:1619677471.211809:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)--
00000800:00000200:24.2:1619677471.211946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:18.0:1619677471.212034:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.212037:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.212038:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.212039:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.212041:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.212045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.212224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (20)++
00000800:00000200:17.0:1619677471.212230:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:17.0:1619677471.212233:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.15@o2ib3
00000800:00000200:1.0:1619677471.212233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000400:00000200:17.0:1619677471.212235:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.15@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.212238:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.212239:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677471.212240:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.212242:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.212243:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.212244:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677471.212247:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.212249:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677471.212251:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.212252:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677471.212253:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677471.212254:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677471.212255:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:17.0:1619677471.212255:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:17.0:1619677471.212256:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)--
00000800:00000200:5.2:1619677471.213316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (20)++
00000800:00000200:1.0:1619677471.213323:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.213326:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.15@o2ib3
00000800:00000200:17.0:1619677471.213327:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000400:00000200:1.0:1619677471.213329:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.15@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.213331:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.213333:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677471.213334:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.213336:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.213336:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.213338:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677471.213341:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.213343:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677471.213345:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.213346:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677471.213347:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677471.213348:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677471.213349:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.213349:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:1.0:1619677471.213350:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)--
00000800:00000200:17.0:1619677471.213441:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.213443:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.213444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.213445:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.213447:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.213450:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.213529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:1.0:1619677471.214281:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.214284:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.214285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.214287:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.214290:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.214292:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677471.214299:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.214300:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.214301:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.214302:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677471.214306:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.214308:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677471.214310:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.214311:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677471.214312:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677471.214314:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677471.214314:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.214316:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.214317:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.214317:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.214330:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.214332:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.214332:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.214333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:5.2:1619677471.214389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (20)++
00000800:00000200:1.0:1619677471.214395:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.214397:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.15@o2ib3
00000800:00000200:17.0:1619677471.214398:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000400:00000200:1.0:1619677471.214400:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.15@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.214403:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.214405:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677471.214406:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.214408:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.214409:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.214409:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677471.214413:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.214415:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677471.214417:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.214418:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677471.214419:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677471.214421:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677471.214421:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.214422:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:1.0:1619677471.214422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)--
00000800:00000200:12.0:1619677471.214434:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.214437:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677471.214438:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.214439:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.214442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.214443:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677471.214452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677471.214452:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.214455:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.214455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.214458:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677471.214458:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677471.214460:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.214461:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.214462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677471.214462:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.214464:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.214465:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.214465:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677471.214466:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.214467:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677471.214468:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.214470:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677471.214470:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677471.214472:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:2.0:1619677471.214472:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677471.214474:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677471.214474:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677471.214475:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.214475:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:2.0:1619677471.214475:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.214476:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677471.214477:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.214477:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.214478:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.214479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:2.0:1619677471.214480:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.214482:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.214486:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.214487:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.214488:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.214491:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.214491:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.214492:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.214493:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.214945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.214987:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.214990:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.214991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.214991:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.214993:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.214995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677471.215363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (20)++
00000800:00000200:17.0:1619677471.215368:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:17.0:1619677471.215371:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.15@o2ib3
00000800:00000200:1.0:1619677471.215372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000400:00000200:17.0:1619677471.215374:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.15@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.215376:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.215377:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677471.215378:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.215380:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.215381:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.215382:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677471.215386:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.215387:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677471.215390:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.215391:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:17.0:1619677471.215391:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677471.215393:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677471.215394:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:17.0:1619677471.215395:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cffdc7c00] (22)++
00000800:00000200:17.0:1619677471.215396:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc7c00] (23)--
00000800:00000200:17.0:1619677471.215396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:5.2:1619677471.215401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.215407:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)++
00000800:00000200:18.0:1619677471.215408:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:1.0:1619677471.215408:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cffdc7c00] (23)--
00000800:00000200:1.0:1619677471.215409:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:1.0:1619677471.215410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)--
00000800:00000200:18.0:1619677471.215411:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.215412:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.215412:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.215415:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.215418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.216183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.216353:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.216356:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.216357:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.216357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.216359:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.216362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.216428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.216434:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.216437:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.216438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.216440:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.216444:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.216445:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.216447:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.216449:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.216450:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.216451:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.216454:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.216457:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.216459:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.216460:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.216461:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.216463:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.216464:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.216465:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.216465:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.216476:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.216479:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.216479:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.216480:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.216481:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.216483:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.216497:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.216499:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.216500:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.216501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.216502:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.216502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.217045:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.217048:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.217049:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.217049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.217051:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.217054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677471.217124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (20)++
00000800:00000200:1.0:1619677471.217129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.217132:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.15@o2ib3
00000800:00000200:17.0:1619677471.217133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000400:00000200:1.0:1619677471.217135:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.15@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.217138:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.217139:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:1.0:1619677471.217139:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:1.0:1619677471.217141:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.217142:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:1.0:1619677471.217142:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677471.217143:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.217143:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.217143:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.217144:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:2.0:1619677471.217146:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677471.217148:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.217150:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:2.0:1619677471.217151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.217153:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.217154:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:1.0:1619677471.217155:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677471.217156:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:1.0:1619677471.217157:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.217158:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:1.0:1619677471.217159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)--
00000800:00000200:24.2:1619677471.217163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:18.0:1619677471.218000:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.218003:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677471.218003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.218005:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.218008:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.218009:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677471.218011:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.218013:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.218015:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.218016:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677471.218019:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.218021:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677471.218025:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.218026:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677471.218026:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677471.218028:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677471.218029:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.218030:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.218031:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:5.2:1619677471.218715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (20)++
00000800:00000200:17.0:1619677471.218721:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:17.0:1619677471.218724:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.15@o2ib3
00000800:00000200:1.0:1619677471.218725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000400:00000200:17.0:1619677471.218727:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.15@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.218730:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.218731:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:17.0:1619677471.218732:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.218734:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.218735:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.218736:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:17.0:1619677471.218739:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.218741:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:17.0:1619677471.218744:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.218745:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:17.0:1619677471.218745:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677471.218747:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:17.0:1619677471.218748:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:17.0:1619677471.218749:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:17.0:1619677471.218749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)--
00000800:00000200:24.2:1619677471.218753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677471.219909:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.219911:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.219913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.219913:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.219915:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.219918:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.220291:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.220296:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.220299:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:17.0:1619677471.220301:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.220304:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.220305:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.220307:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.220308:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.220309:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.220310:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.220314:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.220315:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.220318:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.220319:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.220319:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.220321:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.220321:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.220323:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.220324:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.220324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.220336:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.220338:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.220338:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.220339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677471.220423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (20)++
00000800:00000200:17.0:1619677471.220429:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:17.0:1619677471.220432:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.15@o2ib3
00000800:00000200:1.0:1619677471.220433:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000400:00000200:17.0:1619677471.220434:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.15@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.220437:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.220438:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:17.0:1619677471.220440:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.220441:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.220442:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.220443:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:17.0:1619677471.220447:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.220448:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:17.0:1619677471.220451:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.220452:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:17.0:1619677471.220452:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677471.220454:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:17.0:1619677471.220454:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:17.0:1619677471.220455:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:17.0:1619677471.220455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)--
00000800:00000200:24.2:1619677471.220459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677471.220481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:1.0:1619677471.220622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.220625:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.220626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.220626:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.220629:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.220632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.220643:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.220646:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.220646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.220647:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.220647:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.220648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.220856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.221074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.221082:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.221086:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.221088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.221091:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.221094:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.221096:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.221098:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.221100:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.221101:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.221102:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.221106:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.221108:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.221111:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.221112:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.221113:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.221115:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.221116:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.221116:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.221117:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.221809:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.221812:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.221813:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.221813:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.221815:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.221818:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:13.0:1619677471.222087:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.222090:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.222093:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677471.222094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.222095:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.222097:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677471.222098:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96518714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.222099:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.222100:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.222102:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677471.222105:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.222107:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677471.222110:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.222110:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677471.222111:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677471.222113:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677471.222113:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.222114:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.222115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677471.222317:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.222320:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.222321:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.222322:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.222325:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.222328:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.222336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.222377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.222382:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.222385:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.222386:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.222386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.222388:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.222390:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.222402:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.222405:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.222406:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.222406:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.222407:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.222407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.224322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.224328:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.224330:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.224332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.224333:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.224337:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.224338:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.224339:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.224341:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.224342:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.224343:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.224346:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.224348:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.224350:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.224351:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.224352:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.224354:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.224355:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.224356:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.224356:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.224368:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.224370:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.224371:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.224372:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.224373:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.224374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.224889:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.224892:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.224894:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.224894:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.224896:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.224900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.224908:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.224911:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.224911:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.224912:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.224913:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.224914:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.224961:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.224964:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.224965:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.224965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.224967:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.224970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.225031:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.225034:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.225035:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.225037:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.225040:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.225042:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.225043:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.225045:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.225045:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.225046:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.225049:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.225051:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.225054:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.225055:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.225055:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.225057:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.225057:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.225058:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.225058:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.225525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.225531:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.225534:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.225535:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.225540:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.225542:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.225544:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.225546:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.225548:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.225549:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.225550:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.225554:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.225556:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.225559:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.225560:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.225561:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.225563:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.225563:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.225565:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.225566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.226450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.226784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.226791:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.226794:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.226795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.226797:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.226800:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.226802:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677471.226803:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.226805:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.226806:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.226807:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677471.226811:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.226813:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677471.226817:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.226817:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677471.226818:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677471.226820:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677471.226821:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.226823:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677471.226824:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677471.226824:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677471.226830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.226835:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677471.226838:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677471.226838:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.226839:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677471.227479:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.227482:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.227483:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.227483:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.227485:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.227488:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.228235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.228289:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.228291:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.228292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.228292:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.228299:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.228301:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.229371:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.229374:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.229375:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.229375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.229377:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.229380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.229869:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.229871:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.229873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.229873:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.229875:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.229879:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.230373:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.230376:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.230377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.230380:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.230382:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.230383:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.230385:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.230386:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.230387:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.230388:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.230392:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.230393:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.230398:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.230399:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.230400:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.230401:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.230402:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.230403:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.230403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.230991:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.230995:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.230997:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.230997:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.231000:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.231005:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.231227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.231233:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.231235:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.231236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.231238:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.231241:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.231243:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.231244:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.231247:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.231249:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.231250:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.231253:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.231255:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.231257:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.231258:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.231259:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.231261:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.231262:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.231263:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.231263:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.231274:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.231276:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.231276:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.231277:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.231278:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.231279:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.231399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.231406:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.231409:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.231410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.231412:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.231414:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.231415:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.231417:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.231418:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.231420:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.231420:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.231424:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.231426:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.231429:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.231430:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.231431:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.231433:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.231434:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.231436:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.231437:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.231439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.231443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.231449:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.231452:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.231452:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.231454:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.232432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.233137:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.233139:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.233140:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.233141:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.233143:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.233146:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.233156:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.233159:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.233160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.233160:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.233161:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.233162:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.233819:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.233822:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677471.233823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.233826:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.233829:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.233830:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.233831:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.233833:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.233835:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.233836:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.233840:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.233842:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.233844:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.233845:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.233846:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.233847:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.233848:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.233849:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.233850:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677471.233860:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.233863:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.233863:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.233865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.233865:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.233868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.233877:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.233879:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.233880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:18.0:1619677471.233882:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:17.0:1619677471.233882:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677471.233885:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677471.233885:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.233886:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000400:00000200:17.0:1619677471.233886:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677471.233886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.233887:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.233888:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:17.0:1619677471.233888:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.233889:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.233890:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.233891:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677471.233895:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.233897:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677471.233900:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.233901:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677471.233901:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677471.233903:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677471.233904:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.233904:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.233905:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:12.0:1619677471.233916:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.233919:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.233920:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.233922:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.233924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.233925:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677471.233947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677471.233953:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.233956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677471.233956:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000400:00000200:1.0:1619677471.233959:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677471.233963:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.233964:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677471.233965:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.233967:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.233968:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.233968:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677471.233972:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677471.233974:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677471.233977:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.233977:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677471.233978:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677471.233980:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677471.233980:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677471.233981:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677471.233982:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:20.2:1619677471.234203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.234243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.234249:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.234252:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.234253:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.234253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.234255:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.234258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.235097:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.235100:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.235101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.235102:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.235105:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.235106:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.235108:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.235109:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.235110:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.235111:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.235114:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.235116:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.235119:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.235119:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.235120:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.235122:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.235122:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.235123:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.235124:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.235906:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4c00] (21)++
00000800:00000200:17.0:1619677471.235909:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.68@o2ib3
00000800:00000200:1.0:1619677471.235909:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4c00] (22)--
00000400:00000200:17.0:1619677471.235912:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.68@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.235914:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.235916:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677471.235917:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.235918:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.235920:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.235921:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677471.235924:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.68@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.235926:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677471.235928:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.235929:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677471.235930:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677471.235932:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677471.235932:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4c00] (21)++
00000800:00000200:17.0:1619677471.235933:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4c00] (22)--
00000800:00000200:17.0:1619677471.235934:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4c00] (21)--
00000800:00000200:1.0:1619677471.236496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.236499:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.236500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.236500:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.236502:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.236505:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.236564:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.236567:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.236568:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.236568:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.236570:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.236573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.237389:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677471.237391:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677471.237391:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.237392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677471.237393:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.237395:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:1.0:1619677471.237395:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.237396:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.237396:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.237398:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:2.0:1619677471.237399:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.237402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.238012:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.238015:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.238015:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.238016:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.238018:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.238021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.238035:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.238038:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.238039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.238039:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.238040:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.238041:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.238209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.238215:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.238218:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.238219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.238221:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.238223:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.238224:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.238226:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.238228:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.238230:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.238231:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.238234:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.238236:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.238239:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.238239:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.238241:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.238242:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.238243:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.238244:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.238246:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.238246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.238247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.2:1619677471.238252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.238255:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.238257:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.238258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.238259:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.238260:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.238260:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.238263:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677471.238263:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.238265:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.238266:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.238268:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.238269:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.238270:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.238272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.238273:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.238274:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.238274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.238275:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.238276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.238277:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.238278:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.238279:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.238280:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.238281:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.238282:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.238284:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.238285:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.238286:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.238286:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.238573:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.238576:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.238578:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.238578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.238580:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.238584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.239566:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.239569:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.239569:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.239571:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.239574:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.239575:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677471.239577:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.239578:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.239579:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.239580:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677471.239583:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.239585:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677471.239588:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.239589:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677471.239589:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677471.239591:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677471.239592:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.239592:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.239593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677471.240200:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.240203:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.240204:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.240206:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.240209:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.240210:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.240212:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.240213:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.240214:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.240215:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.240217:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.240219:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.240222:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.240222:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.240223:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.240225:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.240225:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.240227:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.240228:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.240229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.240239:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.240242:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.240242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.240243:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.240422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.240465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.240467:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.240469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.240469:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.240471:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.240473:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.240484:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.240487:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.240488:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.240488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.240489:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.240490:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.242332:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.242334:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.242336:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.242336:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.242338:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.242341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.242920:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.242923:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.242924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.242925:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.242927:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.242930:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.242987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.242995:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.243000:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.243002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.243005:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.243008:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.243010:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.243013:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.243015:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.243017:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.243018:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.243024:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.243026:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.243030:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.243031:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.243032:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.243035:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.243036:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.243037:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.243038:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.243372:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.243375:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.243375:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677471.243377:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.243379:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.243381:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.243382:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.243384:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.243385:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.243386:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.243389:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.243391:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.243393:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.243394:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.243395:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677471.243396:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.243397:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.243397:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.243398:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677471.243761:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.243764:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.243765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.243766:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.243768:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.243771:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.244412:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.244415:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.244416:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.244418:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.244421:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.244422:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.244424:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.244426:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.244427:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.244428:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.244431:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.244434:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.244436:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.244437:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.244437:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.244439:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.244440:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.244440:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.244441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677471.245438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (20)++
00000800:00000200:1.0:1619677471.245445:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.245447:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.15@o2ib3
00000800:00000200:17.0:1619677471.245448:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000400:00000200:1.0:1619677471.245450:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.15@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.245452:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.245454:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.245456:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.245457:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.245458:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.245460:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.245470:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.245473:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.245476:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.245477:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.245478:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677471.245479:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.245480:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.245481:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cffdc7c00] (22)++
00000800:00000200:1.0:1619677471.245482:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc7c00] (23)--
00000800:00000200:1.0:1619677471.245483:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:5.2:1619677471.245488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:17.0:1619677471.245493:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)++
00000800:00000200:17.0:1619677471.245495:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cffdc7c00] (23)--
00000800:00000200:17.0:1619677471.245496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:17.0:1619677471.245497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)--
00000800:00000200:20.2:1619677471.245613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:2.0:1619677471.245681:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.245683:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.245685:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.245685:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.245688:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.245691:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.245699:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.245702:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.245703:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.245703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.245704:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.245705:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.245958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.245963:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.245966:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.245966:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677471.245968:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.245971:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.245972:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.245974:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.245975:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.245976:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.245978:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.245981:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.245983:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.245985:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.245986:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.245988:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.245989:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.245990:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.245990:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.245991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.246013:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.246015:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.246015:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.246016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.246017:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.246019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.246358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.246405:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.246408:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.246409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.246409:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.246411:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.246413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.247344:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.247347:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.247349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.247349:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.247352:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.247355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.247541:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.247544:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.247544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.247545:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.247547:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.247551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.247560:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.247563:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.247564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.247564:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.247565:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.247566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:13.0:1619677471.247763:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.247766:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.247769:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677471.247771:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677471.247771:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.247773:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677471.247774:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96518581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.247775:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.247776:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.247777:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677471.247780:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.247782:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677471.247785:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.247786:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677471.247786:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677471.247788:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677471.247788:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.247789:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.247790:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677471.247803:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.247806:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.247806:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.247807:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.247809:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.247812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.248333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.248343:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.248345:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.248349:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:18.0:1619677471.248350:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677471.248350:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.248352:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.248353:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.248354:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.248354:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677471.248357:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.248360:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.248362:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.248363:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.248367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.248369:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.248369:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.248370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.248372:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.248375:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.248376:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.248377:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.248380:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.248380:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.248381:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.248382:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.248916:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.248920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.248920:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.248922:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.248924:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.248926:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.248927:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.248928:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.248929:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.248930:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.248933:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.248935:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.248937:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.248938:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.248939:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.248941:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.248941:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.248942:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.248942:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.250152:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.250155:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.250156:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.250156:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.250159:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.250165:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.250166:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.250169:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.250170:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.250171:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.250171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.250171:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677471.250257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.250260:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.250261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.250261:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.250264:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.250269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.250271:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.250273:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.250274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.250274:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.250275:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.250276:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.252223:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.252226:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.252227:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.252227:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.252229:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.252232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.252322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.252329:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.252332:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.252332:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.252334:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.252337:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.252338:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.252339:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.252341:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.252342:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.252343:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.252346:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.252348:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.252350:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.252351:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.252352:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.252354:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.252355:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.252356:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.252356:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.252367:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.252369:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.252370:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.252371:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.252371:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.252373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.252439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.252483:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.252486:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.252487:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.252487:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.252489:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.252491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.253431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:1.0:1619677471.253462:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.253465:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.253466:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.253466:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.253468:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.253471:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.253477:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.253480:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.253481:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.253482:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.253485:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.253488:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.253570:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.253573:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.253573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.253576:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.253578:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.253579:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.253580:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.253582:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.253583:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.253584:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.253587:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.253589:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.253591:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.253592:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.253593:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.253594:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.253595:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.253596:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.253596:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.255413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.255419:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.255421:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.255423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.255425:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.255427:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.255429:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.255430:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.255431:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.255433:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.255434:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.255437:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.255439:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.255442:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.255443:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.255444:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.255446:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.255446:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.255448:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.255450:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.255450:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.255456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.255462:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.255464:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.255466:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.255466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.256028:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.256031:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.256032:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.256033:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.256035:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.256039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.257228:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.257231:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.257232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.257233:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.257235:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.257238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.258321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.258357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.258364:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.258367:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.258368:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.258368:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.258370:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.258372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.258513:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.258515:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.258516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.258517:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.258519:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.258522:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.259264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.259267:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.259268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.259268:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.259270:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.259273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.259282:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.259285:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.259286:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.259286:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.259287:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.259288:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.259507:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.259510:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.259511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.259513:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.259515:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.259516:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.259517:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.259518:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.259519:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.259520:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.259524:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.259526:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.259528:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.259529:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.259530:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.259531:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.259532:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.259533:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.259534:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.259534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.259546:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.259548:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.259548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.259549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.260366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677471.260406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:12.0:1619677471.260410:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.260412:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677471.260413:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.260414:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.260416:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.260417:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677471.260426:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677471.260429:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677471.260430:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.260430:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677471.260431:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.260432:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677471.260451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.260458:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.260461:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.260463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.260464:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.260467:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.260469:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.260470:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.260472:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.260473:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.260474:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.260478:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.260480:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.260482:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.260484:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.260484:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.260486:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.260487:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.260488:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.260489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.262111:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.262114:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.262115:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.262115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.262117:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.262121:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.263383:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.263386:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.263387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.263387:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.263390:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677471.263392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677471.263392:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.263398:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.263401:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.263402:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.263404:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.263407:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.263408:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.263410:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.263411:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.263412:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.263414:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.263417:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.263419:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.263421:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.263422:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.263423:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.263425:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.263426:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.263427:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.263427:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.263438:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.263440:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.263442:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.263442:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.263444:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.263446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.263687:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.263689:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.263690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.263691:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.263693:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.263695:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.264659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677471.264768:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.264771:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.264772:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.264773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.264774:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.264778:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.264790:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.264793:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.264794:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.264794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.264795:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.264796:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.264816:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.264819:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.264820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.264822:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.264824:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.264826:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.264827:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.264828:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.264829:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.264830:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.264834:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.264836:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.264838:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.264839:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.264839:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.264841:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.264841:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.264842:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.264843:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.265041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.265047:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.265050:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.265051:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.265052:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.265055:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.265056:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.265058:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.265059:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.265060:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.265061:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.265065:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.265067:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.265070:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.265071:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.265072:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.265074:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.265075:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.265076:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.265076:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677471.265305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.265312:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.265315:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.265315:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.265318:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.265321:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.265323:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.265324:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.265326:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.265328:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.265329:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.265334:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.265338:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.265341:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.265342:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.265343:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.265345:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.265346:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.265347:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.265348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.266360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677471.266801:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.266804:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.266805:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.266806:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.266807:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.266811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.267023:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.267025:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.267027:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.267027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.267029:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.267032:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.269023:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.269024:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.269025:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.269026:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.269027:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.269028:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.269029:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677471.269031:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.269033:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677471.269034:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.269036:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.269037:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.269038:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:17.0:1619677471.269039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.269041:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.269043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:1.0:1619677471.269043:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.269045:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677471.269049:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.269049:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677471.269050:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677471.269052:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677471.269052:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.269053:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.269054:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.269055:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.269067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.269069:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.269070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.269070:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677471.269347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.269351:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.269354:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.269354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.269357:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.269365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.269367:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.269370:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.269372:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.269372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.269374:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.269375:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.270566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.270740:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.270743:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.270744:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.270744:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.270747:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.270749:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.271668:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.271671:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.271672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.271672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.271674:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.271677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.271997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.272003:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.272006:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.272007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.272009:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.272012:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.272014:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.272015:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.272017:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.272019:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.272020:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.272023:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.272025:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.272027:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.272028:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.272029:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.272031:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.272032:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.272034:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.272035:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.272036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.272041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.272045:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.272047:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.272047:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.272048:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.272049:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677471.272049:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.272050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.272051:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.272051:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.272052:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.272063:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.272065:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.272067:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.272067:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.272067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.272068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.272402:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.272405:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.272405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.272408:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.272410:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.272412:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.272414:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.272415:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.272416:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.272417:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.272421:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.272422:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.272424:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.272425:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.272426:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.272427:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.272428:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.272429:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.272429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.272682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.272688:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.272691:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.272692:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.272694:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.272696:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.272698:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.272700:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.272702:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.272703:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.272705:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.272710:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.272712:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.272715:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.272716:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.272717:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.272719:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.272722:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.272723:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.272723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.272748:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.272750:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.272751:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.272752:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.272754:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.272758:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.272832:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.272835:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.272836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.272838:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.272841:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.272843:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677471.272844:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.272845:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.272846:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.272847:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677471.272851:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.272853:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677471.272856:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.272856:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677471.272857:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677471.272859:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677471.272859:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.272860:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.272860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677471.272873:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.272875:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677471.272876:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.272878:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.272881:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.272882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677471.273674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.273709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677471.273989:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.273992:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.273993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.273994:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.273996:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.273999:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.274040:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.274043:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.274043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.274044:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.274046:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.274050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.275717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:13.0:1619677471.275762:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.275765:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.275766:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.275768:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.275770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.275771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:18.0:1619677471.275937:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.275940:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677471.275941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.275943:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.275945:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.275947:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677471.275948:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.275950:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.275951:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.275952:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677471.275955:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.275957:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677471.275960:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.275961:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677471.275962:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677471.275964:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677471.275965:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.275966:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677471.275967:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677471.275968:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.275978:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:1.0:1619677471.275979:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677471.275980:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677471.275981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.275982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:1.0:1619677471.275982:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.275983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.275983:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.275985:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.275987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.276325:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.276328:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.276329:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.276330:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.276332:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.276335:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.276577:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.276580:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.276581:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.276581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.276584:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.276587:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.276598:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.276601:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.276602:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.276602:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.276603:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.276603:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.276977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.277021:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.277024:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.277025:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.277025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.277027:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.277029:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.277043:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.277045:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.277046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.277047:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.277048:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.277048:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.277095:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.277098:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.277099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.277099:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.277101:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.277104:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.277116:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.277119:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.277120:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.277121:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.277122:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.277123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.277459:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.277462:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.277463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.277465:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.277467:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.277469:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.277470:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.277472:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677471.277473:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:17.0:1619677471.277473:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.277474:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.277477:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.277479:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677471.277479:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677471.277481:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.277482:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:2.0:1619677471.277482:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677471.277483:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677471.277484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677471.277485:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.277485:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:2.0:1619677471.277485:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677471.277486:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.277487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:2.0:1619677471.277488:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.277489:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.277491:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.277492:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.277493:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.277494:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.277498:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.277500:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.277503:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.277504:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.277505:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.277507:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.277508:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.277509:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.277510:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.277511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.277516:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.277522:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.277525:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.277525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.277526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677471.277964:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.277967:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.277969:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.277971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.277971:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.277973:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677471.277974:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96518437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.277975:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.277976:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.277977:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677471.277981:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.277983:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677471.277985:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.277986:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677471.277987:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677471.277988:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677471.277989:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.277990:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677471.277991:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677471.277992:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:18.0:1619677471.278002:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.278005:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.278006:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.278006:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:2.0:1619677471.278006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677471.278008:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:18.0:1619677471.278009:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.278009:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.278010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677471.278013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.278607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.280508:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.280511:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.280512:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.280512:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.280514:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.280517:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.280785:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.280789:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.280791:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.280792:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.280794:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.280802:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.280804:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.280808:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.280808:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.280809:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.280811:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.280812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.281376:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.281379:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.281380:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.281380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.281382:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.281385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.281386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.281392:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.281395:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.281395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.281398:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.281400:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.281401:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.281403:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.281404:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.281406:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.281407:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.281410:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.281412:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.281414:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.281415:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.281415:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.281417:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.281418:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.281419:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.281419:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.281431:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.281433:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.281434:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.281435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.281436:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.281437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.282246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.282252:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.282255:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.282256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.282257:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.282259:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.282261:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.282263:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.282264:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.282265:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.282266:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.282270:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.282272:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.282275:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.282276:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.282277:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.282279:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.282279:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.282280:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.282281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.282700:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.282703:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.282704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.282704:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.282706:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.282709:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.283102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.283145:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.283148:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.283149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.283149:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.283151:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.283153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.283715:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.283718:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.283719:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.283719:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.283721:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.283724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.284140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.284162:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.284165:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.284166:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.284168:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.284170:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.284171:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.284173:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.284174:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677471.284175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677471.284175:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.284176:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.284179:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.284181:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.284184:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.284184:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677471.284185:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (23)++
00000800:00000200:1.0:1619677471.284187:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.284188:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.284189:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.284190:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.284191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.284202:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.284204:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.284205:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.284206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677471.285441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cffdc7c00] (20)++
00000800:00000200:1.0:1619677471.285447:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.285451:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.15@o2ib3
00000800:00000200:17.0:1619677471.285452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (22)--
00000400:00000200:1.0:1619677471.285454:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.15@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.285456:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.285458:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:1.0:1619677471.285459:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.285460:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.285461:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.285462:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:1.0:1619677471.285466:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.285468:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:1.0:1619677471.285470:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.285471:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:1.0:1619677471.285472:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677471.285474:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:1.0:1619677471.285474:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc7c00] (21)++
00000800:00000200:1.0:1619677471.285475:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc7c00] (22)--
00000800:00000200:1.0:1619677471.285476:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc7c00] (21)--
00000800:00000200:24.2:1619677471.285480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:18.0:1619677471.286366:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.286370:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.286371:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.286371:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.286373:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.286377:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.286463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:2.0:1619677471.286973:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.286976:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677471.286977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677471.286978:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.286982:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.286983:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677471.286984:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.286986:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.286987:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.286988:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.286992:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.286993:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.286997:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.286998:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677471.286999:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677471.287001:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677471.287001:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.287002:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.287003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:17.0:1619677471.289391:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.289394:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.289396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.289398:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.289401:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.289403:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.289404:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.289405:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.289406:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.289407:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.289411:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.289412:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.289415:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.289416:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.289417:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.289419:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.289419:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.289420:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.289421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.289716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.289760:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.289762:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.289763:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.289763:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.289765:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.289768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.289832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.289838:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.289841:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.289842:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.289844:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.289847:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.289849:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.289850:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.289852:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.289853:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.289854:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.289857:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.289859:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.289862:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.289863:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.289864:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.289866:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.289866:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.289867:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.289868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.290480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.290486:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.290489:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.290490:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.290491:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.290495:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.290496:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.290497:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.290499:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.290500:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.290501:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.290505:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.290508:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.290510:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.290511:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.290512:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.290514:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.290515:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.290516:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.290517:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.290528:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.290530:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.290531:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.290532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.290533:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.290535:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.290547:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.290549:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.290550:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.290550:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.290551:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.290552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.291402:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.291404:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.291405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.291406:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.291408:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.291412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.291426:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.291429:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.291430:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.291432:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.291434:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.291436:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677471.291437:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.291438:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.291439:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.291440:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677471.291444:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.291445:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677471.291448:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.291449:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677471.291449:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677471.291451:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677471.291452:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.291452:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.291453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677471.291509:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.291512:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.291513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.291513:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.291515:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.291518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.291527:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.291530:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.291531:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.291531:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.291532:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.291532:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.293950:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.293953:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.293954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.293955:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.293958:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.293960:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.293961:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.293963:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.293964:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.293966:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.293970:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.293972:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.293975:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.293976:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.293977:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.293978:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.293979:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.293980:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.293980:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.294651:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.294658:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.294663:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.294665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.294668:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.294671:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.294673:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.294676:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.294678:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.294680:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.294682:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.294687:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.294691:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.294695:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.294696:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.294697:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.294700:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.294701:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.294702:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.294703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.296682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.296685:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.296686:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.296686:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.296688:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.296692:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.297160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.297203:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.297206:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.297207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.297207:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.297209:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.297211:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.297597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.297604:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.297607:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.297608:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.297610:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.297613:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.297615:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.297616:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.297618:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.297619:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.297620:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.297624:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.297626:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.297629:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.297630:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.297630:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.297633:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.297634:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.297635:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.297636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.297646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.297648:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.297649:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.297650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.297651:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.297652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.298592:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.298595:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.298595:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.298598:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.298600:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.298601:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.298602:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.298604:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.298605:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.298606:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.298609:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.298610:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.298613:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.298614:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.298614:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.298616:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.298616:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.298617:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.298617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.299188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.299195:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.299198:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.299199:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.299201:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.299203:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.299205:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.299207:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.299208:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.299210:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.299211:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.299215:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.299217:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.299221:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.299222:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.299223:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.299225:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.299227:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.299229:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.299231:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.299231:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.299236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.299243:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.299245:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.299246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.299248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677471.299750:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.299753:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.299753:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.299755:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.299757:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.299759:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:18.0:1619677471.300087:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.300090:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.300091:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.300091:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.300094:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.300097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.300871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.2:1619677471.300911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:2.0:1619677471.301411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.301414:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.301416:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.301416:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.301418:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.301424:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.301426:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.301428:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.301430:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.301430:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.301431:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.301431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.301750:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.301753:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.301754:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.301754:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.301757:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.301760:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.302443:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.302446:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.302447:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.302447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.302449:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.302453:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.303885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.304453:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.304456:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.304456:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.304459:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.304461:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.304462:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677471.304463:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.304465:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.304466:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.304467:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677471.304470:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.304472:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.304474:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.304475:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.304475:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.304477:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.304477:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.304478:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.304479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677471.304492:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.304494:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.304494:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.304495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.304496:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.304498:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.304510:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.304512:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.304513:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.304513:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.304514:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.304514:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.304539:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.304542:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.304543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.304544:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.304545:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.304547:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.304548:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.304549:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.304550:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.304551:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.304553:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.304555:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.304557:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.304558:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.304558:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.304560:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.304560:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.304561:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.304562:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.304563:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.304573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.304576:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.304576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.304577:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.305538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.305544:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.305548:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.305549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.305552:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.305555:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.305557:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.305559:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.305560:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.305561:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.305562:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.305566:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.305568:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.305570:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.305571:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.305572:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.305574:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.305576:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.305577:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.305578:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.305579:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.305584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.305590:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.305592:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.305593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.305594:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.305619:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.305621:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.305622:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.305623:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.305624:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.305626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.305919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.305955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.306077:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.306080:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.306081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.306081:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.306083:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.306087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.306578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.306584:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.306587:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.306588:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.306590:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.306593:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.306594:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.306596:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.306598:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.306599:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.306600:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.306604:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.306606:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.306608:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.306609:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.306610:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.306612:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.306613:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.306614:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.306615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.307059:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.307062:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.307063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.307064:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.307066:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.307070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.307466:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.307469:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.307470:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.307470:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.307473:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.307476:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.307486:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.307488:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.307489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.307490:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.307491:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.307491:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.307622:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.307625:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.307626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.307627:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.307629:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.307632:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.308800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.308803:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.308805:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.308805:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.308807:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.308811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.309558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.309601:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.309603:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.309604:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.309605:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.309607:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.309609:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.2:1619677471.309849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677471.310231:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.310235:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.310235:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.310237:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.310240:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.310241:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.310242:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.310243:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.310245:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.310246:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.310250:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.310251:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.310254:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.310255:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.310255:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.310257:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.310257:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.310258:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.310259:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:12.0:1619677471.310358:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.310361:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677471.310362:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.310364:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.310366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.310367:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:18.0:1619677471.310405:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.310408:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.310409:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.310409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.310411:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.310415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.310662:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.310665:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.310666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.310666:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.310669:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.310672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:13.0:1619677471.310676:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.310678:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.310680:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.310682:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.310682:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.310685:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677471.310693:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677471.310696:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677471.310697:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.310698:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677471.310698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.310700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:17.0:1619677471.310718:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.310720:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.310721:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.310722:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.2:1619677471.310724:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677471.310724:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.310740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.310742:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.310744:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.310744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.310745:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.310746:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.310747:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.311001:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.311004:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.311005:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.311006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.311007:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.311018:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.311039:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.311041:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.311042:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.311043:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.311045:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.311051:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.311105:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.311107:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:1.0:1619677471.311107:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:18.0:1619677471.311109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.311109:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.311110:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:2.0:1619677471.311111:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.311112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.311112:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.311114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:1.0:1619677471.311114:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.311117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.311703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.311710:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.311712:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.311713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.311715:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.311718:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.311720:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.311721:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.311723:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.311724:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.311725:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.311729:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.311730:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.311734:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.311735:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.311736:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.311738:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.311739:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.311740:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.311741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677471.312461:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.312464:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.312466:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677471.312468:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677471.312469:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.312470:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677471.312471:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96518248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.312472:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.312473:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.312474:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677471.312478:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.312481:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677471.312485:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.312486:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677471.312486:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677471.312489:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677471.312489:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.312490:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.312491:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677471.312503:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:17.0:1619677471.312505:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677471.312505:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.312507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.312507:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.312508:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.312509:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.312509:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.312509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.312511:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.312513:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.312515:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.312524:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677471.312525:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677471.312526:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.312527:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.312527:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677471.312528:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677471.312528:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.312528:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.312529:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.312530:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.312531:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.312532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.312788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677471.312834:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.312836:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.312838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.312838:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.312841:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.312844:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.313629:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.313632:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.313633:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.313633:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.313636:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.313639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.313909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.313916:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.313919:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.313919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.313921:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.313924:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.313926:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.313928:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.313930:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.313931:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.313932:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.313936:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.313938:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.313940:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.313941:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.313942:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.313944:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.313945:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.313946:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.313947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.313957:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.313959:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.313960:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.313961:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.313962:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.313964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.314265:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.314268:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.314269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.314270:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.314273:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.314275:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.314277:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.314278:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.314280:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.314281:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.314284:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.314286:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.314288:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.314289:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.314290:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677471.314291:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.314292:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.314298:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.314299:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677471.314845:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.314848:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.314849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.314851:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.314854:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.314855:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.314856:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.314858:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.314859:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.314860:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.314863:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.314865:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.314868:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.314868:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.314869:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.314871:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.314871:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.314872:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.314873:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.315471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.315514:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.315517:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.315518:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.315518:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.315520:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.315522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.315623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.315629:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.315633:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.315634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.315636:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.315638:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.315640:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.315641:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.315643:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.315644:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.315646:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.315650:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.315652:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.315656:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.315656:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.315657:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.315660:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.315661:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.315662:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.315663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677471.316075:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.316079:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.316083:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.316084:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.316086:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.316089:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677471.316091:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96518226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.316093:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.316095:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.316096:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677471.316103:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.316106:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677471.316109:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.316110:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677471.316111:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677471.316113:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677471.316114:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.316115:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.316116:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677471.316677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.316715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.316721:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.316724:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.316725:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.316726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.316727:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.316729:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.317502:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.317504:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.317506:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.317506:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.317508:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.317511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.317511:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.317514:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.317515:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.317515:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.317518:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.317520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.319102:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.319105:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.319106:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.319106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.319109:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.319111:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.319744:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.319747:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.319748:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.319748:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.319751:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.319754:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.319770:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.319773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.319773:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.319774:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.319775:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.319776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.320070:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.320073:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.320074:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.320076:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.320078:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.320079:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.320081:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.320082:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.320083:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.320084:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.320087:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.320089:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.320092:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.320092:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.320093:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.320095:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.320095:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.320096:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.320097:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.320535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.320542:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.320545:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.320546:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.320549:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.320551:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.320553:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.320554:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.320556:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.320558:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.320559:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.320563:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.320565:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.320567:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.320568:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.320569:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.320571:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.320572:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.320573:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.320574:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.320584:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.320587:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.320588:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.320589:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.320590:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.320593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.321049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677471.321093:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.321096:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.321097:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.321097:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:2.0:1619677471.321097:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.321099:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.321100:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.321101:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.321101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:18.0:1619677471.321103:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:17.0:1619677471.321103:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.321106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.321112:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.321115:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.321116:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677471.321116:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:18.0:1619677471.321117:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.321118:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.321118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.321119:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.321120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.321120:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.321121:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.321122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.321185:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.321187:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.321188:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.321189:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.321191:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.321194:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.322378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.322384:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.322387:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.322389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.322390:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.322393:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.322395:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.322396:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.322397:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.322399:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.322400:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.322403:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.322405:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.322408:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.322409:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.322410:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.322412:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.322413:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.322414:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.322414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677471.322614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.322657:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.322660:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.322661:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.322661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.322663:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.322665:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.322847:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.322850:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.322851:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.322851:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.322853:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.322856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.322865:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.322868:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.322869:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.322869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.322870:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.322871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.323500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.323507:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.323510:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.323510:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.323513:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.323516:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.323517:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.323519:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.323521:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.323522:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.323523:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.323527:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.323529:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.323532:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.323533:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.323534:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.323536:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.323537:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.323539:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.323540:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.323541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.323547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.323552:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.323554:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.323555:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.323556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.323736:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.323738:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.323740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.323740:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.323742:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.323744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.324577:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.324580:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677471.324581:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677471.324583:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.324586:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.324587:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677471.324589:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96518182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.324591:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.324592:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.324593:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677471.324597:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.324599:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677471.324602:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.324603:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677471.324604:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677471.324606:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677471.324606:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.324607:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.324608:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:17.0:1619677471.325375:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.325378:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.325379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.325381:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.325383:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.325384:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.325386:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.325387:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.325388:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.325389:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.325392:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.325393:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.325396:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.325396:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.325397:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.325399:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.325399:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.325401:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.325402:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.325403:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.325413:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.325415:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.325415:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.325417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.325950:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.325953:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.325954:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.325954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.325957:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.325960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.326214:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.326217:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.326219:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.326219:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.326221:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.326227:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.326244:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.326247:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.326248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.326249:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.326251:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.326254:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.326269:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.326271:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.326273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.326273:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.326274:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.326274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.326375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:17.0:1619677471.326666:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.326669:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.326670:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.326670:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.326672:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.326675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.327363:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.327366:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.327368:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.327368:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.327370:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.327373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.328177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.328184:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.328187:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.328188:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.328190:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.328192:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.328194:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.328196:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.328198:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.328199:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.328200:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.328204:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.328206:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.328210:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.328211:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.328212:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.328214:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.328215:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.328216:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.328217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677471.328305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.328308:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.328310:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.328312:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.328314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.2:1619677471.328315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677471.328315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:17.0:1619677471.328362:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.328365:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.328366:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.328366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.328368:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.328370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.328732:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.328735:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.328736:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.328736:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.328738:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.328741:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.329950:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.329953:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.329953:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.329956:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.329958:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.329959:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.329960:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.329962:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.329963:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.329964:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.329967:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.329968:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.329970:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.329971:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.329972:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.329974:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.329974:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.329975:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.329975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.330794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.330800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.330803:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.330805:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.330806:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.330810:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.330811:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.330812:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.330814:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.330815:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.330816:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.330819:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.330821:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.330823:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.330824:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.330825:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.330827:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.330828:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.330829:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.330829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.330840:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.330843:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.330843:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.330844:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.330845:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.330847:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.331053:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.331057:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.331058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.331058:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.331060:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.331064:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677471.331094:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.331097:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.331099:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.331102:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.331102:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.331103:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677471.331105:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96518146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.331106:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.331107:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.331108:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677471.331111:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.331112:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677471.331115:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.331116:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:12.0:1619677471.331117:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:12.0:1619677471.331119:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:12.0:1619677471.331119:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.331120:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.331121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677471.331134:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.331137:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.331138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.331138:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.331140:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.331144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.331155:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.331158:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.331159:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.331159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.331160:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.331161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.332331:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.332334:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.332335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.332336:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.332338:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.332342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.332626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.332631:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.332634:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.332635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.332637:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.332639:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.332640:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.332641:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.332643:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.332644:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.332645:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.332649:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.332651:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.332654:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.332655:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.332656:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.332658:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.332659:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.332659:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.332660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.2:1619677471.332798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:18.0:1619677471.333088:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.333091:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.333092:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.333092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.333095:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.333099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.333176:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.333179:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.333180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.333180:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.333183:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.333186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.333196:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.333199:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.333200:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.333200:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.333202:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.333202:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:13.0:1619677471.333227:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.333229:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.333230:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.333232:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.333234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.333236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677471.333272:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.333274:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.333275:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.333277:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.333278:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.333279:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677471.333871:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.333873:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.333874:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.333875:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.333877:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.333879:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.334903:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.334906:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.334907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.334907:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.334909:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.334915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.334917:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.334919:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.334920:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.334920:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.334921:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.334922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677471.335040:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.335042:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.335043:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.335044:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.335047:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.335048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677471.335059:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677471.335062:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677471.335062:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.335063:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677471.335064:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.335065:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677471.335219:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.335222:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.335223:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.335223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.335226:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.335229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.335259:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.335262:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.335263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.335265:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.335267:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.335268:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.335270:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.335271:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.335272:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.335273:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.335276:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.335278:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.335280:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.335281:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.335282:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.335284:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.335284:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.335285:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.335285:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:12.0:1619677471.335956:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.335958:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.335959:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.335960:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.335963:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.335964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:18.0:1619677471.336191:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.336194:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.336195:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.336195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.336198:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.336201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.338071:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.338074:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.338075:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.338075:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.338078:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.338081:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.338671:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.338673:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.338674:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.338675:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.338677:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.338680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.338711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.338718:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.338721:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.338722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.338723:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.338727:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.338729:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.338730:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.338732:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.338733:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.338734:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.338737:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.338739:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.338741:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.338742:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.338743:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.338745:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.338746:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.338747:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.338747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.338924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.338963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677471.339118:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.339121:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.339123:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.339126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.339126:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.339127:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677471.339128:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96518096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.339130:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.339131:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.339132:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677471.339135:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.339136:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677471.339139:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.339140:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677471.339140:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677471.339142:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677471.339142:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.339144:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677471.339145:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677471.339146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.339159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677471.339161:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677471.339162:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.339163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677471.339813:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.339816:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.339817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.339817:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.339819:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.339822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.339831:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.339833:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.339834:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.339834:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.339836:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.339836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.340247:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.340250:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.340251:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.340251:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.340253:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.340256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.340393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.340399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.340402:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.340403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.340405:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.340408:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.340409:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.340411:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.340412:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.340413:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.340415:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.340418:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.340420:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.340423:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.340424:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.340425:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.340427:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.340428:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.340430:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.340431:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.340432:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.340437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.340442:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.340443:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.340443:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677471.340444:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.340444:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.340445:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.340446:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.340447:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.340447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.340448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677471.340861:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.340864:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.340864:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.340867:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.340869:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.340870:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.340871:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.340874:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.340875:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.340876:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.340879:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.340881:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.340884:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.340885:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.340886:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.340887:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.340888:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.340889:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.340889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.341012:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.341015:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.341016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.341017:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.341019:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.341022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.341034:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.341037:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.341038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.341038:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.341039:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.341040:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.341301:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.341303:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.341305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.341305:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.341307:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.341310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.341736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.341743:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.341746:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.341747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.341749:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.341754:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.341755:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677471.341757:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.341758:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.341760:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.341761:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677471.341764:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.341766:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677471.341770:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.341771:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677471.341772:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677471.341773:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677471.341774:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.341775:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.341776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677471.342532:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.342535:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.342536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.342538:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.342540:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.342541:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677471.342543:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.342544:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.342545:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.342546:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677471.342549:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.342551:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.342553:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.342554:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.342555:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.342556:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.342557:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.342557:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.342557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677471.342571:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.342573:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.342574:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.342574:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.342575:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.342577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.343670:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.343672:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.343674:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.343674:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.343676:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.343680:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.344245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677471.344560:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.344563:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.344564:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.344565:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.344567:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.344571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.344580:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.344583:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.344584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.344584:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.344585:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.344586:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.344912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.344919:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.344922:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.344922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.344925:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.344927:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.344929:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.344930:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.344932:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.344933:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.344934:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.344938:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.344940:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.344944:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.344945:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.344946:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.344947:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.344949:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.344950:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.344952:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.344953:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.344958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.344963:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.344966:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.344966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.344968:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.345083:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.345086:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.345086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.345089:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.345090:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.345091:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.345093:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.345094:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.345095:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.345096:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.345100:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.345101:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.345104:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.345104:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.345105:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.345107:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.345107:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.345108:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.345109:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.345110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.345121:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.345122:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.345123:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.345124:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.345365:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.345368:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.345368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.345370:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.345372:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.345374:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677471.345376:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.345377:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.345378:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.345379:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677471.345382:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.345383:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677471.345386:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.345387:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677471.345388:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677471.345390:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677471.345390:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.345391:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.345392:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.345393:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.345405:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.345407:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.345408:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.345409:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677471.346007:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.346010:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.346011:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.346011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.346014:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.346017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.346130:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.346133:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.346134:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.346134:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.346137:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.346140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.347289:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.347292:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.347293:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.347300:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.347302:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.347305:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.347306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.347307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.347307:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.347308:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.347309:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.347310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.347594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.347601:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.347604:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.347605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.347607:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.347610:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.347612:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.347613:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.347615:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.347616:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.347618:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.347621:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.347623:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.347625:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.347626:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.347627:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.347629:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.347630:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.347631:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.347632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.347642:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.347644:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.347645:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.347646:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.347647:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.347649:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.349468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.349475:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.349478:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.349479:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.349481:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.349484:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.349485:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.349487:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.349489:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.349490:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.349492:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.349495:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.349497:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.349500:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.349501:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.349502:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.349504:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.349505:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.349507:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.349507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.349539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.349902:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.349905:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.349905:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.349907:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.349910:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.349912:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.349913:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.349915:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.349916:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.349917:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.349920:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.349922:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.349924:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.349925:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.349926:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.349927:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.349927:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.349928:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.349928:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.350108:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.350111:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.350112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.350112:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.350114:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.350117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.350499:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.350503:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.350504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.350506:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.350508:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.350509:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677471.350510:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.350512:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.350513:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.350513:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677471.350516:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.350518:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.350519:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.350520:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.350521:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.350522:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.350523:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.350523:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.350524:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677471.350536:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.350538:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.350539:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.350540:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.350540:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.350541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.350901:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.350905:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.350906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.350906:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.350909:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.350913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.351786:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.351789:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.351789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.351790:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.351792:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.351795:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.352330:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.352333:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.352334:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.352335:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.352337:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.352341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.352861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.352864:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677471.352865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677471.352867:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.352870:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.352871:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677471.352872:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.352875:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.352876:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.352877:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.352881:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.352883:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.352886:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.352887:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677471.352888:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677471.352890:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677471.352891:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.352892:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.352892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677471.352930:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.352933:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.352934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.352935:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.352937:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.352940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.352954:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.352957:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.352958:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.352958:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.352959:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.352960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.354452:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.354455:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.354456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.354458:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.354460:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.354462:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.354463:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.354464:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.354465:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.354466:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.354469:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.354471:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.354473:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.354475:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.354475:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.354477:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.354478:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.354478:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.354479:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.354620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.354627:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.354630:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.354631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.354633:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.354636:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.354638:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677471.354639:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.354641:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.354642:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.354644:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677471.354647:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.354649:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677471.354652:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.354653:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677471.354654:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677471.354656:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677471.354657:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.354659:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677471.354660:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677471.354661:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677471.354666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.354671:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677471.354674:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677471.354675:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.354677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677471.354823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.354829:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.354832:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.354833:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.354834:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.354837:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.354839:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.354841:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.354842:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.354844:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.354845:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677471.354846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677471.354848:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.354850:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.354853:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.354854:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.354854:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.354856:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.354857:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.354858:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.354858:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.355008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.355013:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.355016:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.355017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.355019:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.355022:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.355024:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.355025:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.355027:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.355028:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.355029:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.355034:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.355037:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.355040:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.355041:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.355042:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.355044:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.355045:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.355046:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.355047:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.355058:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.355061:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.355062:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.355062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.355064:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.355066:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.355691:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.355694:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.355695:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.355695:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.355697:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.355700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.356682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.356720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.356726:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.356728:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.356729:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.356730:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.356731:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.356733:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.358004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.358007:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.358009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.358009:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.358011:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.358014:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.359425:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.359428:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.359429:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.359431:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.359433:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.359435:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677471.359436:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.359437:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.359438:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.359439:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677471.359443:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.359445:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677471.359448:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.359448:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677471.359449:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677471.359451:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677471.359452:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.359452:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.359453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677471.360191:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.360194:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.360195:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.360197:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.360201:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.360202:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.360203:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96518000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.360205:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.360206:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.360207:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.360210:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.360212:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.360214:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.360215:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.360216:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.360217:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.360218:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.360218:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.360219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:13.0:1619677471.360365:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.360367:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.360368:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.360369:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.360372:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.360373:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677471.360474:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.360477:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.360478:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.360479:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.360482:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.360484:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677471.360486:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.360488:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.360489:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.360490:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677471.360495:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.360498:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677471.360501:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.360502:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677471.360502:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677471.360504:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677471.360505:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.360506:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.360507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677471.360684:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.360687:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.360689:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.360689:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.360691:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.360697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.360699:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.360701:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.360702:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.360702:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.360703:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.360704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.360845:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.360848:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.360849:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.360849:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.360851:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.360855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.360864:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.360866:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.360867:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.360867:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.360868:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.360869:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.360987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.360993:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.360995:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.360997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.360998:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.361000:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.361002:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.361004:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.361005:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.361006:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.361007:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.361011:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.361013:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.361016:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.361017:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.361017:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.361019:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.361020:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.361021:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.361021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.361636:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.361639:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.361640:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.361641:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.361643:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.361646:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.362082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677471.362084:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.362087:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.362088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.362088:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.362090:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.362094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.362126:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.362128:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.362129:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.362130:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.362131:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.362132:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677471.362132:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.362134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:17.0:1619677471.362135:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677471.362135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.362138:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.362140:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677471.362141:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.362142:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.362143:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.362144:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677471.362147:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.362149:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677471.362151:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.362152:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677471.362153:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677471.362154:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677471.362155:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.362156:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677471.362157:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677471.362158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677471.362168:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677471.362169:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:18.0:1619677471.362171:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:1.0:1619677471.362171:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:18.0:1619677471.362172:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.362172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.362172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.362173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:18.0:1619677471.362175:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.362177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.362436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.362442:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.362444:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.362446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.362448:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.362450:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.362452:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677471.362453:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.362455:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.362456:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.362457:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.362461:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.362462:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.362466:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.362467:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677471.362468:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677471.362470:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677471.362472:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.362473:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.362473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677471.363473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.363479:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.363482:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.363483:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.363485:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.363488:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.363490:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.363491:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.363492:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.363493:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.363495:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.363498:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.363500:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.363501:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.363502:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.363503:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.363505:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.363506:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.363507:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.363507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.363519:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.363521:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.363522:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.363523:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.363524:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.363526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.363538:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.363541:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.363542:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.363542:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.363542:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.363543:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.2:1619677471.363809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:5.2:1619677471.363909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677471.363916:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.363919:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677471.363920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677471.363923:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677471.363925:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.363927:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677471.363929:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.363930:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.363931:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.363932:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677471.363935:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677471.363937:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677471.363940:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.363941:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677471.363942:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677471.363944:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677471.363944:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.363945:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677471.363945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:12.0:1619677471.364870:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677471.364873:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677471.364877:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677471.364877:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : GET - for me
00000400:00000200:12.0:1619677471.364880:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.103@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677471.364884:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.103@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677471.364887:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677471.364888:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677471.364891:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:12.0:1619677471.364897:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677471.364900:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677471.364903:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.364904:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677471.364905:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.364907:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677471.364908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.364919:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.364922:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.364923:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.364924:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:13.0:1619677471.364926:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.364927:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:2.0:1619677471.365633:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.365636:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.365637:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.365638:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.365640:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.365644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.365774:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.365777:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.365778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.365780:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.365783:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.365784:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.365786:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.365787:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.365788:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.365789:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.365792:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.365793:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.365796:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.365796:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.365797:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.365799:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.365799:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.365800:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.365802:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.365802:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.365813:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.365814:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.365815:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.365817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.366375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.366382:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.366385:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.366386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.366388:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.366390:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.366392:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.366393:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.366395:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.366397:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.366398:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.366401:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.366403:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.366406:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.366407:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.366408:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.366410:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.366410:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.366412:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.366413:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.366414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.366420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.366425:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.366428:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.366428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.366430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.366604:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.366607:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.366607:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677471.366610:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.366612:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.366613:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677471.366615:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.366616:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.366617:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.366618:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677471.366620:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.366622:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677471.366625:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.366625:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677471.366626:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677471.366628:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.366628:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.366629:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.366629:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677471.366642:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.366644:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.366645:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.366645:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.366648:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.366650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.366993:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.366995:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.366996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.366997:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.366999:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.367001:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.2:1619677471.367184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:13.0:1619677471.367294:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.367301:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677471.367301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.367302:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.367305:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.367308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677471.367326:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.367329:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.367330:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.367331:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.367333:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.367334:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677471.367378:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.367381:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.367382:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.367382:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.367384:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.367387:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.367398:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.367401:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.367401:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.367402:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.367403:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.367404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.367620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.367675:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.367677:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.367678:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.367679:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.367680:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.367683:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.367795:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.367798:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.367800:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.367800:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.367802:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.367806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.368861:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.368864:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.368866:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.368866:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.368868:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.368871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.369931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.369937:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.369940:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.369941:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.369943:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.369946:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.369947:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.369949:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.369951:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.369952:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.369953:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.369957:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.369959:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.369961:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.369962:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.369963:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.369965:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.369966:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.369968:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.369969:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.369970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.369975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.369979:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.369981:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.369981:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677471.369982:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.369982:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.369983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.369984:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.369985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.369985:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.369986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.370485:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.370487:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.370489:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.370489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.370491:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.370494:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.370503:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.370506:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.370507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.370507:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.370508:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.370509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.371211:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.371214:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.371214:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.371218:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.371220:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.371221:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.371223:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.371224:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.371225:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.371226:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.371229:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.371231:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.371234:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.371234:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.371235:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.371236:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.371237:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.371238:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.371238:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.371687:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.371690:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.371691:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.371691:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.371693:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.371696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.371782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.371788:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.371791:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.371792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.371794:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.371796:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.371798:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.371799:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.371801:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.371802:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.371803:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.371806:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.371808:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.371811:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.371811:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.371812:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.371814:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.371815:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.371816:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.371816:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.372054:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.372056:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.372057:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.372057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.372059:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.372062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.373694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677471.374666:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.374669:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.374670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.374670:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.374672:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.374676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.374687:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.374690:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.374691:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.374691:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.374692:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.374693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.374955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.374989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.375398:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.375401:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.375401:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.375403:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.375405:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.375406:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:1.0:1619677471.375407:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.375408:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96517910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.375409:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:1.0:1619677471.375410:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677471.375411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.375411:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.375411:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.375413:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677471.375414:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677471.375417:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.375418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.375419:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.375421:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.375422:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.375423:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.375424:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.375425:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.375426:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.375427:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.376251:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.376254:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.376255:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.376255:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.376257:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.376261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.377425:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.377428:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.377429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.377431:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.377433:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.377434:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677471.377436:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.377437:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.377439:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.377440:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677471.377443:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.377445:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677471.377448:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.377449:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677471.377450:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677471.377453:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:1.0:1619677471.377453:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.377454:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.377454:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677471.377545:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.377547:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.377550:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677471.377552:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677471.377552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.377553:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677471.377554:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96517899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.377556:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.377556:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.377557:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677471.377560:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.377562:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677471.377564:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.377565:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677471.377565:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677471.377566:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677471.377567:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.377568:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.377568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677471.377601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.377608:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.377611:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.377612:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.377614:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.377616:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.377617:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.377619:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.377620:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.377622:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.377623:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.377626:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.377628:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.377631:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.377632:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.377633:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.377635:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.377636:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.377637:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.377637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.378357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.378363:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.378366:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.378367:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.378369:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.378372:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.378374:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.378375:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.378377:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.378378:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.378379:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.378383:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.378384:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.378387:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.378387:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.378388:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.378390:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.378391:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.378392:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.378392:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.378404:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.378406:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.378407:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.378408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.378409:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.378410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.379010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.379016:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.379018:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.379020:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.379021:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.379023:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.379025:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677471.379027:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.379028:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.379029:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.379030:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677471.379034:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.379036:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677471.379039:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.379040:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677471.379041:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677471.379043:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677471.379044:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.379045:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.379045:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677471.379058:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.379061:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.379062:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.379062:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.379064:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.379067:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.379076:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.379079:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.379079:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.379080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.379081:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.379081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.2:1619677471.379350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677471.379409:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.379411:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.379412:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.379414:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.379417:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.379418:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.379420:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.379421:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.379422:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.379423:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.379426:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.379427:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.379430:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.379431:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.379431:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.379433:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.379434:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.379434:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.379435:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.379487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677471.380287:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.380290:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.380291:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.380291:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.380300:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.380303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.380372:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.380375:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.380376:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.380376:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.380378:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.380381:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.381214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.381217:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.381218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.381220:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.381222:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.381224:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677471.381225:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.381226:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.381227:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.381228:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677471.381231:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.381233:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677471.381236:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.381237:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:17.0:1619677471.381237:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677471.381239:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:17.0:1619677471.381240:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.381240:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.381241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:20.2:1619677471.381330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.381380:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.381383:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.381384:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.381385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (24)--
00000400:00000200:1.0:1619677471.381386:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.381389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.381397:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.381400:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.381401:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:1.0:1619677471.381401:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:17.0:1619677471.381402:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (25)--
00000800:00000200:1.0:1619677471.381402:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (24)--
00000800:00000200:17.0:1619677471.381403:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.381403:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.381404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.381404:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.381406:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.381968:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.381971:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.381973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.381973:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.381975:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.381978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.382778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.382785:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.382788:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.382788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.382790:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.382794:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.382795:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.382797:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.382798:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.382799:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.382800:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.382805:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.382808:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.382811:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.382812:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.382812:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.382815:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.382816:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.382817:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.382817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.383280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.383286:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.383289:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.383290:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.383292:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.383302:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.383305:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677471.383306:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677471.383307:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:2.0:1619677471.383308:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677471.383310:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:2.0:1619677471.383310:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677471.383310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:2.0:1619677471.383312:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677471.383314:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.383316:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.383317:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677471.383318:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.383319:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96517863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.383319:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:17.0:1619677471.383322:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.383323:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677471.383323:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.383324:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:17.0:1619677471.383325:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677471.383325:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677471.383327:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677471.383329:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:17.0:1619677471.383330:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.383330:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.383331:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677471.383333:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.383335:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.383336:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.383337:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.383338:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.383339:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.383340:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.383341:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.383342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.383353:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.383355:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.383356:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.383357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.383361:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.383363:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.383365:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.383365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.383368:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.383372:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.385056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677471.385174:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.385177:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.385178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.385178:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.385180:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.385184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.385492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.385498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.385501:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.385502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.385504:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.385506:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.385508:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.385509:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.385511:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.385512:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.385514:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.385517:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.385519:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.385521:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.385522:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.385523:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.385525:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.385526:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.385527:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.385528:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.385538:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.385541:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.385541:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.385542:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.385544:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.385545:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.385555:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.385558:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.385559:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.385559:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.385560:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.385560:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.385682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.385685:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.385686:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.385686:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.385689:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.385692:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.385977:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.385980:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.385982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.385982:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.385984:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.385987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.387288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.387328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.387335:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.387338:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.387339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.387339:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.387341:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.387343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.387698:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.387701:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.387702:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.387702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.387704:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.387707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.387861:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.387864:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.387865:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.387867:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.387869:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.387870:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.387871:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.387873:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.387874:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.387875:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.387878:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.387880:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.387883:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.387883:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.387884:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.387886:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.387886:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.387887:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.387888:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.387909:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.387911:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.387912:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.387914:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.387916:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.387918:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677471.387919:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.387921:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.387922:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.387923:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677471.387926:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.387927:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677471.387929:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.387930:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677471.387930:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677471.387932:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677471.387932:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.387933:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.387933:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677471.388356:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.388359:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.388360:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.388360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.388363:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.388366:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.389058:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.389061:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.389062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.389063:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.389065:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.389068:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.389404:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.389407:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.389407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.389408:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.389410:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.389413:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.389525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.2:1619677471.389930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.389936:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.389939:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.389940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.389942:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.389945:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.389946:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.389948:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.389950:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.389951:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.389952:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.389956:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.389957:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.389961:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.389962:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.389963:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.389965:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.389966:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.389968:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.389969:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.389969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.389974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.389983:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.389985:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.389986:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.389987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.390051:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.390054:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.390055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.390056:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.390058:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.390062:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.390071:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.2:1619677471.390073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:2.0:1619677471.390074:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.390075:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.390075:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.390076:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.390077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.390310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.391549:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.391551:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.391552:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:1.0:1619677471.391552:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.391553:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.391554:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.391554:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677471.391556:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.391558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.391559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.391559:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.391560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.391561:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.391562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.391562:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.391563:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.391566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.391568:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.391570:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.391571:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.391571:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.391572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.391572:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.392084:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.392087:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.392087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.392090:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.392092:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.392093:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677471.392094:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.392096:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.392097:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.392098:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677471.392101:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.392102:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.392104:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.392105:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.392106:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.392108:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.392108:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.392110:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677471.392111:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677471.392111:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.392123:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677471.392123:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.392123:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.392124:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:1.0:1619677471.392124:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.392125:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.392125:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.392127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.392127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677471.392127:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.392258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.392265:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.392268:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.392269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.392271:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.392273:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.392275:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.392276:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.392278:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.392279:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.392280:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.392283:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.392286:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.392288:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.392289:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.392289:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.392291:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.392292:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.392297:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.392297:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.392306:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.392309:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.392309:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.392310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.392311:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.392312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.392557:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.392560:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.392561:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.392562:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.392564:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.392567:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.392576:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.392579:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.392580:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.392580:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.392581:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.392582:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.393327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.393334:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.393337:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.393338:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.393341:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.393342:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.393344:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.393346:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.393348:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.393349:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.393350:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.393357:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.393360:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.393363:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.393364:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.393365:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.393367:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.393368:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.393368:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.393369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.393386:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.393388:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.393389:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.393390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.393392:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.393395:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.393980:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.393983:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.393984:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.393984:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.393986:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.393989:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.394803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.394810:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.394813:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.394814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.394817:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.394819:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.394821:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.394822:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.394824:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.394825:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.394826:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.394831:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.394834:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.394836:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.394838:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.394839:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.394840:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.394841:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.394842:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.394844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.394855:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.394857:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.394858:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.394859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.394862:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.394866:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.395242:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.395245:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.395246:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.395246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.395249:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.395252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.398287:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.398290:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.398290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.398298:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.398300:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.398301:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.398302:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.398304:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.398305:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.398306:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.398309:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.398311:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.398314:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.398314:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.398315:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.398317:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.398317:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.398318:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.398318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:12.0:1619677471.398372:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.398377:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.398380:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.398383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.398383:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.398386:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:12.0:1619677471.398388:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96517786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.398390:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.398392:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.398393:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:12.0:1619677471.398399:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.398402:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677471.398405:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.398407:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:12.0:1619677471.398411:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:12.0:1619677471.398413:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:12.0:1619677471.398414:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.398416:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.398417:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677471.398570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:1.0:1619677471.398844:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.398847:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.398848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.398848:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.398851:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.398853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.399446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.399491:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.399494:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.399495:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.399495:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.399497:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.399499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.399666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.399672:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.399675:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.399675:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.399676:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.399677:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:18.0:1619677471.399680:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.399682:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677471.399683:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.399686:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.399688:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.399690:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.399692:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.399693:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.399695:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.399696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.399697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.399699:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.399702:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.399705:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.399705:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.399706:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.399708:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.399709:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.399710:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.399711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.399715:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.399718:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.399719:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.399719:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.399722:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.399724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.399733:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.399736:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.399737:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.399737:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.399738:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.399739:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.401254:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.401257:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.401259:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.401259:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.401261:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.401264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.401925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.401932:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.401935:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.401936:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.401938:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.401940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.401942:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.401943:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.401948:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.401949:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.401950:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.401953:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.401955:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.401957:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.401958:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.401958:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.401960:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.401961:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.401962:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.401963:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.401964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.401969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.401975:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.401976:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.401976:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.401977:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.401977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.401977:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.401979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.401979:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.401980:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.401980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.402040:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.402043:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.402043:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.402044:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.402046:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.402049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.402058:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.402061:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.402062:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.402062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.402063:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.402064:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.403481:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.403487:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.403489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.403493:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.403496:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.403499:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.403501:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.403505:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.403507:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.403509:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.403515:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.403518:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.403522:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.403524:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.403525:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.403528:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.403528:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.403531:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.403532:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.403534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.403546:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.403550:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.403552:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.403554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.403990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.403996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.403999:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.404000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.404002:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.404005:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.404006:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.404007:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.404009:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.404010:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.404011:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.404014:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.404016:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.404019:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.404021:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.404021:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.404024:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.404024:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.404025:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.404026:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.404602:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.404605:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.404606:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.404606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.404608:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.404612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.405368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.405404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.405411:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.405414:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.405415:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.405415:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.405417:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.405419:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.406847:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.406849:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.406850:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.406851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.406853:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.406856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.408068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.408071:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.408072:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.408074:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.408076:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.408077:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.408079:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.408080:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.408081:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.408082:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.408085:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.408087:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.408089:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.408090:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.408091:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.408093:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.408093:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.408094:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.408094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.409466:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.409469:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.409470:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.409470:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.409473:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.409478:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.409480:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.409482:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.409483:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.409484:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.409485:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.409485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.409924:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.409926:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.409927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.409928:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.409930:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.409934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.410038:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.410041:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.410042:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.410042:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.410044:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.410047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.410071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.410929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.410935:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.410938:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.410939:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.410941:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.410944:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.410945:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.410946:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.410948:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.410949:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.410950:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.410953:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.410955:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.410957:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.410957:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.410959:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.410960:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.410961:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.410962:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.410962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.410974:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.410977:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.410977:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.410978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.410979:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.410981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.410994:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.410996:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.410997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.410997:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.410998:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.410998:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.411814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:18.0:1619677471.411821:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677471.411824:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:2.0:1619677471.411825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:18.0:1619677471.411827:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.411829:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.411831:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677471.411832:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.411834:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.411835:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.411836:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677471.411840:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.411842:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677471.411845:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.411846:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677471.411847:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677471.411850:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677471.411850:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677471.411852:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:18.0:1619677471.411852:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:20.2:1619677471.411924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.411969:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.411973:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.411974:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.411975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.411978:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.411983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.411984:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.411987:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.411989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.411989:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.411990:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.411991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.412047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.412052:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.412055:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.412056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.412058:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.412061:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.412063:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.412064:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.412065:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.412066:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.412068:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.412071:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.412073:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.412076:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.412076:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.412077:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.412079:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.412080:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.412081:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.412083:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.412083:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.412088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.412094:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.412096:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.412097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.412098:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.413041:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.413044:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.413045:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.413048:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.413050:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.413051:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.413052:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.413054:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.413055:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.413056:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.413059:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.413060:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.413063:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.413063:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.413064:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.413066:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.413066:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.413067:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.413067:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.413268:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.413271:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.413272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.413272:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.413274:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.413278:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.414154:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.414157:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.414157:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.414158:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.414160:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.414163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.414790:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.414793:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.414794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.414795:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.414797:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.414801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.414985:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.414988:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.414989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.414991:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.414993:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.414995:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677471.414997:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.414999:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.415000:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.415001:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677471.415005:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.415007:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677471.415011:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.415012:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:1.0:1619677471.415013:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677471.415015:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677471.415015:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.415016:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.415017:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.415018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:13.0:1619677471.415028:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:17.0:1619677471.415029:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:13.0:1619677471.415030:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:17.0:1619677471.415031:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:13.0:1619677471.415031:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.415032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.415033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:13.0:1619677471.415033:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.415035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.415036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677471.415043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677471.415047:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677471.415050:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677471.415051:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.415052:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677471.415052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.415054:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677471.416401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.416407:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.416410:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.416412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.416414:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.416417:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.416419:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.416421:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96517697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.416423:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.416424:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.416425:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.416429:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.416432:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.416435:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.416436:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.416437:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.416440:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.416441:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.416442:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.416442:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.417761:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.417764:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.417765:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.417765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.417768:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.417771:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.417819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.417866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.417873:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.417875:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.417877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.417877:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.417881:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.417884:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.418305:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.418308:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.418308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.418311:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.418313:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.418315:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.418316:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.418318:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.418319:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.418320:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.418323:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.418325:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.418327:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.418328:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.418329:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.418330:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.418331:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.418331:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.418332:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.419216:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.419219:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.419220:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.419220:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.419223:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.419226:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.419464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677471.419513:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.419516:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.419517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.419518:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.419520:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.419524:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.419682:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.419685:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.419686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.419687:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.419689:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.419695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.419697:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.419699:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.419700:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.419700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.419702:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.419703:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.420153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677471.420160:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.420163:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677471.420164:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677471.420166:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.420168:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.420170:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677471.420171:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.420173:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.420174:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.420175:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677471.420179:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.420180:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677471.420184:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.420185:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677471.420185:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677471.420187:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677471.420187:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.420189:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677471.420190:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677471.420191:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:5.2:1619677471.420197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677471.420203:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677471.420205:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677471.420206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677471.420207:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:20.2:1619677471.420316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.421252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.421258:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.421261:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.421262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.421264:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.421267:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.421269:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.421270:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.421272:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.421273:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.421274:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.421277:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.421279:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.421282:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.421283:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.421284:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.421286:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.421287:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.421288:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.421289:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.421825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.421832:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.421835:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.421836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.421838:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.421841:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.421843:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.421844:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.421846:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.421847:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.421848:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.421851:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.421853:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.421854:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.421855:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.421856:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.421857:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.421858:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.421859:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.421859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.421871:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.421873:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.421873:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.421874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.421875:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.421877:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.423779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.423820:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.423822:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.423823:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.423823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.423825:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.423828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.424640:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.424642:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.424643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.424644:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.424646:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.424649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.424720:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.424723:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.424724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.424724:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.424726:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.424729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.424743:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.424745:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.424746:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.424746:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.424747:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.424748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.424991:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.424993:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.424994:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.424996:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.424998:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.424999:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.425000:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.425001:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.425003:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.425004:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.425007:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.425009:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.425011:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.425012:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.425013:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.425015:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.425015:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.425017:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.425017:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.425018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.425029:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.425031:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.425031:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.425032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.425585:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.425588:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.425589:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.425589:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.425592:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.425595:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.426078:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.426080:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.426081:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.426082:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.426083:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.426085:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.426478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677471.426987:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.426991:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.426995:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677471.426997:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677471.426997:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.427000:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677471.427003:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96517643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.427005:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.427007:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.427008:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677471.427013:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.427016:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677471.427020:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.427021:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677471.427022:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677471.427025:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677471.427026:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.427027:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.427027:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677471.427040:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.427042:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.427044:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.427044:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.427046:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.427049:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.427363:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.427366:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.427368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.427368:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.427371:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.427374:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.428625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.428633:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.428635:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.428636:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.428638:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.428640:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.428642:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.428643:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.428644:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.428646:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.428646:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.428650:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.428652:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.428655:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.428656:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.428657:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.428658:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.428659:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.428660:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.428661:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.429501:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.429505:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.429506:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.429506:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.429508:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.429512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.429693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.429729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.429736:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.429738:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.429740:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.429740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.429742:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.429745:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.429754:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.429757:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.429757:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.429758:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.429759:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.429759:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.429875:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.429878:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.429879:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.429881:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.429883:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.429885:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.429886:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.429887:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.429888:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.429889:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.429892:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.429894:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.429896:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.429897:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.429898:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.429899:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.429900:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.429901:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.429901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.430097:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.430100:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.430101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.430101:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.430103:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.430106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.430373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.430380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.430383:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.430383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.430385:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.430388:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.430389:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.430390:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.430392:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.430393:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.430394:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.430397:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.430399:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.430401:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.430401:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.430402:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.430404:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.430404:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.430405:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.430406:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.430417:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.430419:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.430420:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.430421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.430422:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.430424:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.430720:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.430723:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.430724:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.430724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.430726:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.430731:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.430732:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.430734:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.430735:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.430736:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.430736:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.430736:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677471.431232:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.431235:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.431235:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.431236:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.431238:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.431241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.431249:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.431252:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.431253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.431253:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.431254:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.431255:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.432031:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.432034:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.432035:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.432035:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.432037:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.432040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.433005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.433012:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.433015:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.433016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.433018:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.433020:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.433021:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.433023:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.433024:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.433025:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.433026:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.433029:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.433031:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.433034:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.433034:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.433035:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.433037:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.433038:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.433040:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.433041:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.433041:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.433047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.433052:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.433054:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.433055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.433056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.434347:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.434349:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.434350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.434351:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.434353:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.434356:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.435603:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.435606:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.435606:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.435608:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.435610:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.435612:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.435613:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.435614:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.435616:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.435617:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.435620:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.435622:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.435625:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.435625:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.435626:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.435628:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.435628:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.435629:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.435629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.435807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.435839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:1.0:1619677471.435847:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.435850:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.435851:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.435851:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.435854:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.435856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.435882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:12.0:1619677471.435886:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.435889:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677471.435889:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.435891:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.435893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.435894:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677471.437931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.437937:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.437940:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.437942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.437943:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.437946:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.437948:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.437949:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.437951:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.437952:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.437953:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.437956:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.437958:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.437960:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.437961:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.437962:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.437964:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.437965:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.437966:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.437967:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.437967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.437974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.437980:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.437981:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.437981:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677471.437982:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.437982:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677471.437983:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.437983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.437984:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.437985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.437985:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677471.438745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.438752:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.438755:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.438756:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.438757:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.438761:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.438762:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.438764:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.438766:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.438766:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.438768:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.438771:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.438773:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.438775:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.438776:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.438777:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.438779:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.438779:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.438780:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.438781:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.439707:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.439710:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.439711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.439711:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.439714:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.439719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.439721:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.439723:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.439724:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.439725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.439725:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.439726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.440022:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.440025:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.440026:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.440026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.440028:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.440031:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.440880:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.440885:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.440885:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.440887:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.440890:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.440891:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.440892:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.440894:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.440895:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.440896:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.440899:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.440900:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.440903:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.440904:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.440904:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.440906:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.440906:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.440907:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.440908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.441664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.441705:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.441707:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.441708:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.441708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.441711:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.441712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:12.0:1619677471.443059:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.443062:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.443062:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.443064:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.443066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.443066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677471.443529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.443948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.443955:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.443958:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.443959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.443961:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.443963:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.443964:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.443966:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.443967:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.443968:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.443969:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.443973:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.443975:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.443978:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.443979:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.443980:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.443982:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.443983:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.443985:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677471.443986:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677471.443986:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677471.443992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.443997:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677471.444000:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677471.444001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.444002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:1.0:1619677471.444152:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.444155:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.444156:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.444158:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.444160:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.444162:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.444163:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.444164:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.444165:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.444166:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.444170:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.444171:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.444174:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.444175:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.444176:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677471.444178:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.444178:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.444179:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.444179:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677471.444540:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.444542:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.444542:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:18.0:1619677471.444543:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.444543:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.444544:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.444545:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:2.0:1619677471.444546:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.444549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.444550:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.444550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.444551:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.445121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.445127:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.445130:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.445131:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.445133:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.445136:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.445138:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.445139:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.445140:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.445141:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.445142:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.445145:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.445147:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.445149:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.445150:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.445151:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.445152:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.445153:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.445154:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.445154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.445165:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.445168:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.445168:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.445169:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.445170:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.445172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.445642:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.445645:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.445646:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.445646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.445649:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.445652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.445736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.445743:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.445746:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.445747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.445748:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.445751:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.445752:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.445753:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.445755:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.445756:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.445757:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.445760:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.445762:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.445765:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.445766:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.445767:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.445769:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.445769:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.445770:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.445771:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.445925:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.445928:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.445929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.445931:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.445933:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.445934:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.445936:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.445937:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.445938:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.445939:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.445942:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.445944:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.445946:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.445947:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.445947:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.445949:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.445950:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.445951:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.445952:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.445952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.445964:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.445965:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.445966:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.445967:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.446738:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.446741:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.446742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.446743:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.446745:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.446747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.446757:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.446760:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.446760:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.446761:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.446762:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.446763:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.447245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.447281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.447286:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.447289:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.447290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.447291:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.447293:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.447300:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.448207:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.448210:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.448211:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.448213:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.448215:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.448217:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677471.448218:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.448219:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.448220:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.448222:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677471.448225:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.448227:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677471.448230:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.448231:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677471.448231:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677471.448233:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677471.448233:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.448234:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.448235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:2.0:1619677471.448307:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.448310:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677471.448312:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:20.2:1619677471.448313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:2.0:1619677471.448313:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.448316:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.448318:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677471.448320:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.448322:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.448323:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.448324:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677471.448328:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.448330:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677471.448333:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.448334:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677471.448335:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677471.448337:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677471.448338:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.448339:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677471.448341:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677471.448341:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.448353:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677471.448354:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677471.448355:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.448357:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677471.449352:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.449355:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.449356:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.449357:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.449359:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.449362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.450190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.450196:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.450199:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.450200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.450202:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.450204:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.450206:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.450207:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.450209:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.450210:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.450211:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.450214:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.450216:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.450219:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.450220:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.450221:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.450223:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.450224:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.450225:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.450225:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.450555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.450558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.450558:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.450561:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.450563:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.450565:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.450566:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.450567:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.450568:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.450570:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.450573:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.450575:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.450577:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.450578:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.450579:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.450580:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.450581:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.450581:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.450582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.451415:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.451417:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.451418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.451419:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.451421:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.451424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677471.452519:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.452523:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.452527:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.452530:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.452530:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.452533:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677471.452535:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96517535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.452537:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.452539:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.452540:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677471.452546:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.452550:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677471.452553:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.452554:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677471.452555:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677471.452557:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677471.452558:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.452560:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677471.452562:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677471.452563:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.452578:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677471.452581:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677471.452582:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.452583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677471.452827:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.452830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.452830:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677471.452831:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.452833:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.452836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.452848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.452890:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.452893:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.452894:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.452894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.452897:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.452899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.452909:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.452912:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.452912:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.452913:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.452914:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.452914:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.452932:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.452934:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.452935:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.452937:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.452939:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.452941:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677471.452942:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.452943:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.452944:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.452945:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677471.452948:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.452950:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677471.452953:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.452954:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677471.452955:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677471.452957:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677471.452957:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.452958:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.452959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677471.452971:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.452975:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.452976:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.452979:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.452980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.452982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677471.452992:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677471.452996:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677471.452997:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.452998:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677471.452999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.453001:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:2.0:1619677471.453032:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.453036:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.453038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.453038:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.453042:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.453047:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.453050:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.453053:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.453054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.453055:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.453056:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.453057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.454083:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.454086:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.454087:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.454087:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.454089:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.454093:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.454099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.454147:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.454150:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.454151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.454151:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.454153:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.454155:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.454247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.454252:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.454255:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.454256:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.454259:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.454262:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.454263:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.454264:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96517523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.454266:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.454268:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.454269:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.454272:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.454274:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.454277:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.454278:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.454279:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.454281:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.454281:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.454282:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.454283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.454294:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.454299:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.454301:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.454301:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.454303:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.454304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.455859:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.455862:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.455863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.455865:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.455868:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.455869:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.455871:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.455872:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.455873:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.455876:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.455879:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.455881:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.455883:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.455884:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.455885:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.455887:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.455887:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.455888:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.455888:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.456744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.456751:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.456754:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.456755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.456756:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.456759:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.456761:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.456763:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.456765:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.456767:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.456768:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.456772:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.456775:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.456778:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.456779:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.456780:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.456782:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.456782:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.456784:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.456785:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.456786:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.456791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.456797:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.456799:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.456800:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.456801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.456813:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.456816:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.456817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.456817:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.456819:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.456822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.458248:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.458250:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.458251:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.458253:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.458256:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.458257:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677471.458259:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.458260:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.458261:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.458262:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677471.458265:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.458267:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677471.458270:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.458271:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677471.458272:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677471.458273:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677471.458274:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.458275:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.458275:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677471.458386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.458421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677471.459057:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.459060:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.459062:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.459062:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.459064:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.459070:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.459072:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.459074:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.459075:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.459076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.459076:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.459077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.459606:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.459609:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.459610:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.459612:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.459615:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.459616:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677471.459617:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.459618:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.459619:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.459620:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677471.459623:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.459625:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.459627:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.459628:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.459628:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.459630:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.459630:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.459631:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.459631:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677471.459643:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.459645:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.459646:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.459647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.459648:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.459649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.459830:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.459832:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.459833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.459833:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.459835:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.459838:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.459849:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.459852:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.459853:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.459853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.459854:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.459855:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.460159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.460203:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.460206:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.460207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.460207:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.460209:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.460211:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.460588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.460595:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.460597:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.460598:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.460600:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.460602:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.460603:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.460605:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.460607:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.460608:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.460609:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.460612:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.460614:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.460616:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.460617:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.460617:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.460619:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.460619:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.460620:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.460621:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.460633:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.460635:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.460636:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.460637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.460638:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.460639:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.462128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.462131:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.462131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.462134:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.462136:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.462137:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.462138:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.462139:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.462140:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.462141:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.462144:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.462146:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.462148:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.462149:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.462150:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.462152:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.462152:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.462153:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.462153:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.462375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.462382:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.462385:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.462386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.462388:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.462390:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.462391:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.462392:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.462394:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.462395:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.462396:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.462399:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.462401:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.462404:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.462404:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.462405:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.462407:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.462407:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.462408:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.462409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.462545:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.462548:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.462549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.462549:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.462551:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.462555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.464385:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.464388:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.464389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.464390:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.464393:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.464397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.466661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.466704:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.466707:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.466708:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.466708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.466710:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.466712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.466723:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.466725:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.466726:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.466726:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.466727:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.466728:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.466778:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.466781:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.466782:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.466782:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.466784:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.466787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.467120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.467127:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.467130:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.467131:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.467133:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.467136:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.467137:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.467138:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.467140:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.467141:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.467142:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.467145:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.467147:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.467150:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.467151:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.467152:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.467154:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.467154:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.467155:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.467155:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.468091:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.468094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.468094:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.468096:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.468099:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.468100:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.468101:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.468103:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.468103:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.468104:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.468107:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.468109:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.468112:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.468112:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.468113:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.468115:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.468116:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.2:1619677471.468117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677471.468117:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.468118:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.468119:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:2.0:1619677471.468123:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.468127:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.468128:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677471.468130:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.468132:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000400:00000200:2.0:1619677471.468132:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677471.468133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.468134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:2.0:1619677471.468135:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.468137:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.468139:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.468142:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.468144:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.468145:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.468150:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.468154:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.468157:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.468158:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.468159:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.468161:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.468162:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.468164:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.468165:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.468166:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.468171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.468177:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.468178:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:1.0:1619677471.468178:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:18.0:1619677471.468179:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.468179:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.468180:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.468181:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677471.468181:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.468182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677471.468182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:1.0:1619677471.468182:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.468183:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.468184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:1.0:1619677471.468185:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.468186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.468188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.468245:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.468248:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677471.468249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677471.468251:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.468253:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.468255:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.468256:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.468258:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.468259:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.468261:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.468264:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.468266:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.468268:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.468269:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.468270:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.468272:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.468273:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.468274:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.468274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677471.468284:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.468286:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.468287:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.468287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.468288:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.468290:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.468304:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.468307:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.468309:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.468309:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.468310:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.468310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.469212:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.469215:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.469216:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.469216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.469219:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.469223:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.469710:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.469713:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.469714:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.469714:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.469717:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.469720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.471915:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.471918:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.471919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.471919:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.471922:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.471925:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.472684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.472730:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.472732:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.472733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.472733:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.472735:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.472737:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.473121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.473127:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.473130:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.473131:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.473133:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.473135:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.473137:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677471.473138:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.473140:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.473141:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.473142:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677471.473142:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.473145:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:18.0:1619677471.473146:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.473146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:18.0:1619677471.473148:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:17.0:1619677471.473149:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.473151:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.473152:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.473153:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677471.473153:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:17.0:1619677471.473153:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.473154:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96517447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677471.473155:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:17.0:1619677471.473155:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677471.473156:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:17.0:1619677471.473156:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677471.473157:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:17.0:1619677471.473157:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677471.473158:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:17.0:1619677471.473160:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.473162:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.473164:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.473165:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.473166:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.473168:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.473168:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.473169:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.473170:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.473398:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.473400:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.473401:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.473401:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.473403:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.473407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.473417:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.473419:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.473420:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.473420:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.473421:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.473422:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.474030:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.474033:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.474034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.474035:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.474037:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.474041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.474420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.2:1619677471.474457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:5.2:1619677471.474461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:13.0:1619677471.474466:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:1.0:1619677471.474468:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:13.0:1619677471.474469:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.474471:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.474471:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677471.474472:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:12.0:1619677471.474474:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:13.0:1619677471.474475:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:1.0:1619677471.474475:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677471.474478:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.474479:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:1.0:1619677471.474480:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677471.474481:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.474483:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.474484:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.474485:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677471.474489:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677471.474491:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677471.474494:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.474495:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677471.474495:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677471.474497:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677471.474498:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677471.474499:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677471.474499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:20.2:1619677471.475045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.475053:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.475056:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.475056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.475059:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.475061:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.475063:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.475064:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.475065:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.475066:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.475067:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.475071:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.475073:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.475077:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.475078:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.475078:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.475080:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.475081:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.475082:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.475082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.475227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677471.475834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.475841:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.475844:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.475845:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.475847:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.475850:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.475852:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.475853:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.475855:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.475856:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.475857:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.475861:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.475863:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.475865:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.475866:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.475867:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.475869:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.475870:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.475871:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.475871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.475882:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.475884:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.475885:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.475885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.475887:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.475888:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.476461:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.476463:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.476464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.476465:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.476467:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.476470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.478449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.478487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.478493:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.478495:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.478496:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.478497:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.478498:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.478498:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.478500:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677471.478500:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.478503:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.478504:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.478505:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.478506:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.478507:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.478508:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.478509:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.478510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:1.0:1619677471.478511:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.478513:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.478516:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.478517:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.478517:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.478519:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.478519:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.478520:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.478521:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677471.478700:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.478703:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.478705:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.478705:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.478707:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.478710:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.478843:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.478845:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.478846:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.478846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.478849:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.478852:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.478861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.478864:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.478865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.478865:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.478866:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.478867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.479116:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.479119:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.479119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.479120:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.479122:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.479126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.479135:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.479137:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.479138:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.479138:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.479140:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.479140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.479262:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.479265:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.479266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.479268:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.479271:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.479272:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677471.479273:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.479275:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.479275:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.479277:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677471.479280:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.479281:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677471.479284:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.479285:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677471.479286:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677471.479287:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677471.479288:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.479289:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.479289:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677471.479301:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.479304:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.479305:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.479305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.479307:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.479310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.479343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.479349:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.479352:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.479353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.479355:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.479357:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.479359:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.479360:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.479362:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.479363:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.479364:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.479368:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.479370:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.479374:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.479375:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.479376:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.479378:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.479379:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.479380:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.479382:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.479382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.479387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.479393:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.479395:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.479396:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.479397:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.482314:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.482316:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.482317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.482319:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.482322:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.482323:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677471.482325:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.482326:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.482327:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.482328:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677471.482331:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.482333:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677471.482336:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.482336:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677471.482337:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677471.482339:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677471.482339:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.482341:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.482342:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.482343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.482354:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.482356:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.482356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.482357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677471.482453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.482459:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.482462:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.482463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.482465:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.482469:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.482470:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.482472:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.482474:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.482475:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.482476:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.482480:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.482483:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.482485:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.482486:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.482487:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.482489:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.482490:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.482491:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.482492:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.482502:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.482505:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.482505:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.482506:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.482507:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.482509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.483433:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.483437:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.483438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.483438:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.483441:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.483444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.483877:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.483880:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.483881:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.483881:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.483883:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.483887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.484117:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.484120:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.484121:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.484122:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.484124:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.484127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.484288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.484294:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.484299:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.484302:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677471.484305:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.484307:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.484309:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.484310:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.484312:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.484313:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.484314:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.484318:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.484320:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.484323:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.484324:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.484325:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.484327:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.484328:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.484329:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.484330:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.484391:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.484394:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.484394:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.484397:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.484399:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.484401:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.484402:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.484403:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.484405:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.484405:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.484409:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.484411:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.484414:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.484415:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.484415:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.484417:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.484417:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.484418:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.484419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.484933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.484977:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.484980:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.484981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.484981:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.484982:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.484985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.484997:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.485000:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.485001:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.485001:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.485002:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.485003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.485212:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.485215:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.485216:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.485216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.485219:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.485222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.485374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.485377:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.485378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.485378:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.485381:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.485385:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.488616:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.488618:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.488619:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.488621:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.488624:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.488625:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.488627:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.488628:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.488629:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.488630:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.488634:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.488636:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.488638:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.488639:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.488640:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.488642:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.488642:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.488644:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.488645:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.488646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.488657:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.488660:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.488661:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.488662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.488974:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.488977:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.488977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.488978:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.488980:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.488984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.488995:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.488998:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.488999:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.489000:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.489001:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.489002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.491012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.491057:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.491060:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.491061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.491061:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.491063:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.491065:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.491130:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.491133:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.491133:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.491134:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.491136:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.491140:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.491149:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.491152:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.491153:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.491153:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.491154:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.491155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.491775:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.491778:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.491779:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.491780:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.491782:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.491785:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.491999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.492004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.492007:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.492009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.492010:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.492012:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.492014:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.492015:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.492017:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.492018:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.492019:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.492022:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.492024:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.492027:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.492027:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.492028:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.492030:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.492031:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.492032:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.492032:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.492044:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.492046:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.492046:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.492047:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.492048:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.492050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.492069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.492075:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.492077:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.492078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.492080:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.492082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.492083:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.492085:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.492087:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.492088:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.492089:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.492092:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.492094:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.492097:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.492098:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.492099:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.492101:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.492102:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.492103:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.492103:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.493239:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.493242:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.493243:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.493245:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.493247:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.493248:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.493250:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.493252:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.493253:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.493254:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.493257:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.493260:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.493262:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.493263:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.493264:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.493266:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.493266:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.493267:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.493268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.493890:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.493893:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.493893:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.493894:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.493896:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.493899:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.495404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.495410:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.495413:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.495415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.495417:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.495420:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.495422:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677471.495424:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96517354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.495425:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.495426:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.495428:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.495432:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.495434:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.495437:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.495438:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677471.495439:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677471.495441:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677471.495442:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.495443:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.495444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:12.0:1619677471.495957:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677471.495961:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677471.495964:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.111@o2ib7 : GET - for me
00000800:00000200:13.0:1619677471.495965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677471.495968:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.111@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677471.495970:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.111@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677471.495971:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677471.495972:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677471.495974:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677471.495981:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677471.495984:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677471.495986:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.495987:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677471.495988:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.495989:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677471.495990:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.496002:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.496005:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.496006:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.496008:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: REPLY: OK
00000800:00000200:12.0:1619677471.496008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.496010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677471.496272:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.496275:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.496276:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.496276:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.496278:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.496282:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.496795:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.496798:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.496798:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.496801:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.496803:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.496805:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677471.496806:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.496807:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.496809:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.496809:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677471.496812:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.496814:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677471.496816:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.496817:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677471.496818:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677471.496820:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677471.496820:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.496821:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677471.496822:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677471.496823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.496833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:18.0:1619677471.496834:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.496836:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:1.0:1619677471.496836:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677471.496837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677471.496837:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.496837:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:18.0:1619677471.496838:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.496840:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.496843:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.496917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.496962:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.496965:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.496966:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.496966:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.496968:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.496971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.497196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.497202:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.497204:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.497206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.497208:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.497210:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.497212:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.497213:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.497215:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.497216:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.497217:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.497221:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.497223:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.497226:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.497227:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.497228:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.497230:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.497230:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.497232:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.497232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.497383:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.497386:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.497387:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.497387:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.497390:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.497393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:12.0:1619677471.497572:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.497575:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.497577:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677471.497579:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.497580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.497580:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677471.497581:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96517343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.497583:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.497583:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.497584:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677471.497587:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.497589:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:12.0:1619677471.497589:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677471.497592:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:12.0:1619677471.497592:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.497592:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:17.0:1619677471.497593:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677471.497593:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677471.497593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.497595:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.497595:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677471.497595:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.497596:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.497597:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677471.497598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.497608:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677471.497609:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677471.497611:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.497612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677471.497612:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:1.0:1619677471.497612:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677471.497613:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.497614:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.497614:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.497614:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:2.0:1619677471.497617:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.497624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.497642:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.497645:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.497646:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.497646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.497648:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.497651:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.497660:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.497663:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.497664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.497664:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.497665:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.497666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.497887:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.497889:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.497890:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.497891:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.497893:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.497897:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.497906:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.497908:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.497909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.497909:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.497911:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.497912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.498663:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.498666:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.498667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.498669:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.498672:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.498673:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677471.498674:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96517339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.498676:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.498677:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.498679:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.498681:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.498683:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:17.0:1619677471.498684:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.498686:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.498686:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.498688:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.498689:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.498690:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:1.0:1619677471.498690:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677471.498691:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677471.498691:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96517338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677471.498693:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:1.0:1619677471.498693:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.498693:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.498694:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.498695:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.498697:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.498697:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677471.498698:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.498699:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.498701:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.498703:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.498704:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.498704:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.498706:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.498707:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.498707:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.498708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.498777:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.498780:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.498781:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.498781:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.498783:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.498787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:13.0:1619677471.499996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.499999:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.500001:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677471.500003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.500004:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.500006:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677471.500007:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96517334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.500008:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.500009:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.500010:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677471.500013:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.500015:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677471.500018:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.500018:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:13.0:1619677471.500019:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:13.0:1619677471.500021:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677471.500022:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.500023:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.500023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677471.500037:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.500040:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.500042:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.500042:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.500044:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.500048:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.500451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.500458:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.500460:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.500461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.500463:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.500467:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.500468:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.500470:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96517332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.500471:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.500472:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.500473:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.500476:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.500478:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.500480:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.500481:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.500482:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.500483:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.500484:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.500486:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.500487:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.500487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.500493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.500499:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.500500:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.500500:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.500501:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677471.500501:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.500503:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.500504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.500505:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.500506:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.500506:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.502242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.502248:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.502251:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.502252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.502254:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.502257:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.502258:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.502260:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.502262:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.502263:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.502264:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.502268:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.502270:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.502272:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.502273:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.502274:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.502276:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.502278:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.502279:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.502280:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.502280:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.502286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.502291:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.502295:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.502298:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.502299:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.503764:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.503767:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.503768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.503768:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.503770:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.503774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.504028:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.504031:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.504032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.504034:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.504037:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.504039:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.504040:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.504041:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.504042:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.504043:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.504047:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.504049:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.504051:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.504052:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.504052:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.504054:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.504055:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.504055:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.504056:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.504483:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.504486:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.504487:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.504487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.504489:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.504493:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.504539:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.504542:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.504543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.504545:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.504547:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.504549:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.504550:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.504552:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.504553:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.504554:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.504557:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.504559:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.504561:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.504562:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.504563:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677471.504564:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.504565:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.504566:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.504566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:2.0:1619677471.505858:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.505860:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.505862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.505862:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.505864:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677471.505867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.507202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:17.0:1619677471.507780:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.507783:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.507783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.507786:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.507788:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.507790:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677471.507791:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.507792:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.507793:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.507795:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677471.507798:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.507800:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.507802:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.507802:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.507803:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.507805:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.507805:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.507806:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.507806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677471.507820:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.507822:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.507822:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.507823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.507825:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.507826:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.508642:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.508645:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.508646:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.508646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.508649:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.508656:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.508657:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.508659:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.508660:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.508661:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.508661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.508662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.508677:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.508680:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.508680:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.508681:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.508683:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.508689:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.508691:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.508693:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.508694:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.508694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.508695:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.508696:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.508698:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.508701:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.508702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.508704:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.508705:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.508707:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.508708:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.508709:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.508710:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.508711:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.508715:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.508717:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.508719:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.508720:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.508721:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.508722:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.508723:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.508725:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.508726:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.508726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.508737:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.508739:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.508739:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.508740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.508947:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.508950:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.508951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.508951:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.508953:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.508956:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.508976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.508981:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.508984:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.508986:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.508987:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.508990:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.508991:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.508993:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.508994:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.508995:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.508997:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.509000:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.509002:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.509005:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.509006:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.509006:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.509008:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.509010:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.509010:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.509011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.509054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.509060:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.509063:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.509064:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.509066:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.509069:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.509070:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.509071:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.509072:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.509074:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.509074:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.509077:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.509079:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.509081:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.509082:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.509082:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.509084:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.509085:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.509086:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.509086:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.509098:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.509100:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.509100:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.509101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.509102:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.509104:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.509118:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.509121:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.509121:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.509122:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.509122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.509123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.510461:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.510464:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.510466:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.510466:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.510468:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.510471:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.513167:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:18.0:1619677471.513171:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:17.0:1619677471.513171:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.513172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:18.0:1619677471.513174:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000400:00000200:17.0:1619677471.513174:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677471.513174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:17.0:1619677471.513176:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.513177:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.513178:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677471.513179:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.513179:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.513180:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677471.513181:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.513181:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.513182:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.513183:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.513184:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.513184:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677471.513186:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677471.513188:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677471.513191:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.513191:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677471.513193:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677471.513194:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.513194:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.513195:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677471.513197:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.513197:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677471.513197:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.513197:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:18.0:1619677471.513198:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677471.513198:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.513199:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.513200:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677471.513200:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.513202:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.513203:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677471.514002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.514005:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.514006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.514006:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.514008:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.514013:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.514336:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.514339:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.514340:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.514340:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.514342:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.514345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.514812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.514818:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.514820:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.514822:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.514824:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.514826:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.514828:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.514830:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96517277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.514832:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.514833:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.514834:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.514837:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.514839:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.514843:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.514844:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.514845:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.514847:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.514848:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.514848:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.514849:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.516232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.516239:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.516242:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.516243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.516246:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.516249:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.516250:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.516251:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.516253:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.516254:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.516255:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.516258:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.516259:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.516261:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.516262:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.516263:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.516264:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.516265:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.516266:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.516266:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.516277:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.516279:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.516280:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.516281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.516283:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.516284:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.516944:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.516947:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.516947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.516948:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.516950:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.516956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.516958:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.516960:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.516961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.516962:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.516963:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.516963:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.517077:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.517079:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.517080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.517083:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.517085:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.517086:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.517088:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.517089:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.517090:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.517091:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.517094:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.517096:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.517098:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.517099:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.517100:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.517101:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.517102:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.517103:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.517103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.517298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.517333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.517340:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.517342:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.517344:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.517344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.517346:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.517348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.518897:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.518900:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.518902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.518902:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.518904:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.518908:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.519527:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.519529:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.519530:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.519530:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.519532:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.519535:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.519708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.519713:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.519716:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.519717:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.519718:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.519720:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.519722:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.519723:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.519725:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.519726:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.519727:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.519730:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.519732:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.519735:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.519736:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.519738:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.519740:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.519741:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.519741:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.519742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.520612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.520619:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.520622:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.520623:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.520625:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.520628:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.520630:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677471.520631:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.520633:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.520634:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.520635:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677471.520638:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.520640:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677471.520643:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.520644:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677471.520645:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677471.520646:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677471.520647:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.520648:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.520649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:1.0:1619677471.522260:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.522263:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.522264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.522266:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.522268:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.522270:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.522271:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.522272:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.522273:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.522274:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.522278:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.522280:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.522282:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.522283:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.522284:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.522285:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.522286:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.522286:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.522287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.522615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.522621:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.522624:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.522626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.522628:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.522631:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.522632:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.522634:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.522635:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.522637:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.522638:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.522641:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.522643:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.522645:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.522646:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.522647:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.522649:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.522650:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.522651:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.522652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.522662:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.522665:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.522665:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.522666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.522668:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.522669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.523032:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.523035:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.523036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.523037:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.523039:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.523042:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.523495:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.523498:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.523499:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.523499:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.523501:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.523504:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.523515:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.523518:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.523519:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.523519:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.523521:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.523522:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.523747:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.523750:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.523751:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.523751:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.523754:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.523756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.524874:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.524877:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.524878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.524878:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.524881:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.524883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.526206:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.526209:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.526210:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.526212:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.526214:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.526215:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.526217:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.526218:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.526219:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.526221:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.526224:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.526226:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.526228:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.526229:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.526230:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.526231:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.526232:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.526233:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.526234:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.526234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.526245:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.526246:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.526247:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.526248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.526459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.526466:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.526469:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.526470:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.526471:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.526474:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.526475:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.526477:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.526478:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.526479:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.526480:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.526484:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.526485:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.526488:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.526489:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.526490:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.526492:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.526493:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.526495:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.526496:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.526497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.526502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.526507:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.526510:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.526511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.526511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677471.528007:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.528010:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.528011:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.528011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.528013:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.528016:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.528613:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.528616:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.528618:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.528618:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.528620:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.528624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.528636:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.528639:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.528640:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.528640:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.528641:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.528641:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.529531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.530012:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.530015:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.530016:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.530016:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.530018:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.530021:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.530531:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.530534:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.530535:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.530535:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.530537:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.530540:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.530689:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.530693:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.530694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.530696:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.530697:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.530699:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677471.530700:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.530701:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.530702:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.530703:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677471.530706:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.530707:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677471.530710:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.530711:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677471.530711:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677471.530713:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.530714:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.530714:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.530715:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677471.530728:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.530731:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.530732:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.530733:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.530734:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.530737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.531716:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.531718:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.531719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.531720:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.531722:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.531724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.532051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.532057:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.532060:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.532060:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677471.532062:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.532065:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.532067:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.532068:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.532070:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.532071:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.532073:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.532076:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.532078:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.532080:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.532081:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.532082:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.532084:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.532085:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.532086:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.532087:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.532088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.532094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.532098:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.532099:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.532100:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.532100:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677471.532102:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.532102:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.532102:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.532103:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.532104:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.532105:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677471.532335:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.532338:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.532338:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.532342:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.532345:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.532346:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677471.532349:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.532351:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.532352:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.532354:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677471.532359:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.532362:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.532364:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.532365:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.532366:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.532367:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.532367:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.532368:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.532369:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677471.532381:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.532383:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677471.532384:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.532385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.532387:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.532390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.532399:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.532402:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.532402:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.532403:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.532404:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.532404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.532495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.2:1619677471.532615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.532621:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.532624:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.532626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.532627:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.532630:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.532631:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.532633:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.532634:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.532636:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.532637:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.532641:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.532642:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.532646:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.532647:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.532648:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.532650:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.532651:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.532652:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.532653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.533228:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.533230:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.533231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.533232:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.533234:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.533237:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.533245:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.533248:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.533250:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.533250:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.533251:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.533252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.533978:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:18.0:1619677471.533979:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677471.533981:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:18.0:1619677471.533982:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:17.0:1619677471.533982:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:18.0:1619677471.533983:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.533983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:1.0:1619677471.533984:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.533985:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.533986:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677471.533987:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.533988:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677471.533989:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:1.0:1619677471.533989:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.533991:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.533992:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.533995:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.533997:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.534000:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.534001:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.534001:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.534003:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.534003:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.534004:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.534005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.535025:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.535027:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.535028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.535030:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.535032:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.535034:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677471.535035:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.535037:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.535038:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.535039:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677471.535042:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.535044:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677471.535047:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.535048:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677471.535049:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677471.535051:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677471.535051:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.535052:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.535055:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.535055:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.535068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.535070:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.535071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.535072:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677471.535763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.535808:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.535811:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.535812:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.535812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.535814:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.535816:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.536455:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.536458:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.536459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.536459:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.536461:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.536464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.537287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.537293:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.537298:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.537301:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677471.537303:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.537306:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.537307:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.537308:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.537310:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.537311:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.537312:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.537315:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.537317:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.537320:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.537321:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.537322:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.537324:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.537324:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.537325:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.537325:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.538989:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.538992:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.538993:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.538993:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.538996:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.538998:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.539019:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.539023:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.539024:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.539024:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.539026:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.539030:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.540771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.540778:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.540781:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.540782:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.540784:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.540786:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.540787:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.540789:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.540791:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.540792:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.540793:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.540796:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.540798:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.540800:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.540801:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.540802:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.540803:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.540804:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.540805:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.540805:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.540816:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.540819:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.540819:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.540820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.540821:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.540822:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.541029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.541034:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.541037:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.541038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.541039:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.541041:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.541042:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677471.541044:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.541045:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.541046:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.541046:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677471.541050:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.541051:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677471.541054:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.541054:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677471.541055:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677471.541056:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677471.541057:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.541058:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677471.541059:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677471.541060:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677471.541064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.541070:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677471.541073:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677471.541073:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.541074:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677471.541095:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.541098:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.541099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.541100:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.541102:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.541105:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.541350:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.541353:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.541354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.541356:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.541359:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.541360:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.541361:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.541363:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.541364:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.541365:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.541368:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.541369:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.541372:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.541373:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.541374:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.541376:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.541376:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.541377:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.541377:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:12.0:1619677471.542439:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.542442:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.542444:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.542447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.542447:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.542449:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677471.542450:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96517178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.542451:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.542452:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.542453:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677471.542456:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.542457:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677471.542460:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.542461:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677471.542461:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677471.542463:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677471.542463:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.542464:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.542465:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677471.543378:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.543382:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677471.543383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.543384:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.543387:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.543389:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677471.543390:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677471.543391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:18.0:1619677471.543391:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.543392:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.543394:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677471.543398:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.543398:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.543400:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.543400:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677471.543402:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677471.543404:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.543405:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:2.0:1619677471.543405:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.543406:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:2.0:1619677471.543406:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.543407:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96517175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677471.543408:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:2.0:1619677471.543409:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.543409:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.543411:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.543412:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.543413:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.543414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677471.543415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.543415:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.543417:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.543420:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.543420:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.543421:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.543423:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.543424:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.543424:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.543425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677471.543919:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.543922:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.543922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.543923:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.543925:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.543929:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.543938:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.543941:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.543941:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.543942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.543943:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.543943:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.545124:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.545127:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.545128:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.545128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.545131:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.545133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.545990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.546034:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.546037:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.546038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.546038:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.546040:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.546042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.546052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.546054:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.546055:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.546055:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.546056:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.546057:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.546645:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.546648:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.546649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.546651:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.546653:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.546654:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.546656:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.546657:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.546658:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.546659:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.546662:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.546664:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.546666:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.546667:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.546668:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.546669:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.546670:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.546671:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.546671:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:13.0:1619677471.547077:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.547080:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.547081:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.547083:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.547084:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.547085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677471.547391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.547398:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.547401:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.547402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.547404:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.547407:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.547408:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.547410:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.547411:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.547413:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.547414:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.547417:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.547419:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.547421:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.547422:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.547423:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.547425:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.547425:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.547426:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.547427:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677471.547789:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.547791:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.547792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.547793:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.547795:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.547798:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.547808:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.547810:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.547811:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.547811:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.547812:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.547813:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.548728:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.548731:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.548732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.548733:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.548735:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.548738:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.549187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.549193:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.549196:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.549197:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.549199:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.549201:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.549202:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.549204:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.549206:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.549207:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.549208:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.549211:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.549213:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.549215:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.549216:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.549216:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.549218:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.549219:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.549219:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.549220:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.549243:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.549245:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.549245:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.549246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.549248:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.549249:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.549903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.549909:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.549911:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.549912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.549913:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.549916:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.549917:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.549918:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.549919:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.549920:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.549921:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.549923:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.549925:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.549927:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.549928:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.549928:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.549930:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.549930:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.549931:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.549932:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.549933:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.549938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.549944:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.549946:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.549947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.549947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.551283:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.551286:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.551287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.551289:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.551292:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.551299:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.551300:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.551302:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.551303:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.551304:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.551307:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.551308:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.551311:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.551311:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.551312:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.551314:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.551314:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.551316:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.551317:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.551317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.551329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.551331:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.551332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.551332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.551550:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.551552:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.551553:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.551553:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.551555:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.551557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.552874:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.552876:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.552877:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.552878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.552880:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.552882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.553347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.553350:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.553351:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.553352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.553353:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.553357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.553534:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.553537:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.553538:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.553539:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.553541:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.553544:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.553560:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.553564:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.553565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.553565:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.553566:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.553567:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.554237:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.554240:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.554241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.554241:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.554243:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.554246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.554478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.554485:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.554488:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.554488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.554490:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.554492:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.554494:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.554495:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.554497:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.554498:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.554499:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.554502:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.554503:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.554506:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.554507:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.554508:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.554510:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.554510:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.554511:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.554512:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.555140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.555176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677471.555291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b389600] (20)++
00000800:00000200:20.2:1619677471.555331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b389600] (21)++
00000800:00000200:17.0:1619677471.555640:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.555643:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.555643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.555646:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.555648:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.555649:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.555650:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.555651:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.555652:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.555653:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.555656:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.555658:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.555661:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.555662:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.555662:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.555664:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.555665:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.555665:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.555666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.556586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.556592:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.556595:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.556596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.556598:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.556601:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.556602:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.556603:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.556605:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.556607:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.556608:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.556611:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.556613:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.556614:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.556615:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.556616:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.556617:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.556618:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.556618:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.556619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.556631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.556634:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.556634:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.556635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.556636:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.556638:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.556653:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.556655:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.556656:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.556656:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.556657:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.556657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.558500:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.558503:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.558503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.558504:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.558507:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.558510:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677471.558601:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.558605:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.558609:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.558611:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.558612:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.558614:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677471.558616:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96517122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.558619:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.558621:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.558623:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677471.558629:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.558631:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677471.558635:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.558636:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:12.0:1619677471.558637:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:12.0:1619677471.558639:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:12.0:1619677471.558640:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.558643:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677471.558645:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677471.558646:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.558661:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677471.558665:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677471.558666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.558667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677471.560437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.560535:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.560537:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.560538:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.560539:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.560541:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.560544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.560816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.560823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.560826:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.560828:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.560830:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.560832:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.560834:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677471.560835:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:2.0:1619677471.560836:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.560837:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.560839:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677471.560840:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:2.0:1619677471.560840:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677471.560841:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:2.0:1619677471.560846:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677471.560847:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677471.560848:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677471.560850:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.560852:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677471.560853:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677471.560853:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.560855:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:17.0:1619677471.560856:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96517115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.560857:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.560858:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677471.560859:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677471.560859:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677471.560860:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677471.560860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:17.0:1619677471.560862:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.560868:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.560872:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.560876:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.560877:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.560878:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.560881:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.560882:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.560883:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.560884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.562652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.562659:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.562662:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.562662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.562665:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.562668:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.562670:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.562671:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.562673:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.562674:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.562675:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.562679:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.562680:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.562686:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.562687:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.562688:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.562690:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.562692:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.562693:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.562694:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.562695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.562700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.562706:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000800:00000200:18.0:1619677471.562708:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.562708:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000800:00000200:18.0:1619677471.562709:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677471.562709:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.562710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.562710:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000400:00000200:2.0:1619677471.562711:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.562712:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000400:00000200:18.0:1619677471.562713:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.562713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (24)--
00000800:00000200:2.0:1619677471.562714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:18.0:1619677471.562715:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.562715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.562718:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.562831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677471.562879:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.562881:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.562882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.562882:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.562885:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.562888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.563336:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.563338:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.563339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.563340:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.563342:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.563345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.564466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.2:1619677471.564514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:2.0:1619677471.564991:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.564993:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.564995:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.564995:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.564997:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.565001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.565317:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.565322:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.565323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.565327:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.565330:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.565332:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.565335:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.565337:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.565338:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.565340:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.565345:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.565348:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.565352:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.565353:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.565354:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.565357:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.565357:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.565358:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.565359:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.565551:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.565554:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.565555:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.565555:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.565558:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.565564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.565566:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.565569:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.565569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.565570:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.565571:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.565572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.566687:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.566690:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.566691:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.566691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.566693:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.566697:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.567362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.567369:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.567371:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.567372:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.567374:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.567376:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.567377:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.567380:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.567381:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.567382:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.567383:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.567391:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.567393:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.567396:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.567397:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.567398:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.567399:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.567400:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.567401:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.567401:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.2:1619677471.567410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:18.0:1619677471.567484:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.567487:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.567488:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.567488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.567490:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.567494:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:13.0:1619677471.567534:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.567537:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.567538:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.567540:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.567541:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.567548:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:1.0:1619677471.567553:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.567556:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.567557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.567557:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.567559:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.567562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.567573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.567574:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.567574:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000400:00000200:17.0:1619677471.567575:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.567576:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677471.567577:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.567577:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.567578:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.567579:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.567581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.567581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.567582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677471.567722:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.567724:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.567725:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.567726:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.567728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.567729:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677471.568097:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.568100:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.568101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.568101:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.568103:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.568107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.568769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:18.0:1619677471.568815:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.568818:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.568819:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.568819:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.568821:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.568825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.568835:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.568837:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.568838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.568838:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.568839:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.568840:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.569961:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.569964:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.569964:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.569967:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.569969:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.569970:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677471.569972:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.569973:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.569974:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.569975:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677471.569979:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.569980:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677471.569983:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.569984:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677471.569985:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677471.569986:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677471.569987:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.569987:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.569988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677471.570185:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.570189:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.570190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.570192:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.570195:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.570196:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.570198:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96517066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.570200:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.570201:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.570202:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.570206:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.570208:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.570210:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.570212:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.570212:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.570214:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.570214:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.570216:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.570217:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.570217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.570229:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.570231:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.570231:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.570232:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.570432:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.570436:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.570436:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.570437:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.570439:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.570442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.571751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.571757:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.571760:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.571761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.571762:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.571765:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.571766:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.571768:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.571770:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.571771:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.571772:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.571775:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.571777:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.571780:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.571781:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.571782:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.571783:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.571784:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.571785:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.571785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.571797:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.571799:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.571799:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.571800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.571801:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.571802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.572135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.572140:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.572142:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.572144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.572144:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.572146:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.572148:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.572149:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.572150:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.572151:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.572152:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.572154:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.572156:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.572159:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.572159:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.572160:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.572162:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.572162:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.572163:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.572164:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.572164:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.572170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.572175:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.572177:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.572178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.572180:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.572298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.572326:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.572329:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.572330:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.572330:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.572332:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.572336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.572338:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.572339:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.572340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.572340:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.572341:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.572343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.572464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.572467:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.572468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.572470:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.572472:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.572474:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677471.572475:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.572476:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.572477:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.572478:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677471.572482:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.572484:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677471.572487:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.572488:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677471.572489:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677471.572491:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677471.572491:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.572492:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.572492:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:2.0:1619677471.572968:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.572970:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.572972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.572972:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.572974:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.572977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.572987:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.572990:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.572990:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.572990:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.572991:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.572992:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.574212:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.574215:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.574216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.574216:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.574218:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.574222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.574710:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.574713:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677471.574714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677471.574715:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.574719:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.574720:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677471.574721:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96517043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.574723:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.574724:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.574725:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677471.574729:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.574731:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677471.574733:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.574734:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677471.574735:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677471.574736:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677471.574737:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.574738:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677471.574739:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677471.574740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.574752:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677471.574754:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677471.574754:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.574755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:17.0:1619677471.576144:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.576147:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.576147:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.576150:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.576152:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.576154:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.576155:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.576156:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.576157:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.576158:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.576161:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.576162:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.576165:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.576166:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.576167:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.576169:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.576169:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.576170:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.576171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.576912:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.576915:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.576916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.576918:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.576921:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.576922:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677471.576923:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.576925:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.576926:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.576927:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677471.576930:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.576932:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677471.576935:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.576936:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677471.576937:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677471.576938:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.576939:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.576940:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.576940:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677471.576952:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.576955:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.576956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.576956:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.576958:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.576961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.577126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677471.577714:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.577716:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.577717:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.577717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.577720:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.577723:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.577804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.577810:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.577813:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.577814:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.577816:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.577818:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.577820:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.577821:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.577822:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.577824:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.577825:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.577829:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.577831:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.577833:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.577834:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.577835:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.577837:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.577837:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.577838:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.577839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.578082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.578088:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.578091:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.578091:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.578093:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.578096:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.578097:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.578099:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.578100:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.578101:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.578102:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.578104:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.578106:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.578107:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.578108:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.578108:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.578109:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.578110:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.578110:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.578111:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.578123:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.578125:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.578126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.578126:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.578128:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.578128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.578538:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.578541:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.578542:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.578542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.578545:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.578548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.579084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.579126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.579133:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.579135:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.579136:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.579137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.579140:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.579142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.579156:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.579159:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.579160:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.579160:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.579161:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.579162:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.579329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.579332:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.579333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.579335:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.579337:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.579338:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677471.579339:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.579341:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.579342:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.579343:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677471.579346:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.579348:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677471.579351:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.579351:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677471.579352:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677471.579354:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677471.579354:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.579356:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677471.579357:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677471.579358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.579368:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677471.579370:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677471.579371:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.579372:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677471.580610:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.580613:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.580614:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.580614:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.580616:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.580620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.581629:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.581632:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.581632:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.581635:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.581637:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.581638:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.581640:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96517007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.581641:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.581642:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.581643:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.581646:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.581648:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.581651:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.581652:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.581652:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.581654:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.581654:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.581655:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.581655:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677471.582477:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.582479:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.582481:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.582481:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.582484:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.582488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.582705:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.582708:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.582709:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.582709:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.582711:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.582714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.582725:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.582728:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.582729:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.582729:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.582730:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.582731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.582756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.583482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.583489:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.583492:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.583493:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.583495:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.583497:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.583499:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677471.583501:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.583503:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.583504:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.583505:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677471.583508:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.583510:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677471.583514:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.583515:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677471.583516:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677471.583518:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677471.583519:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.583520:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.583520:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:13.0:1619677471.583532:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.583535:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.583536:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.583538:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.583540:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.583541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677471.584602:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.584605:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.584606:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.584606:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.584608:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.584612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.584621:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.584623:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.584624:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.584625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.584626:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.584626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.584858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.584864:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.584867:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.584868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.584870:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.584873:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.584875:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.584876:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.584878:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.584879:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.584880:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.584883:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.584885:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.584887:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.584888:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.584888:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.584890:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.584891:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.584891:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.584892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.584955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.584997:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.585000:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.585001:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.585001:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.585003:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.585005:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.585408:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.585411:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.585412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.585412:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.585414:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.585416:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.585511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.585518:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.585521:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.585522:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.585524:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.585527:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.585528:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.585529:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.585531:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.585532:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.585533:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.585536:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.585538:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.585540:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.585541:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.585542:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.585544:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.585544:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.585545:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.585546:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.585557:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.585559:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.585559:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.585560:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.585561:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.585563:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.585585:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.585588:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.585589:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.585589:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.585589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.585590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.585899:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.585902:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.585903:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.585904:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.585906:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.585908:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677471.586212:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.586215:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.586217:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.586219:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.586219:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.586220:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677471.586222:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96516982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.586224:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.586226:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.586227:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677471.586232:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.586235:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677471.586237:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.586238:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677471.586239:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677471.586240:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677471.586241:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.586242:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.586242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677471.586657:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.586659:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.586660:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.586662:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.586665:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.586666:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677471.586668:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.586669:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.586670:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.586671:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677471.586675:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.586677:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677471.586680:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.586680:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:1.0:1619677471.586681:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677471.586683:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677471.586684:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.586685:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.586685:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677471.586697:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.586700:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.586701:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.586703:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.586704:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.586705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677471.586714:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677471.586716:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677471.586717:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.586718:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677471.586718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.586719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677471.586893:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.586896:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.586897:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.586899:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.586901:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.586903:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677471.586904:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.586906:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.586907:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.586908:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677471.586910:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.586912:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677471.586914:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.586915:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677471.586915:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677471.586917:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677471.586917:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.586918:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.586919:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677471.586931:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.586933:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.586934:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.586935:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.586936:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.586939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.587242:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.587244:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.587245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.587246:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.587248:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.587252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.587320:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.587323:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.587324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.587326:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.587328:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.587329:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.587331:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.587332:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.587333:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.587334:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.587338:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.587339:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.587342:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.587343:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.587344:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.587345:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.587346:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.587347:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.587347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677471.588067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677471.588463:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.588466:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.588468:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.588470:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.588470:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.588471:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:12.0:1619677471.588472:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96516969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.588474:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.588475:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.588476:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:12.0:1619677471.588479:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.588480:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677471.588482:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.588483:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:12.0:1619677471.588484:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:12.0:1619677471.588485:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:12.0:1619677471.588486:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.588486:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.588487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:5.2:1619677471.589192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677471.589199:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.589202:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677471.589202:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677471.589205:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677471.589207:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.589209:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677471.589210:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.589211:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.589213:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.589214:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677471.589217:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677471.589219:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677471.589222:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.589223:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677471.589223:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677471.589225:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677471.589226:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.589226:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677471.589227:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:20.2:1619677471.589813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.589819:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.589822:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.589823:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.589825:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.589827:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.589829:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.589830:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.589832:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.589833:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.589834:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.589838:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.589839:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.589842:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.589843:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.589844:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.589845:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.589846:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.589847:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.589847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.590298:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.590301:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.590302:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.590304:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.590306:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.590308:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677471.590309:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.590310:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.590311:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.590312:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677471.590315:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.590317:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677471.590320:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.590320:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677471.590321:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677471.590323:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677471.590323:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.590324:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.590324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677471.590808:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.590811:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.590812:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.590812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.590815:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.590819:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.591332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.591369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.591376:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.591379:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.591380:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.591380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.591382:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.591385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:13.0:1619677471.591564:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.591567:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.591569:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677471.591571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.591572:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.591573:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677471.591574:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96516955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.591575:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.591576:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.591577:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677471.591580:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.591582:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677471.591584:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.591585:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677471.591586:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677471.591587:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677471.591588:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.591589:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.591590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677471.591991:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.591993:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.591995:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.591995:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.591997:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.592001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.593018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.593025:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:1.0:1619677471.593027:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677471.593028:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.593028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:1.0:1619677471.593030:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:18.0:1619677471.593031:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677471.593031:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.593031:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.593033:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.593034:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677471.593035:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677471.593037:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96516947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677471.593038:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:18.0:1619677471.593039:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.593040:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.593041:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677471.593045:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.593047:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677471.593049:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.593051:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677471.593052:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677471.593054:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677471.593055:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.593056:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.593056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677471.593068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.593071:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.593072:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.593072:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.593074:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.593076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.593085:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.593088:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.593089:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.593089:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.593090:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.593091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.593401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.594716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.594722:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.594725:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.594726:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.594728:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.594731:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.594732:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.594733:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.594735:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.594736:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.594737:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.594741:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.594743:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.594746:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.594747:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.594749:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.594750:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.594752:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.594753:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.594754:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.594754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.594760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.594766:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.594768:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.594769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.594770:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677471.595114:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.595117:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.595121:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.595122:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.595124:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.595126:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677471.595128:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96516939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.595130:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.595132:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.595134:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677471.595139:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.595142:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677471.595145:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.595146:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677471.595147:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677471.595149:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677471.595150:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.595152:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677471.595154:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677471.595155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.595171:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677471.595173:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677471.595174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.595174:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677471.595392:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.595395:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.595395:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.595397:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.595399:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.595401:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:18.0:1619677471.595482:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.595485:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.595486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.595487:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.595489:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.595494:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.595495:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.595497:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.595498:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.595499:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.595500:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.595500:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.596714:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.596717:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.596718:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.596718:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.596721:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.596724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.596919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677471.596963:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.596965:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.596966:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.596966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.596968:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.596970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.597664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.597671:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.597674:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.597674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.597677:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.597679:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.597681:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.597682:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.597684:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.597685:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.597686:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.597689:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.597691:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.597693:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.597694:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.597695:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.597697:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.597698:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.597699:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.597701:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.597701:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.597706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.597711:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.597712:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.597713:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677471.597713:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.597715:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677471.597715:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.597716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.597716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.597717:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.597718:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677471.598174:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.598177:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.598178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.598178:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.598180:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.598183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.598650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677471.598691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:12.0:1619677471.598696:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.598701:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677471.598702:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.598706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677471.598706:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.598711:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.598713:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677471.598716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.598716:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677471.598717:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.598718:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.598719:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:17.0:1619677471.598924:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.598927:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.598928:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.598928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.598930:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.598933:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.598947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.599126:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.599129:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.599130:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.599132:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.599134:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.599135:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677471.599137:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.599138:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.599139:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.599140:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677471.599144:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.599145:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677471.599148:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.599149:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677471.599150:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677471.599152:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677471.599152:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.599154:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.599155:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.599155:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.599168:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.599170:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.599171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.599172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677471.599599:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.599601:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.599603:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.599603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.599605:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.599608:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.599666:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.599669:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.599670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.599670:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.599673:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.599676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:13.0:1619677471.600345:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.600348:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.600350:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677471.600352:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677471.600352:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.600354:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677471.600355:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96516911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.600356:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.600357:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.600358:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677471.600362:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.600363:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677471.600366:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.600367:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677471.600367:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677471.600369:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677471.600369:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.600370:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.600371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677471.600384:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.600387:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.600388:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.600388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.600390:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.600393:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.600404:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.600407:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.600408:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.600408:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.600409:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.600409:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.601463:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.601466:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.601467:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.601467:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.601469:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.601473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.601907:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.601910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.601911:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.601926:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.601929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.601929:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.601931:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.601933:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.601933:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.2:1619677471.601943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.601950:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.601953:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.601954:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.601954:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.601956:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.601958:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.602328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.602334:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.602338:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.602338:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.602340:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.602343:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.602345:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.602347:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.602348:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.602349:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.602351:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.602357:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.602360:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.602363:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.602363:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.602364:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.602366:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.602367:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.602368:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.602369:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.602372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.602390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:12.0:1619677471.602559:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.602561:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677471.602562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.602575:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.602578:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677471.602580:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)++
00000800:00000200:13.0:1619677471.602582:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (24)--
00000800:00000200:12.0:1619677471.602583:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (24)--
00000800:00000200:12.0:1619677471.602583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.602603:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.602606:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.602607:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.602609:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:12.0:1619677471.602609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.602612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677471.602760:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.602762:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677471.602763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.602780:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.602783:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677471.602785:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)++
00000800:00000200:13.0:1619677471.602787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (24)--
00000800:00000200:12.0:1619677471.602789:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677471.602789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.2:1619677471.602811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:13.0:1619677471.602841:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.602844:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.602845:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.602847:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: REPLY: OK
00000800:00000200:12.0:1619677471.602847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.602850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677471.602926:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.602929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.602930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677471.602944:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677471.602946:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:12.0:1619677471.602946:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.602947:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.602948:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.602949:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[3] from 10.0.10.110@o2ib7
00000800:00000200:13.0:1619677471.602950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677471.602950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.602951:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:12.0:1619677471.602951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:18.0:1619677471.602954:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.602954:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.602971:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.602973:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.602975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.602975:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.602977:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.602980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.603134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.603166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.603174:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.603176:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.603177:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.603178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.603179:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.603182:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.603186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.603193:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.603195:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.603196:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.603196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.2:1619677471.603197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.603197:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.603198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.604588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.604622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.604780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.604787:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.604790:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.604791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.604793:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.604795:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.604797:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.604798:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.604800:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.604801:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.604802:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.604805:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.604807:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.604809:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.604810:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.604810:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.604812:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.604813:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.604814:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.604814:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.604826:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.604829:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.604829:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.604830:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.604831:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.604832:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.605328:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.605331:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.605332:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.605332:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.605335:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.605338:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.605552:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.605555:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.605556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.605556:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.605558:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.605561:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.605699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.605707:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.605710:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.605711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.605711:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.605713:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.605716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.605871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.606174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.606182:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.606185:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.606185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.606186:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.606188:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.606191:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.606300:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.606303:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.606304:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:18.0:1619677471.606305:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.606306:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677471.606307:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.606307:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (23)++
00000800:00000200:2.0:1619677471.606308:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (24)--
00000800:00000200:2.0:1619677471.606309:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.606310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.606311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.606580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.606586:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.606589:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.606590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.606592:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.606594:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.606596:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.606597:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.606599:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.606600:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.606601:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.606604:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.606606:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.606609:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.606609:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.606610:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.606612:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.606613:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.606614:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.606615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.606619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.607772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.607780:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.607783:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.607784:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.607784:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.607786:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.607790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.609234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.609267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677471.610106:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.610109:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.610110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.610111:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.610113:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.610117:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.611013:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.611016:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.611018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.611018:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.611019:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.611022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.611245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.611256:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.611260:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.611262:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.611262:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:8.2:1619677471.611264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:17.0:1619677471.611267:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.611272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.611273:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.611276:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.611277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.611278:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.611280:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.611281:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.611767:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.611770:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.611772:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.611772:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.611775:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.611779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.611787:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.611790:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.611791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.611791:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.611793:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.611793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.613136:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.613139:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.613140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.613142:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677471.613145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:1.0:1619677471.613145:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.613147:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.613148:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.613149:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.613150:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.613151:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.613155:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.613156:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.613159:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.613160:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.613161:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677471.613163:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677471.613164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.613164:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.613165:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.613166:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.613167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.613178:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.613180:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.613181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.613182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.0:1619677471.613629:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.613633:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.613633:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.613634:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.613636:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.613636:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.613637:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.613637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.613665:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.613668:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.613669:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.613669:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.613672:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.613675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.614213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.614546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.614552:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.614555:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.614556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.614558:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.614561:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.614562:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.614564:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.614565:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.614566:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.614567:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.614570:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.614572:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.614574:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.614574:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.614575:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.614577:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.614578:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.614579:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.614579:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.614590:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.614592:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.614593:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.614593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.614595:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.614596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.614638:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.614641:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.614642:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.614642:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.614644:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.614648:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.614703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.614748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.614756:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.614759:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.614760:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.614760:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.614762:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.614765:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.615038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.615044:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.615047:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.615048:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.615050:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.615052:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.615053:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.615055:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.615057:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.615059:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.615060:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.615063:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.615065:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.615069:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.615070:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.615071:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.615073:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.615074:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.615075:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.615075:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677471.615130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.615315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.615317:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.615318:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.615320:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.615322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.615333:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.615583:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.615586:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.615588:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.615591:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.615593:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.615593:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.615595:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.615596:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.615596:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.615599:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.615601:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.615604:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.615604:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.615605:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.615607:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.615607:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.615608:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.615608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.615621:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.615624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.615624:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.615625:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.615627:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.615630:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.615639:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.615642:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.615643:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.615643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.615645:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.615645:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.615661:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.615664:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.615665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.615665:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.615668:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677471.615670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.615891:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.2:1619677471.616561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.616593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.616596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.617351:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.617353:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.617354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.617356:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.617358:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.617360:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.617361:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.617362:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.617363:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.617364:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.617367:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.617369:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.617371:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.617372:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.617373:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.617374:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.617375:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.617376:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.617376:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.617380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.617683:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.617685:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.617688:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.617691:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.617692:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.617694:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.617695:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.617696:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.617697:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.617699:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.617701:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.617702:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.617703:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.617703:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.617705:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.617705:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.617706:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.617706:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.617736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.617744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677471.617744:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.617747:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.617748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.617749:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.617751:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.617754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.618243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.618246:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.618247:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.618249:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.618250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677471.618360:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.618362:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.618363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.618364:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.618366:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.618369:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.619123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.619143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677471.619770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.619776:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.619779:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.619780:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.619782:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.619785:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.619786:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.619788:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.619789:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.619790:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.619791:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.619795:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.619796:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.619799:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.619800:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.619801:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.619803:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.619804:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.619805:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.619807:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.619807:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677471.619810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.619812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.619818:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.619820:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.619821:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.619822:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677471.619943:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.2:1619677471.620473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.620489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.620507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.620515:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.620518:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.620519:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.620519:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.620521:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.620523:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.620670:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.620672:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.620673:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.620675:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.620677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.620691:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.620771:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.620774:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.620776:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.620778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.620779:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.620780:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.620782:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.620783:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.620784:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.620786:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.620788:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.620791:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.620791:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.620792:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.620794:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.620794:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.620795:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.620797:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.620797:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.620800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.620807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.620810:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.620811:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.620811:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.620814:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.620817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.620904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.620910:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.620913:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.620914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.620916:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.620919:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.620920:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.620922:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.620924:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.620926:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.620927:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.620930:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.620932:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.620934:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.620935:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.620936:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.620938:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.620939:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.620940:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.620941:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.620951:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.620953:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.620954:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.620954:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.620955:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.620957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.621738:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.621741:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.621742:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.621744:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.621747:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.621748:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.621749:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.621751:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.621752:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.621753:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.621756:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.621757:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.621760:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.621761:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.621761:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.621763:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.621764:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.621764:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.621765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.621770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.621846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.621880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.622269:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.622271:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.622274:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.622278:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.622279:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.622280:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.622281:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.622283:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.622283:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.622286:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.622288:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.622289:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.622290:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.622291:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.622292:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.622298:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.622299:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.622301:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.622303:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.622304:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.622305:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.622306:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.622307:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.622587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.623304:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.623307:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.623310:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.623311:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.623313:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.623314:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.623316:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.623316:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.623317:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.623320:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.623322:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.623323:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.623324:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.623325:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.623326:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.623327:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.623327:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.623328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.623342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.2:1619677471.624306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:8.2:1619677471.624344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.624352:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.624355:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.624356:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.624359:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.624361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.624733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.624740:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.624743:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.624744:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.624746:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.624748:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.624750:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.624752:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.624754:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.624755:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.624756:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.624760:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.624762:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.624765:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.624767:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.624768:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.624770:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.624771:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.624772:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.624773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.625010:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.625012:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.625013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.625014:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.625016:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.625020:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.625029:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.625032:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.625033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.625034:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.625035:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.625035:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.625291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.625785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677471.625973:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.625975:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.625976:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.625978:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.625981:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.625982:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.625983:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.625985:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.625986:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.625987:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.625990:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.625991:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.625994:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.625994:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.625995:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.625997:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.625998:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.625999:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.625999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.626003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.626656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.626689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.627294:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.627304:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.627307:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.627309:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.627310:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.627311:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.627313:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.627314:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.627315:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.627318:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.627319:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.627321:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.627322:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.627322:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.627324:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.627324:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.627325:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.627325:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.627342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.627360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.2:1619677471.627375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.627973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.628009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.628011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.628017:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.628020:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.628021:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.628022:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.628023:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.628025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.628031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.628039:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.628041:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.628042:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.628042:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.628043:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.628044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.628241:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.628244:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.628246:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.628248:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.628249:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.628250:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.628252:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.628253:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.628254:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.628257:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.628259:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.628260:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.628261:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.628262:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.628263:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.628263:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.628264:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.628265:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.628266:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.628280:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.628281:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.628281:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.628282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.628889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.628904:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.628906:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677471.628907:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.628908:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.628909:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677471.629956:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.629959:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.629960:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.629961:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.629963:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.629965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.630033:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:8.0:1619677471.630034:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:8.0:1619677471.630035:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.630036:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.630038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677471.630407:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.630409:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.630410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.630413:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.630415:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.630416:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.630418:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.630419:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.630420:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.630421:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.630424:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.630426:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.630428:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.630429:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.630430:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.630432:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.630432:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.630433:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.630434:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.630435:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.630438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.630445:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.630447:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.630448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.630449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.0:1619677471.630638:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8b2b000] (21)++
00000800:00000200:8.0:1619677471.630641:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.49@o2ib3
00000400:00000200:8.0:1619677471.630643:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.49@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.630645:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.630647:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677471.630648:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.630650:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.630651:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.630652:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677471.630656:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.630658:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677471.630661:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.630662:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:8.0:1619677471.630663:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:8.0:1619677471.630665:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:8.0:1619677471.630665:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8b2b000] (21)++
00000800:00000200:8.0:1619677471.630667:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8b2b000] (22)++
00000800:00000200:8.0:1619677471.630668:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8b2b000] (23)--
00000800:00000200:8.0:1619677471.630669:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8b2b000] (22)--
00000800:00000200:18.0:1619677471.630680:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.630683:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.630684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.630685:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.630687:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.630690:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8b2b000] (22)--
00000800:00000200:18.0:1619677471.630691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.630724:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8b2b000] (21)++
00000800:00000200:8.0:1619677471.630726:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.49@o2ib3
00000800:00000200:13.0:1619677471.630728:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:8.0:1619677471.630729:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.49@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.630730:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.630731:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.630732:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.630732:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677471.630733:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96516799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.630734:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677471.630734:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.630735:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677471.630736:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:8.0:1619677471.630736:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677471.630738:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:8.0:1619677471.630739:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.630741:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677471.630744:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.630744:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677471.630745:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677471.630746:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677471.630747:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8b2b000] (21)++
00000800:00000200:8.0:1619677471.630748:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8b2b000] (22)--
00000800:00000200:8.0:1619677471.630748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8b2b000] (21)--
00000800:00000200:8.0:1619677471.630791:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8b2b000] (22)--
00000800:00000200:8.0:1619677471.630832:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8b2b000] (21)++
00000800:00000200:8.0:1619677471.630834:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.49@o2ib3
00000400:00000200:8.0:1619677471.630836:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.49@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.630838:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.630840:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677471.630841:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.630842:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.630843:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.630844:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677471.630847:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.630848:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677471.630850:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.630851:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677471.630851:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677471.630853:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677471.630853:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8b2b000] (21)++
00000800:00000200:8.0:1619677471.630854:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8b2b000] (22)++
00000800:00000200:8.0:1619677471.630856:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8b2b000] (23)--
00000800:00000200:8.0:1619677471.630856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8b2b000] (22)--
00000800:00000200:8.0:1619677471.630870:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8b2b000] (23)++
00000800:00000200:8.0:1619677471.630871:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8b2b000] (24)--
00000800:00000200:8.0:1619677471.630872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8b2b000] (23)--
00000800:00000200:8.0:1619677471.630874:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8b2b000] (22)--
00000800:00000200:8.0:1619677471.630913:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8b2b000] (21)++
00000800:00000200:8.0:1619677471.630916:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.49@o2ib3
00000400:00000200:8.0:1619677471.630918:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.49@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.630921:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.630923:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.630924:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.630925:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.630926:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.630927:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.630929:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.630931:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.630934:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.630935:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.630935:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.630937:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.630937:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8b2b000] (21)++
00000800:00000200:8.0:1619677471.630938:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8b2b000] (22)--
00000800:00000200:8.0:1619677471.630938:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8b2b000] (21)--
00000800:00000200:8.2:1619677471.630943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.630951:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.630954:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.630955:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.630955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.630958:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.630961:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.631090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8b2b000] (22)--
00000800:00000200:8.0:1619677471.631092:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.631093:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.631094:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.631095:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.631101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.631102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.631359:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8b2b000] (21)++
00000800:00000200:8.0:1619677471.631361:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.49@o2ib3
00000400:00000200:8.0:1619677471.631364:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.49@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.631366:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.631368:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677471.631369:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.631370:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.631371:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.631372:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677471.631375:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.631376:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677471.631379:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.631380:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677471.631380:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.631382:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677471.631383:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8b2b000] (21)++
00000800:00000200:8.0:1619677471.631384:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8b2b000] (22)++
00000800:00000200:8.0:1619677471.631385:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8b2b000] (23)--
00000800:00000200:8.0:1619677471.631386:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8b2b000] (22)--
00000800:00000200:8.2:1619677471.631388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.631401:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8b2b000] (21)++
00000800:00000200:8.0:1619677471.631404:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.49@o2ib3
00000800:00000200:8.2:1619677471.631407:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:8.0:1619677471.631411:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.49@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.631414:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.631415:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.631417:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96516789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.631418:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.631419:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.631420:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.631424:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.631426:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.631429:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.631430:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.631430:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.631432:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.631433:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8b2b000] (21)++
00000800:00000200:8.0:1619677471.631433:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8b2b000] (22)--
00000800:00000200:8.0:1619677471.631434:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8b2b000] (21)--
00000800:00000200:2.0:1619677471.631447:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.631450:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.631451:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.631451:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.631454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8b2b000] (22)--
00000400:00000200:2.0:1619677471.631454:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.631457:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.632128:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.632131:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.632131:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.632132:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.632134:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.632134:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.632163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.632169:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.632171:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.632172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.632174:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.632177:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.632179:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.632180:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.632182:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.632183:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.632184:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.632187:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.632189:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.632192:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.632192:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.632193:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.632195:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.632196:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.632196:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.632197:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677471.632757:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.632760:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.632763:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.632765:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.632767:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.632768:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96516785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.632770:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.632771:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.632772:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.632776:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.632778:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.632780:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.632782:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.632782:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.632784:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.632784:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.632785:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.632786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.632790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.632798:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.632801:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.632802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.632803:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.632805:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.632808:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.632809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.632816:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.632819:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.632820:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.632820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.632821:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.632821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.632885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.633134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.633165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.633389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.633395:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.633398:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.633399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.633402:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.633405:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.633406:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.633408:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.633410:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.633411:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.633412:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.633415:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.633417:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.633420:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.633421:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.633422:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.633424:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.633424:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.633426:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.633427:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.633428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.633433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.633438:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.633440:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.633440:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.633440:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.633441:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.633442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.633442:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.633443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.633444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.633444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.633596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.633627:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.633629:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.633629:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.633631:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.633633:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.633981:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.633998:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.634000:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677471.634001:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.634002:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.634003:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677471.634686:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.634689:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.634690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.634691:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.634694:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.634695:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.634697:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.634698:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.634699:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.634700:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.634703:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.634705:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.634708:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.634709:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.634709:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677471.634711:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.634712:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.634713:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.634713:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677471.634879:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.634882:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.634883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.634885:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.634887:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.634889:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.634890:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.634892:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.634894:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.634895:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.634899:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.634900:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.634903:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.634904:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.634904:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.634906:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.634906:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.634907:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.634908:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.634913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677471.634974:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.634977:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.634978:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.634978:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.634981:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.634985:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.635461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.2:1619677471.635496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.635514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677471.636403:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.636406:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.636407:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.636409:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.636411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.636744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.636751:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.636754:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.636755:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.636757:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.636759:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.636761:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.636762:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.636764:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.636765:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.636766:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.636770:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.636772:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.636775:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.636776:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.636776:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.636778:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.636779:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.636780:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.636781:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.636784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.636859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.636880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677471.636906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.636938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.636946:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.636949:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.636951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.636951:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.636953:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.636956:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.637213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.637228:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.637232:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.637232:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.637233:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.637235:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.637235:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.637236:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.637236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.637248:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.637251:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.637252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.637252:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.637255:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.637258:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.638206:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.638209:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.638210:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.638212:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.638215:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.638217:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.638218:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.638219:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.638220:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.638221:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.638224:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.638226:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.638228:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.638228:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.638229:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.638230:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.638231:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.638232:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.638233:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.638234:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.638244:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.638246:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.638246:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.638247:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.638248:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.638249:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.638249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.638250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.638250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.638251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677471.638449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.2:1619677471.638491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.2:1619677471.638491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.638496:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677471.638499:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677471.638500:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.638500:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677471.638502:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.638505:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677471.638513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677471.638516:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677471.638519:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677471.638520:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677471.638520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677471.638521:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677471.638522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.2:1619677471.638825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.638833:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.638836:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.638837:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.638837:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.638839:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.638843:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.639757:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.639760:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.639761:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.639761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.639764:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.639768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.641046:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.641049:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.641050:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.641050:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.641052:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.641054:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677471.641055:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.641058:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.641059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.641061:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.641063:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.641065:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.641066:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.641067:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.641068:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.641069:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.641073:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.641074:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.641077:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.641077:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.641078:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.641080:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.641080:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.641081:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.641082:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.641086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.641436:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.641438:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.641439:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.641441:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.641443:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.641445:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.641446:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.641448:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.641449:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.641450:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.641453:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.641455:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.641458:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.641458:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.641459:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.641461:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.641461:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.641462:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.641462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.641467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.641475:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.641478:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.641479:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.641479:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.641481:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.641485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.641510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.641682:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.641684:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.641685:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.641685:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.641687:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.641689:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.641699:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.641701:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.641702:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.641702:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.641703:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.641703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677471.642130:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.642133:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.642134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.642136:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.642138:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.642139:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677471.642141:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.642142:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.642143:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.642144:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677471.642147:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.642149:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677471.642152:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.642152:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677471.642153:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677471.642155:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677471.642155:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.642156:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.642157:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677471.642520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.642526:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.642529:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.642530:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.642532:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.642534:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.642536:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.642537:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.642538:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.642539:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.642540:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.642543:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.642545:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.642547:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.642548:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.642549:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.642550:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.642551:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.642552:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.642553:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.642554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677471.642557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.642559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.642566:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.642569:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.642570:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.642570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.642691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.642726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.642730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677471.642747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677471.642880:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.642883:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.642883:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.642884:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.642886:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.642888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.643270:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.643272:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.643273:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.643275:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.643277:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.643278:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.643279:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.643280:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.643281:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.643282:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.643285:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.643286:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.643288:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.643289:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.643289:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.643290:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.643291:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.643291:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.643291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.643302:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.643304:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.643305:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.643306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.643306:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.643308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.643324:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.643326:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.643327:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.643327:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.643327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.643328:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.643396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.643403:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.643406:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.643407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.643409:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.643411:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.643413:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.643414:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.643416:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.643417:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.643418:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.643421:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.643423:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.643425:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.643426:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.643426:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.643428:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.643429:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.643430:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.643431:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.643442:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.643443:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.643444:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.643445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.643446:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.643447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.643920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.643926:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.643928:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.643929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.643930:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.643932:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.643934:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.643935:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.643936:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.643938:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.643939:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.643942:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.643943:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.643946:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.643947:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.643947:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.643949:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.643950:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.643950:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677471.643951:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677471.643952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677471.643957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.643962:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677471.643965:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677471.643965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.643966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677471.644192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677471.644428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.644463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677471.644514:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.644517:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.644518:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.644518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.644520:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.644524:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.644535:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.644538:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.644538:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.644539:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.644541:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.644541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.644874:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.644877:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.644878:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.644879:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.644881:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.644884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.645074:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.645077:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.645078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.645080:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.645082:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.645083:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.645085:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.645086:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.645087:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.645088:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.645092:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.645093:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.645096:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.645097:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.645098:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.645099:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.645100:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.645100:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.645101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.645113:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.645115:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.645117:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.645117:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.645119:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.645122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.645137:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.645139:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.645140:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.645140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.645141:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.645142:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.645482:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677471.645485:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677471.645486:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677471.645487:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.645490:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.645492:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.645493:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.645494:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.645495:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.645496:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.645500:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.645501:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.645504:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.645505:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.645505:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.645507:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.645507:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677471.645508:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677471.645509:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:2.0:1619677471.645521:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.645524:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.645525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.645525:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.645527:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.645531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.646196:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.646199:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.646200:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.646202:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.646205:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.646207:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677471.646208:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.646210:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.646211:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.646212:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677471.646215:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.646217:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677471.646219:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.646220:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677471.646221:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677471.646222:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677471.646223:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.646224:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.646224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677471.646237:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.646239:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.646240:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.646240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.646242:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.646245:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.646290:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.646293:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.646299:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.646301:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.646303:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.646305:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.646306:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.646307:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.646308:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.646309:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.646311:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.646313:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.646315:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.646316:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.646316:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.646318:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.646318:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.646319:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.646319:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.646324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.646488:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.646490:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.646491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.646491:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.646493:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.646495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.646982:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.646984:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.646985:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.646986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.646987:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.646989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.647234:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.647236:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.647237:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:12.0:1619677471.647237:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:8.0:1619677471.647237:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.647239:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.647240:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:8.0:1619677471.647241:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:12.0:1619677471.647242:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677471.647245:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.647246:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.647246:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:12.0:1619677471.647247:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96516726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.647249:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.647250:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.647251:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677471.647252:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.647254:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:12.0:1619677471.647254:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.647255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.647256:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:12.0:1619677471.647256:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677471.647257:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.647257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677471.647258:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.647259:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:12.0:1619677471.647260:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:12.0:1619677471.647262:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:12.0:1619677471.647262:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.647263:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.647264:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.0:1619677471.647712:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.647714:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.647715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.647717:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.647720:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.647721:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.647722:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.647723:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.647724:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.647725:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.647728:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.647729:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.647731:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.647731:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.647732:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.647733:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.647734:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.647735:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.647736:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.647736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.647749:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.647751:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.647752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.647753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.647755:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.647756:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.647756:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.647757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.647758:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.647759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.647939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677471.648771:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.648774:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.648775:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.648778:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.648782:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.648783:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.648784:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96516720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.648785:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.648786:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.648787:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.648790:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.648792:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.648794:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.648795:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.648796:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.648797:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.648798:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.648799:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.648800:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.648800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.648801:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677471.648801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.648802:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.648803:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.648806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.648807:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.648808:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.648809:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.648810:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.648810:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.648811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.648936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.648973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.648982:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.648984:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.648985:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.648986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.648988:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.648990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.2:1619677471.649173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677471.649176:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677471.649178:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677471.649179:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.649180:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677471.649181:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.649183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:2.0:1619677471.649328:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.649332:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.649333:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.649333:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.649336:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.649339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.649600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.649801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.649808:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.649810:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.649811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.649813:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.649816:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.649818:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.649819:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.649821:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.649822:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.649823:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.649826:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.649828:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.649830:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.649831:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.649832:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.649834:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.649834:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.649835:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.649836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677471.649934:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.649938:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.649939:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.649941:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.649942:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.649945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677471.650124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677471.650132:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677471.650135:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677471.650136:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677471.650137:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677471.650140:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.650142:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677471.650143:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.650144:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.650145:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.650146:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677471.650150:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677471.650152:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.650154:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.650154:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.650155:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.650156:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.650157:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677471.650157:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677471.650158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.2:1619677471.650162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.650170:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.650171:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.650172:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.650173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.650174:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677471.650176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.650183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.650190:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.650192:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.650193:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.650194:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.650194:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.650195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.650392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.650429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.651095:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677471.651098:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677471.651099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677471.651101:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.651103:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.651105:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677471.651106:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.651107:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.651108:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.651109:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677471.651112:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.651114:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677471.651117:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.651118:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677471.651118:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677471.651120:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677471.651121:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677471.651121:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677471.651122:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:17.0:1619677471.651812:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.651815:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.651816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.651818:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.651820:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.651821:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.651822:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.651824:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.651825:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.651826:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.651829:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.651831:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.651833:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.651834:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.651834:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.651836:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.651837:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.651838:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.651839:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.651840:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.651843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.651851:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.651853:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.651854:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.651855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.651861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:24.0:1619677471.651998:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.652000:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.652001:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.652001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.652003:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.652006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.652742:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.652746:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.652747:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.652748:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.652751:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.652752:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.652754:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.652755:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.652756:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.652757:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.652759:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.652761:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.652762:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.652763:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.652763:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.652765:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.652765:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.652766:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.652766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.652787:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.652789:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.652790:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.652791:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.652791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.652792:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.652865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.652873:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.652876:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.652877:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.652877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.652879:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.652883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.652939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.653432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.653438:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.653441:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.653442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.653444:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.653447:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.653448:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.653450:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.653451:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.653453:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.653454:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.653457:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.653459:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.653461:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.653462:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.653462:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.653464:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.653465:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.653465:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.653466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.653478:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.653480:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.653481:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.653481:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.653483:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.653484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.653577:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.653579:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.653580:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.653580:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.653582:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.653584:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.653628:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.653630:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.653631:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.653633:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.653634:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.653636:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.653637:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.653638:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.653639:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.653639:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.653642:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.653644:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.653645:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.653646:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.653646:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.653647:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.653648:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.653648:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.653648:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.653659:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.653660:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.653661:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.653662:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.653662:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.653664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.653681:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.653683:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.653684:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.653684:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.653684:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.653685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.653758:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.653760:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.653761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.653762:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.653764:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.653766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.653863:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.653865:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.653866:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.653867:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.653868:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.653870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.653885:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.653887:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.653888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.653888:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.653889:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.653890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677471.654069:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.654072:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.654073:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.654073:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.654075:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.654078:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.654087:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.654090:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.654090:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.654091:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.654092:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.654092:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.654232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.654237:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.654240:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.654241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.654243:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.654245:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.654246:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677471.654247:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.654249:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.654250:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.654251:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.654254:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.654256:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.654259:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.654260:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677471.654261:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677471.654263:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677471.654263:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.654264:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.654265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677471.654363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.654369:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.654372:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.654373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.654375:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.654378:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.654379:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.654381:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.654382:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.654383:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.654384:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.654388:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.654389:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.654392:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.654392:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.654393:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.654395:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.654395:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.654396:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.654396:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.654761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.654866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.654899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.654901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.654907:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.654910:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.654911:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.654911:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.654913:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.654916:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.655121:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.655124:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.655125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.655126:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.655128:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.655129:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.655131:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.655132:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.655133:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.655134:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.655138:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.655139:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.655142:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.655143:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.655143:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.655145:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.655146:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.655147:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.655148:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.655149:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.655162:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.655166:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.655168:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.655168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.655170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677471.655171:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.655174:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.655174:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.655175:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.655177:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.655180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.655548:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.655550:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.655551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.655553:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.655555:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.655556:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677471.655557:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.655559:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.655560:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.655563:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677471.655566:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.655568:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.655570:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.655571:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.655572:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.655573:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.655574:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.655575:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.655575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677471.655591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.655599:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.655601:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.655601:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.655602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.655603:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.655604:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.656106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.656140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.656586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.656592:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.656595:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.656596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.656598:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.656602:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.656603:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677471.656605:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.656606:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.656607:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.656608:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677471.656611:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.656613:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677471.656616:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.656617:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677471.656618:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677471.656620:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677471.656620:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.656621:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.656622:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:24.0:1619677471.656633:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.656636:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.656637:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.656637:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.656639:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.656642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677471.657025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.657034:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.657036:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.657037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.657038:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.657040:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.657043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.657045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.657052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.657054:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.657055:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.657056:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.657057:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.657058:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.657144:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.657147:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.657148:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.657149:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.657151:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.657152:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.657153:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.657154:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.657155:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.657156:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.657159:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.657161:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.657163:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.657164:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.657165:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.657167:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.657167:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.657168:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.657168:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.657173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.657448:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.657451:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.657452:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.657452:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.657454:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.657456:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677471.658827:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.658829:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.658830:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.658831:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.658833:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.658836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.658841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.658846:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.658849:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.658850:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.658852:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.658854:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.658856:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.658857:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.658858:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.658859:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.658860:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.658864:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.658865:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.658868:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.658869:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.658870:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.658872:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.658873:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.658873:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.658874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.658879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677471.658894:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.658897:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.658898:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.658900:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.658903:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.658905:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.658907:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.658908:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.658909:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.658911:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.658914:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.658916:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.658918:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.658919:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.658919:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.658921:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.658922:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.658923:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.658923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.658934:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.658936:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.658937:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.658938:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.658939:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.658941:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.659384:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.659387:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.659388:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.659388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.659389:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.659392:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677471.659398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677471.659964:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.659967:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.659968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.659970:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.659972:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.659973:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.659975:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.659976:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.659977:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.659978:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.659981:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.659982:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.659985:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.659986:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.659986:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.659988:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.659988:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.659989:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.659989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.659994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.660002:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.660005:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.660006:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.660006:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.660008:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.660011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.660343:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.660345:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.660346:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.660347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.660348:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.660351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.660364:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.660366:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.660366:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677471.660367:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.660368:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.660368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.660976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.661015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.661034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:24.0:1619677471.661274:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.661276:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.661278:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.661278:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.661280:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.661282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.661525:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.661528:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.661529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.661530:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.661532:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.661534:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.661535:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.661536:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.661537:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.661538:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.661541:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.661542:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.661545:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.661546:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.661546:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.661548:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.661549:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.661549:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.661550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.661554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.661563:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.661565:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.661566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.661567:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.661569:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.661572:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.661934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.661941:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.661944:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.661944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.661947:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.661949:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.661951:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.661952:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.661954:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.661955:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.661956:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.661959:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.661961:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.661963:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.661963:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.661964:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.661966:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.661966:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.661967:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.661968:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.661990:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.661992:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.661993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.661993:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.661995:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.661996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.662419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.662463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.662472:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.662474:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.662475:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.662475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.662478:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.662480:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.662625:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.662627:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.662628:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.662629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.662630:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.662632:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677471.662730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.662737:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.662740:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.662741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.662741:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.662743:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.662747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.662909:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.662912:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.662913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.662915:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.662917:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.662918:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.662920:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.662921:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.662922:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.662923:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.662926:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.662928:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.662931:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.662931:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.662932:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.662934:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.662934:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.662935:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.662936:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.662940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.663191:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677471.663194:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677471.663194:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:8.0:1619677471.663196:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.663199:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.663201:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677471.663202:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.663204:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.663205:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.663206:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677471.663209:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.663211:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677471.663214:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.663214:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677471.663215:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677471.663217:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677471.663217:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677471.663218:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677471.663218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677471.663223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.663231:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.663233:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.663234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.663235:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.663237:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.663240:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.663243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.663250:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.663253:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.663254:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.663254:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.663255:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.663256:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:12.0:1619677471.663321:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.663324:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.663326:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677471.663328:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.663330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.663330:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677471.663331:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96516659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.663333:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.663334:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.663334:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677471.663338:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.663339:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677471.663342:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.663343:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677471.663343:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677471.663346:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677471.663346:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.663347:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.663348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677471.663373:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.663375:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.663377:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.663377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.663379:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.663382:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.663396:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.663399:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.663400:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.663400:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.663401:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.663401:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.663558:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.663561:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.663562:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.663562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.663564:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.663568:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.663805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.663813:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.663816:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.663817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.663817:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.663819:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.663822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.663823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.663831:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.663833:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.663834:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.663834:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.663835:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.663836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.663863:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.663866:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.663867:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.663868:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.663871:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.663872:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677471.663874:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.663876:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.663877:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.663878:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677471.663882:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.663884:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677471.663887:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.663888:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677471.663889:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677471.663891:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677471.663891:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.663893:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677471.663894:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677471.663894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:13.0:1619677471.663903:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:17.0:1619677471.663905:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:13.0:1619677471.663906:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:17.0:1619677471.663907:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:13.0:1619677471.663907:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.663908:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:13.0:1619677471.663908:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.663909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677471.663910:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.663912:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:24.0:1619677471.664282:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.664285:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.664286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.664290:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.664293:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.664300:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.664302:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.664304:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.664305:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.664306:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.664311:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.664313:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.664315:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.664316:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.664317:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.664318:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.664319:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.664320:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.664321:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.664322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.664334:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.664335:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.664336:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677471.664336:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.664338:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.664338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.664339:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.664340:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.664341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.664343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.664439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:8.2:1619677471.664474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.664495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.664587:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.664589:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.664591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.664591:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.664593:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.664596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.664763:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.664766:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.664767:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.664769:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.664772:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.664773:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.664774:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.664776:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.664777:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.664778:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.664781:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.664783:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.664786:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.664786:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.664787:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677471.664788:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.664789:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.664790:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.664791:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.664792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.664803:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.664804:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.664805:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.664806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677471.665053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.2:1619677471.665057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677471.665061:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.665064:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.665065:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.665065:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.665067:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.665070:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.665250:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.665252:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.665253:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.665254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.665255:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.665257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677471.666320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.666660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.666694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.666714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677471.666839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.666846:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.666849:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.666850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.666852:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.666854:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.666855:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.666856:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.666858:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.666859:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.666860:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.666864:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.666866:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.666869:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.666870:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.666871:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.666873:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.666873:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.666875:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.666876:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.666877:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.666882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.666888:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.666890:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.666891:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.666891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677471.667154:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.667157:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.667158:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.667160:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.667162:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.667163:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.667165:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.667166:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.667167:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.667168:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.667171:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.667172:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.667175:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.667176:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.667177:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.667178:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.667179:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.667180:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.667180:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.667185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.667193:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.667196:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.667197:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.667197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.667199:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.667203:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.667512:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.667515:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.667516:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.667518:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.667520:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.667521:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.667523:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.667524:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.667525:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.667526:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.667529:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.667531:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.667533:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.667534:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.667535:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.667536:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.667537:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.667537:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.667538:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.667543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.667802:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.667805:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.667806:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.667807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.667808:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.667811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.667826:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.667829:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.667829:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677471.667829:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.667830:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.667831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677471.667877:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.667880:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.667881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.667881:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.667883:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.667889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.667891:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.667894:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.667894:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.667895:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.667896:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.667897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.668318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.668352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.668360:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.668363:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.668364:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.668364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.668366:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.668368:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.668698:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.668701:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.668702:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.668704:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.668706:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.668708:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.668709:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.668710:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.668711:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.668712:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.668716:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.668718:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.668721:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.668721:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.668722:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.668724:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.668724:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.668725:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.668725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.668729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.668738:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.668740:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.668741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.668742:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.668744:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.668746:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.668827:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.668829:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.668830:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.668831:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.668833:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.668836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.669102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.669111:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.669113:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.669114:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.669115:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.669117:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.669120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.669129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.669136:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.669138:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.669139:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.669140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.669140:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.669141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.670683:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.670685:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.670686:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.670688:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.670691:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.670692:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.670694:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.670695:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.670696:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.670698:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.670701:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.670703:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.670705:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.670706:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.670707:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.670708:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.670709:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.670709:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.670710:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.670721:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.670723:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.670724:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.670724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.670726:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.670727:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.670742:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.670744:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.670745:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.670745:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.670746:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.670747:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.670985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.671010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677471.671240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.671248:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.671252:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.671254:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.671257:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.671261:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677471.671263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:2.0:1619677471.671263:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.671266:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.671269:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677471.671271:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677471.671271:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677471.671273:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677471.671274:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.671279:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.671283:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.671284:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677471.671287:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677471.671288:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.671289:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:18.0:1619677471.671290:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:2.0:1619677471.671290:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:18.0:1619677471.671293:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.671293:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.671304:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.671305:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.671307:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.671308:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.671311:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.671313:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677471.671314:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.671315:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.671317:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.671317:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.671319:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.2:1619677471.671320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.671320:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.671322:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677471.671322:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.671324:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.671325:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:2.0:1619677471.671325:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.671327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.671329:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.671329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.671330:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.671331:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.671332:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.671332:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.671333:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.671334:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.671335:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677471.671480:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.671483:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.671484:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.671485:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.671486:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.671488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677471.672071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.672080:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.672083:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.672084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.672085:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.672087:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.672090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.672196:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.672199:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.672200:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.672201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.672202:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.672205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.672241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.672454:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.672457:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.672458:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.672458:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.672459:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.672462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677471.672664:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.672667:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.672667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.672668:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.672670:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.672673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.672754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.672785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.672812:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.672815:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.672815:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.672817:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.672820:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.672821:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.672822:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.672824:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.672825:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.672826:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.672829:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.672831:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.672833:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.672834:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.672835:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.672837:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.672837:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.672839:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.672840:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.672840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.672842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.672851:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.672853:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.672853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.672854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.672953:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.672956:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.672957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.672958:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.672960:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.672962:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677471.672964:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.672965:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.672966:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.672967:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677471.672970:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.672972:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677471.672975:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.672976:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677471.672976:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677471.672978:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677471.672978:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.672979:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.672979:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677471.673038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677471.673084:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.673087:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.673088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.673088:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.673090:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.673094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.673649:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.673652:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.673652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.673653:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.673655:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.673660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.673662:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.673665:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.673665:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.673666:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.673667:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.673667:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.673989:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.673991:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.673992:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.673993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.673994:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.673996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.674014:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.674016:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.674017:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.674019:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.674021:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.674022:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.674024:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.674025:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.674026:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.674027:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.674030:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.674032:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.674035:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.674036:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.674036:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.674038:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.674038:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.674040:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.674041:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.674041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.674055:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.2:1619677471.674055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.674056:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.674057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.674058:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677471.674064:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.674066:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.674067:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.674068:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.674070:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.674072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.674242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677471.674815:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.674817:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.674818:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.674818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.674821:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677471.674823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.674832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.674877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.674885:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.674887:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.674888:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.674889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.674890:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.674893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.675280:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.675283:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.675284:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.675285:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.675288:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.675290:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.675291:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.675293:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.675299:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.675300:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.675303:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.675305:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.675306:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.675307:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.675308:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.675309:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.675310:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.675311:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.675311:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.675322:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.675324:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.675324:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.675325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.675326:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.675327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.675465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.675473:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.675475:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.675476:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.675477:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.675479:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.675481:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.675614:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677471.675617:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677471.675618:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677471.675619:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.675621:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.675623:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677471.675624:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.675626:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.675627:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.675628:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677471.675630:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.675632:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677471.675634:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.675635:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677471.675635:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677471.675637:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677471.675637:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677471.675639:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677471.675640:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677471.675640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677471.675653:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677471.675655:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677471.675656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677471.675657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:20.2:1619677471.675785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.675792:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.675794:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.675796:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.675797:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.675800:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.675801:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.675802:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.675804:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.675805:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.675806:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.675810:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.675811:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.675814:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.675814:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.675815:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.675817:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.675818:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.675818:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.675819:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.675823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.675842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677471.676150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.676153:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.676154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.676154:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.676156:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.676159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.676167:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.676170:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.676171:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.676171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.676172:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.676172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.676698:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.676701:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.676702:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.676704:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.676706:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.676708:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.676709:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.676710:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.676711:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.676712:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.676716:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.676718:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.676720:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.676721:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.676722:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.676724:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.676724:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.676725:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.676726:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.676738:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.676741:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.676742:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.676742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.676744:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.676748:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.677031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.677202:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.677205:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.677206:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.677206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.677207:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.677210:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.677571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.677577:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.677580:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.677581:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.677583:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.677585:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.677587:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.677588:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.677590:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.677590:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.677592:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.677594:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.677596:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.677598:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.677598:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.677599:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.677601:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.677601:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.677602:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.677602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.677615:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.677617:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.677617:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.677618:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.677619:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.677620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.677634:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.677637:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.677638:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.677638:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.677638:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.677641:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.677648:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.677651:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677471.677651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.677654:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.677656:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.677657:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677471.677658:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.677660:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.677661:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.677662:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677471.677666:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.677668:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.677669:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.677671:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.677671:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.677672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.677673:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677471.677673:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:2.0:1619677471.677673:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.677676:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677471.677676:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.677676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.677678:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.677678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677471.677678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.678321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677471.678445:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.678448:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.678450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.678450:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.678452:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.678456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.678621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.678656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.678658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.678664:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.678667:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.678668:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.678669:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.678670:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.678672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.678679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.678686:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.678688:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.678689:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.678689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.678690:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.678691:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.679010:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.679013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.679013:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.679015:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.679018:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.679019:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.679020:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.679021:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.679022:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.679024:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.679026:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.679028:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.679031:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.679032:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.679032:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.679034:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.679035:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.679035:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.679036:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.679040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.680188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.680197:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.680199:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.680200:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.680201:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.680203:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.680206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.680219:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.680222:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.680223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.680224:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.680226:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.680228:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.680229:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.680230:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.680231:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.680232:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.680235:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.680236:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.680239:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.680240:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.680240:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.680242:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.680242:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.680243:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.680244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.680248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.680256:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.680259:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.680259:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677471.680260:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.680262:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.680264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.680266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.680273:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.680275:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.680276:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.680276:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.680277:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.680278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.680890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.680922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.680930:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.680933:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.680934:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.680934:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.680936:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.680942:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.681217:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.681219:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.681220:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.681220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.681222:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.681225:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.681356:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.681359:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.681360:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.681362:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.681365:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.681366:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.681368:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.681369:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.681370:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.681371:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.681374:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.681375:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.681377:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.681378:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.681379:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.681380:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.681381:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.681382:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.681383:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.681384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.681395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.681396:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.681397:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.681398:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.681399:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.681399:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.681400:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.681400:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.681401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.681401:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.682532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677471.682595:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.682598:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.682599:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.682599:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.682601:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.682606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.682607:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.682610:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.682611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.682611:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.682612:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.682612:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.682686:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.682688:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.682689:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.682690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.682691:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.682693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.682708:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.682711:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.682712:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.682712:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.682713:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.682713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.683414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.683420:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.683423:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.683424:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.683426:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.683429:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.683430:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.683431:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.683434:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.683436:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.683437:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.683440:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.683442:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.683445:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.683446:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.683446:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.683448:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.683449:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.683450:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.683451:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677471.683564:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.683567:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.683567:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.683568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.683569:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.683571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677471.683602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.683606:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.683610:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.683612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.683613:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.683615:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677471.683617:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96516560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.683619:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.683621:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.683622:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677471.683628:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.683630:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677471.683634:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.683635:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677471.683636:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677471.683639:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677471.683640:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.683642:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677471.683644:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677471.683645:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:2.0:1619677471.683652:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:8.0:1619677471.683653:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677471.683655:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.683656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:8.0:1619677471.683656:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:2.0:1619677471.683656:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.683657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.683659:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.683659:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.683660:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:8.0:1619677471.683661:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.683663:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:8.0:1619677471.683663:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677471.683663:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:13.0:1619677471.683664:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:8.0:1619677471.683664:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677471.683665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:8.0:1619677471.683666:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.683667:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.683667:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.683670:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.683672:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.683675:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.683676:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.683676:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.683678:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.683678:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.683679:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.683680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.683691:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.683694:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.683695:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.683695:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.683697:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.683701:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.683765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.683786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677471.684487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.684520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.684644:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.684646:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.684647:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.684647:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.684649:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.684651:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.684784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.684790:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.684793:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.684794:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.684796:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.684799:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.684800:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.684802:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.684803:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.684804:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.684805:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.684808:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.684809:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.684811:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.684812:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.684813:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.684815:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.684815:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.684816:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.684817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.684828:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.684830:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.684831:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.684831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.684832:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.684833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:23.0:1619677471.685344:0:19929:0:(o2iblnd_cb.c:3440:kiblnd_check_conns()) conn[ffff91a478b7e200] (20)++
00000800:00000200:23.0:1619677471.685353:0:19929:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a478b7e200] (21)++
00000800:00000200:23.0:1619677471.685358:0:19929:0:(o2iblnd_cb.c:3475:kiblnd_check_conns()) conn[ffff91a478b7e200] (22)--
00000800:00000200:13.0:1619677471.685374:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a478b7e200] (22)++
00000800:00000200:13.0:1619677471.685377:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a478b7e200] (23)--
00000800:00000200:13.0:1619677471.685378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a478b7e200] (22)--
00000800:00000200:13.0:1619677471.685379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a478b7e200] (21)--
00000800:00000200:17.0:1619677471.685597:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.685600:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.685601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.685604:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.685606:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.685608:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.685609:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.685611:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.685612:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.685613:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.685618:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.685620:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.685623:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.685623:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677471.685624:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.685625:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.685626:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.685627:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677471.685627:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.685628:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.685628:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:8.0:1619677471.685628:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677471.685629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.685633:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.685637:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:8.0:1619677471.685641:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.685642:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.685643:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96516549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.685645:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677471.685645:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:8.0:1619677471.685646:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.685647:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677471.685647:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.685649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.685649:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.685651:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677471.685652:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.685654:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677471.685655:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.685656:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.685656:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.685657:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.685659:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.685659:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.685659:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.685660:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.685670:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.685672:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.685673:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.685674:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.685675:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.685677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677471.686968:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.686971:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.686971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.686974:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.686976:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.686978:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677471.686980:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.686981:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.686983:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.686984:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677471.686988:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.686991:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.686993:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.686994:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.686995:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.686996:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.686997:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.686998:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.686998:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677471.687003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.2:1619677471.687007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677471.687010:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.687012:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677471.687013:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.687014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.687015:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.687017:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.687097:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677471.687100:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677471.687100:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:8.0:1619677471.687103:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.687105:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.687106:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677471.687108:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.687109:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.687110:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.687111:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677471.687114:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.687116:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677471.687119:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.687120:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677471.687120:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677471.687122:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677471.687122:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677471.687123:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677471.687124:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:2.0:1619677471.687542:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.687545:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.687546:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.687546:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.687548:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.687552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.687653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.687691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.687699:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.687702:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.687703:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.687703:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.687705:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.687707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.687858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.687864:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.687867:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.687868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.687870:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.687872:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.687873:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.687874:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.687876:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.687878:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.687879:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.687882:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.687884:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.687887:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.687888:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.687888:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.687890:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.687891:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.687892:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.687894:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.687894:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.687899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.687905:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.687907:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.687908:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.687909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677471.687938:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.687941:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.687943:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.687945:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.687947:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.687948:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.687950:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.687951:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.687952:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.687953:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.687956:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.687958:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.687961:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.687962:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.687962:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.687964:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.687965:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.687965:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.687966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.687970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.687978:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.687981:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.687982:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.687982:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.687984:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.687987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.688105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.688641:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.688643:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.688644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.688644:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.688646:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.688649:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677471.688857:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.688860:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.688860:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.688861:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.688864:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.688868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.688895:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.688898:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.688899:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.688899:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.688900:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.688902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.690005:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.690007:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.690008:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.690010:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.690012:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.690013:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.690014:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.690016:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.690016:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.690017:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.690020:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.690021:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.690024:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.690025:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.690026:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.690027:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.690028:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.690029:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.690030:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.690030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677471.690042:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677471.690044:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677471.690045:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677471.690046:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:18.0:1619677471.690046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.690046:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.690047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.690048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677471.690048:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.690052:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.690062:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.690065:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.690065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.690067:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.690068:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.690068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.690268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.690305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.690308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677471.690325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.690637:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.690640:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.690641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.690642:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.690644:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.690646:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.690647:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.690648:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.690649:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.690650:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.690654:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.690655:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.690658:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.690659:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.690659:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.690661:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.690661:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.690662:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.690663:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.690667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.691393:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.691396:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.691397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.691399:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.691401:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.691403:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677471.691405:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.691406:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.691407:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.691408:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677471.691412:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.691413:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677471.691416:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.691417:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677471.691418:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677471.691420:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677471.691420:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.691421:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.691421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677471.691433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.691436:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.691437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.691437:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.691439:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.691442:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.691453:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.691455:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.691456:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.691456:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.691457:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.691458:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.691488:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.691491:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.691492:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.691494:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.691497:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.691499:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.691500:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96516522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.691502:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.691503:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.691503:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.691506:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.691508:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.691510:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.691511:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.691511:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.691513:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.691513:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.691514:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.691514:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.691521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.691525:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.691527:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.691528:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.691529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.691530:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.691530:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677471.691531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677471.691533:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.691534:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.691534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.691537:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.691539:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.691543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.691551:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.691553:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.691554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.691554:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.691555:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.691556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.692091:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.692094:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.692095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.692097:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.692100:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.692101:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677471.692102:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.692104:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.692105:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.692105:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677471.692109:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.692111:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.692112:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.692113:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.692114:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.692115:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.692116:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.692116:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.692117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677471.692132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.692139:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.692141:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677471.692142:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.692143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.692144:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.692145:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.692284:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.692287:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.692289:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.692289:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.692291:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.692299:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.693233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.693240:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.693242:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.693243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.693245:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.693248:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.693250:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.693251:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.693253:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.693254:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.693255:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.693259:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.693261:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.693263:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.693264:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.693265:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.693266:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.693267:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.693268:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.693268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.693273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.693361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.693367:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.693370:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.693371:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.693372:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.693375:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.693376:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.693378:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.693379:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.693380:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.693381:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.693383:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.693385:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.693386:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.693387:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.693387:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.693389:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.693390:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.693390:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.693391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.693405:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.693407:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.693408:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.693408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.693409:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.693410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.693491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.693500:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.693503:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.693504:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.693504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.693506:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.693509:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.693636:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.693639:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.693639:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.693640:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.693641:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.693644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.693653:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.693656:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.693657:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.693657:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.693657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.693658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.693936:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.693938:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.693940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.693941:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.693942:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.693944:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.693945:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.693946:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.693947:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.693948:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.693950:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.693952:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.693955:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.693955:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.693956:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.693958:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.693958:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.693958:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.693959:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.693964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.693971:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.693974:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.693975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.693975:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.693977:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.693980:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.694603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.694637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.694742:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.694745:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.694746:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.694746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.694748:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.694750:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.695543:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.695545:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.695547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.695547:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.695549:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.695552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.695907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.695942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.695951:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.695953:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.695954:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.695954:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.695957:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.695959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.696226:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.696229:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.696230:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.696232:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.696234:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.696236:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.696237:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.696238:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.696239:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.696240:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.696243:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.696245:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.696247:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.696247:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.696248:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.696249:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.696250:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.696251:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.696251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.696261:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.696263:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.696264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.696264:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.696265:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.696267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677471.697031:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.697034:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.697035:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.697035:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.697038:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.697041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.697049:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.697052:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.697053:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.697053:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.697054:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.697055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.697367:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.697369:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.697370:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.697371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.697372:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.697375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677471.698264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.698267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.698267:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.698270:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.698271:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.698273:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.698274:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.698275:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.698276:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.698277:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.698281:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.698282:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.698285:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.698286:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.698287:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.698289:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.698289:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.698291:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.698292:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.698292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.698297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.698304:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.698306:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.698306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.698307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.698908:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.698911:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.698914:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.698915:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677471.698916:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:8.0:1619677471.698918:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.698919:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.698919:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.698920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.698920:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.698921:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677471.698923:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677471.698924:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96516495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.698926:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.698927:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677471.698927:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.698929:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000400:00000200:8.0:1619677471.698933:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677471.698936:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000400:00000200:8.0:1619677471.698938:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.698939:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.698940:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.698940:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.698941:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.698941:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677471.698942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.698944:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.698945:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.698945:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.698947:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.698948:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.698949:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.698950:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.698951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.698961:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.698963:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.698964:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.698965:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677471.698966:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:24.0:1619677471.698967:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.698967:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.698967:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677471.698967:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.698969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.698969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:2.0:1619677471.698969:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.698970:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.698971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677471.698971:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.698975:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.699200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3a3000] (20)++
00000800:00000200:18.0:1619677471.699207:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3a3000] (21)++
00000800:00000200:18.0:1619677471.699210:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.22@o2ib3
00000800:00000200:2.0:1619677471.699210:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)--
00000400:00000200:18.0:1619677471.699212:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.22@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.699215:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.699216:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677471.699217:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.699219:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.699220:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.699221:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677471.699224:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.699226:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 968 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677471.699229:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.699230:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677471.699231:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677471.699233:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677471.699233:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3a3000] (21)++
00000800:00000200:18.0:1619677471.699235:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3a3000] (22)--
00000800:00000200:18.0:1619677471.699235:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (21)--
00000800:00000200:8.2:1619677471.700069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.700088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677471.700111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.700117:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.700120:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.700121:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.700123:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.700126:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.700127:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.700128:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.700130:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.700131:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.700132:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.700136:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.700137:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.700139:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.700140:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.700140:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.700142:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.700143:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.700144:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.700145:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.700146:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.700151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.700157:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.700158:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.700158:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677471.700159:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.700159:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.700159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.700160:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.700160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.700161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.700162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.700183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.700185:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.700187:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677471.700189:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677471.700189:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677471.700190:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677471.700192:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.700193:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.700193:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.700194:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.700195:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.700196:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.700197:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.700199:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.700201:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.700202:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.700203:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677471.700204:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:2.0:1619677471.700204:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.700204:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:18.0:1619677471.700205:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.700206:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.700207:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.700207:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.700208:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.700209:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677471.700210:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677471.700211:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.700212:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.700213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677471.700214:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.700214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.700215:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.700216:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.700217:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677471.700217:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.700218:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.700218:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:2.0:1619677471.700219:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.700220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.700221:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.700221:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.700221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.700222:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.700222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.700239:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.700242:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.700243:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.700243:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.700244:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.700244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.700271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.700340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.700377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.700385:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.700388:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.700389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.700389:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.700391:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.700393:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.700397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.700404:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.700407:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.700407:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.700407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.700408:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.700409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.700421:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.700424:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.700426:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.700427:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.700430:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.700432:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.700433:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.700434:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.700435:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.700436:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.700439:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.700441:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.700443:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.700444:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.700445:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.700447:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.700447:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.700448:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.700448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.700452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.700460:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.700463:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.700464:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.700464:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.700466:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.700469:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.700569:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.700571:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.700572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.700572:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.700574:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.700576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.700588:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.700590:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.700591:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.700591:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.700592:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.700593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677471.701107:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.701110:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.701111:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.701113:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.701115:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.701117:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677471.701118:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.701120:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.701121:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.701122:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677471.701125:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.701127:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.701129:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.701129:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.701130:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.701131:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.701132:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.701132:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.701133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677471.701138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.701145:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.701147:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.701148:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.701149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.701149:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.701151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.701776:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.701779:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.701780:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.701780:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.701783:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.701787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.701790:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.701792:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.701793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.701795:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.701798:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.701799:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.701801:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.701802:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.701803:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.701804:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.701807:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.701808:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.701810:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.701811:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.701812:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.701813:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.701814:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.701815:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.701815:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.701825:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.701827:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.701827:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.701828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.701829:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.701830:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.701844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677471.701846:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.701848:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.701849:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.701850:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.701850:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.701850:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.701876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.701879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.701884:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.701887:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.701888:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.701888:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.701890:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.701892:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.702366:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.702369:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.702370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.702370:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.702372:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.702376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.703452:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.703455:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.703456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.703458:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.703459:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.703461:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.703462:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.703463:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.703464:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.703465:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.703468:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.703470:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.703473:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.703474:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.703475:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.703477:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.703478:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.703478:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.703479:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.703484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.703884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.703906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.704017:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.704019:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677471.704020:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.704022:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.704025:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.704027:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677471.704028:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.704030:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.704031:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.704032:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677471.704035:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.704037:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677471.704039:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.704040:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677471.704041:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677471.704043:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677471.704043:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.704044:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.704045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:17.0:1619677471.704058:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.704061:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.704061:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.704063:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.704066:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.704067:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677471.704068:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.704070:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.704071:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.704072:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677471.704075:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.704077:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677471.704080:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.704080:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677471.704081:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677471.704083:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677471.704084:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.704084:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.704085:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677471.704249:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.704251:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.704252:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.704252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.704254:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.704256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.704370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:2.0:1619677471.704376:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677471.704379:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:18.0:1619677471.704380:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:2.0:1619677471.704382:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.704384:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.704386:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677471.704387:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.704389:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.704390:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.704391:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677471.704394:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.704396:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677471.704400:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.704401:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677471.704402:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677471.704404:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677471.704405:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677471.704407:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907cbe3400] (22)++
00000800:00000200:2.0:1619677471.704408:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (23)--
00000800:00000200:2.0:1619677471.704408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:20.2:1619677471.704414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677471.704420:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)++
00000800:00000200:18.0:1619677471.704422:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907cbe3400] (23)--
00000800:00000200:18.0:1619677471.704423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:18.0:1619677471.704424:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:20.2:1619677471.704441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.704446:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.704449:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.704450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.704452:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.704455:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.704456:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.704457:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.704459:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.704460:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.704461:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.704464:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.704465:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.704468:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.704469:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.704469:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.704471:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.704471:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.704472:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.704473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.704655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677471.705053:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.705056:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.705056:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.705058:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.705060:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.705061:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.705063:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.705064:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.705065:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.705066:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.705069:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.705071:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.705074:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.705074:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.705075:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.705077:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.705077:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.705078:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.705078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.705083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.705091:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.705093:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.705094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.705095:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.705097:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.705100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.705405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.706225:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.706228:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.706229:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.706229:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.706231:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.706234:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.706408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677471.706434:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.706437:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.706438:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.706438:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.706440:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.2:1619677471.706442:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677471.706448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677471.706450:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.706453:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.706454:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.706454:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.706456:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.706458:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.706478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:8.2:1619677471.706513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.706642:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.706644:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.706645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.706647:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.706649:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.706651:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.706652:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96516450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.706653:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.706654:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.706655:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.706658:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.706659:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.706661:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.706662:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.706662:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.706664:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.706664:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.706665:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.706667:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.706667:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677471.706670:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.706673:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.706674:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.706674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.706677:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.706678:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.706679:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.706680:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.706681:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677471.706681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.706682:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677471.706682:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.706683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.706683:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.706684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.706684:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.707226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.707234:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.707236:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.707237:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.707238:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.707240:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.707242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.707245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.707252:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.707255:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.707256:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.707256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.707257:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.707258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.707933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.707967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.707976:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.707978:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.707980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.707980:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.707981:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.707984:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.708061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.708067:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.708070:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.708071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.708073:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.708075:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.708077:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.708079:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.708081:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.708082:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.708084:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.708087:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.708089:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.708091:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.708092:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.708092:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.708094:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.708095:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.708095:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.708096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.708108:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.708110:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.708110:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.708111:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.708113:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.708114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.708686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677471.709036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.709039:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.709040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.709042:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.709044:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.709046:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.709047:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.709049:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.709049:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.709051:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677471.709052:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677471.709054:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.709055:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.709056:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.709056:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677471.709056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.709059:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.709059:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.709060:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.709060:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.709061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677471.709062:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.709063:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.709064:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.709065:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.709068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.709088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.2:1619677471.709943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677471.710085:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.710088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.710088:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.710090:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.710091:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.710093:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.710094:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.710095:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.710096:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.710097:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.710100:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.710102:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.710104:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.710105:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.710106:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.710107:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.710108:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.710109:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.710109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.710123:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.710125:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.710127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.710127:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.710129:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.710132:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.710182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.710188:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.710191:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.710191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.710193:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.710195:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.710197:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.710198:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.710200:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.710201:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.710201:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.710204:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.710206:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.710208:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.710209:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.710210:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.710211:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.710212:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.710214:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.710215:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.710215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677471.710218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.710221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.710226:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.710228:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.710229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.710230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.710655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.710664:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.710666:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.710668:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.710668:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.710670:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.710673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.711386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.711394:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.711397:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.711398:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.711398:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.711400:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677471.711403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.711406:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.711407:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.711409:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.711410:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.711410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.711411:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.711412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.711523:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.711525:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.711526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.711527:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.711529:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.711532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.711559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.711561:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.711563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.711564:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.711567:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.711568:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.711569:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.711570:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.711571:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.711572:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.711575:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.711576:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.711578:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.711578:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.711579:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.711580:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.711580:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.711581:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.711582:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.711592:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.711594:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.711594:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.711595:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.711596:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.711597:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.712200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.712236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.712243:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.712246:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.712247:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.712247:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.712249:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.712251:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.712254:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.712256:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.712258:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.712258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.712260:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.712262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.712276:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.712279:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.712280:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.712280:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.712281:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.712281:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677471.712383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677471.712390:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.712393:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677471.712393:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677471.712395:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.712398:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.712399:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677471.712400:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.712402:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.712403:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.712404:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677471.712408:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.712409:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677471.712412:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.712413:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677471.712414:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677471.712416:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677471.712416:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.712418:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677471.712419:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677471.712420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:5.2:1619677471.712426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:24.0:1619677471.712429:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.712431:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677471.712431:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000400:00000200:24.0:1619677471.712432:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.712432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677471.712433:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000400:00000200:24.0:1619677471.712434:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.712434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677471.712435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:24.0:1619677471.712437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.712458:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.712461:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.712462:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.712463:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.712465:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.712467:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.712468:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.712469:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.712470:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.712471:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.712474:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.712476:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.712477:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.712478:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.712479:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.712480:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.712481:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.712481:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.712482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.712491:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.712493:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.712494:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.712495:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.712495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.712496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.712513:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.712516:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.712517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.712517:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.712518:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.712518:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.713241:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677471.713244:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677471.713245:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677471.713247:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.713249:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.713250:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.713251:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.713253:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.713254:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.713255:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.713257:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.713259:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.713261:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.713262:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.713262:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.713263:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.713264:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677471.713265:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677471.713265:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677471.713275:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.713277:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.713278:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.713278:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.713280:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.713281:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677471.713557:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.713560:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.713561:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.713562:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.713564:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.713565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677471.713582:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677471.713585:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677471.713586:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.713586:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677471.713587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.713588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677471.713712:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.713715:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.713716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.713718:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.713721:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.713722:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.713723:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.713725:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.713726:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.713727:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.713730:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.713732:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.713734:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.713735:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.713736:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.713738:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.713738:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.713739:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.713739:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.713745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.714297:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.714300:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.714301:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.714301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.714302:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.714305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677471.714338:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b38fa00] (21)++
00000800:00000200:2.0:1619677471.714341:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.8@o2ib3
00000800:00000200:18.0:1619677471.714342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38fa00] (22)--
00000400:00000200:2.0:1619677471.714344:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.15.8@o2ib3 : GET - for me
00000400:00000200:2.0:1619677471.714348:0:19932:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.15.8@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:2.0:1619677471.714351:0:19932:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.15.8@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:2.0:1619677471.714352:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907b38fa00] (21)++
00000800:00000200:2.0:1619677471.714353:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907b38fa00] (22)--
00000400:00000200:2.0:1619677471.714355:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.15.8@o2ib3 local destination
00000400:00000200:2.0:1619677471.714360:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.15.8@o2ib3(10.51.15.8@o2ib3:10.51.15.8@o2ib3) : REPLY try# 0
00000800:00000200:2.0:1619677471.714362:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.15.8@o2ib3
00000800:00000200:2.0:1619677471.714365:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91906ffa8a80] -> 10.51.15.8@o2ib3 (2) version: 12
00000800:00000200:2.0:1619677471.714366:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91907b38fa00] (21)++
00000800:00000200:2.0:1619677471.714366:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907b38fa00] (22)++
00000800:00000200:2.0:1619677471.714368:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91907b38fa00] (23)--
00000800:00000200:2.0:1619677471.714369:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38fa00] (22)--
00000800:00000200:18.0:1619677471.714379:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b38fa00] (22)++
00000800:00000200:18.0:1619677471.714382:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907b38fa00] (23)--
00000400:00000200:18.0:1619677471.714383:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.714383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38fa00] (22)--
00000400:00000200:18.0:1619677471.714385:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.15.8@o2ib3: REPLY: OK
00000800:00000200:18.0:1619677471.714386:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38fa00] (21)--
00000800:00000200:20.2:1619677471.714414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.714450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.714452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.714458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.714460:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.714461:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.714462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.714464:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.714466:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.715439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.715592:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.715595:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.715596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.715596:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.715599:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.715605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.715607:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.715609:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.715610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.715611:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.715612:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.715613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.715990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:18.0:1619677471.716243:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.716246:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:8.2:1619677471.716247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.716247:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.716248:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.716250:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.716253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.716374:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.716377:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.716378:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.716380:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.716382:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.716383:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.716384:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.716386:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.716387:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.716388:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.716391:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.716392:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.716395:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.716396:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.716396:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.716398:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.716399:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.716400:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.716401:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.716401:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.716404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.716412:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677471.716415:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677471.716415:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.716416:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.716416:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.716417:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.716418:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677471.716418:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.716419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677471.716421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.716772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.716837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.716843:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.716846:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.716847:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.716849:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.716851:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.716853:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.716854:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.716856:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.716858:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.716858:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.716862:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.716864:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.716867:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.716868:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.716869:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.716872:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.716873:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.716874:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.716874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.717208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.717216:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.717219:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.717220:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.717220:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.717222:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.717225:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.717229:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.717232:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.717233:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.717235:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.717237:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.717239:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.717240:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.717242:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.717243:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.717244:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.717247:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.717249:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.717252:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.717252:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.717253:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.717254:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.717255:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.717256:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.717256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.717260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.717269:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.717271:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.717272:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.717272:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.717275:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.717277:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.717392:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.717395:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.717395:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.717396:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.717397:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.717400:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.718035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.718068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.718418:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.718421:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.718422:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.718424:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.718427:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.718428:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.718430:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.718431:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.718432:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.718433:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.718436:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.718438:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.718441:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.718442:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.718442:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.718444:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.718444:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.718445:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.718445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.718450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.718458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.718461:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.718462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.718462:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.718464:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.718467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.718468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.718476:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.718479:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.718480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.718481:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.718481:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.718482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.718873:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.718876:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.718877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.718880:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.718882:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.718883:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.718885:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.718886:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.718887:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.718888:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.718892:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.718894:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.718896:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.718897:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.718898:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.718900:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.718900:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.718902:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.718903:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.718904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.718906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.718915:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.718917:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.718917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.718918:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.718933:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.718935:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.718937:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.718937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.718939:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.718942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.718959:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.718961:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.718962:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.718963:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.718964:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.718964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.719055:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.719057:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.719058:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.719058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.719060:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.719062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.719073:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.719075:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.719076:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.719076:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.719077:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.719078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677471.719707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677471.719712:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.719715:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.719716:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.719716:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.719718:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.719720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677471.719726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.2:1619677471.720307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.720341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.720349:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.720352:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.720353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.720353:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.720355:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.720358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.720363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.720371:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.720373:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.720374:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.720375:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.720375:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.720376:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.720452:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.720455:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.720456:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.720456:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.720458:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.720462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.721135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.721140:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.721143:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.721144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.721146:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.721149:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.721150:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.721151:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.721153:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.721154:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.721155:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.721159:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.721161:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.721164:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.721165:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.721166:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.721167:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.721168:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.721170:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.721170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.721316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.721335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.2:1619677471.722089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.722110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677471.722147:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.722149:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.722150:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.722151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.722152:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.722154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.722181:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.722183:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.722184:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.722185:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.722188:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.722189:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.722191:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.722192:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.722193:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.722193:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.722196:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.722197:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.722199:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.722200:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.722200:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.722201:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.722202:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.722203:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.722203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.722213:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.722215:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.722215:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.722217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.722217:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.722218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.722981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.722989:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.722992:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.722993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.722993:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.722996:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.722999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.723425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.723432:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677471.723434:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677471.723436:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:24.0:1619677471.723437:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677471.723437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677471.723438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.723441:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.723441:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.723443:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.723444:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.723445:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.723447:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96516377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.723447:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.723449:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.723449:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.723450:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.723450:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.723451:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.723452:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.723453:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.723457:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.723459:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.723462:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.723464:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.723465:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.723466:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.723467:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.723467:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.723468:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677471.723469:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (23)++
00000800:00000200:18.0:1619677471.723469:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.723470:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.723471:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677471.723472:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677471.723472:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677471.723472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677471.723474:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.723475:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.723475:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677471.723484:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000800:00000200:8.0:1619677471.723488:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677471.723488:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000800:00000200:18.0:1619677471.723489:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000400:00000200:2.0:1619677471.723489:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.723490:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000800:00000200:8.0:1619677471.723490:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677471.723491:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.723491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.723492:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677471.723493:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677471.723493:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.723496:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.723497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.723498:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.723784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677471.723788:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.723791:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.723792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.723793:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.723796:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.723801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.723822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.724536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.724540:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.724540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.724543:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.724545:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.724546:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.724547:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.724549:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.724550:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.724550:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.724553:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.724555:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.724558:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.724559:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.724559:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.724561:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.724562:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.724562:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.724563:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.724568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.725048:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.725051:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.725052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.725054:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677471.725057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:8.0:1619677471.725057:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.725059:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.725060:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.725061:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.725062:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.725063:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677471.725063:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.725066:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.725067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:8.0:1619677471.725067:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.725069:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677471.725069:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677471.725072:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677471.725072:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.725073:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.725073:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:2.0:1619677471.725074:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677471.725075:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.725076:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.725076:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677471.725076:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677471.725077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677471.725078:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.725079:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.725080:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677471.725081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:2.0:1619677471.725083:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.725085:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677471.725088:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.725089:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.725089:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677471.725090:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677471.725092:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677471.725092:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.725092:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677471.725093:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:1.0:1619677471.725093:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.725094:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.725095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:1.0:1619677471.725095:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.725098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.725391:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.725394:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.725395:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.725395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.725397:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.725401:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.726539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.726827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.726862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.726865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.726870:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.726872:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.726874:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.726874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.726875:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.726878:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.727184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.727190:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.727193:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.727194:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.727196:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.727199:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.727201:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.727202:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.727204:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.727205:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.727206:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.727210:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.727212:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.727215:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.727216:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.727217:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.727219:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.727220:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.727221:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.727221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.727225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677471.727789:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.727791:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.727792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.727794:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.727797:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.727798:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.727800:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.727801:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.727802:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.727803:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.727806:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.727808:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.727810:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.727810:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.727811:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.727812:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.727813:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.727813:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.727814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.727824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.727826:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.727826:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.727827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.727828:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.727830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.727906:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.727909:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.727909:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.727910:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.727911:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.727913:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677471.727947:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.727950:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677471.727951:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.727953:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.727956:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.727957:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677471.727959:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.727960:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.727961:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.727962:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677471.727966:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.727968:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677471.727972:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.727973:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677471.727974:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677471.727977:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677471.727977:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.727979:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677471.727980:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677471.727981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.727996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677471.727998:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677471.727999:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.728001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:8.2:1619677471.728024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.728047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.2:1619677471.729099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.729107:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.729110:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.729111:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.729111:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.729113:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677471.729115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:1.0:1619677471.729116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.729170:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.729173:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.729174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.729174:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.729179:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.729182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.729197:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.729200:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.729201:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.729202:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.729203:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.729203:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.729236:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.729239:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.729240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.729240:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.729242:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.729244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.729255:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.729257:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.729258:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.729259:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677471.729259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.729260:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.729439:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.729442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.729442:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.729444:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.729445:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.729446:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.729447:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.729448:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.729449:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.729450:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.729453:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.729454:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.729457:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.729458:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.729458:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.729460:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.729461:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.729461:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.729462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.729475:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.729478:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.729479:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.729479:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.729482:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.729485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.729644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.729677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.729708:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.729711:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.729712:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.729716:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.729718:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.729719:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.729721:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.729722:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.729723:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.729724:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.729728:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.729730:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.729732:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.729733:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.729733:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.729735:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.729736:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.729736:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.729737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.729741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.730061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.730068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.730071:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.730072:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.730073:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.730075:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.730077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.730149:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.730152:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.730153:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.730154:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.730156:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.730161:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.730163:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.730165:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.730166:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.730167:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.730168:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.730168:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.730774:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.730777:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.730778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.730780:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.730782:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.730784:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.730785:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.730786:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.730787:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.730788:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.730791:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.730793:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.730795:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.730796:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.730797:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.730798:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.730799:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.730800:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.730801:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.730802:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677471.730814:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677471.730815:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.730817:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677471.730817:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.730818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.730818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677471.730818:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.730820:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677471.730820:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.730823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.731772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677471.731982:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.731985:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.731986:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.731986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.731989:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.731993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.732130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.732136:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.732138:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.732139:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.732142:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.732148:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.732149:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.732151:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.732153:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.732154:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.732155:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.732159:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.732160:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.732163:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.732164:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.732165:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.732166:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.732167:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.732168:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.732169:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.732180:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.732182:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.732183:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.732184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.732185:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.732187:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.732913:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.732916:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.732917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.732918:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.732920:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.732924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.733198:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.733200:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.733202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.733203:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.733206:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.733207:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.733209:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.733210:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.733211:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.733212:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.733215:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.733216:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.733218:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.733218:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.733219:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.733220:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.733220:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.733221:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.733221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.733232:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.733234:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.733234:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.733235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.733236:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.733237:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.733255:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.733257:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.733258:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.733258:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.733258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.733259:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.733885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.733892:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.733895:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.733895:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.733898:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.733901:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.733903:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.733904:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96516329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.733906:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.733907:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.733908:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.733912:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:8.2:1619677471.733913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677471.733914:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.733917:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.733918:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.733918:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.733920:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:8.0:1619677471.733920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677471.733921:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677471.733921:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677471.733922:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:18.0:1619677471.733923:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:8.0:1619677471.733923:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.733924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677471.733924:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.733924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677471.733924:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.733925:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.733925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:8.0:1619677471.733926:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:17.0:1619677471.733927:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677471.733929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677471.733929:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.733932:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.733933:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.733933:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677471.733935:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677471.733936:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677471.733937:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677471.733937:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.733938:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.733938:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.733939:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.733939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.733940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677471.734007:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.734010:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.734010:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.734011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.734012:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.734015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.734609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.2:1619677471.734658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:13.0:1619677471.734662:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.734665:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.734666:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.734667:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.734669:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.734670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677471.734683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677471.734686:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677471.734687:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.734688:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677471.734689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.734690:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677471.734756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.734792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.734800:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.734803:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.734804:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.734804:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.734806:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.734808:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.734905:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.734908:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.734909:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.734909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.734912:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.734915:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.734944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677471.734988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:12.0:1619677471.734992:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.734995:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677471.734996:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.734999:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.735001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.735002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:17.0:1619677471.735314:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.735318:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.735318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.735320:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.735322:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.735324:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.735325:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.735327:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.735328:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.735329:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.735332:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.735334:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.735336:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.735337:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.735338:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.735340:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.735340:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.735341:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.735341:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.735346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.735367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677471.735385:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.735388:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.735389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.735391:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.735394:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.735395:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.735396:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.735398:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.735398:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.735399:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.735402:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.735404:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.735407:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.735408:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.735408:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.735410:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.735411:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.735411:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.735412:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.735416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.735423:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.735426:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.735427:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.735427:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.735430:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.735432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.735836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.736129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.736160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.736167:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.736170:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.736171:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.736171:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.736173:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.736175:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.736178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.736185:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.736188:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.736190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.736190:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.736191:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.736191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.736773:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.736776:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.736777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.736779:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.736782:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.736783:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.736784:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.736786:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.736787:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.736788:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.736792:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.736793:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.736796:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.736797:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.736798:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.736799:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.736800:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.736800:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.736801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.736805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.736814:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.736816:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.736817:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.736817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.736820:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.736822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.737369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.738117:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.738120:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.738121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.738123:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.738126:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.738127:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.738128:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.738130:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.738130:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.738132:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.738135:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.738137:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.738138:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.738139:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.738140:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.738141:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.738141:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.738143:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.738144:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.738144:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.738155:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.738156:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.738156:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.738158:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:24.0:1619677471.738159:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.738159:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.738159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.738160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.738161:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.738161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.738797:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.738800:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.738801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.738801:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.738802:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.738805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.738921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.738928:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.738931:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.738932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.738934:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.738937:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.738938:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.738940:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.738942:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.738943:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.738944:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.738947:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.738949:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.738950:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.738951:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.738952:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.738953:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.738954:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.738956:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.738957:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.738958:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.738963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.738968:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.738969:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.738969:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677471.738970:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.738970:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.738971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.738972:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.738973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.738974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.738974:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.738989:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.738991:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.738992:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.738992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.738993:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.738993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.739081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.739086:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.739089:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.739090:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.739091:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.739093:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.739095:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.739096:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.739097:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.739098:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.739099:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.739103:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.739105:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.739107:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.739108:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.739109:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.739110:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.739111:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.739112:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.739112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.739117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.739135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.739690:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.739693:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.739694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.739695:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.739697:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.739701:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.739703:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.739705:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.739706:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.739706:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.739708:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.739708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.739897:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.739899:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.739900:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.739901:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.739902:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.739904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.739916:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.739918:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.739919:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.739919:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.739920:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.739921:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677471.740196:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.740200:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.740200:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.740203:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.740205:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.740207:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.740209:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.740210:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.740211:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.740212:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.740216:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.740218:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.740221:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.740222:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.740222:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.740224:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.740225:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.740227:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.740228:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.740228:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.740230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.740239:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.740241:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.740242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.740243:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677471.740484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.740490:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.740493:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.740494:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.740496:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.740498:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.740499:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.740501:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.740502:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.740503:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.740504:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.740507:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.740508:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.740510:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.740511:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.740511:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.740513:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.740513:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.740514:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.740514:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:5.2:1619677471.740517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677471.740519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.740522:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.740524:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.740526:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.740526:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.740528:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.740530:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677471.740531:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.740533:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.740534:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.740534:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677471.740538:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.740539:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.740541:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.740541:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.740542:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.740543:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.740544:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.740545:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.740545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677471.740550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.740557:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.740559:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.740560:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.740560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.740562:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.740564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.740695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:24.0:1619677471.741159:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.741162:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.741163:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.741163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.741165:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.741167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.741569:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.741571:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.741572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.741574:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.741575:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.741577:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.741578:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.741579:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.741580:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.741581:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.741584:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.741585:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.741588:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.741589:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.741589:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.741591:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.741592:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.741592:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.741593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.741607:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.741610:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.741611:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.741612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.741613:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.741617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.741739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.741775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677471.741783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.741790:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.741793:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.741794:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.741795:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.741797:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.741799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.742168:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.742171:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.742171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.742172:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.742175:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.742178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.742338:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.742340:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.742342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.742342:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.742344:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.742347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.742368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.742732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:8.2:1619677471.742765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.742958:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.742961:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.742962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.742964:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.742966:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.742968:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.742970:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.742971:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.742972:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.742973:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.742977:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.742979:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.742982:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.742983:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.742984:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.742986:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.742986:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.742987:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.742987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.742992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.743000:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.743003:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.743004:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.743004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.743007:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.743011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.743014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.743020:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.743023:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.743024:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.743025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.743025:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.743026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.743035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.743069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:5.2:1619677471.743087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.743093:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.743095:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.743096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.743098:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.743100:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.743101:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.743103:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.743104:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.743105:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.743106:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.743110:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.743111:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.743114:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.743115:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.743116:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.743117:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.743118:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.743119:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677471.743120:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677471.743121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677471.743125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.743131:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677471.743133:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677471.743134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.743135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.743146:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.743149:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.743149:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.743150:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.743152:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.743155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677471.743433:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.743436:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.743437:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.743439:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.743441:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.743443:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677471.743444:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.743446:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.743447:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.743448:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677471.743451:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.743455:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677471.743458:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.743459:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677471.743460:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677471.743462:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677471.743462:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.743463:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.743464:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.743465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:13.0:1619677471.743476:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.743479:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.743480:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.743482:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.743484:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.743485:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677471.743691:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.743693:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.743694:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.743695:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677471.743894:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.743897:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.743898:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.743899:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.743901:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.743904:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.744230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.744235:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.744238:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.744239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.744241:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677471.744243:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.744245:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.744246:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.744247:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.744248:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.744249:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.744252:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677471.744254:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.744256:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.744257:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.744258:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.744259:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.744260:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.744261:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.744261:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.744345:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.744347:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.744348:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.744349:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.744350:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.744353:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677471.744630:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.744634:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.744635:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.744636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.744637:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.744643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.744644:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.744647:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.744648:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.744648:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.744649:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.744650:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.744773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.744779:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.744783:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.744783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.744785:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677471.744788:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.744789:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.744790:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.744792:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.744793:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.744794:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.744797:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677471.744798:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.744801:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.744801:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.744802:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.744803:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.744804:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.744804:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.744805:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.745252:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.745255:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.745256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.745258:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.745261:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.745262:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.745263:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.745265:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.745266:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.745267:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.745269:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.745271:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.745273:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.745274:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.745275:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.745276:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.745276:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.745277:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.745277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.745287:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.745289:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.745290:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.745291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.745292:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.745302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.745309:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.745311:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.745312:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.745312:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.745313:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.745314:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.745317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677471.745351:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.745353:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.745354:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.745355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.745356:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.745358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.745399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.745406:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.745410:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.745410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.745412:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.745415:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.745416:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.745418:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.745419:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.745421:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.745422:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.745425:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.745427:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.745430:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.745431:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.745432:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.745434:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.745435:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.745436:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.745437:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677471.746083:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.746086:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.746089:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.746091:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.746091:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.746093:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677471.746094:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96516264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.746096:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.746097:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.746098:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677471.746101:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.746102:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677471.746105:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.746106:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677471.746106:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677471.746108:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677471.746108:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.746110:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.746110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677471.746534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.746543:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.746546:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.746547:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.746547:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.746549:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.746553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.746554:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.746556:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.746557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.746559:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.746561:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.746562:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.746563:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.746565:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.746566:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.746567:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.746569:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.746571:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.746573:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.746574:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.746575:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.746576:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.746577:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.746577:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.746578:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.746583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.746600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.746833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.746839:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.746842:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.746844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.746845:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.746848:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.746850:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.746851:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.746853:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.746854:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.746855:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.746858:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.746860:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.746862:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.746863:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.746864:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.746866:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.746867:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.746868:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.746868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.746880:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.746882:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.746882:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.746883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.746884:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.746886:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.747196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:5.2:1619677471.747384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.747391:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.747394:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.747395:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.747397:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.747399:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.747401:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.747402:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.747403:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.747404:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.747405:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.747409:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.747410:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.747413:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.747414:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.747414:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.747416:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.747416:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.747417:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.747417:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677471.747485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.748042:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.748045:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.748046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.748046:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.748048:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.748050:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.748264:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.2:1619677471.748266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677471.748267:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.748268:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.748269:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.748272:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.748273:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.748275:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.748277:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.748278:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.748279:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.748283:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.748285:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.748289:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.748289:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.748290:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.748292:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.748292:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.748298:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.2:1619677471.748298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.748300:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.748301:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677471.748306:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.748309:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.748310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.748310:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677471.748312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:1.0:1619677471.748312:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.748315:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.748319:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.748321:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.748322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.748322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.748444:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.748446:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.748447:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.748447:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.748449:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.748452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677471.748646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.748648:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.748650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.748650:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.748652:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.748656:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.749390:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.749392:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.749393:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.749394:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.749396:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.749399:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.750165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.750172:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.750174:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.750175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.750177:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.750179:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.750180:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.750182:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.750183:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.750184:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.750185:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.750188:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.750190:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.750192:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.750193:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.750194:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.750196:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.750196:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.750197:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677471.750198:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677471.750199:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677471.750204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.750209:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677471.750211:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677471.750212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.750213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.750708:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.750711:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.750712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.750714:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.750717:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.750718:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.750720:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.750721:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.750722:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.750723:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.750726:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.750728:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.750730:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.750731:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.750731:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.750733:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.750733:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.750734:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.750734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.750744:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.750746:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.750747:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.750747:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.750749:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.750750:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677471.750932:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.750935:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.750935:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.750938:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.750940:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.750941:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.750942:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.750944:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.750945:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.750946:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.750949:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.750951:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.750954:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.750954:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.750955:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.750957:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.750957:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.750958:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.750958:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.750964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.751343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.751349:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.751352:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.751353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.751355:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.751357:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.751359:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.751360:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.751362:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.751363:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.751364:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.751367:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.751369:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.751371:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.751373:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.751374:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.751376:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.751377:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.751378:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.751378:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.751383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677471.751636:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.751639:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.751639:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.751640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.751641:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.751643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.751653:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.751656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.751656:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677471.751657:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.751658:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.751658:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.751930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.751937:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.751940:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.751941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.751943:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.751946:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.751948:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.751949:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.751951:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.751952:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.751953:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.751956:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.751958:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.751961:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.751962:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.751962:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.751964:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.751965:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.751966:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677471.751967:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677471.751968:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.2:1619677471.751970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.751973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.751979:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677471.751981:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677471.751982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.751983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677471.751993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.2:1619677471.752117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677471.752520:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.752522:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.752524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.752525:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.752528:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.752529:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.752530:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.752532:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.752533:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.752534:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.752537:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.752539:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.752542:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.752542:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.752543:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.752545:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.752545:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.752546:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.752546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.752551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.752559:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.752562:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.752563:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.752563:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.752565:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.752569:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.753030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.753126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.753169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.753177:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.753179:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.753180:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.753180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.753183:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.753185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677471.753314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.753319:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.753322:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.753323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.753325:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.753327:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.753328:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.753330:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.753331:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.753332:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.753333:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.753336:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.753338:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.753341:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.753342:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.753342:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.753344:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.753344:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.753345:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.753346:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.753725:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.753727:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.753728:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.753728:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.753730:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.753732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677471.753785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.753793:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.753796:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.753796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.753798:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.753801:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677471.753804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.753807:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.753810:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.753812:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.753812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.753813:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.753814:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.753815:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.753962:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.753965:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.753966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.753968:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.753971:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.753972:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.753974:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.753976:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.753977:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.753978:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.753981:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.753983:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.753986:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.753987:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.753987:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.753989:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.753989:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.753990:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.753990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.753994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.754003:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.754005:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.754007:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.754007:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.754009:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.754012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.754419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.754446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.754453:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677471.754455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677471.754456:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.754457:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.754459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:18.0:1619677471.754459:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677471.754460:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:18.0:1619677471.754462:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.754463:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.0:1619677471.754463:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.754464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:18.0:1619677471.754464:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96516217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.754464:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.754466:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.754466:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:18.0:1619677471.754467:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.754468:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.754469:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:18.0:1619677471.754472:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.754474:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.754476:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677471.754476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:18.0:1619677471.754478:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.754478:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.754481:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.754481:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.754482:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.754483:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.754494:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.754496:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.754497:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.754498:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.754499:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.754501:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.755061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.755067:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.755070:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.755071:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.755073:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.755075:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.755077:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677471.755078:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.755080:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.755081:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.755082:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677471.755086:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.755087:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677471.755091:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.755092:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677471.755093:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677471.755095:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677471.755096:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.755097:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.755097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677471.755914:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.755917:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.755918:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.755921:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.755923:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.755925:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.755926:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.755928:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.755929:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.755931:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.755935:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.755937:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.755940:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.755941:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.755942:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.755943:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.755944:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.755944:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.755945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.755950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:5.2:1619677471.756078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.756083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.756086:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.756087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.756089:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.756091:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.756092:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.756094:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.756095:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.756096:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.756097:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.756100:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.756102:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.756104:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.756105:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.756106:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.756107:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.756107:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.756108:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.756109:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.756215:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.756218:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.756219:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.756220:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.756223:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.756225:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.756226:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96516210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.756227:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.756228:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.756229:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.756231:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.756233:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.756234:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.756235:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.756236:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.756237:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.756237:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.756239:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.756240:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.756240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.756251:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.756253:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.756254:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.756255:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.756256:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677471.756256:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.756257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.756257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.756258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.756258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.756911:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.756914:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.756915:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.756915:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.756917:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.756919:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.757416:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.757419:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.757420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.757420:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.757422:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.757423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.757432:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.757434:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.757435:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.757435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.757436:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.757436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.757568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.757575:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.757578:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.757579:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.757581:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.757584:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.757585:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.757587:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.757589:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.757590:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.757591:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.757595:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.757597:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.757601:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.757602:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.757603:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.757605:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.757606:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.757608:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.757609:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.757610:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.757615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.757621:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.757623:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.757624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.757625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.758128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677471.758576:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.758578:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.758579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.758579:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.758581:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.758583:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.758976:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.758979:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.758980:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.758982:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.758983:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.758985:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.758986:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.758988:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.758989:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.758990:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.758993:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.758994:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.758998:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.758998:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.758999:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.759001:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.759001:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.759002:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.759002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.759007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.759015:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.759017:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.759019:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.759019:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.759021:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.759024:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.759145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.759164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:5.2:1619677471.759314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.759322:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.759326:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.759328:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.759331:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.759334:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.759336:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:1.0:1619677471.759336:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677471.759338:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:1.0:1619677471.759338:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677471.759339:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.759339:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.759340:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.759343:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:1.0:1619677471.759343:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677471.759345:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:1.0:1619677471.759345:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.759351:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.759354:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.759357:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.759359:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.759360:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.759362:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.759363:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.759364:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.759365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.760179:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.760181:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.760183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.760184:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.760187:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.760188:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.760190:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.760191:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.760192:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.760193:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.760196:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677471.760197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677471.760198:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.760200:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.760200:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.760201:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.760202:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.760203:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.760203:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.760204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.760214:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.760216:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.760217:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.760218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.760219:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.760221:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.760236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677471.760243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677471.760245:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:24.0:1619677471.760246:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.760247:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:17.0:1619677471.760247:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:8.0:1619677471.760247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.760248:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:17.0:1619677471.760248:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.760248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677471.760249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:17.0:1619677471.760251:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.760253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.760478:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.760481:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.760482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.760483:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.760486:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.760487:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677471.760489:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.760490:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.760491:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.760492:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677471.760495:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.760497:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677471.760500:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.760501:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677471.760502:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677471.760503:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677471.760504:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.760505:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.760505:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677471.760514:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.760516:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.760517:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.760519:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.760520:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.760522:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.760523:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.760525:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.760526:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.760527:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.760530:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.760532:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.760534:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.760535:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.760535:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.760537:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.760537:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.760539:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.760540:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.760541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.760544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.760551:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.760553:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.760555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.760555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677471.760934:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.760936:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.760938:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.760938:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.760940:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.760944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.760957:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.760959:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.760960:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.760960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.760962:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.760962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.761032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.761040:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.761043:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.761044:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.761044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.761046:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.761049:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.761150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.761153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.761159:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.761162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.761162:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677471.761165:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677471.761167:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677471.761168:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.761169:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.0:1619677471.761170:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.761171:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677471.761171:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.761173:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.761174:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.761175:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.761176:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.761176:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.761178:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.761180:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96516182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.761180:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677471.761181:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.761182:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677471.761182:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.761183:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677471.761185:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.761186:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:8.0:1619677471.761187:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.761187:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.761189:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677471.761189:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.761189:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.761190:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677471.761191:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.761191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677471.761192:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.761192:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.761193:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.761194:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.761195:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.761195:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.761202:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.761204:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677471.761205:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:18.0:1619677471.761205:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.761206:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.761207:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.761207:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.761207:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.761208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.761209:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.761209:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.761211:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677471.761990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.761997:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.762000:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.762001:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.762003:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.762005:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.762006:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.762008:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.762009:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.762010:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.762011:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.762014:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.762016:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.762019:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.762020:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.762020:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.762022:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.762022:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.762024:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677471.762025:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677471.762025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677471.762030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.762035:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677471.762038:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677471.762038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.762039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.762450:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677471.762453:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677471.762454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677471.762456:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.762458:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.762459:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677471.762461:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.762462:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.762463:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.762464:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677471.762467:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.762469:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677471.762472:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.762473:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677471.762473:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677471.762475:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677471.762476:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677471.762476:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677471.762477:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:24.0:1619677471.762533:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.762536:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.762537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.762538:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.762540:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677471.762543:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.763126:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.763129:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.763130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.763130:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.763132:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.763135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.763274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.763280:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.763284:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.763285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.763287:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.763289:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.763291:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.763293:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.763299:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.763300:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.763301:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.763305:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.763307:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.763309:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.763310:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.763311:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.763313:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.763314:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.763315:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.763315:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.763320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.763423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.763536:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.763538:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.763539:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.763539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.763540:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.763542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677471.763583:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.763586:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.763587:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.763587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.763590:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.763593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.764251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677471.764395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.764398:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.764399:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.764399:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.764401:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.764403:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.764413:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.764416:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.764417:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.764417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.764418:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.764419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677471.764849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.764854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.764857:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.764858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.764860:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.764862:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.764864:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.764865:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.764867:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.764867:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.764869:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.764872:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.764874:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.764877:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.764878:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.764879:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.764880:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.764881:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.764882:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.764882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.764910:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.764913:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.764914:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.764915:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.764919:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.764920:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.764922:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.764923:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.764924:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.764925:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.764928:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.764930:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.764933:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.764934:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.764934:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.764936:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.764937:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.764938:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.764939:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.764940:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.764942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.764950:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677471.764952:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677471.764953:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.764954:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.764954:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.764955:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.764955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.764956:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677471.764957:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.764960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.764963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.764970:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.764973:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.764974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.764974:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.764975:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.764976:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.765688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.765708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677471.765787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.765793:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.765796:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.765797:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.765798:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.765801:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.765803:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677471.765804:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.765806:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.765807:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.765808:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677471.765812:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.765813:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677471.765817:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.765818:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677471.765819:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677471.765821:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677471.765822:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.765824:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.765824:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:13.0:1619677471.765836:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.765839:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.765840:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.765842:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.765845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.765846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677471.765863:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.765868:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.765868:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.765873:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.765877:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.765879:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.765881:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.765884:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.765886:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.765888:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.765894:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.765897:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.765901:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.765902:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.765903:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.765905:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.765906:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.765907:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.765908:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.765911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.766215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.766253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.766260:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.766262:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.766264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.766264:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.766266:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.766268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.766272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.766279:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.766282:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.766283:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.766283:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.766285:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.766285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.766568:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.766571:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.766572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.766574:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.766577:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.766578:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.766580:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.766581:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.766582:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.766583:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.766586:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.766588:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.766591:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.766592:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.766593:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.766594:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.766595:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.766595:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.766596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.766610:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.766613:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.766614:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.766614:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.766617:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.766620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.766965:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.766968:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.766969:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.766969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.766971:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.766974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.766996:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.766998:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.766999:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.766999:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.767000:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.767001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.767073:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.767075:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.767076:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.767076:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.767078:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.767080:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.767585:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.767588:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.767589:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.767590:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.767592:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.767593:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.767594:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.767596:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.767597:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.767597:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.767600:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.767601:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.767603:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.767604:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.767604:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.767605:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.767606:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.767607:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.767607:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.767618:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.767619:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.767620:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.767621:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.767621:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.767623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.767702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.767708:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.767711:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.767712:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.767713:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.767716:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.767718:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.767720:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.767721:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.767722:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.767724:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.767727:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.767729:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.767732:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.767733:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.767734:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.767736:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.767737:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.767738:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.767738:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.767742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.767762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:5.2:1619677471.767949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.767955:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.767958:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.767959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.767960:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.767963:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.767964:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.767965:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.767967:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.767968:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.767969:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.767972:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.767973:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.767976:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.767977:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.767978:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.767979:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.767980:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.767980:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.767981:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677471.768331:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.768334:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.768335:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.768335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.768338:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.768341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.768350:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.768353:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.768354:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.768354:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.768355:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.768356:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.768986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.768992:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.768995:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.768996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.768998:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.769001:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.769002:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.769004:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.769005:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.769007:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.769007:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.769011:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.769013:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.769015:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.769016:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.769016:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.769018:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.769019:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.769021:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.769022:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.769022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.769027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.769033:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.769035:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.769035:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.769035:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.769036:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.769036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.769038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.769038:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.769039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.769039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.769523:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.769526:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.769527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.769527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.769529:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.769530:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.769673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.769675:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.769676:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.769677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.769678:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.769680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677471.769926:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.769928:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.769930:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.769933:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.769933:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.769934:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:12.0:1619677471.769936:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96516145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.769937:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.769938:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.769939:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:12.0:1619677471.769941:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.769943:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677471.769945:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.769946:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:12.0:1619677471.769947:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:12.0:1619677471.769948:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:12.0:1619677471.769949:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.769949:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.769950:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.0:1619677471.769960:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.769962:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.769963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.769963:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.769964:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.769967:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.769983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.769985:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.769986:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.769986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.769987:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.769988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677471.769989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677471.770040:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.770043:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.770044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.770046:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.770048:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.770050:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677471.770051:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.770053:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.770053:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.770055:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677471.770058:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.770060:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677471.770062:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.770063:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677471.770064:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677471.770065:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677471.770066:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.770067:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677471.770068:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677471.770069:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.770079:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:18.0:1619677471.770080:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677471.770081:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677471.770081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677471.770082:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:1.0:1619677471.770082:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:18.0:1619677471.770084:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.770084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.770086:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.770088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.770604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.770611:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.770613:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.770614:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.770616:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.770619:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.770620:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.770621:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.770623:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.770624:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.770625:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.770628:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.770630:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.770633:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.770633:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.770634:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.770636:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.770636:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.770637:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.770638:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.770889:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.770892:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.770893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.770893:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.770895:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.770898:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677471.771032:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.771035:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.771036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.771038:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.771040:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.771042:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.771043:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.771044:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.771045:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.771046:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.771049:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.771051:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.771053:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.771054:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.771055:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.771057:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.771057:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.771058:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.771058:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.771063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.771108:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677471.771111:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677471.771112:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677471.771114:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.771116:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.771117:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677471.771118:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.771120:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.771121:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.771122:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677471.771125:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.771127:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677471.771130:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.771131:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677471.771131:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677471.771133:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677471.771134:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677471.771135:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677471.771136:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677471.771137:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677471.771150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677471.771152:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677471.771153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677471.771153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677471.771466:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.771469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.771469:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.771471:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.771474:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.771475:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.771477:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.771478:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.771479:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.771480:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.771483:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.771484:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.771486:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.771486:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.771486:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.771488:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.771488:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.771489:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.771490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.771500:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.771502:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.771503:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.771504:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.771504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.771505:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.771844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.771853:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.771855:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.771856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.771857:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.771859:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.771862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.771973:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.771976:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.771977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.771977:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.771980:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.771983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:13.0:1619677471.772124:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.772127:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.772129:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677471.772131:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677471.772131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.772133:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677471.772134:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96516133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.772135:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.772137:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.772138:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677471.772141:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.772143:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677471.772146:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.772147:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677471.772147:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677471.772149:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677471.772150:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.772150:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.772151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677471.772155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.772163:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.772165:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.772166:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.772167:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.772169:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.772172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.772334:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.772337:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.772338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.772339:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.772341:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.772343:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.772344:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.772345:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.772346:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.772347:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.772350:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.772352:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.772355:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.772355:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.772356:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.772358:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.772358:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.772359:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.772360:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.772361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677471.772373:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677471.772374:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677471.772375:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677471.772376:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677471.772376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677471.772377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677471.772377:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.772379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677471.772380:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.772383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.772391:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.772394:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.772395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.772395:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.772396:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.772397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.772816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.772853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.772861:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.772864:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.772864:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.772865:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.772867:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.772869:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.773173:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.773176:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.773177:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.773177:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.773180:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.773183:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.773315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.773321:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.773324:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.773325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.773326:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.773329:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.773330:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.773331:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.773333:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.773334:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.773335:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.773338:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.773339:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.773342:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.773343:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.773343:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.773345:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.773345:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.773347:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677471.773348:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677471.773348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677471.773354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.773359:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677471.773361:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677471.773362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.773363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.773907:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.773910:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.773911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.773914:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.773917:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.773918:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.773919:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.773921:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.773922:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.773923:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.773926:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.773928:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.773931:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.773932:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.773933:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.773934:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.773935:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.773935:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.773936:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.773949:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.773951:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.773953:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.773953:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.773955:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.773958:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.774465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.774471:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.774474:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.774475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.774477:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.774479:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.774481:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.774482:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.774485:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.774486:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.774487:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.774491:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.774493:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.774497:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.774498:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.774499:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.774501:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.774502:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.774503:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.774503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677471.774519:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.774521:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.774522:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.774522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.774524:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.774526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.775002:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.775005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.775005:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.775006:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.775008:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.775009:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677471.775319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.775531:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.775534:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.775535:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.775537:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.775539:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.775541:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.775542:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.775543:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.775544:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.775545:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.775549:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.775550:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.775553:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.775554:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.775555:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.775556:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.775557:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.775558:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.775558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.775562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.775581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677471.776078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:2.0:1619677471.776084:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677471.776087:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:18.0:1619677471.776088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:2.0:1619677471.776091:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : GET - for me
00000400:00000200:2.0:1619677471.776094:0:19932:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.2.6@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:2.0:1619677471.776096:0:19932:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.2.6@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:2.0:1619677471.776098:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677471.776099:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000400:00000200:2.0:1619677471.776101:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.2.6@o2ib3 local destination
00000400:00000200:2.0:1619677471.776105:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.2.6@o2ib3(10.51.2.6@o2ib3:10.51.2.6@o2ib3) : REPLY try# 0
00000800:00000200:2.0:1619677471.776107:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.2.6@o2ib3
00000800:00000200:2.0:1619677471.776109:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91906ffa8180] -> 10.51.2.6@o2ib3 (2) version: 12
00000800:00000200:2.0:1619677471.776110:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677471.776111:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919078fb5400] (22)++
00000800:00000200:2.0:1619677471.776112:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919078fb5400] (23)--
00000800:00000200:2.0:1619677471.776113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000800:00000200:20.2:1619677471.776118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677471.776124:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (22)++
00000800:00000200:18.0:1619677471.776127:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919078fb5400] (23)--
00000400:00000200:18.0:1619677471.776128:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.776128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:18.0:1619677471.776130:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.2.6@o2ib3: REPLY: OK
00000800:00000200:18.0:1619677471.776132:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:5.2:1619677471.776439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.776445:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.776448:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.776448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.776450:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.776452:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.776453:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.776455:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.776456:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.776457:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.776458:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.776461:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.776462:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.776465:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.776466:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.776466:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.776468:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.776468:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.776469:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.776469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.777028:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.777031:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.777032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.777033:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.777037:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.777038:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.777040:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.777041:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.777042:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.777043:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.777046:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.777048:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.777051:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.777051:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.777052:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.777054:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.777054:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.777055:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.777055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.777060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.777068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.777071:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.777072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.777072:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.777074:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.777077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.777443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.777452:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.777454:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.777455:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.777455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.777457:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677471.777460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.777463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.777465:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.777467:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.777468:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.777468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.777469:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.777469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.777945:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.777948:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.777950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.777950:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.777952:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.777955:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.777957:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677471.777957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:24.0:1619677471.777958:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677471.777959:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000400:00000200:8.0:1619677471.777960:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677471.777961:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.777961:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677471.777963:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.777963:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000400:00000200:8.0:1619677471.777964:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:2.0:1619677471.777964:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.777964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:8.0:1619677471.777965:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.777967:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.777968:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.777968:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.777971:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.777973:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.777974:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.777974:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.777975:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.777976:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.777977:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.777977:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.777978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.777988:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.777990:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.777991:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.777992:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.777993:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.777994:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.778012:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.778014:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.778015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.778015:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.778016:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.778017:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.778190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.778196:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.778199:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.778200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.778202:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.778204:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.778206:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.778207:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.778209:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.778210:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.778211:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.778215:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.778217:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.778219:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.778220:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.778221:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.778223:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.778224:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.778225:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.778225:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.778236:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.778238:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.778239:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.778240:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.778241:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.778242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.778438:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.778441:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.778442:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.778443:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.778446:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.778447:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677471.778449:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.778450:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.778451:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.778452:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677471.778455:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.778457:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677471.778460:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.778461:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677471.778462:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677471.778464:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677471.778464:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.778465:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.778465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677471.778741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.778777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.778779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.778786:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.778788:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.778789:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.778790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.778791:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.778794:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677471.779015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.779021:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.779023:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.779024:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.779026:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.779028:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.779030:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.779031:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.779032:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.779033:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.779034:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.779037:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.779038:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.779041:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.779041:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.779042:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.779043:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.779044:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.779044:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.779045:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.779807:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.779809:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.779810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.779811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.779812:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.779814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677471.779819:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.779822:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.779823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.779825:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.779827:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.779829:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677471.779831:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.779832:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.779833:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.779834:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677471.779837:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.779839:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677471.779842:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.779843:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677471.779843:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677471.779845:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677471.779846:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.779846:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.779847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677471.780062:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.780065:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.780065:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.780068:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.780069:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.780071:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.780072:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.780073:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.780074:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.780075:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.780079:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.780081:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.780083:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.780084:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.780085:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.780087:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.780087:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.780088:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.780089:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.780090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.780102:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.780104:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.780105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.780106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.780116:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.780118:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.780119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.780120:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.780122:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.780126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.780392:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.780395:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.780396:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.780396:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.780399:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.780402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.780422:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.780425:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.780426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.780427:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.780428:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.780429:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.780920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.2:1619677471.780923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677471.780923:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.780924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.780924:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.780927:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.780928:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677471.780929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677471.780931:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.780932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.780934:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.780936:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.780938:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:2.0:1619677471.780938:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.780939:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.780941:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677471.780942:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677471.780942:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.780943:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:2.0:1619677471.780943:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677471.780944:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:2.0:1619677471.780944:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677471.780945:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:2.0:1619677471.780948:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.780950:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.780952:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.780953:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.780954:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.780956:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.780957:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.780958:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.780959:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.780960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677471.780962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.780965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.780971:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.780974:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.780975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.780975:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.781173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:5.2:1619677471.781688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.781694:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.781697:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.781698:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.781700:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.781702:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.781704:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.781705:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.781706:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.781707:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.781708:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.781711:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.781713:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.781716:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.781717:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.781718:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.781719:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.781720:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.781720:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.781720:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.782093:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.782095:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.782096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.782098:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.782100:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.782102:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.782103:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.782104:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.782105:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.782106:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.782109:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.782111:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.782113:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.782113:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.782114:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.782115:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.782116:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.782116:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.782117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.782126:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.782128:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.782129:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.782130:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.782131:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.782132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677471.782576:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.782579:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.782580:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.782582:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.782584:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.782586:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.782587:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.782589:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.782590:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.782591:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.782594:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.782596:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.782598:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.782599:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.782600:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.782601:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.782602:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.782603:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.782605:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.782605:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.782609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.782617:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.782619:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.782620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.782621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.782712:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.782716:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.782717:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.782717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.782720:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.782724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.783082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.783091:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.783093:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.783094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.783095:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.783097:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.783100:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.783486:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.783488:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.783489:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.783491:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.783493:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.783495:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.783496:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.783497:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.783499:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.783499:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.783502:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.783504:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.783507:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.783508:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.783508:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.783510:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.783510:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.783511:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.783511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.783516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.783524:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.783526:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.783527:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.783527:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.783530:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.783532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677471.784362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.784367:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.784370:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.784371:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.784372:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.784374:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.784376:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.784377:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.784378:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.784379:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.784380:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.784383:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.784385:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.784387:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.784388:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.784389:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.784390:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.784391:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.784392:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677471.784393:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677471.784393:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677471.784398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.784403:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677471.784405:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677471.784405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.784406:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677471.784850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677471.785172:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.785174:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.785175:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.785176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.785177:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.785180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.785349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.785380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.2:1619677471.785382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:2.0:1619677471.785387:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.785389:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:17.0:1619677471.785390:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:18.0:1619677471.785391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.785392:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677471.785393:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.785394:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.785394:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:2.0:1619677471.785395:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.785396:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677471.785396:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677471.785398:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:2.0:1619677471.785398:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.785399:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.785401:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.785401:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.785405:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.785407:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.785409:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.785410:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.785411:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.785412:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.785413:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.785414:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.785415:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.785427:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.785430:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.785431:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.785431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.785433:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.785435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.786157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.786163:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.786166:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.786168:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.786169:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.786171:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.786172:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.786174:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.786175:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.786176:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.786178:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.786181:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.786183:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.786186:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.786187:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.786188:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.786190:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.786191:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.786192:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.786193:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.786312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.786775:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.786777:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.786778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.786778:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.786780:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.786782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.786890:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.786893:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.786894:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.786895:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.786898:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.786899:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.786900:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.786901:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.786902:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.786903:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.786906:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.786907:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.786909:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.786909:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.786910:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.786911:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.786911:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.786912:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.786912:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.786923:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.786924:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.786925:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.786925:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.786926:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.786927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677471.787431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.787437:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.787440:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.787441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.787443:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.787446:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.787448:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.787449:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96516077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.787451:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.787452:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.787453:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.787457:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.787459:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.787462:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.787463:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.787463:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.787465:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.787466:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.787466:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.787467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677471.787471:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.787474:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.787475:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.787475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.787478:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.787482:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.787937:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.787941:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.787942:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.787943:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.787945:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.787947:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.787948:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.787949:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.787950:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.787951:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.787955:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.787957:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.787960:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.787960:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.787961:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.787963:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.787963:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.787964:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.787966:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.787966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.787969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.787977:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677471.787979:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677471.787980:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.787980:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.787981:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.787981:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.787982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.787982:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677471.787983:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.787986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.788119:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.788122:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.788123:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.788125:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.788127:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.788128:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.788129:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.788130:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.788131:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.788132:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.788135:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.788137:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.788139:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.788140:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.788141:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.788143:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.788143:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.788144:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.788144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.788149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.788194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.788202:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.788205:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.788205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.788206:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.788208:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.788211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.788213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.788220:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.788223:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.788224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.788224:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.788225:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.788226:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.789176:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.789178:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.789179:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.789180:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.789182:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.789184:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677471.789207:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.789210:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.789210:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.789212:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.789214:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.789216:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677471.789217:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.789218:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.789219:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.789220:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677471.789223:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.789225:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677471.789228:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.789229:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:17.0:1619677471.789229:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677471.789231:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677471.789232:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.789232:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.789233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:24.0:1619677471.789921:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.789923:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.789925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.789926:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.789929:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.789931:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.789932:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.789934:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.789934:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.789936:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.789939:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.789940:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.789943:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.789944:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.789945:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.789946:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.789947:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.789947:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.789948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.789952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.789961:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.789963:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.789965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.789965:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.789967:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.789970:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.790039:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.790041:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.790042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.790042:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.790044:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.790046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.790059:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.790062:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677471.790062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.790063:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.790064:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.790064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677471.790122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.790127:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.790130:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.790131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.790133:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.790136:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.790137:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.790139:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.790140:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.790141:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.790142:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.790145:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.790147:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.790150:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.790150:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.790151:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.790153:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.790153:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.790154:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.790154:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677471.790592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.790625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.790850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.790858:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.790861:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.790862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.790862:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.790864:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.790868:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.792258:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.792261:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.792263:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.792263:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.792265:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.792269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.792512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.792519:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.792522:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.792523:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.792526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:18.0:1619677471.792526:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.792528:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.792530:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.792532:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.792533:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.792534:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:18.0:1619677471.792535:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.792536:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.792537:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.792541:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.792543:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.792544:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677471.792546:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677471.792547:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.792548:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.792549:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:2.0:1619677471.792550:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677471.792551:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.792551:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.792553:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.792554:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.792555:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677471.792556:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.792557:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.792559:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677471.792559:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.792561:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.792561:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677471.792563:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.792564:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.792565:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.792565:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.792565:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677471.792566:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:18.0:1619677471.792567:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.792568:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.792568:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677471.792569:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.792569:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.792570:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.792570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677471.792575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.792584:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.792587:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.792588:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.792588:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677471.792588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.792590:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.792591:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.792592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677471.792592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:8.0:1619677471.792594:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.792597:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.792599:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.792600:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.792601:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.792603:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.792604:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.792607:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.792608:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.792611:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.792612:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.792612:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.792614:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.792615:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.792616:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.792616:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.792628:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.792631:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.792632:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.792632:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.792634:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.792637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.792706:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.792709:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.792710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.792710:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.792712:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.792716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.792726:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.792729:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.792730:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.792730:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.792731:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.792732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.792991:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.792993:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.792994:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.792995:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.792997:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677471.793000:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.793035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:5.2:1619677471.793271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.793277:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.793280:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.793281:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.793283:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.793285:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.793286:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.793288:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.793289:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.793290:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.793291:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.793299:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677471.793300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:1.0:1619677471.793301:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.793303:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.793304:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.793305:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677471.793306:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677471.793306:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.793307:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.793307:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.793308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677471.793309:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.793311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.793313:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.793315:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.793317:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.793318:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.793320:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.793321:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.793322:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.793326:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.793328:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.793332:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.793332:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.793333:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.793335:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.793336:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677471.793337:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:2.0:1619677471.793337:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.793338:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.793340:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.793340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.793342:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.793345:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.793347:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.793348:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.793349:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.793350:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.793351:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.793354:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.793356:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.793359:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.793360:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.793360:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.793362:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.793362:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.793363:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.793364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.793368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.793443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.793461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677471.793475:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.793477:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.793478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.793480:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.793483:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.793484:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.793486:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.793487:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.793488:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.793489:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.793492:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.793493:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.793496:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.793497:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.793497:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.793499:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.793499:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.793500:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.793501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.793513:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.793516:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.793517:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.793518:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.793520:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.793523:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.793772:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.793775:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.793776:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.793776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.793778:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.793781:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.793792:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.793794:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.793795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.793795:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.793796:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.793797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.794332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677471.794380:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.794382:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.794383:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.794383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.794385:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.794387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.794792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.794827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.794846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.795047:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.795050:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.795052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.795053:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.795056:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.795057:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677471.795058:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.795060:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.795060:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.795062:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677471.795065:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.795066:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677471.795069:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.795070:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677471.795070:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677471.795072:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.795072:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.795073:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.795073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677471.795086:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.795088:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.795089:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.795090:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.795092:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.795094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.795105:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.795108:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.795108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.795109:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.795110:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.795111:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.795424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.795433:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.795436:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.795437:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.795437:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.795440:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.795443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.795444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.795451:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677471.795451:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.795454:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677471.795454:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.795455:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677471.795455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:8.0:1619677471.795455:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.795455:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.795456:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:8.0:1619677471.795457:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.795457:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.795460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.795668:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.795670:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.795672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.795673:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.795676:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.795678:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.795679:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96516040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.795680:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.795681:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.795682:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.795685:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.795687:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.795689:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.795690:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.795691:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.795692:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.795693:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.795694:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.795695:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.795696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.795698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.795707:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.795709:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:8.0:1619677471.795709:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677471.795711:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.795711:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677471.795711:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.795712:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.795712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677471.795713:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.795716:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677471.796031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.796036:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.796039:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.796040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.796042:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.796044:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.796045:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.796046:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.796048:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.796051:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.796052:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.796056:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.796057:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.796060:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.796060:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.796061:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.796063:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.796063:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.796064:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677471.796065:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677471.796066:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677471.796071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.796077:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677471.796079:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677471.796079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.796080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677471.796272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.796312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.796316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677471.796486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677471.797425:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.797428:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.797429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.797431:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.797433:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.797434:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.797436:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.797437:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.797438:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.797439:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.797442:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.797444:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.797446:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.797447:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.797448:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.797450:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.797450:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.797451:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.797452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.797457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677471.797482:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.797486:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.797487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.797487:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.797489:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.797493:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.797711:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.797713:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.797714:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.797716:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.797718:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.797720:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677471.797721:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.797723:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.797724:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.797725:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677471.797728:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.797729:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677471.797732:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.797732:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677471.797733:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677471.797735:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677471.797735:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.797736:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.797736:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677471.797748:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.797751:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.797752:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.797752:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.797755:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.797757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.797777:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.797779:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.797781:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.797781:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.797782:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.797786:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.798141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:8.2:1619677471.798177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.798251:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.798254:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.798255:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.798255:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.798257:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.798261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.798408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.798414:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.798417:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.798418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.798420:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.798422:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.798424:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.798426:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.798428:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.798429:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.798430:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.798434:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.798436:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.798439:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.798440:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.798441:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.798443:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.798444:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.798444:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.798445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.798449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.798466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:5.2:1619677471.798661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.798667:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.798670:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.798672:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.798673:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.798676:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.798677:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.798678:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.798679:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.798680:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.798681:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.798684:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.798686:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.798688:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.798689:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.798690:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.798691:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.798692:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.798692:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.798692:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.798887:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.798890:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.798891:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.798891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.798893:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677471.798896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.799015:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677471.799017:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677471.799018:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677471.799020:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.799023:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.799025:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677471.799026:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.799027:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.799029:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.799030:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677471.799032:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.799034:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677471.799036:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.799036:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677471.799037:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.799038:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.799039:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677471.799040:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:8.0:1619677471.799042:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677471.799042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677471.799053:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.799053:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.799054:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.799055:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.799056:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677471.799057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.799057:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677471.799058:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.799058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677471.799059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:24.0:1619677471.799078:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.799080:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.799081:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.799081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.799082:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.799083:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.799129:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.799131:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.799132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.799133:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.799135:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.799137:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.799137:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.799139:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.799140:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.799141:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.799143:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.799145:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.799146:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.799146:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.799147:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.799148:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.799149:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.799149:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.799150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.799172:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.799173:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.799174:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.799175:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.799175:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.799176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677471.799484:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.799487:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.799488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.799490:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.799492:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.799494:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677471.799495:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.799496:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.799497:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.799498:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677471.799502:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.799503:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677471.799506:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.799507:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677471.799508:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677471.799509:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677471.799509:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.799511:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.799512:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.799512:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.799524:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.799526:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.799526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.799527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677471.799533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.799535:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.799536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.799537:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.799539:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.799541:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.799796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.800078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.800114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.800131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677471.800167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.800174:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.800176:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.800178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.800180:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.800183:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.800185:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.800186:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.800188:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.800189:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.800190:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.800194:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.800196:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.800200:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.800201:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.800201:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.800204:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.800204:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.800206:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677471.800207:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677471.800208:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677471.800213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.800219:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677471.800221:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677471.800222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.800223:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:24.0:1619677471.800674:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.800676:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.800677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.800678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.800679:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.800681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.800714:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.800716:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.800717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.800717:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.800719:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.800721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677471.800787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:5.2:1619677471.801278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.801284:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.801288:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.801288:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.801291:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.801299:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.801300:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.801301:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.801302:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.801303:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.801304:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.801307:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.801309:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.801312:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.801312:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.801313:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.801314:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.801315:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.801316:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.801316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.801475:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.801479:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.801480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.801481:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.801484:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.801485:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.801487:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.801488:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.801489:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.801490:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.801493:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.801495:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.801498:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.801498:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.801499:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.801501:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.801501:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.801502:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.801502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.801507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.801515:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.801518:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.801519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.801519:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.801521:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.801525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.801715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.801723:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.801726:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.801727:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.801727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.801729:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.801732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.802206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.802213:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.802216:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.802217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.802221:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.802223:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.802225:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.802226:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96516009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.802228:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.802229:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.802230:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.802234:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.802236:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.802239:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.802240:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.802240:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.802241:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.802242:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.802243:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.802243:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.802244:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:18.0:1619677471.802245:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.802246:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.802247:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.802248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.802249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.802253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.802255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.802256:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.802257:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.802257:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.802259:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.802259:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.802260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.802260:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.802261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.802262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.802263:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677471.802558:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.802561:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.802562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.802564:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.802566:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.802568:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.802569:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.802570:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.802571:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.802572:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.802575:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.802576:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.802579:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.802580:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.802580:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.802582:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.802582:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.802584:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.802585:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.802586:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.802589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.802597:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.802599:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.802599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.802600:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677471.802621:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.802624:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.802625:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.802626:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.802628:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.802633:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.802635:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.802637:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.802637:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.802638:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.802640:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.802640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.803064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.803070:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.803073:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.803074:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.803075:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.803077:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.803079:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.803080:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.803082:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.803083:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.803084:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.803088:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.803090:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.803092:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.803093:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.803094:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.803096:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.803097:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.803098:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.803099:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.803099:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.803105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.803110:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.803113:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.803114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.803115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.803315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677471.803499:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.803502:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.803503:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.803505:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.803508:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.803509:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.803510:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96516004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.803511:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.803512:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.803513:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.803516:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.803517:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.803519:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.803520:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.803520:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.803522:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.803522:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.803523:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.803523:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.803534:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.803535:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.803536:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.803537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.803538:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.803539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.803554:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.803556:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.803556:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.803557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.803557:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.803557:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677471.803862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.803867:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.803870:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.803871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.803873:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.803875:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.803877:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.803878:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96516001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.803879:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.803880:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.803881:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.803884:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.803885:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.803888:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.803889:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.803889:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.803891:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.803891:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.803892:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.803893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.804475:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.804478:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.804479:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.804480:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.804482:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.804483:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.804484:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.804486:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.804487:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.804488:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.804490:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.804492:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.804494:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.804494:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.804495:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.804496:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.804496:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.804498:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.804499:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.804499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.804509:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.804511:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.804511:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.804513:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677471.804513:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.804514:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.804515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.804515:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.804516:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.804516:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.804890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.804926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.804935:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.804937:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.804938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.804939:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.804941:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.804944:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.804949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.804956:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.804959:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.804960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.804960:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.804961:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.804961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.805992:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.805994:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.805996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.805996:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.805997:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.806000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677471.806202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.806238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.806272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.806289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:5.2:1619677471.806493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.806499:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.806502:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.806503:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.806505:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.806507:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.806508:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.806509:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.806510:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.806512:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.806513:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.806516:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.806517:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.806520:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.806521:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.806521:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.806523:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.806523:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.806525:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677471.806526:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677471.806526:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677471.806531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.806536:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677471.806538:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677471.806539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.806540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677471.806996:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.806999:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.807000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.807001:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.807003:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.807007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677471.807626:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.807629:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.807631:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.807633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.807634:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.807635:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677471.807636:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.807638:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.807638:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.807639:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677471.807642:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.807644:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677471.807646:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.807647:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677471.807647:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677471.807649:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.807649:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.807651:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677471.807653:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677471.807653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.807667:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677471.807669:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677471.807669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.807671:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.807672:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677471.807672:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:13.0:1619677471.807673:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677471.807673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:13.0:1619677471.807674:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.807676:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677471.808046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.808055:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.808058:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.808059:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.808060:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.808062:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677471.808064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.808067:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.808068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.808070:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.808071:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.808071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.808072:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.808072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.808622:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.808625:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.808626:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.808627:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.808628:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.808631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.808644:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.808646:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.808648:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.808648:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.808649:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.808650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677471.809066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.809072:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.809075:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.809076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.809077:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.809079:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.809081:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.809082:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.809083:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.809084:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.809086:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.809089:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.809090:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.809093:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.809093:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.809094:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.809096:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.809096:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.809097:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.809098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.809158:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.809160:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.809161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.809163:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.809165:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.809167:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.809168:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.809169:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.809170:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.809172:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.809175:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.809176:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.809178:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.809179:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.809180:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.809181:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.809181:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.809182:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.809182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.809193:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.809194:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.809195:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.809196:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.809197:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.809198:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.809296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.809667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.809676:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.809679:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.809680:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.809680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.809682:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.809685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.809895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.809901:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.809903:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.809905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.809906:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.809909:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.809911:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.809913:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.809915:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.809916:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.809917:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.809921:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.809923:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.809926:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.809927:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.809928:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:24.0:1619677471.809930:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677471.809930:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.809931:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.809932:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677471.809933:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677471.809933:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677471.809934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.809936:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.809938:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.809939:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.809940:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.809942:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.809942:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.809944:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.809946:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.809948:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.809949:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.809950:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.809951:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.809952:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.809953:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.809953:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.809954:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.809966:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.809968:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.809969:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.809970:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.809971:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.809972:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.810715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677471.810749:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.810752:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.810753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.810754:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677471.810755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:24.0:1619677471.810756:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.810759:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.810760:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.810762:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.810763:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.810764:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.810767:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.810769:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.810772:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.810773:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.810773:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.810775:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.810776:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.810776:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.810777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.810781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677471.810790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677471.810790:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.810792:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.810793:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.810794:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.810796:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.810799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.810823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.810825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.810830:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.810833:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.810834:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.810834:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.810836:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.810838:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.811148:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.811151:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.811152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.811152:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.811154:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.811156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.811410:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.811413:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.811414:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.811414:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.811414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.811415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.811578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.811585:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.811588:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.811589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.811591:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.811594:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.811596:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.811597:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.811599:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.811600:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.811601:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.811605:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.811607:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.811609:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.811610:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.811611:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.811613:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.811614:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.811615:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.811615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.811627:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.811629:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.811630:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.811631:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.811632:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.811634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677471.811729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.811734:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.811737:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.811738:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.811740:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.811742:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.811743:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.811745:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.811746:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.811747:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.811748:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.811751:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.811753:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.811755:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.811756:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.811757:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.811758:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.811759:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.811759:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.811760:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.812045:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.812047:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.812048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.812048:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.812052:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.812055:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.812139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.812177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.812185:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.812188:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.812189:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.812189:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.812191:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.812193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.812392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:12.0:1619677471.813303:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.813307:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.813308:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677471.813310:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.813311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.813312:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677471.813313:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.813315:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.813316:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.813316:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677471.813319:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.813321:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677471.813324:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.813325:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677471.813325:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677471.813327:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677471.813328:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.813329:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.813329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677471.813343:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.813346:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.813347:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.813347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.813350:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.813353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677471.814402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.814408:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.814411:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.814412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.814413:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.814416:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.814417:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.814419:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.814420:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.814421:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.814422:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.814425:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.814427:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.814429:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.814430:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.814431:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.814432:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.814433:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.814433:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.814434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677471.814572:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.814575:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.814577:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677471.814579:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.814580:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.814582:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677471.814583:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.814584:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.814585:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.814586:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677471.814589:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.814591:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677471.814594:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.814595:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677471.814596:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677471.814598:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677471.814598:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.814599:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.814600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677471.814612:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.814615:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.814617:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.814617:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.814619:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.814622:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.814783:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.814786:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.814787:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.814788:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.814791:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.814793:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.814794:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.814795:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.814796:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.814797:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.814800:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.814802:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.814805:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.814805:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.814806:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.814808:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.814808:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.814809:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.814811:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.814811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677471.814821:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.814824:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677471.814824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:18.0:1619677471.814825:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.814825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.814827:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.814827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677471.814828:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.814829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.814833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.814835:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.814837:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.814838:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.814839:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.814840:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.814840:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.814857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.814865:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.814868:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.814869:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.814869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.814872:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.814875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.815721:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.815724:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.815725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.815727:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.815730:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.815732:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.815733:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.815734:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.815735:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.815736:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.815740:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677471.815740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677471.815741:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.815744:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.815745:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.815745:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677471.815746:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677471.815747:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.815747:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.815748:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.815748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.815749:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.815750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.815752:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677471.815753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:2.0:1619677471.815755:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.815757:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.815758:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.815761:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677471.815761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:2.0:1619677471.815762:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.815763:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677471.815763:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.815764:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.815765:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.815767:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677471.815768:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.815770:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677471.815771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.815773:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.815774:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.2:1619677471.815775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677471.815775:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.815777:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.815778:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.815779:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.815780:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677471.815782:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.2:1619677471.815784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677471.815785:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.815786:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.815786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.815787:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.815788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.816003:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.816006:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.816007:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.816007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.816009:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.816013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677471.816170:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.816173:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.816173:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.816175:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.816177:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.816178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677471.816191:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677471.816194:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677471.816194:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.816195:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677471.816196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.816198:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677471.816716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.816721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.816754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.817012:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.817014:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.817015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.817015:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.817017:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.817020:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.817030:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.817032:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.817033:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.817033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.817034:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.817035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.817234:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.817236:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.817237:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.817237:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.817239:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.817241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677471.817446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.817452:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.817454:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.817455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.817457:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.817459:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.817460:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.817462:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.817463:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.817464:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.817465:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.817468:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.817470:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.817473:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.817473:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.817474:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.817476:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.817477:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.817478:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677471.817479:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677471.817479:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677471.817485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.817491:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677471.817493:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677471.817493:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.817494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.818026:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.818029:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.818030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.818031:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.818035:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.818036:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.818037:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.818038:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.818040:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.818040:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.818044:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.818046:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.818049:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.818049:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.818050:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.818052:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.818052:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.818053:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.818053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.818057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.818066:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.818068:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.818069:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.818070:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.818072:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.818075:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.818271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.818308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.818327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.2:1619677471.818587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.818607:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.818609:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.818610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.818610:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.818612:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.818614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.819303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.819309:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.819311:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.819313:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.819314:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.819317:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.819319:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677471.819320:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.819322:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.819323:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.819325:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677471.819328:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.819330:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677471.819334:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.819335:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677471.819335:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677471.819338:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677471.819339:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.819340:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.819341:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:24.0:1619677471.819351:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.819354:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.819355:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.819355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.819357:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.819360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.819372:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.819375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.819375:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677471.819376:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.819377:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.819377:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677471.820117:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.820120:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.820121:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.820121:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.820124:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:5.2:1619677471.820124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677471.820127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.820130:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.820134:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.820134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.820136:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.820139:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.820140:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.820141:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.820143:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.820144:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.820145:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.820147:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.820149:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.820152:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.820153:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.820153:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.820155:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.820156:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.820156:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.820157:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.820223:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.820226:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.820227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.820229:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.820232:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.820233:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.820234:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.820235:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.820237:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.820238:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.820241:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.820243:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.820244:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.820245:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.820246:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.820247:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.820248:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.820248:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.820249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.820259:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.820261:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.820261:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.820262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.820263:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.820265:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.820507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:2.0:1619677471.820513:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677471.820516:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:18.0:1619677471.820517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:2.0:1619677471.820519:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.820521:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.820523:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677471.820524:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.820526:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.820527:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.820528:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677471.820531:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.820533:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677471.820536:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.820537:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677471.820537:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677471.820539:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677471.820540:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677471.820541:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:2.0:1619677471.820542:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:20.2:1619677471.820679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.820685:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.820687:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.820688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.820690:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.820693:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.820694:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.820695:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.820696:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.820697:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.820698:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.820702:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.820703:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.820705:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.820706:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.820707:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.820708:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.820709:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.820709:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.820710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.820715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.820810:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.820812:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.820814:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.820814:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.820816:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.820820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.820829:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.820832:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.820833:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.820833:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.820834:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.820834:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.820854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.820859:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.820862:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.820863:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.820864:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.820867:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.820869:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.820870:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.820872:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.820873:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.820874:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.820877:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.820879:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.820881:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.820882:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.820882:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.820884:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.820885:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.820885:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.820886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.820898:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.820899:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.820900:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.820901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.820902:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.820904:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.821063:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.821065:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.821067:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.821068:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.821071:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.821073:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677471.821074:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.821075:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.821076:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.821077:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677471.821080:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.821081:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.821083:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.821084:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.821085:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.821086:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.821087:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.821087:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.821088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677471.821092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.821100:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.821102:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.821103:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.821103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.821105:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.821106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.821832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.821841:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.821843:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.821844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.821844:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.821847:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.821850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.822115:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.822118:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.822119:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.822119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.822121:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.822123:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.822554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.822587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.822590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677471.822598:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.822601:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.822602:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.822604:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.822605:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.822607:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.822608:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.822609:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.822610:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.822611:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.822614:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.822616:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.822619:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.822620:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.822620:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.822622:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.822623:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.822624:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.822626:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.822626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.822628:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.822637:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677471.822639:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677471.822639:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.822640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.822641:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677471.822641:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.822642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.822643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677471.822643:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.822645:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.822647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.822655:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.822657:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.822658:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.822658:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.822659:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.822660:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677471.822713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.822719:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.822721:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.822722:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.822724:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.822726:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.822727:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.822728:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.822729:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.822730:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.822731:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.822735:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.822736:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.822739:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.822740:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.822740:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.822742:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.822742:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.822743:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.822743:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677471.823367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:2.0:1619677471.823374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:8.0:1619677471.823377:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:2.0:1619677471.823377:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:18.0:1619677471.823378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:8.0:1619677471.823379:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.823380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.823380:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.823380:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.823382:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:2.0:1619677471.823382:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.823385:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:8.0:1619677471.823386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:2.0:1619677471.823386:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.823388:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.823389:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.823390:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677471.823394:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.823395:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677471.823398:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.823399:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677471.823400:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677471.823402:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677471.823403:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677471.823404:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:2.0:1619677471.823405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:24.0:1619677471.823414:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.823417:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.823418:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.823418:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.823421:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.823423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.823586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.824011:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.824013:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.824014:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.824016:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.824019:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.824020:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.824021:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.824022:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.824023:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.824024:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.824027:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.824029:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.824031:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.824032:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.824033:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.824034:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.824035:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.824036:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.824036:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.824059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.824067:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.824070:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.824071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.824071:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.824073:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.824076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677471.824133:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.824135:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.824136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.824138:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.824139:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.824141:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677471.824142:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.824143:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.824145:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.824146:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677471.824150:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.824152:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677471.824155:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.824156:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677471.824156:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677471.824158:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677471.824159:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.824160:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.824160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677471.824476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.824509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.824533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:24.0:1619677471.825107:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.825110:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.825111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.825113:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.825116:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.825117:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.825118:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.825120:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.825121:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.825122:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.825125:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.825126:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.825128:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.825129:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.825130:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.825131:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.825132:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.825132:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.825133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.825144:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.825146:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.825147:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.825148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.825149:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.825150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.825165:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.825167:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.825168:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.825168:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.825168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.825169:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677471.825277:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.825280:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.825281:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.825282:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.825285:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.825286:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677471.825288:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.825289:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.825290:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.825291:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677471.825298:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.825300:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.825302:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.825302:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.825303:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.825305:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.825306:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.825307:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677471.825308:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677471.825309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677471.825312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.825321:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.825322:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.825322:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.825322:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677471.825323:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677471.825323:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.825324:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.825324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.825325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.825326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:5.2:1619677471.825383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.825389:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.825391:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.825392:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.825393:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.825395:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.825396:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.825397:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.825398:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.825399:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.825399:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.825402:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.825403:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.825406:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.825407:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.825407:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.825408:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.825409:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.825409:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.825410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677471.825851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.825858:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.825861:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.825862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.825862:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.825864:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.825868:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.826027:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.826031:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.826032:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.826033:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.826034:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.826038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.826997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.827003:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.827006:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.827007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.827008:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.827011:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.827012:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.827014:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677471.827016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000400:00000200:2.0:1619677471.827016:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.827017:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.827018:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.827021:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677471.827022:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:2.0:1619677471.827023:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677471.827024:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677471.827025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677471.827027:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677471.827027:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.827027:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.827028:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:17.0:1619677471.827030:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.827030:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:17.0:1619677471.827031:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:2.0:1619677471.827031:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677471.827033:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677471.827033:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:17.0:1619677471.827034:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677471.827034:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.827034:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677471.827035:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.827036:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.2:1619677471.827039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677471.827039:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.827041:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677471.827044:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.827044:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:17.0:1619677471.827045:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677471.827046:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677471.827047:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677471.827047:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:18.0:1619677471.827048:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:17.0:1619677471.827048:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:18.0:1619677471.827049:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677471.827049:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:18.0:1619677471.827050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.827192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.827197:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.827200:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.827201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.827203:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.827207:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.827208:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.827209:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.827211:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.827212:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.827213:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.827217:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.827218:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.827221:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.827222:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.827223:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.827225:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.827225:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.827226:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.827227:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.827238:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.827240:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.827241:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.827241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.827243:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.827244:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.827423:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.827426:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.827426:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.827427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.827428:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.827431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.827500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:5.2:1619677471.827986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.827992:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.827995:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.827996:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.827998:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.828001:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.828002:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.828003:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.828004:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.828006:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.828007:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.828010:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.828012:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.828014:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.828015:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.828016:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.828017:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.828018:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.828019:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677471.828020:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677471.828020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677471.828026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.828032:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677471.828034:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677471.828035:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.828036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677471.828063:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.828065:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.828067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.828067:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.828069:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.828073:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.828098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.828105:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.828109:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.828110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.828110:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.828112:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.828115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.828418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.828466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.828483:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.828485:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.828486:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.828486:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.828491:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.828495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677471.828508:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.828511:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.828512:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.828514:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.828515:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.828516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677471.828543:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.828546:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.828547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.828549:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.828550:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.828551:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.828553:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.828554:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.828555:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.828556:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.828560:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.828562:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.828565:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.828566:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.828567:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.828569:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.828570:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677471.828570:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.828571:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677471.828571:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.828572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.828573:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677471.828574:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.828576:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.828577:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677471.828579:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96515902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.828580:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.828581:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.828582:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677471.828585:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:24.0:1619677471.828586:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.828587:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677471.828588:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677471.828589:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.828589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.828591:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.828592:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:18.0:1619677471.828592:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.828593:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677471.828594:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677471.828596:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677471.828596:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:18.0:1619677471.828596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.828597:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:2.0:1619677471.828607:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.828610:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.828611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.828611:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.828612:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.828613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.829771:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.829774:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.829775:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.829777:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.829780:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.829781:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.829783:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.829784:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.829785:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.829786:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.829789:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.829790:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.829793:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.829794:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.829795:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.829796:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.829797:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.829798:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.829799:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.829800:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.829802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.829810:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677471.829813:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677471.829813:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.829814:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.829814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.829815:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.829816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.829817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677471.829817:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.829820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.829822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.829829:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.829831:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.829832:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.829832:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.829833:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.829834:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.829915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.829936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:12.0:1619677471.830160:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.830163:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.830164:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.830165:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.830168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677471.830168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677471.830178:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.830181:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677471.830182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.830184:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.830186:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.830188:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677471.830189:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.830191:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.830192:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.830193:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677471.830197:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.830198:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677471.830202:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.830204:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677471.830205:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677471.830207:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677471.830208:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.830209:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.830209:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.2:1619677471.830284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.830318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.830386:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.830389:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.830389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.830392:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.830394:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.830395:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.830396:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.830398:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.830399:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.830400:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.830403:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.830405:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.830407:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.830408:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.830408:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.830410:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.830411:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.830411:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.830412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.830416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:5.2:1619677471.830567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.830573:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.830576:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.830577:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.830578:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.830581:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.830582:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.830583:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.830585:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.830586:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.830587:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.830590:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.830591:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.830594:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.830594:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.830595:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.830596:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.830597:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.830597:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.830598:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.830667:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.830669:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.830670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.830672:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.830675:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.830677:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.830678:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.830679:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.830680:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.830682:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.830685:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.830687:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.830689:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.830690:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.830691:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.830693:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.830693:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.830694:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.830695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.830706:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.830709:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.830710:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.830711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.830713:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.830717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.830755:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.830758:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.830759:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.830759:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.830761:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.830765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.831187:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.831190:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677471.831191:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677471.831193:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.831195:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.831196:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677471.831198:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.831199:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.831200:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.831202:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.831205:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.831207:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.831211:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.831212:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677471.831212:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677471.831215:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677471.831216:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.831216:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.831217:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:24.0:1619677471.831293:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.831301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.831303:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.831303:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.831305:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.831308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.832170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.832177:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.832179:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.832180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.832182:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.832185:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.832187:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.832188:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.832190:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.832191:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.832193:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.832196:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.832199:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.832202:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.832203:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.832204:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.832206:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.832207:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.832208:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.832209:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677471.832857:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.832859:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.832860:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.832861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.832862:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.832865:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.832876:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.832879:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.832880:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.832880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.832881:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.832881:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677471.833095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.833101:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.833104:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.833105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.833107:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.833109:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.833111:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.833112:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.833113:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.833114:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.833115:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.833119:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.833120:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.833123:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.833124:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.833124:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.833126:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.833126:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.833127:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.833127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677471.833158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.833164:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.833167:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.833169:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.833170:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.833174:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.833176:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677471.833178:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.833179:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.833180:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.833182:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677471.833185:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.833187:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677471.833190:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.833192:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677471.833192:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677471.833194:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677471.833195:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.833197:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.833197:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677471.833292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.833301:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.833304:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.833305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.833305:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.833307:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.833311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.833312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.833320:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.833323:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.833324:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.833324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.833325:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.833325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.833730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.833738:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.833741:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.833742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.833742:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.833744:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.833747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.833749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.833785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.833788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677471.833804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677471.834104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.834110:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.834114:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.834115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.834116:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.834120:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.834122:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.834123:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.834125:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.834126:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.834128:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.834131:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.834133:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.834135:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.834136:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.834137:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.834139:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.834140:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.834142:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.834144:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.834144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.834150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.834154:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.834156:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.834156:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.834157:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.834158:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.834158:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.834159:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.834160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.834161:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.834161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.834381:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.834384:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.834385:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.834385:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.834387:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.834389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.834622:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.834624:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.834626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.834627:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.834629:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.834631:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.834632:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.834634:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.834635:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677471.834635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:24.0:1619677471.834636:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.834640:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.834642:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.834645:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.834646:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.834646:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.834648:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.834648:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.834649:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.834650:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.834654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.834662:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.834665:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.834666:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.834666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.834668:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.834671:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.835339:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.835342:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.835343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.835344:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.835348:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.835349:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.835350:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.835352:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.835353:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.835353:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.835356:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.835357:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.835358:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.835359:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.835359:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.835360:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.835361:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.835362:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.835362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.835372:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.835375:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.835375:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.835376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.835377:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.835378:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677471.835654:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.835656:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.835658:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.835658:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.835660:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.835664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.835704:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.835707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.835707:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.835709:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.835711:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.835713:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.835714:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.835716:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.835717:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.835718:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.835721:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.835723:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.835725:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.835726:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.835727:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.835728:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.835729:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.835729:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.835730:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677471.835735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.835740:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.835742:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.835743:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.835745:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.835746:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677471.835746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:17.0:1619677471.835748:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.835749:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.835750:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.835751:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.835752:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.835755:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.835756:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.835759:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.835760:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.835760:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.835762:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.835762:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.835763:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.835763:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677471.835883:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.835886:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.835887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.835889:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.835891:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.835892:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677471.835894:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.835895:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.835896:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.835897:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677471.835901:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.835902:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677471.835904:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.835905:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677471.835905:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677471.835907:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.835907:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.835908:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.835909:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677471.835920:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.835923:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.835924:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.835924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.835927:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.835929:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.835942:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.835945:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.835946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.835946:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:8.0:1619677471.835948:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:2.0:1619677471.835948:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.835949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.835951:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.835952:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.835952:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.835954:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.835956:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.836244:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.836247:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.836248:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.836249:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.836251:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.836252:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.836253:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.836255:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.836255:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.836257:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.836260:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.836261:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.836264:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.836265:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.836265:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.836267:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.836268:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.836269:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.836270:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.836271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677471.836281:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.836283:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677471.836283:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677471.836284:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.836285:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.836285:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.836286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.836287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677471.836288:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.836291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.836500:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.836502:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.836503:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.836503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.836504:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.836507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.836518:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.836520:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.836521:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.836521:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.836522:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.836523:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.837626:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677471.837629:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677471.837630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677471.837632:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.837634:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.837636:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677471.837637:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.837639:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.837639:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.837640:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677471.837643:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.837645:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677471.837648:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.837649:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677471.837649:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677471.837651:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677471.837651:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677471.837652:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677471.837653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:20.2:1619677471.837657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.2:1619677471.837657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:2.0:1619677471.837663:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677471.837666:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:2.0:1619677471.837666:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.837667:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677471.837669:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:2.0:1619677471.837669:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677471.837670:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.837670:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.837672:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677471.837672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.837673:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.837675:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677471.837676:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:2.0:1619677471.837677:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.837678:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.837679:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.837683:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.837684:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.837687:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.837688:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.837689:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.837691:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.837692:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.837693:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.837694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.837697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677471.838197:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.838199:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.838200:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.838200:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.838203:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.838205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677471.838393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.838399:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.838402:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.838403:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.838405:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.838407:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.838408:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.838410:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.838411:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.838412:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.838413:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.838416:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.838418:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.838421:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.838421:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.838422:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.838424:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.838424:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.838425:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677471.838427:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677471.838427:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677471.838433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.838439:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677471.838441:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677471.838441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.838443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677471.838808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.838848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.838856:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:24.0:1619677471.838859:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677471.838859:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.838860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.838860:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.838861:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.838862:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.838862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:1.0:1619677471.838862:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:24.0:1619677471.838864:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.838865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.838866:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677471.840019:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.840022:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.840023:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.840023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.840025:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.840029:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.840118:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.840121:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.840122:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.840122:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.840124:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.840126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.840212:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.840214:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.840216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.840217:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.840220:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.840221:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.840222:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.840223:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.840224:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.840225:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.840228:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.840229:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.840232:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.840233:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.840233:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.840235:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.840235:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.840236:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.840236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.840241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.840250:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.840253:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.840254:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.840254:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.840257:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.840261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.840482:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.840486:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.840488:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.840489:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.840492:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.840496:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.840611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.840654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:5.2:1619677471.840662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.840670:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.840671:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.840673:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.840674:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677471.840677:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.840678:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.840682:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.840682:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.840684:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677471.840687:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.840689:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.840691:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.840693:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677471.840696:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.840698:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.840700:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677471.840702:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.840704:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677471.840704:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:17.0:1619677471.840706:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.840710:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.840712:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:17.0:1619677471.840713:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.840713:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677471.840716:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.840716:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.840718:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677471.840720:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.840720:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677471.840721:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.840722:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.840723:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.840723:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677471.840724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.840725:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.840726:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.840728:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.840729:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677471.840732:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.840736:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.840738:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.840738:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.840742:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.840745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.841034:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.841037:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.841038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.841040:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.841043:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.841044:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.841046:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.841047:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.841048:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.841049:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.841053:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.841055:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.841058:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.841059:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.841059:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.841061:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.841061:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.841062:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.841063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.841067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.841077:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.841081:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.841083:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.841083:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.841086:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677471.841089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.841095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.841098:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.841100:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.841101:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.841103:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.841103:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.841104:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.841105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.841107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:17.0:1619677471.841108:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677471.841108:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.841112:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.841114:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.841116:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.841119:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.841120:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677471.841121:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677471.841122:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677471.841124:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.841125:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.841125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.841128:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:17.0:1619677471.841128:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.841131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677471.841131:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.841135:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.841136:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.841137:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.841140:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.841141:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.841142:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.841143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.841157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.842017:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.842019:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.842020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.842020:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.842021:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.842024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677471.842030:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.842033:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.842035:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.842035:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.842037:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.842041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.842053:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.842056:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.842057:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.842057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.842059:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.842059:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.842685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.842691:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.842693:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.842694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.842696:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.842698:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.842699:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.842701:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.842703:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.842704:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.842705:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.842709:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.842711:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.842713:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.842714:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.842715:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.842717:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.842718:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677471.842718:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.842718:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.842719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677471.842720:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.842722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.842723:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.842726:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.842727:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.842729:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.842730:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.842731:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677471.842731:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:24.0:1619677471.842732:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.842733:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:24.0:1619677471.842734:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677471.842734:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.842735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.842736:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.842736:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.842738:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.842739:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.842740:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.842740:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.842742:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.842743:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.842743:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.842744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.842755:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.842759:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.842760:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.842760:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.842762:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.842765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677471.843289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.843296:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.843298:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.843299:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.843301:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.843303:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.843305:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.843306:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.843307:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.843308:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.843309:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.843313:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.843314:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.843316:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.843317:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.843318:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.843319:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.843320:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.843320:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.843321:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.844160:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.844162:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.844163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.844163:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.844165:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.844168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.844180:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.844183:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.844184:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.844184:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.844185:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.844185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.844223:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.844225:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.844226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.844226:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.844228:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.844231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.844247:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.844250:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.844251:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.844251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.844252:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.844252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.844634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.844641:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.844644:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.844644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.844647:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.844649:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.844650:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.844652:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.844654:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.844655:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.844656:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.844660:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.844662:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.844665:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.844666:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.844667:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.844670:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.844670:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.844671:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.844672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.845068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.845104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.845113:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.845116:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.845117:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.845117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.845119:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.845122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.845225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.845228:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.845229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.845229:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.845232:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.845235:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.845244:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.845247:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.845248:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.845248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.845250:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.845251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677471.845467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.845472:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.845475:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.845476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.845478:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.845480:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.845481:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.845483:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.845484:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.845486:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.845487:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.845490:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.845491:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.845494:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.845495:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.845495:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.845497:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.845497:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.845498:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.845498:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.845565:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.845567:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.845568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.845570:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.845573:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.845575:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.845576:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.845577:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.845578:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.845579:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.845582:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.845584:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.845585:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.845586:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.845587:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.845588:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.845589:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.845590:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.845592:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.845592:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.845603:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.845604:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.845605:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.845606:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:24.0:1619677471.845607:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.845607:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.845607:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.845608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.845609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.845609:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677471.845871:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.845874:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.845875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.845877:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.845879:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.845881:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.845882:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.845883:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.845885:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.845886:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.845889:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.845890:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.845893:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.845893:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.845894:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.845896:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.845896:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.845897:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.845898:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.845899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.845902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.845909:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.845911:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.845912:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.845913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.845926:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677471.845931:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.845934:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.845935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.845935:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.845937:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.845940:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.846553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.846589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.846590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.846597:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.846600:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.846601:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.846601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.846603:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.846605:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.846608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.846615:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.846618:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.846619:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.846619:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.846620:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.846621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.847419:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.847422:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.847423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.847424:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.847427:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.847429:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.847430:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.847431:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.847432:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.847433:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.847436:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.847438:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.847441:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.847441:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.847442:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.847444:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.847444:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.847445:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.847446:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.847458:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.847461:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.847462:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.847463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.847464:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.847468:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677471.847941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.847947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.847950:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.847951:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.847953:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.847955:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.847956:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.847957:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.847959:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.847960:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.847961:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.847966:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.847968:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.847971:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.847972:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.847973:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.847974:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.847975:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.847976:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677471.847977:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677471.847978:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677471.847983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.847988:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677471.847990:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677471.847991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.847993:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677471.847999:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.848001:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.848002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.848002:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.848004:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.848007:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.848406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.848415:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.848418:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.848419:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.848419:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.848421:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.848424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.848633:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.848635:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.848636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.848638:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.848640:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.848641:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677471.848643:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.848645:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.848646:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.848647:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677471.848650:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.848652:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677471.848654:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.848655:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677471.848656:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677471.848657:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.848658:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.848658:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.848659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677471.848673:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.848675:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.848676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.848677:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.848679:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.848682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.848823:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.848825:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.848826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.848828:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.848831:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.848832:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.848833:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.848834:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.848835:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.848836:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.848839:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.848841:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.848844:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.848844:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.848845:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.848847:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.848847:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.848848:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.848848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.848861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.848863:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.848864:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.848865:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.848867:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.848870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.848880:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.848883:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.848884:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.848884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.848885:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.848886:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.848932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.848937:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.848940:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.848941:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.848943:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.848946:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.848947:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.848949:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.848951:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.848952:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.848953:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.848957:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.848960:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.848964:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.848965:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.848966:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.848968:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.848968:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.848970:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.848972:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.848972:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.848977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.848983:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.848986:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.848987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.848987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677471.849331:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.849334:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.849335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.849335:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.849337:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.849340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.849352:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.849354:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.849355:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.849355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.849357:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.849357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.849440:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.849442:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.849443:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.849444:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.849445:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.849447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.849458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.849465:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.849468:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.849469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.849472:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.849476:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.849477:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.849479:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.849481:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.849482:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.849483:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.849486:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.849488:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.849491:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.849492:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.849493:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.849494:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.849495:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.849496:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.849497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.849508:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.849510:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.849511:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.849512:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.849513:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.849514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.849685:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.849688:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.849689:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.849689:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.849691:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.849693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677471.849992:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.849995:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.849996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.849997:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.849999:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.850002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.850056:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.850059:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.850060:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.850060:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.850062:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.850066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.850133:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.850136:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.850137:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.850137:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.850139:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.850141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.850490:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.850492:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.850494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.850495:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.850497:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.850499:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.850500:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.850501:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.850502:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.850503:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.850505:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.850507:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.850508:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.850509:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.850510:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.850511:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.850511:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.850512:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.850512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.850523:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.850525:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.850525:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.850526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.850527:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.850528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.851641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.851678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.851687:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.851690:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.851691:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.851691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.851693:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.851696:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677471.851717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.851722:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.851725:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.851726:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.851728:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.851730:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.851731:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.851732:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.851733:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.851734:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.851735:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677471.851737:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:1.0:1619677471.851738:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.851740:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677471.851740:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677471.851741:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.851741:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677471.851743:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677471.851744:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.851744:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.851744:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677471.851746:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677471.851746:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.851747:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.851747:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.851748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677471.851795:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.851798:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.851798:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.851800:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.851803:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.851804:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.851806:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.851807:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.851808:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.851809:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.851811:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.851813:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.851815:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.851815:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.851816:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.851818:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.851818:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.851819:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.851819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.851824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.852770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:8.2:1619677471.852802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.852805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:8.0:1619677471.852904:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677471.852907:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677471.852908:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677471.852910:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.852912:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.852914:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.852915:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.852917:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.852918:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.852919:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.852922:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.852924:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.852927:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.852928:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.852929:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.852930:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.852931:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677471.852932:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677471.852932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:20.2:1619677471.852936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:18.0:1619677471.852946:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.852949:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.852950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.852951:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.852953:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.852958:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.852973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.852981:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.852984:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.852985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.852985:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.852987:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.852989:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.853210:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.853213:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.853214:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.853215:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.853218:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.853220:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.853221:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.853222:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.853223:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.853225:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.853228:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.853230:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.853233:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.853233:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.853234:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.853236:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.853236:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.853238:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.853239:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.853239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.853242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.853250:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677471.853252:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677471.853253:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.853253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.853254:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.853254:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.853255:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677471.853256:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.853256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677471.853259:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.853702:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.853704:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.853705:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.853705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.853707:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.853709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.853718:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.853720:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.853721:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.853721:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.853722:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.853722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.854126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.854134:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.854137:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.854138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.854138:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.854140:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.854143:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.854638:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.854641:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.854642:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.854642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.854644:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.854646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.854883:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.854886:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.854887:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.854888:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.854891:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.854892:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.854894:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.854895:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.854896:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.854897:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.854900:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.854901:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.854904:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.854905:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.854905:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.854907:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.854908:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.854908:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.854908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.854921:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.854924:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.854926:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.854926:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.854928:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.854931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.854941:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.854943:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.854944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.854945:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.854946:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.854949:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.855117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.855123:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.855125:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.855127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.855128:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.855131:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.855133:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.855134:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.855136:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.855137:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.855138:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.855142:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.855143:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.855146:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.855147:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.855147:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.855149:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.855150:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.855151:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.855151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.855155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677471.855428:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.855431:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.855431:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.855433:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.855435:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.855437:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677471.855438:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.855439:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.855440:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.855441:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677471.855444:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.855446:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677471.855448:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.855449:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677471.855450:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677471.855451:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677471.855452:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.855452:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.855453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677471.855809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.855830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677471.855904:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.855907:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.855908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.855909:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.855912:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.855914:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.855915:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.855916:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.855917:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.855918:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.855920:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.855922:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.855923:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.855924:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.855925:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.855926:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.855926:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.855927:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.855928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.855938:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.855940:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.855940:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.855941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.855942:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.855944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677471.856217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.856223:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.856225:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.856226:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.856228:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.856231:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.856232:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.856233:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.856234:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.856235:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.856236:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.856240:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.856241:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.856244:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.856244:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.856245:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.856247:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.856247:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.856248:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.856248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677471.856827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.856833:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.856837:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.856837:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.856840:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.856842:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.856843:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.856845:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.856847:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.856848:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.856849:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.856853:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.856855:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.856857:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.856858:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.856859:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.856861:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.856862:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.856863:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.856863:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.856875:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.856877:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.856877:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.856878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.856879:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.856880:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.857333:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.857335:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.857336:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.857336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.857338:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.857341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.857884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.857915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677471.858029:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.858032:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.858033:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.858033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.858036:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.858039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.858373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677471.858423:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.858425:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.858427:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.858427:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.858429:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.858433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.858435:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.858438:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.858439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.858439:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.858440:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.858441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.858809:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.858812:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.858812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.858815:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.858818:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.858819:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.858821:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.858823:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.858824:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677471.858824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:17.0:1619677471.858825:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.858829:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.858831:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.858834:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.858834:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.858835:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.858835:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.858835:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.858837:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:1.0:1619677471.858837:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.858838:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.858839:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.858839:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.858840:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677471.858841:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.858842:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.858842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.858843:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)++
00000800:00000200:8.2:1619677471.858843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.858843:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.858844:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.858844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.859045:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.859047:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.859048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.859049:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.859051:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.859054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.859254:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.859256:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.859257:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.859258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.859259:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.859261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.859274:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.859276:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.859277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.859277:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.859278:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.859279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.859483:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.859485:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.859487:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.859488:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.859489:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.859491:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.859492:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.859493:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.859494:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.859495:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.859498:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.859499:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.859502:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.859503:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.859504:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.859505:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.859506:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.859506:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.859507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.859511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.859520:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.859523:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.859524:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.859524:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.859526:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.859529:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.859688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.859722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.859725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677471.860077:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.860080:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.860081:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.860081:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.860083:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.860084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.860102:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.860104:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.860105:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.860105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.860106:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.860107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.860595:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.860597:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.860598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.860600:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.860602:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.860603:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.860604:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.860606:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.860607:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.860607:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.860610:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.860612:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.860614:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.860615:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.860615:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.860616:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.860617:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.860618:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.860619:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.860620:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.860631:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.860632:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.860632:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.860634:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.860635:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.860636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.860636:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.860636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.860637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.860637:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.860885:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.860887:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.860888:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.860889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.860890:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.860893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.860933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:5.2:1619677471.861037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.861044:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.861047:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.861048:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.861049:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.861052:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.861053:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.861054:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.861055:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.861057:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.861057:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.861061:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.861062:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.861065:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.861066:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.861066:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.861068:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.861069:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.861069:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677471.861070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677471.861252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.861258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.861262:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.861263:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.861265:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.861267:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.861269:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.861271:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.861273:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.861274:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.861275:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.861279:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.861281:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.861284:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.861285:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.861286:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.861289:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.861290:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.861291:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.861291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677471.861574:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.861576:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.861577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.861578:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.861580:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.861583:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.862085:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.862087:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.862088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.862090:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.862091:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.862093:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.862094:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.862095:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.862096:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.862097:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.862101:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.862102:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.862105:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.862106:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.862107:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.862109:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.862109:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.862110:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.862110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.862122:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.862125:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.862126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.862126:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.862129:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.862133:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.862191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677471.862921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677471.862994:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.862996:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.862997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.862997:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.862999:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.863002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.863018:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.863021:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.863022:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.863022:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.863022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.863023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.863293:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.863297:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.863301:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.863302:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.863304:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.863307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.863682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.863726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.864402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.864404:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.864406:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.864406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677471.864407:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:24.0:1619677471.864408:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.864410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677471.864410:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.864411:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.864412:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.864414:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.864417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.865075:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.865077:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.865078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.865080:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.865082:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.865084:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.865085:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.865087:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.865088:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.865089:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.865092:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.865094:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.865097:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.865098:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.865099:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.865100:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.865101:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.865102:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.865102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.865106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.865115:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.865118:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.865119:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.865119:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.865121:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.865124:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677471.865395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677471.865404:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.865409:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677471.865410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677471.865414:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.865417:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.865420:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677471.865422:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.865425:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.865427:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.865429:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677471.865435:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.865438:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677471.865442:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.865443:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677471.865444:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677471.865447:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677471.865448:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677471.865450:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677471.865451:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677471.865453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677471.865458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.865466:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677471.865470:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677471.865472:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.865474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677471.865478:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.865480:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.865481:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.865482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.865484:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.865486:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677471.865839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.865841:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.865843:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.865846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.865846:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.865847:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677471.865849:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.865850:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.865851:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.865852:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677471.865855:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.865857:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677471.865860:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.865860:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677471.865861:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677471.865863:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677471.865863:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.865864:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.865865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677471.865884:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.865887:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.865889:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.865889:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.865891:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.865894:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.865916:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.865920:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.865921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.865925:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.865928:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.865930:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.865932:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.865935:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.865936:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.865940:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.865945:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.865948:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.865952:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.865953:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.865954:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.865956:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.865957:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.865958:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.865959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.865962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.866065:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.866068:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.866068:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.866070:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.866073:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.866074:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.866076:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.866077:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.866078:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.866079:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.866082:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.866084:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.866085:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.866086:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.866087:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.866088:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.866088:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.866089:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.866089:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.866099:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.866101:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.866101:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.866103:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.866103:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.866104:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.866106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.866112:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.866115:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.866116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.866118:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.866121:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.866123:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.866124:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.866126:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.866127:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.866128:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.866131:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.866134:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.866137:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.866138:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.866139:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.866141:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.866142:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.866143:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.866144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.866333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.866378:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677471.866381:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.866384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.866384:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.866385:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.866387:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.866389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677471.866389:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.866393:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.866395:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.866396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.866399:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.866403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.866541:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.866544:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.866545:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.866545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.866547:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.866549:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.866562:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.866564:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.866565:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.866565:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.866566:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.866566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.868374:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.868377:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.868379:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.868380:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.868382:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.868385:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.868416:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.868420:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.868421:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.868424:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.868426:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.868429:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.868431:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677471.868434:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:8.0:1619677471.868440:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.868441:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.868442:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677471.868447:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:8.0:1619677471.868448:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.868451:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:8.0:1619677471.868451:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:17.0:1619677471.868453:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.868453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.868455:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.868456:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:17.0:1619677471.868457:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.868457:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.868460:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.868461:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677471.868462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.868462:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.868464:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.868465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677471.868477:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677471.868481:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677471.868482:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677471.868484:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:2.0:1619677471.868484:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.868485:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677471.868485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.868488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677471.868488:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.868495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.868498:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.868501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.868502:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.868504:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.868507:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.868507:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677471.869584:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.869586:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.869587:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.869588:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.869591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.869592:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677471.869702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677471.869708:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.869711:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677471.869712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677471.869714:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.869716:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.869718:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677471.869719:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.869721:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.869722:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.869723:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677471.869726:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.869728:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677471.869730:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.869731:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677471.869732:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677471.869733:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677471.869734:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677471.869734:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677471.869735:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677471.869906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.869939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.869961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677471.870210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.870217:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.870220:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.870221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.870223:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.870226:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.870227:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.870229:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.870231:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.870232:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.870233:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.870237:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.870239:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.870241:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.870242:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.870243:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.870245:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.870246:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.870247:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.870249:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.870249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.870254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.870259:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.870260:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.870261:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.870262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.870263:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677471.870263:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.870264:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.870265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.870265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.870266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677471.870440:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.870442:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.870443:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.870443:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.870446:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.870449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.870843:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.870846:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.870847:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.870848:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.870851:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.870852:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.870853:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.870855:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.870856:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.870857:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.870860:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.870862:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.870865:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.870866:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.870866:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.870868:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.870868:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.870869:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.870870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.870874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.870883:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.870885:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.870886:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.870887:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.870889:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677471.870890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.870894:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.870895:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.870897:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.870898:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.870899:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.870899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.870899:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:8.0:1619677471.871049:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.871051:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.871052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.871052:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.871054:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.871056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.872144:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.872146:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.872147:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.872147:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.872149:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.872151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677471.872179:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.872183:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.872186:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.872187:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.872189:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.872192:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677471.872194:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.872199:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.872201:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.872203:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677471.872208:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.872211:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677471.872215:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.872216:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677471.872217:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677471.872219:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677471.872221:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.872223:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677471.872225:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677471.872226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.872239:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677471.872241:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677471.872242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.872243:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.0:1619677471.872446:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.872449:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.872449:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.872451:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.872454:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.872455:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.872456:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.872457:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.872458:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.872459:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.872462:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.872464:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.872466:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.872467:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.872468:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.872470:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.872470:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.872471:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.872471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.872475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.872484:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.872487:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.872488:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677471.872488:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.872490:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.872493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.872581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.872588:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.872591:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.872592:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.872595:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.872597:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.872599:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.872600:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.872602:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.872603:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.872604:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.872608:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.872610:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.872613:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.872614:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.872615:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.872617:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.872617:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.872619:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.872620:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.872620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677471.872623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.872626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.872631:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.872634:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.872635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.872636:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.872642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.2:1619677471.873786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.873818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.873820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677471.873827:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.873829:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.873831:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.873831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.873833:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.873835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.873839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.873846:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.873849:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.873850:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.873850:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.873851:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.873851:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.874183:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.874186:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.874187:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.874187:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.874189:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.874191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.874205:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.874207:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.874208:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.874208:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.874209:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.874209:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.874341:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.874343:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.874344:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.874344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.874346:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.874347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.874366:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.874368:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.874369:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.874369:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.874370:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.874371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.874572:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.874575:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.874576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.874577:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.874580:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.874581:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.874582:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.874584:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.874584:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.874585:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.874588:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.874590:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.874592:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.874593:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.874594:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.874595:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.874596:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.874597:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.874597:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.874611:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.874614:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.874616:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.874616:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.874618:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.874622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677471.875033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677471.875040:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.875042:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677471.875043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677471.875045:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.875048:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.875050:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677471.875051:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.875052:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.875053:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.875054:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677471.875058:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.875060:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677471.875062:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.875063:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677471.875064:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677471.875066:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677471.875066:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.875067:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677471.875067:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.0:1619677471.875080:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.875083:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.875083:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.875084:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.875086:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.875089:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.876500:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.876503:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.876504:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.876504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.876505:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.876507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.877062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.877068:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.877071:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.877072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.877074:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.877077:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.877079:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.877080:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.877082:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.877083:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.877084:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.877088:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.877090:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.877092:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.877093:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.877094:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.877096:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.877097:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.877098:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.877098:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.877109:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.877112:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.877112:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.877113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.877114:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.877116:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.877315:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.877317:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.877318:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.877320:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.877323:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.877324:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.877326:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.877327:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.877328:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.877329:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.877332:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.877334:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.877337:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.877338:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.877338:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.877340:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.877340:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.877342:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.877343:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.877344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.877346:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.877354:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677471.877357:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677471.877357:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.877358:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.877358:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.877359:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.877360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677471.877360:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.877361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677471.877363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.877663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.877669:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.877672:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.877673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.877675:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.877678:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.877679:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.877680:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.877682:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.877683:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.877684:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.877688:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.877691:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.877694:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.877696:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.877696:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.877699:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.877701:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.877702:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.877703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677471.877725:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.877727:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.877728:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.877728:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.877730:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.877732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.877999:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677471.878002:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677471.878003:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677471.878004:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.878007:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.878008:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677471.878009:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.878011:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.878012:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.878012:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677471.878015:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.878017:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677471.878019:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.878020:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677471.878020:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677471.878022:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677471.878023:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677471.878024:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677471.878025:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677471.878026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677471.878038:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677471.878040:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677471.878041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677471.878042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:24.0:1619677471.878540:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.878543:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.878544:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.878545:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.878548:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.878549:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.878550:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.878551:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.878552:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.878553:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.878556:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.878557:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.878560:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.878561:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.878561:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.878562:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.878563:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.878564:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.878564:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.878569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.878578:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.878580:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.878581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.878582:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.878584:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.878586:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.878898:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.878901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.878901:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.878902:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.878904:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.878906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677471.879077:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.879080:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.879081:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.879082:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.879084:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.879087:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.879726:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.879729:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.879730:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.879730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.879733:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677471.879735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677471.880222:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.880224:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.880226:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.880226:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.880228:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.880232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.880282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.880333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.880342:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.880345:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.880346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.880346:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.880348:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.880351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.880634:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.880637:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677471.880637:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677471.880640:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.880642:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.880644:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677471.880645:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.880647:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.880648:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.880649:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677471.880652:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.880653:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677471.880656:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.880657:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677471.880657:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677471.880659:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677471.880660:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677471.880661:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677471.880662:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677471.880663:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677471.880673:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677471.880673:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677471.880676:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677471.880676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677471.880677:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:1.0:1619677471.880677:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:18.0:1619677471.880678:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.880678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.880680:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.880683:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.880693:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.880696:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.880697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.880698:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.880699:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.880700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.880720:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.880722:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.880723:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.880724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.880725:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.880727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.881215:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.881217:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.881218:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.881218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.881220:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.881222:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.881236:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.881238:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.881239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.881239:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.881240:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.881241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.881913:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.881915:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.881916:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.881916:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:24.0:1619677471.881918:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.881918:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677471.881919:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:8.0:1619677471.881919:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (24)--
00000800:00000200:8.0:1619677471.881920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.881921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.881921:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.881929:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.881932:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.881933:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.881934:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.881937:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.881939:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.881940:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.881941:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.881942:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.881943:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.881946:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.881948:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.881951:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.881952:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.881952:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.881954:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.881954:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.881955:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.881956:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.881968:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.881971:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.881972:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.881973:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.881975:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.881978:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.881986:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.881989:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.881990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.881990:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.881992:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.881992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.882902:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.882904:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.882905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.882905:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.882907:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.882909:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.883180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.883187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.883190:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.883191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.883193:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.883196:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.883197:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.883199:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.883201:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.883202:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.883203:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.883207:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.883209:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.883213:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.883213:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.883215:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.883217:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.883217:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.883218:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.883219:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677471.883940:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.883942:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.883944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.883945:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.883948:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.883950:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.883951:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.883952:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.883953:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.883954:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.883957:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.883958:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.883961:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.883962:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.883962:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.883964:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.883964:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.883965:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.883966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.883970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.883979:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.883982:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.883983:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.883983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.883985:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.883988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.883990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.883998:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.884001:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.884002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.884002:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.884003:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.884004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.884116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.884122:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.884125:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.884126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.884128:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.884131:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.884133:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.884134:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.884136:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.884137:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.884139:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.884142:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.884144:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.884146:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.884147:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.884148:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.884150:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.884151:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.884152:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.884152:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.884163:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.884165:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.884166:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.884166:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.884168:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.884169:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.884893:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.884895:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.884896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.884898:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.884901:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.884902:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.884903:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.884905:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.884906:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.884907:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.884910:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.884911:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.884914:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.884915:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.884915:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.884917:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.884918:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.884919:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.884920:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.884921:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.884923:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.884931:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677471.884934:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677471.884934:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.884935:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.884935:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.884936:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.884936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.884937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677471.884937:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.884940:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.884984:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.884986:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.884987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.884987:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.884989:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.884991:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677471.885173:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.885176:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.885177:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.885178:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.885181:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.885183:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677471.885184:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.885185:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.885186:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.885187:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.2:1619677471.885190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677471.885191:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.885192:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677471.885196:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.885196:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677471.885197:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677471.885199:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677471.885199:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.885201:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.885202:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677471.885202:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.885214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.885216:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.885216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.885218:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677471.885226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.885233:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.885236:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677471.885237:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.885238:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.885240:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.885242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.885549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.885557:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.885560:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.885561:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.885561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.885563:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.885566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.885591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:8.2:1619677471.885625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677471.886157:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.886160:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.886161:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.886161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.886163:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677471.886166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.886730:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.886732:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.886733:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.886733:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.886735:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.886737:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.886747:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.886751:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.886751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.886752:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.886753:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.886753:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.887221:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.887224:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.887225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.887225:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.887227:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.887228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.887281:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.887283:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.887284:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.887285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.887286:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.887288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.888415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.888447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.888618:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.888621:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.888622:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.888624:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.888626:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.888627:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.888629:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.888630:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.888632:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.888632:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.888636:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.888637:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.888640:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.888641:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.888641:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.888643:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.888644:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.888645:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.888645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.888649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.888658:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.888660:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.888662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.888662:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.888664:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.888667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.889485:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.889488:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.889489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.889490:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.889493:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.889494:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.889496:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.889497:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.889498:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.889498:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.889501:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.889502:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.889503:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.889504:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.889504:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.889506:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.889506:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.889507:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.889507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.889518:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.889520:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.889521:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.889521:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.889522:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.889524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.890084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.890092:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.890097:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.890099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.890101:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.890105:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.890107:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.890110:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.890113:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.890115:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.890116:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.890122:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.890125:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.890129:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.890131:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.890132:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.890134:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.890136:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.890137:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.890138:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.890140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.890496:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.890499:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.890500:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.890501:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.890503:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.890507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.890695:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.890698:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.890699:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.890701:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.890702:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.890704:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.890705:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.890706:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.890707:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.890708:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.890711:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.890712:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.890715:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.890716:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.890716:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.890718:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.890718:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.890719:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.890720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.890726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.890734:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.890737:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.890738:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.890738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.890740:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.890743:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.890934:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:8.0:1619677471.890937:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:17.0:1619677471.890938:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.890938:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:8.0:1619677471.890939:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.890940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.890940:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.890940:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.890943:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:1.0:1619677471.890943:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.890944:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:8.0:1619677471.890946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:1.0:1619677471.890946:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.890947:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.890948:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.890949:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.890952:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.890953:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.890956:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.890957:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.890957:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.890959:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.890960:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.890961:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.890962:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.890963:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.890966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.890973:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.890975:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.890976:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.890977:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:24.0:1619677471.891223:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.891225:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.891226:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.891227:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.891229:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677471.891231:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.891483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.891517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.891519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677471.891526:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.891528:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.891529:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.891529:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.891532:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.891534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.891535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.891543:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.891545:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.891546:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.891546:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.891547:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.891548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.892039:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677471.892042:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677471.892043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677471.892045:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.892048:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.892049:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677471.892050:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.892052:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.892053:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.892054:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677471.892057:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.892059:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677471.892061:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.892062:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:8.0:1619677471.892063:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:8.0:1619677471.892065:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:8.0:1619677471.892065:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677471.892066:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677471.892067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:13.0:1619677471.892079:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.892081:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.892082:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.892083:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.892085:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.892086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:24.0:1619677471.892312:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.892315:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.892316:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.892316:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.892318:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.892320:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.892333:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.892335:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.892336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.892336:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.892337:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.892338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.892495:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.892497:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.892498:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.892499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.892500:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.892502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.892930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.892936:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.892939:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.892941:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.892942:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.892945:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.892946:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.892948:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.892951:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.892952:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.892953:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.892957:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.892959:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.892961:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.892962:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.892963:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.892964:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.892965:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.892967:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.892967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.892978:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.892980:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.892981:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.892981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.892983:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.892984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.893989:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.893991:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.893992:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.893992:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.893994:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.893996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.894026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.894061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.894084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:24.0:1619677471.894216:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.894219:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.894220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.894222:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.894225:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.894226:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.894228:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.894229:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.894230:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.894231:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.894234:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.894236:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.894237:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.894238:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.894239:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.894240:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.894241:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.894242:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.894243:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.894244:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.894254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.894255:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.894256:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.894258:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677471.894258:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.894259:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.894259:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.894260:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.894260:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.894261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.894378:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.894380:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.894382:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.894382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.894383:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.894385:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.894397:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.894400:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.894401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.894401:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.894401:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.894402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.894588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.894594:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.894597:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.894598:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.894600:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.894603:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.894604:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.894606:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.894608:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.894609:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.894610:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.894614:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.894617:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.894620:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.894621:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.894622:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.894624:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.894625:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.894627:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.894628:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.894629:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.894634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.894640:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.894642:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.894643:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.894644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.894947:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.894950:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.894951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.894953:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.894955:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.894957:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677471.894958:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.894960:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.894961:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.894961:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677471.894965:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.894966:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677471.894969:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.894970:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677471.894971:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677471.894973:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677471.894973:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.894974:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.894974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677471.895734:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.895737:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.895738:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.895740:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.895741:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.895743:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.895744:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.895745:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.895746:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.895747:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.895751:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.895752:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.895755:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.895756:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.895756:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.895758:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.895759:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.895760:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.895760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.895774:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.895777:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.895778:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.895778:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.895780:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.895784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.896059:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.896061:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.896062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.896062:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.896064:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.896066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.897238:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.897240:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.897241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.897242:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.897245:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.897246:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.897247:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.897249:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.897249:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.897250:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.897253:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.897255:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.897258:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.897259:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.897259:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.897261:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.897261:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.897262:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.897262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.897276:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.897279:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.897280:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.897280:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.897283:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.897286:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.897294:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.897299:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.897302:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.897303:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.897304:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.897305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.897727:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.897729:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.897730:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.897730:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.897732:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.897734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.897745:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.897747:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.897748:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.897748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.897749:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.897749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.897817:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677471.897819:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:8.0:1619677471.897820:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.897821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.897821:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.897823:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:8.0:1619677471.897823:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.897824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.0:1619677471.897825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:17.0:1619677471.897828:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.897832:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.897835:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.897837:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.897839:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.897840:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.897842:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.897848:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.897851:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.897855:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.897856:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.897857:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.897860:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.897861:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.897863:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.897864:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.897866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.898014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677471.898066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.898075:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.898079:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.898081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.898081:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.898084:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.898088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.899059:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.899062:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.899063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.899064:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.899067:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.899069:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.899070:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.899072:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.899073:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.899074:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.899077:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.899079:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.899080:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.899081:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.899082:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.899083:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.899083:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.899084:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.899084:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.899095:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.899097:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.899097:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.899098:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.899099:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.899101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.899776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.899782:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.899785:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.899786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.899787:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.899791:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.899792:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.899794:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.899795:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.899796:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.899798:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.899801:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.899802:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.899805:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.899805:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.899806:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.899808:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.899809:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.899810:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.899811:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.899812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.899817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.899824:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.899826:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.899826:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.899826:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677471.899827:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.899827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.899828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.899829:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.899829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.899830:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.899971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:20.0:1619677471.899981:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.899983:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677471.899984:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.899986:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.899987:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.899988:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677471.899991:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.899993:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677471.899994:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.899995:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.899996:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.899998:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.899998:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.899999:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677471.900000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677471.900004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.900022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677471.900054:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.900056:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.900057:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.900058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.900059:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.900061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.900264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.900269:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.900273:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.900274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.900275:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.900278:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.900279:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.900281:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.900283:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.900285:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.900286:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.900289:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.900290:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.900291:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677471.900292:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.900298:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.900300:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.900302:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.900303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.900304:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.900305:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.900306:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.900309:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.900310:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.900311:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.900311:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677471.900861:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677471.900864:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677471.900866:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.900868:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.900870:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677471.900871:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.900873:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.900874:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.900875:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677471.900879:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.900880:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677471.900883:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.900884:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677471.900885:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677471.900887:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677471.900887:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677471.900889:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677471.900889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677471.901711:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.901714:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.901716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.901717:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.901720:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.901721:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.901722:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.901724:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.901725:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.901726:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.901729:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.901730:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.901733:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.901734:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.901735:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.901736:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.901737:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.901738:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.901739:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.901740:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677471.901752:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677471.901753:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.901755:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.901755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677471.901756:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677471.901757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677471.901757:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.901757:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.901759:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.901763:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.901807:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.901809:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.901810:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.901810:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.901812:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.901814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677471.902723:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.902726:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.902728:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.902729:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.902732:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.902733:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.902734:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.902735:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.902736:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.902737:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.902741:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.902742:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.902745:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.902745:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.902746:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.902748:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.902748:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.902749:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.902749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.902755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.902769:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677471.903112:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.903114:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.903115:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.903115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.903117:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.903119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.903188:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.903191:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.903192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.903193:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.903196:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.903197:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.903198:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.903199:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.903200:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.903201:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.903204:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.903206:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.903208:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.903209:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.903210:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.903212:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.903212:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.903212:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.903213:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.903226:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.903229:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.903230:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.903231:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.903232:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.903236:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677471.903270:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.903273:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.903274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.903276:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.903278:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.903280:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677471.903282:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.903283:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.903284:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.903285:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677471.903288:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.903289:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.903291:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.903292:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.903293:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.903298:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.903298:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.903299:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.903299:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677471.903305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.903312:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.903314:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.903315:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.903316:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.903317:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.903318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.903850:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.903852:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.903853:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.903853:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.903855:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.903858:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.903869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.903871:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.903872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.903872:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.903873:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.903874:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.904465:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.904467:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.904468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.904470:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.904472:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.904474:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.904475:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.904476:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.904477:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.904478:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.904481:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.904483:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.904484:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.904485:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.904485:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.904486:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.904487:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.904488:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.904488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.904499:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.904500:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.904501:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.904502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.904503:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.904504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.904574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.904577:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.904579:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677471.904582:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.904584:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.904586:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677471.904587:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.904589:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.904590:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.904591:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677471.904594:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.904596:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677471.904598:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.904599:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.904599:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.904601:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.904602:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.904602:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677471.904604:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677471.904604:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.904607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.904610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.904613:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.904614:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.904615:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.904616:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.904618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.904620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677471.904627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.904641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677471.904873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.904881:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.904883:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.904884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.904885:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.904887:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.904890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.905372:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.905375:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.905376:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.905376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.905378:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.905380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.905521:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.905524:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.905525:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.905525:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.905527:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.905529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.905794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.905806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.905830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.905837:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.905839:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.905840:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.905842:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.905844:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677471.906813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.906821:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.906826:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.906828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.906831:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.906835:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.906837:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.906840:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.906842:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.906844:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.906846:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.906852:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.906855:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.906859:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.906860:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.906862:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.906864:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.906865:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.906867:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.906868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.907190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.907198:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.907202:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.907204:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.907207:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.907211:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.907214:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677471.907216:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.907219:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.907221:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.907222:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677471.907228:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.907230:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677471.907234:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.907235:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677471.907236:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677471.907239:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677471.907240:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.907241:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.907242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677471.907282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.907290:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.907297:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.907300:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677471.907304:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.907307:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.907310:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.907312:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.907314:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.907316:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.907317:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.907322:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.907325:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.907327:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.907328:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.907329:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.907332:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.907333:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.907334:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.907335:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.907348:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.907352:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.907353:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.907355:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.907356:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.907359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.907370:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.907375:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.907377:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.907377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.907378:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.907379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.907426:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.907428:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.907429:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.907431:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.907433:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.907435:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.907436:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.907437:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.907438:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.907439:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.907442:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.907443:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.907446:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.907447:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.907447:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.907449:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.907449:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.907450:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.907450:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.907455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.907471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677471.907560:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.907563:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.907564:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.907564:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.907566:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.907568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.907785:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677471.907787:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677471.907788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677471.907790:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.907792:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.907794:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677471.907795:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.907796:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.907797:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.907798:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677471.907801:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.907802:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677471.907805:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.907806:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677471.907806:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677471.907808:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677471.907809:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677471.907809:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677471.907810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:1.0:1619677471.907837:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.907840:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.907841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.907843:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.907845:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.907846:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677471.907848:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.907849:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.907850:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.907851:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677471.907854:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.907856:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677471.907858:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.907860:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677471.907860:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677471.907862:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.907862:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.907863:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677471.907864:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677471.907876:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.907878:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.907880:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.907880:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.907882:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.907885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.908122:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.908125:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.908126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.908128:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.908130:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.908132:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.908133:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.908134:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.908135:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.908136:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.908139:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.908141:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.908144:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.908145:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.908145:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.908147:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.908147:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.908148:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.908148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.908160:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.908163:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.908164:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.908165:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.908167:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.908170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.908412:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.908414:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.908415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.908415:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.908417:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.908419:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.908429:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.908431:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.908432:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.908432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.908433:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.908433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.908860:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.908862:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.908863:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.908865:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.908867:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.908869:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.908870:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.908871:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.908872:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.908873:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.908876:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.908877:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.908880:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.908881:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.908881:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.908883:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.908883:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.908885:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.908886:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.908886:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677471.908897:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677471.908899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677471.908900:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677471.908901:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.908902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677471.908902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.908902:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.908903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677471.908904:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.908908:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.909673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.909676:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.909677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.909677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.909678:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.909681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.909691:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.909693:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.909694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.909694:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.909695:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.909696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.909724:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.909726:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.909727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.909728:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.909731:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.909732:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.909733:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.909734:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.909735:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.909736:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.909739:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.909741:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.909743:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.909744:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.909745:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.909746:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.909747:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.909747:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.909748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.909760:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.909763:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.909764:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.909764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.909767:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.909770:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.909949:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.909951:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.909953:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.909953:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.909955:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.909959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.909968:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.909970:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.909972:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.909972:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.909973:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.909974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.910328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.910337:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.910339:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.910341:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.910342:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.910345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677471.910608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.910618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.910641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.910648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.910650:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.910651:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.910653:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.910655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677471.910659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.910672:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677471.911324:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.911327:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.911328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.911328:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.911330:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.911332:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.911741:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.911743:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.911744:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.911744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.911746:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.911747:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.911824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.2:1619677471.911828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.0:1619677471.911834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677471.911834:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.911837:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.911838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.911841:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.911843:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.911844:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.911846:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.911849:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.911850:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.911851:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.911855:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.911857:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677471.911860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677471.911860:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.911861:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.911862:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.911864:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.911865:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.911866:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677471.911867:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:2.0:1619677471.911867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677471.911870:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.911870:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677471.911870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:20.0:1619677471.911872:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.911874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.911876:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677471.911879:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677471.911879:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.911881:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677471.911884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677471.912421:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.912424:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.912425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.912427:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.912429:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.912431:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.912432:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.912433:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.912434:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.912435:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.912438:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.912440:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.912442:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.912443:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.912444:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.912445:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.912446:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.912448:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.912449:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.912449:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.912452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.912460:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.912462:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.912462:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.912463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.0:1619677471.912469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677471.913562:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.913565:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.913566:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.913568:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.913570:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.913572:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.913573:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.913574:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.913575:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.913576:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.913580:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.913581:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.913584:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.913585:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.913586:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.913588:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.913588:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.913589:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.913589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.913594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.913603:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.913606:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.913606:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.913607:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.913609:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.913612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.913615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.913622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.913624:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.913625:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.913626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.913627:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.913627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.913814:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.913816:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.913817:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.913818:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.913818:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.913819:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.913820:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.913821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.913822:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:8.0:1619677471.913822:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.913823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.913824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.914374:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.914376:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.914378:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.914379:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.914381:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.914383:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.914384:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.914385:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.914386:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.914387:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.914390:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.914391:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.914393:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.914393:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.914394:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.914395:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.914395:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.914396:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.914397:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677471.914403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677471.914406:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:8.0:1619677471.914408:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677471.914409:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:8.0:1619677471.914409:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.914410:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.914411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677471.914411:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.914412:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677471.914413:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.914414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:20.0:1619677471.914415:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677471.914416:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.914418:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.914419:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.914420:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677471.914424:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.914425:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677471.914428:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.914429:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:20.0:1619677471.914430:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677471.914432:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677471.914432:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677471.914433:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677471.914434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:24.0:1619677471.914579:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.914581:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.914582:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.914582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.914584:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.914586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.914597:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.914599:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.914600:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.914600:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.914601:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.914602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677471.914990:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.914993:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.914994:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.914995:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.914997:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.915000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.915011:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.915013:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.915014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.915014:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.915016:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.915016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.915350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.915356:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.915359:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.915360:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.915362:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.915364:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.915365:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.915366:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.915368:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.915369:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.915370:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.915373:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.915375:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.915377:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.915378:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.915378:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.915380:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.915381:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.915382:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.915382:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.915394:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.915396:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.915396:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.915398:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.915398:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.915399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.915445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.915458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677471.915544:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.915547:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.915548:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.915548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.915550:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.915552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.915613:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.915615:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.915616:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.915617:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.915619:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.915621:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.915622:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.915623:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.915624:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.915625:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.915628:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.915629:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.915632:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.915633:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.915633:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.915635:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.915635:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.915637:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.915638:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.915639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677471.915649:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677471.915651:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677471.915652:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677471.915653:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677471.915653:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.915653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.915654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.915655:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677471.915655:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.915659:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.916391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.916394:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.916397:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677471.916399:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.916402:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.916403:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677471.916404:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.916405:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.916406:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.916407:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677471.916410:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.916412:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677471.916414:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.916415:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.916415:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.916417:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.916418:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.916420:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677471.916421:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677471.916421:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.916424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.916427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.916431:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.916432:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.916432:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.916433:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.916435:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.916437:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677471.916445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.916459:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677471.916724:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.916726:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.916727:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.916728:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.916729:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.916731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677471.916841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:24.0:1619677471.917549:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.917551:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.917552:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.917552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.917554:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.917556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.917647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.917650:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.917653:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677471.917655:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.917657:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.917659:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677471.917660:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.917661:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.917662:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.917663:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677471.917665:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.917667:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677471.917669:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.917669:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.917670:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.917672:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.917672:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.917673:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677471.917674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677471.917677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.917693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677471.917725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.917738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.2:1619677471.917767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.917773:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.917777:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.917778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.917779:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.917782:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.917783:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.917785:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.917787:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.917788:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.917789:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.917793:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.917795:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.917798:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.917799:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.917800:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.917802:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.917803:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.917805:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.917806:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.917807:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677471.917808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.917813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677471.917819:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:18.0:1619677471.917819:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:20.0:1619677471.917821:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:18.0:1619677471.917822:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:20.0:1619677471.917823:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.917823:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.917823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:20.0:1619677471.917825:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677471.917827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677471.917830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677471.917844:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677471.917853:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.917856:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.917858:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.917859:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677471.917859:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.917860:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:17.0:1619677471.917862:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.917863:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.917864:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.917866:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.917867:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.917868:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.917869:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.917871:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.917873:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677471.917874:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677471.917874:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.917875:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677471.917876:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.917876:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.917877:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677471.917880:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.917881:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.917883:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.917883:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.917884:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.917885:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.917885:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.917885:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.917887:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.917887:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.917888:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.917888:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677471.917889:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.917889:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677471.917890:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677471.917890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677471.917891:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677471.917893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677471.917897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.917905:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:20.0:1619677471.917906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.917907:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677471.917908:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.917909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.917910:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.917912:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.918307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.918314:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.918317:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.918318:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.918320:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.918322:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.918324:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677471.918326:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.918327:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.918328:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.918329:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677471.918333:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.918335:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677471.918337:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.918338:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677471.918339:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677471.918341:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.918341:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.918342:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677471.918343:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677471.918344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.2:1619677471.918347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677471.918349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.918354:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:20.0:1619677471.918356:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677471.918356:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677471.918357:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.0:1619677471.918358:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:2.0:1619677471.918358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:20.0:1619677471.918359:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.918361:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.918363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677471.918553:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.918555:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.918556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.918556:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.918558:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.918560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.918818:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.918820:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.918822:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.918823:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.918825:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.918827:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.918828:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.918829:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.918830:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.918830:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.918833:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.918835:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.918838:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.918838:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.918839:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.918841:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.918841:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.918842:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.918842:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.918847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.918856:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.918858:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.918859:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.918859:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.918862:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.918865:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677471.919019:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.919022:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.919023:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.919023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.919025:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.919029:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.919782:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.919784:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.919786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.919786:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.919789:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.919793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.919810:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.919812:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.919813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.919813:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.919815:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.919817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.919829:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.919832:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.919832:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.919833:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.919833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.919834:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.920132:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.920134:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.920135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.920137:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.920139:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.920141:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.920142:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.920143:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.920144:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.920145:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.920148:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.920149:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.920152:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.920153:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.920153:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.920155:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.920155:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.920156:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.920156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.920161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.920169:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.920172:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.920173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.920173:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.920175:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.920178:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677471.920653:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677471.920684:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677471.920686:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677471.920687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677471.920688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677471.921131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.921138:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.921141:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.921142:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.921143:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.921146:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677471.921341:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677471.921358:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677471.921361:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677471.921362:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677471.921363:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677471.921363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.2:1619677471.921860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.921867:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.921871:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.921872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.921874:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.921877:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.921878:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.921880:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.921882:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.921883:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.921884:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.921887:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.921889:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.921892:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.921893:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.921894:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.921896:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.921897:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.921898:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.921899:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.921909:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.921911:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.921913:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.921913:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.921915:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.921917:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.921925:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.921927:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.921928:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.921929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.921931:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.921933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.922175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.922177:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.922180:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677471.922182:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.922184:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.922186:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677471.922187:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.922188:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.922190:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.922190:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677471.922194:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.922196:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677471.922198:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.922200:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.922200:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.922202:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.922203:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.922203:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677471.922204:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677471.922208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.922222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677471.922227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677471.922233:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677471.922235:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.922236:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.922237:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677471.922237:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677471.922238:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.922239:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000400:00000200:8.0:1619677471.922239:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677471.922240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.922241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:8.0:1619677471.922241:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.922243:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.922244:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.922246:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.922247:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.922248:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.922251:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.922253:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.922256:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.922257:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.922257:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.922259:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.922260:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.922260:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.922261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.922273:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.922276:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.922277:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.922277:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.922280:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.922284:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.922294:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.922301:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.922302:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.922303:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.922304:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.922305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.922649:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.922653:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.922653:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.922656:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.922657:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.922659:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.922660:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.922661:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.922662:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.922663:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.922667:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.922668:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.922671:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.922671:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.922672:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.922674:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.922674:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.922675:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.922675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.922681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.922695:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.923026:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.923028:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.923030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.923031:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.923033:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.923035:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677471.923036:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.923037:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.923039:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.923039:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677471.923042:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.923043:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677471.923046:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.923046:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677471.923047:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677471.923048:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677471.923049:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.923049:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.923050:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677471.923601:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.923603:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.923604:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.923604:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.923606:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.923608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.923617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.923625:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677471.923627:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677471.923629:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.923631:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.923634:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677471.923750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.923757:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.923760:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.923761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.923763:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.923765:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.923766:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.923768:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.923770:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.923771:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.923772:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.923776:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.923778:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.923780:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.923781:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.923782:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.923784:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.923785:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.923786:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.923787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.924069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.924079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.924104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.924112:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.924114:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.924115:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.924115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.924117:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.924120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.924160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.924167:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.924169:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.924171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.924171:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.924172:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.924172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.924224:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.924227:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.924228:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.924230:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.924233:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.924234:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.924235:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.924236:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.924237:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.924238:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.924241:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.924243:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.924244:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.924245:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.924246:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.924247:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.924248:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.924250:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.924251:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.924251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.924262:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.924264:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.924264:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.924266:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.924266:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.924267:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.924267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.924268:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.924268:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.924269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.924507:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.924510:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.924511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.924511:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.924512:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.924514:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.924525:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.924528:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.924529:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.924529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.924530:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.924530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677471.925005:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.925008:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677471.925009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677471.925011:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.925014:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.925016:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.925017:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.925019:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.925020:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.925021:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.925025:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.925027:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.925029:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.925030:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.925031:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.925033:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.925034:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.925035:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677471.925035:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677471.925046:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.925049:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.925049:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.925050:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.925051:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.925053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.925107:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.925110:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.925111:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.925111:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.925112:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.925114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.925627:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.925629:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.925630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.925631:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.925633:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.925635:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.925636:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.925637:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.925638:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.925639:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.925641:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.925643:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.925646:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.925646:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.925647:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.925649:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.925649:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.925650:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.925650:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.925655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.925664:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.925667:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.925667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.925668:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.925670:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677471.925672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.925676:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.925677:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.925681:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.925681:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.925682:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.925683:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.925684:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.925700:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.925702:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677471.925703:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677471.925705:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677471.925706:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:1.0:1619677471.925707:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.925708:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.925709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.925709:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.925709:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677471.925710:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.925712:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:1.0:1619677471.925712:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.925714:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677471.925715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:1.0:1619677471.925715:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677471.925718:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.925720:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677471.925723:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.925724:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677471.925725:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.925727:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:1.0:1619677471.925727:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677471.925727:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677471.925728:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677471.925729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677471.925730:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.925731:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)++
00000800:00000200:8.0:1619677471.925731:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677471.925732:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (25)--
00000800:00000200:24.0:1619677471.925733:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)--
00000800:00000200:8.0:1619677471.925733:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (25)--
00000800:00000200:24.0:1619677471.925734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.925734:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.925736:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.925739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677471.926546:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.926548:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677471.926550:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.926553:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.926554:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677471.926555:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.926557:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.926558:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.926559:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677471.926562:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.926564:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677471.926566:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.926567:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677471.926568:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677471.926570:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677471.926570:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.926571:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677471.926572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677471.926950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.926964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677471.927288:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.927290:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.927291:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.927291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.927297:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.927299:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.927430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.927433:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.927436:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677471.927438:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.927440:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.927441:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677471.927443:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.927444:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.927445:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.927446:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677471.927449:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.927451:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677471.927453:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.927454:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.927455:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.927456:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.927457:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.927458:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677471.927459:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677471.927460:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.927463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.927466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.927469:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.927470:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.927471:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.927472:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.927473:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.927475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677471.927946:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677471.927949:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677471.927949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677471.927952:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.927955:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.927956:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677471.927958:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.927959:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.927960:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.927961:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677471.927964:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.927966:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677471.927968:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.927969:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677471.927970:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677471.927972:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677471.927972:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677471.927973:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677471.927973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:20.0:1619677471.927991:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:1.0:1619677471.928360:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.928363:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.928364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.928366:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.928368:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.928369:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.928371:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.928372:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.928374:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.928374:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.928378:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.928379:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.928381:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.928382:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.928382:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.928383:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.928384:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.928385:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.928385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.928390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.928398:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.928401:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.928402:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.928403:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.928406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677471.928410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.928423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677471.928571:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.928573:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.928574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.928576:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.928579:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.928580:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.928582:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.928583:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.928584:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.928585:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.928588:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.928590:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.928592:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.928592:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.928593:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.928595:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.928595:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.928596:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.928596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.928607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.928609:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.928610:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.928611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.928612:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.928613:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.929226:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.929229:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.929230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.929230:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.929231:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.929233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.929247:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.929249:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.929250:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.929250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.929251:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.929252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.929459:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.929462:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.929463:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.929464:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.929466:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.929467:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.929468:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.929469:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.929470:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.929471:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.929474:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.929475:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.929477:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.929477:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.929478:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.929479:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.929479:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.929480:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.929480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.929491:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.929492:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.929493:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.929494:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.929494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.929496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.929773:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.929775:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.929776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.929776:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.929778:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.929779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.929890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.929897:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.929900:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.929901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.929903:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.929905:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.929907:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.929909:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.929911:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.929912:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.929913:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.929917:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.929919:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.929921:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.929922:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.929923:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.929925:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.929926:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.929927:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.929928:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.929932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.929946:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.2:1619677471.930183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.930189:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.930192:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.930193:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.930195:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.930197:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.930198:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.930200:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.930201:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.930202:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.930203:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.930206:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.930207:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.930209:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.930210:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.930210:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.930212:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.930213:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.930214:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.930216:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.930216:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.930221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.930227:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.930228:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677471.930228:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.930229:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.930230:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.930230:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.930231:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.930232:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.930233:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.930233:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.930492:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.930494:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.930495:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.930496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.930497:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.930499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.930732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.930736:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.930738:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677471.930740:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.930742:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.930743:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677471.930745:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.930746:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.930747:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.930748:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677471.930751:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.930753:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677471.930755:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.930756:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677471.930756:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677471.930758:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677471.930759:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.930759:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677471.930760:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677471.930764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.930773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.930775:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.930776:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.930777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.930778:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.930781:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.930825:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.930828:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.930829:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.930829:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.930831:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.930833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.931293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677471.931305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677471.931336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677471.931343:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.0:1619677471.931344:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:24.0:1619677471.931346:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:20.0:1619677471.931346:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:24.0:1619677471.931347:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.931347:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.931347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.931350:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:20.0:1619677471.931350:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.931352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677471.931353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677471.931359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677471.931364:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.931367:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.931368:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.931368:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.931369:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.931370:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677471.931373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677471.931475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677471.931506:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.931509:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.931510:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.931510:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.931512:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677471.931515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.931643:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.931646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.931646:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.931649:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.931650:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.931651:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.931652:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.931653:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.931654:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.931655:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.931659:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.931661:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.931663:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.931664:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.931665:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.931666:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.931667:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.931668:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.931669:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.931670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.931672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.931681:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677471.931682:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677471.931683:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677471.931684:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.931685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677471.931685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.931685:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.931687:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677471.931687:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.931690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677471.931752:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677471.931754:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677471.931757:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.931759:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.931761:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677471.931762:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.931764:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.931765:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.931766:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677471.931769:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.931771:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677471.931774:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.931775:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677471.931776:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677471.931778:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677471.931778:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677471.931779:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677471.931780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677471.932298:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.932301:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.932302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.932302:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.932304:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.932306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.932318:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.932320:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.932321:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.932321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.932322:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.932323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.932803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677471.932809:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.932812:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.932814:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.932814:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.932816:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.932817:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:2.0:1619677471.932820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677471.933135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.933138:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.933140:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677471.933142:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.933145:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.933147:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677471.933148:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.933149:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.933150:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.933151:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.2:1619677471.933155:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:20.0:1619677471.933159:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.933160:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677471.933163:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.933163:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.933164:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.933166:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.933166:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.933167:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677471.933168:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677471.933171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.933184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:13.0:1619677471.933200:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.933202:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.933203:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.933205:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.933207:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677471.933208:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677471.933217:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677471.933220:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677471.933221:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677471.933222:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677471.933222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.933224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677471.933809:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.933812:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.933813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.933814:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.933817:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.933818:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.933820:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.933821:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.933822:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.933823:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.933826:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.933827:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.933830:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.933831:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.933831:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.933833:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.933834:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.933835:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.933835:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.933849:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.933852:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.933854:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.933854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.933856:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.933860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.933919:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.933922:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.933923:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.933925:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.933928:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.933929:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677471.933931:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.933932:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.933933:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.933934:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677471.933938:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.933939:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677471.933943:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.933944:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677471.933944:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677471.933946:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677471.933947:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.933949:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677471.933950:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.933950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:12.0:1619677471.933972:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.933976:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677471.933977:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.933980:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677471.933982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.933983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677471.934014:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677471.934017:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677471.934018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677471.934018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:24.0:1619677471.934166:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.934169:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.934170:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.934170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.934172:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.934174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.934537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.934543:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.934545:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.934547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.934549:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.934551:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.934553:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.934555:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.934558:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.934559:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.934560:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.934564:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.934566:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.934569:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.934570:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.934571:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.934573:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.934574:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.934575:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.934576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677471.934633:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.934635:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.934636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.934636:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.934638:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.934640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677471.934909:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.934912:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.934912:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677471.934915:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.934917:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.934919:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.934920:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.934921:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.934922:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.934923:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.934927:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.934929:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.934932:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.934933:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.934933:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.934935:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.934935:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.934936:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.934937:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.934937:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.934941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.934948:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677471.934949:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677471.934950:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.934950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677471.934951:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677471.934951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677471.934952:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.934954:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.934956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677471.935002:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.935005:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.935006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.935008:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.935010:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.935012:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.935013:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.935015:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.935016:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.935017:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.935020:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.935022:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.935025:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.935026:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.935026:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.935028:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.935029:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.935030:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.935030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.935042:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.935045:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.935046:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.935047:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.935050:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.935053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.935061:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.935064:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.935065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.935065:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.935067:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.935067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.935798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.935807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.935810:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.935811:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.935811:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.935813:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.935816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.936164:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.936166:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.936167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.936168:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.936170:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.936172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.936644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.936655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.936679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.936688:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.936691:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.936691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.936692:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.936694:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.936696:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677471.936921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.936928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.936931:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.936932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.936934:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.936937:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.936938:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.936940:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.936942:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.936944:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.936945:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.936948:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.936950:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.936952:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.936953:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.936955:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.936956:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.936957:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.936958:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.936959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.936970:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.936972:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.936973:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.936974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.936975:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.936976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.937426:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.937428:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.937429:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.937431:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.937433:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.937435:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.937436:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.937438:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.937439:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.937440:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.937443:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.937445:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.937448:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.937448:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.937449:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.937451:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.937451:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.937452:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.937452:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.937457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.937465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.937468:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.937469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.937469:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.937471:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.937474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.937484:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.937486:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.937487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.937487:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.937489:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.937492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.937809:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.937811:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.937812:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.937813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.937814:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.937816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677471.938452:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.938454:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677471.938457:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677471.938459:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.938460:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677471.938461:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.938462:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.938463:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.938464:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677471.938467:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677471.938469:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677471.938471:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.938472:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677471.938473:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677471.938474:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677471.938475:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.938475:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677471.938476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677471.938882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.938892:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.938917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677471.938921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.938928:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677471.938929:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677471.938931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677471.938933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677471.938940:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677471.938944:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677471.938947:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677471.938947:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.938948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.938949:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677471.938950:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677471.938951:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.938951:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677471.938952:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:8.0:1619677471.938952:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.938953:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.938955:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.938956:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.938957:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.938958:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.938961:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.938962:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.938964:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.938965:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.938965:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.938967:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.938967:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.938969:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.938970:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.938971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.938981:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.938982:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.938983:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.938985:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.938985:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.938986:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.938986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.938987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.938987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.938988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.939000:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.939002:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.939003:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.939003:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.939004:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.939004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677471.939654:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.939657:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.939658:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.939658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.939661:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.939666:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.939668:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.939670:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.939671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.939672:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.939673:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.939674:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.939859:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.939862:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.939863:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.939863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.939865:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.939867:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.940097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.940106:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.940108:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.940109:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.940111:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.940114:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677471.940449:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.940452:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.940454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.940454:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.940456:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.940460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677471.940598:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.940601:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.940601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.940604:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.940606:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.940608:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.940609:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.940610:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.940611:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.940612:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.940615:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.940617:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.940619:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.940620:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.940621:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.940623:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.940623:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.940624:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.940624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.940629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.940644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.2:1619677471.940771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.940778:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.940781:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.940782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.940784:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.940786:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.940788:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.940790:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.940791:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.940792:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.940794:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.940798:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.940800:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.940803:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.940804:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.940805:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.940807:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.940808:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.940810:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.940811:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.940811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.940817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.940822:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.940825:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.940826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677471.940826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677471.941219:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.941222:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.941223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.941224:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.941226:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.941228:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.941229:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.941230:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.941231:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.941232:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.941235:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.941237:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.941240:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.941240:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.941241:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.941243:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.941243:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.941244:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.941244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.941257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677471.941259:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:2.0:1619677471.941260:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:20.0:1619677471.941261:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:18.0:1619677471.941261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.941261:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.941264:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.941264:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:20.0:1619677471.941267:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.941267:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:20.0:1619677471.941268:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677471.941270:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.941271:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.941272:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.941273:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677471.941275:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677471.941277:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677471.941279:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.941280:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677471.941281:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677471.941282:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677471.941283:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.941284:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677471.941285:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677471.941286:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677471.941567:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.941570:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.941571:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.941571:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.941573:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.941575:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677471.942733:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.942736:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.942737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677471.942739:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.942742:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.942743:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677471.942744:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.942746:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.942746:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.942747:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677471.942750:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.942752:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677471.942755:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.942756:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677471.942756:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677471.942758:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.942758:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.942760:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677471.942761:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677471.942761:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677471.942773:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:17.0:1619677471.942773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677471.942775:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677471.942775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677471.942776:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:17.0:1619677471.942776:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:18.0:1619677471.942777:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.942777:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.942780:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.942782:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.943424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.943427:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.943428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.943429:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.943432:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.943434:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.943435:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.943436:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.943437:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.943438:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.943442:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.943444:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.943446:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.943447:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.943448:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.943449:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.943450:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.943450:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.943451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.943455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677471.943463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677471.943464:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.943467:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.943468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.943468:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.943469:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677471.943470:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.943472:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.943473:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677471.943473:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.943474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000400:00000200:2.0:1619677471.943475:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.943478:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.943479:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.943481:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677471.943483:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000400:00000200:2.0:1619677471.943483:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.943484:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677471.943485:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677471.943485:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677471.943486:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.943486:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.943488:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.943488:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:2.0:1619677471.943489:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.943491:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.943493:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.943494:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.943494:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.943496:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.943497:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.943498:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.943499:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.943511:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.943513:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.943514:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.943515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.943516:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.943518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.943667:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.943669:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.943670:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.943670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.943672:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.943674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.943695:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.943697:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.943698:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.943698:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.943699:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.943702:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.943976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.943987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677471.943987:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.943990:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.943993:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677471.943995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.943996:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.943999:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677471.944000:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96515523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.944002:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.944003:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.944004:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677471.944009:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.944011:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677471.944013:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.944014:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677471.944015:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677471.944016:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.2:1619677471.944016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:13.0:1619677471.944017:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.944018:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677471.944018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677471.944024:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.944026:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.944027:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.944027:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.944029:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.944032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.944349:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.944352:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.944353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.944354:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.944357:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.944358:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.944360:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.944361:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.944362:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.944363:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.944366:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.944368:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.944369:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.944370:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.944370:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.944371:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.944372:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.944373:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.944373:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.944383:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.944385:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.944386:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.944387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.944387:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.944389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.944402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.944405:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.944405:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.944406:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.944406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.944407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.944625:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39ea00] (21)++
00000800:00000200:8.0:1619677471.944628:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.17@o2ib3
00000800:00000200:24.0:1619677471.944629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39ea00] (22)--
00000400:00000200:8.0:1619677471.944630:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.17@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.944632:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.944634:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677471.944635:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.944636:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.944637:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.944637:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677471.944640:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.944642:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677471.944645:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.944646:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.0:1619677471.944646:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677471.944648:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677471.944649:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39ea00] (21)++
00000800:00000200:8.0:1619677471.944649:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39ea00] (22)--
00000800:00000200:8.0:1619677471.944650:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39ea00] (21)--
00000800:00000200:20.0:1619677471.944662:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677471.944664:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677471.944666:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.944667:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.944670:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677471.944841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.2:1619677471.945140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.945147:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.945150:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.945151:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.945152:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.945155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677471.945201:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.945204:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677471.945205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677471.945207:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.945209:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.945211:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677471.945213:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.945216:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.945217:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.945218:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677471.945223:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.945225:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677471.945229:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.945230:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677471.945231:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677471.945233:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677471.945234:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677471.945236:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677471.945237:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677471.945238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:24.0:1619677471.945248:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677471.945249:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:24.0:1619677471.945251:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:18.0:1619677471.945251:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000400:00000200:24.0:1619677471.945252:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.945252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:8.0:1619677471.945253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677471.945253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:24.0:1619677471.945254:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.945257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.945268:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.945271:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.945272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.945272:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.945273:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.945274:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.945506:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.945508:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.945509:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.945509:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.945511:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.945513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677471.946014:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.946016:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.946019:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.946021:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.946021:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.946022:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677471.946023:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.946024:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.946025:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.946026:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677471.946029:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.946031:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677471.946034:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.946034:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677471.946035:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677471.946036:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677471.946037:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.946038:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.946038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.0:1619677471.946465:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:17.0:1619677471.946824:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.946827:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.946828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.946830:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.946832:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.946833:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.946834:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.946835:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.946837:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.946838:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.946841:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.946842:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.946845:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.946846:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.946846:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.946848:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.946849:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.946849:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.946850:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.946855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.946863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.946865:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.946866:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.946868:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.946871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677471.946873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.946887:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677471.947178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.947185:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.947188:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.947188:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.947191:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.947193:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.947195:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.947196:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.947198:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.947199:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.947200:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.947204:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.947206:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.947209:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.947210:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.947211:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.947213:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.947214:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.947215:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.947215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677471.947961:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.947964:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.947965:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.947966:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.947968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.947969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677471.947982:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677471.947984:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677471.947985:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677471.947986:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677471.947986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677471.947988:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677471.948209:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.948211:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.948212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.948214:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.948217:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.948218:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.948220:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.948221:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.948222:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.948223:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.948226:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.948228:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.948231:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.948232:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.948232:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.948234:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.948234:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.948236:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.948237:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.948237:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.948240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.948249:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677471.948250:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.948252:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.948252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677471.948252:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.948253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677471.948253:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.948254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677471.948256:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.948259:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677471.948600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677471.948734:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.948737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.948737:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.948738:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.948740:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.948743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.949463:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.949466:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.949467:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.949468:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.949471:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.949473:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.949474:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.949475:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.949476:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.949477:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.949480:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.949482:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.949485:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.949485:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.949486:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.949488:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.949488:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.949489:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.949490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.949505:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.949509:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.949511:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.949513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.949516:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.949521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.949533:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.949538:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.949539:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.949540:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.949542:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.949543:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.949688:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.949692:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.949694:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.949695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.949698:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.949703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.949865:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.949868:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.949869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.949869:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.949871:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.949873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677471.950104:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.950107:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677471.950110:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677471.950112:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.950114:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677471.950115:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.950116:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.950117:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.950118:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677471.950122:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677471.950123:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677471.950126:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.950127:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677471.950128:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677471.950130:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677471.950130:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.950132:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677471.950133:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677471.950133:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677471.950141:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.950144:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.950145:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.950145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.950147:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677471.950150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.950413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.950424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.950449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.950459:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.950462:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.950464:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.950464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.950466:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.950468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.950470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677471.950477:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.950481:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677471.950482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.950482:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.950483:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677471.950484:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677471.950563:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.950566:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.950567:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.950567:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.950570:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.950573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677471.950582:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677471.950585:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677471.950586:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.950587:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.950588:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677471.950589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.950640:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.950642:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.950643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.950644:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.950648:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.950649:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677471.950651:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.950652:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.950653:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.950654:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677471.950656:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.950658:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.950659:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.950660:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677471.950661:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.950662:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.950662:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.950663:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.950663:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677471.950673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.950675:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.950676:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.950677:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.950677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.950678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.950695:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.950698:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.950698:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.950701:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.950702:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.950703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.950712:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.950714:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.950715:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.950715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.950717:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.950719:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677471.950996:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.950999:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.951000:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.951000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.951002:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.951004:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:18.0:1619677471.951006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.951006:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.951007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.951007:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.951010:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.951011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677471.951339:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.951342:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.951344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.951344:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.951346:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.951351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677471.951580:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677471.951582:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677471.951584:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.951587:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.951589:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677471.951590:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.951591:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.951592:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.951593:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677471.951597:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.951598:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677471.951601:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.951602:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677471.951603:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677471.951605:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677471.951605:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677471.951607:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677471.951608:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677471.951608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677471.951639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.951646:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.951648:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.951649:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.951651:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677471.951654:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.951656:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.951657:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.951659:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.951660:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.951661:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.951665:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677471.951667:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.951670:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.951671:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.951672:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677471.951674:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.951675:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.951676:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.951677:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677471.952095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.952100:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.952103:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.952104:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.952106:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.952110:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.952111:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.952112:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.952114:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.952115:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.952116:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.952119:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.952121:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.952122:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.952123:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.952124:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.952126:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.952127:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.952128:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.952128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.952155:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.952157:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.952157:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.952158:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.952159:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.952161:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.952824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.952833:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.952836:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.952836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.952837:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.952839:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.952841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.952946:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.952948:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.952949:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.952949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.952951:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.952953:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677471.953470:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.953473:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.953474:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.953476:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.953479:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.953480:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.953482:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.953483:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.953484:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.953485:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.953488:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.953489:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.953492:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.953493:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.953494:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.953495:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.953496:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.953497:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.953497:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.953502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.953517:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677471.953799:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677471.953802:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677471.953803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677471.953805:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.953807:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.953809:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677471.953810:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.953812:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.953813:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.953814:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677471.953817:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.953819:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677471.953822:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.953823:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677471.953824:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677471.953826:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677471.953826:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677471.953827:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677471.953827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:20.2:1619677471.954104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677471.954110:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.954113:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677471.954114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677471.954116:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.954118:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.954120:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677471.954122:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.954124:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.954125:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.954127:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677471.954130:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.954132:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677471.954136:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.954137:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677471.954138:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677471.954141:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677471.954142:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677471.954143:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677471.954143:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:13.0:1619677471.954155:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677471.954158:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677471.954159:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.954161:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.954163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677471.954164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:18.0:1619677471.954305:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.954308:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677471.954309:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677471.954311:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.954314:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.954316:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677471.954318:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.954320:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.954321:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.954322:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677471.954326:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.954327:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677471.954331:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.954332:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677471.954332:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677471.954334:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677471.954335:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677471.954336:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677471.954336:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:24.0:1619677471.954734:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.954736:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.954737:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.954739:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.954742:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.954744:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.954745:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.954746:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.954747:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.954748:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.954751:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.954753:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.954756:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.954756:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.954757:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.954759:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.954759:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.954760:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.954760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.954772:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.954775:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.954776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.954776:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.954779:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.954782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.954973:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.954976:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.954976:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.954976:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.954978:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.954981:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.954992:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.954995:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.954996:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.954996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.954997:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.954997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677471.955267:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.955270:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677471.955272:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.955275:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.955276:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677471.955277:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.955279:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.955280:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.955281:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677471.955285:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.955286:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677471.955289:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.955290:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677471.955291:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677471.955293:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677471.955299:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.955300:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677471.955300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677471.955306:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.955309:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.955310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.955310:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.955312:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.955315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.955521:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.955523:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.955524:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.955525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.955526:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.955528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.955624:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.955626:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.955627:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.955628:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.955631:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.955633:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.955634:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.955635:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.955636:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.955637:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.955640:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.955641:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.955644:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.955645:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.955645:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.955647:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.955647:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.955649:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.955650:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.955650:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677471.955662:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677471.955663:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.955665:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:18.0:1619677471.955665:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677471.955666:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677471.955666:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.955666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.955667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677471.955668:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.955672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.955681:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.955684:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.955685:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.955685:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.955686:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.955687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.955790:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.955793:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.955794:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.955794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.955796:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.955798:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.956284:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677471.956286:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677471.956287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677471.956289:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.956291:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.956292:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677471.956299:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.956300:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.956301:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.956302:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677471.956305:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.956306:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677471.956309:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.956309:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677471.956310:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677471.956311:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677471.956312:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677471.956312:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677471.956313:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:20.2:1619677471.956406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.956416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.956442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.956451:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.956454:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.956455:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.956455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.956457:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.956459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.956809:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.956812:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.956813:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.956813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.956814:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.956817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.957669:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677471.957671:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677471.957672:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677471.957674:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677471.957676:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.957677:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677471.957678:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.957680:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.957680:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.957681:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677471.957684:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677471.957686:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677471.957689:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.957689:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677471.957690:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677471.957691:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677471.957692:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677471.957693:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677471.957694:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677471.957695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677471.957707:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677471.957708:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677471.957709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677471.957710:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:12.0:1619677471.958222:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.958226:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:13.0:1619677471.958230:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.958230:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677471.958233:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.958236:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:12.0:1619677471.958238:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.958240:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.958242:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.958243:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:12.0:1619677471.958249:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.958251:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677471.958255:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.958256:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:12.0:1619677471.958257:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:12.0:1619677471.958259:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:12.0:1619677471.958260:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.958261:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.958262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677471.958539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.958546:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.958549:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.958550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.958551:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.958555:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.958556:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.958557:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.958559:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.958560:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.958561:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.958564:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.958566:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.958568:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.958569:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.958570:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.958571:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.958572:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.958574:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677471.958575:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.958576:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.958581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.958586:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.958588:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.958589:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677471.958589:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.958590:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677471.958591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.958591:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.958593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.958593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677471.958594:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677471.958876:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.958878:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.958879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.958879:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.958881:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.958883:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677471.959348:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.959350:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.959351:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.959353:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.959356:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.959357:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.959358:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.959360:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.959361:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.959362:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677471.959363:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:1.0:1619677471.959365:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.959366:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:1.0:1619677471.959367:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677471.959368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.959368:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.959369:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677471.959370:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.959370:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.959371:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.959373:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:2.0:1619677471.959374:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.959374:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.959375:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.959376:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.959377:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.959380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.959387:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677471.959388:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677471.959389:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.959390:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677471.959391:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677471.959391:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677471.959392:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.959394:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.959396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677471.959544:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.959547:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.959548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.959550:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.959552:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.959554:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.959555:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.959557:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.959558:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.959559:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.959562:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.959564:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.959567:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.959568:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.959568:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.959570:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.959570:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.959571:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.959571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.959576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.959584:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.959587:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.959588:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.959588:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.959590:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.959593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.959968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.959982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.2:1619677471.960001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.960007:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.960010:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.960011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.960013:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.960015:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.960017:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677471.960018:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.960020:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.960021:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.960022:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677471.960026:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.960028:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677471.960031:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.960033:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677471.960033:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677471.960035:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:18.0:1619677471.960036:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677471.960036:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677471.960037:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:18.0:1619677471.960037:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.960038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:20.0:1619677471.960039:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677471.960041:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.960043:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677471.960044:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.960045:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.960046:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.960047:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677471.960050:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677471.960052:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677471.960055:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.960056:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677471.960056:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677471.960058:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677471.960059:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.960060:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677471.960060:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677471.960071:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.960073:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.960074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.960074:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.960076:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.960079:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.960086:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.960089:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.960089:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.960090:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.960091:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.960092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.960292:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.960298:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.960298:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.960301:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.960304:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.960305:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.960306:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.960308:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.960309:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.960309:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.960313:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.960315:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677471.960318:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.960318:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.960319:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.960321:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.960321:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.960322:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.960323:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.960336:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.960339:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.960340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.960341:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.960343:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.960347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.960554:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.960556:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.960557:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.960557:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.960559:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.960561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.960572:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.960574:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.960575:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.960575:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.960576:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.960577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.960674:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.960676:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.960677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.960677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.960679:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.960681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.960973:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.960976:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.960977:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.960977:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.960979:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.960980:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.961505:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.961508:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.961509:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.961510:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.961513:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.961514:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.961515:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.961517:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.961518:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.961518:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.961521:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.961523:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.961526:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.961527:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.961527:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.961529:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.961529:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.961530:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.961530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.961545:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.961547:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.961548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.961549:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.961551:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.961554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677471.961687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.2:1619677471.961728:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.961732:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.961735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.961735:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.961736:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.961737:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:8.0:1619677471.961738:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:17.0:1619677471.961740:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.961740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.961741:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.961741:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:17.0:1619677471.961743:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.961746:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.961787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.961795:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677471.961798:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677471.961799:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.961801:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.961804:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677471.962074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.962084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.962109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.962112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.962114:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677471.962116:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677471.962117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677471.962117:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677471.962118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677471.962120:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.962121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.962121:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.962123:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.962125:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.962190:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677471.962193:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677471.962194:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677471.962196:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677471.962198:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.962200:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677471.962201:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.962202:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.962203:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.962204:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677471.962208:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677471.962209:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677471.962212:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.962213:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677471.962213:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677471.962215:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677471.962216:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677471.962216:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677471.962217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:13.0:1619677471.962446:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.962449:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677471.962451:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677471.962453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677471.962454:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.962455:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677471.962457:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677471.962458:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677471.962459:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677471.962460:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677471.962463:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677471.962465:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677471.962468:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677471.962468:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677471.962469:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677471.962471:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677471.962471:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677471.962472:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677471.962474:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677471.962474:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677471.962477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.962486:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:12.0:1619677471.962488:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:17.0:1619677471.962489:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.962489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.962490:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677471.962490:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677471.962491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:17.0:1619677471.962492:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.962492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677471.962496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.962496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.962504:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.962506:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.962507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.962507:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.962509:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.962509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.963355:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.963358:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.963359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.963359:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.963361:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.963363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677471.963458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677471.963464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.963467:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677471.963468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677471.963469:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.963472:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.963474:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677471.963475:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.963476:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.963477:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.963478:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677471.963481:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.963483:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677471.963486:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.963487:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677471.963488:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677471.963490:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677471.963491:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677471.963491:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677471.963492:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:24.0:1619677471.963502:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.963505:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.963506:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.963506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.963508:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.963511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677471.963533:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.963536:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.963538:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677471.963540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.963541:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.963542:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677471.963543:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.963545:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.963546:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.963547:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677471.963551:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.963553:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677471.963555:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.963556:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677471.963556:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677471.963558:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677471.963558:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.963559:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.963560:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677471.963694:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.963697:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.963698:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.963700:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.963702:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.963704:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.963705:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.963707:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.963708:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.963709:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.963712:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.963714:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.963716:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.963719:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.963720:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.963722:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.963722:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.963723:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.963723:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.963728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.963736:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.963738:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.963739:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.963740:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.963743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677471.964543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.964549:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.964552:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.964553:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.964555:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.964558:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.964560:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677471.964562:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.964564:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.964565:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.964566:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677471.964570:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.964572:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677471.964575:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.964577:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677471.964578:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677471.964580:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677471.964580:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.964582:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.964584:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.964584:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.964590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.964596:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.964599:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.964600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.964601:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677471.965360:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.965363:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.965364:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.965365:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.965368:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.965370:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.965371:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.965372:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.965373:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.965374:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.965378:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.965379:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.965382:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.965383:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.965383:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.965385:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.965386:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.965387:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.965388:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.965389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.965391:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.965399:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677471.965402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677471.965402:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.965403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.965404:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677471.965404:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.965405:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.965406:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677471.965406:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.965409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.965836:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.965838:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.965839:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.965840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.965841:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.965844:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.965856:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.965858:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.965859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.965860:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.965861:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.965861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.965918:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.965920:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.965921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.965921:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.965923:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.965925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.966166:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.966168:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.966169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.966171:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.966173:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.966174:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.966175:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.966177:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.966177:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.966178:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.966181:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.966182:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.966184:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.966184:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.966185:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.966186:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.966186:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.966187:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.966187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.966197:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.966199:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.966200:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.966201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.966201:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.966203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.966313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.966328:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677471.966334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.966342:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.966344:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677471.966345:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.966346:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677471.966347:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677471.966413:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.966415:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.966416:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.966416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.966418:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.966420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.966431:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.966433:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.966434:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.966434:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.966435:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.966435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.966819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.966826:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.966829:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.966830:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.966832:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.966835:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.966836:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.966838:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.966841:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.966842:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.966843:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.966846:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.966848:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.966850:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.966851:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.966852:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.966854:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.966855:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.966855:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.966856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.966867:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.966869:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.966870:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.966871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.966872:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.966873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677471.966887:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.966890:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.966891:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.966891:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.966891:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.966892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.967175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.967178:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.967179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.967180:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.967182:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.967184:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.967185:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.967186:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.967187:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.967188:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.967191:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.967193:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.967195:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.967196:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.967197:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.967198:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.967199:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.967200:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.967200:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.967212:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.967215:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.967216:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.967216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.967218:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.967222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.967538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.967541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.967541:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.967542:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.967543:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.967546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677471.967886:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.967888:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677471.967891:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.967893:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.967895:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677471.967896:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.967897:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.967898:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.967899:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677471.967902:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.967904:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677471.967907:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.967908:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677471.967908:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677471.967910:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677471.967911:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.967913:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677471.967914:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677471.967914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677471.967922:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.967925:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.967926:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.967926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.967928:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.967931:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.967957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.967967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.967992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.968001:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.968003:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.968004:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.968004:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.968006:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.968009:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677471.968011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677471.968019:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.968022:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677471.968022:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.968023:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.968024:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.968024:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677471.968058:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.968061:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.968062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.968064:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.968066:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.968067:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.968069:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.968070:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.968071:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.968072:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.968075:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.968077:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.968080:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.968080:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.968081:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.968083:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.968083:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.968084:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.968085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.968089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.968096:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.968099:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.968100:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.968102:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.968104:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677471.968110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.968118:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.968121:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.968121:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.968122:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.968124:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.968127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677471.969043:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.969046:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.969047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.969048:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.969050:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.969054:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677471.969200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.969208:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.969211:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.969212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.969213:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.969215:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677471.969218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:1.0:1619677471.969218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677471.969229:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677471.969257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677471.969260:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677471.969262:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:13.0:1619677471.969262:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:20.0:1619677471.969263:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677471.969263:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:13.0:1619677471.969265:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677471.969266:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677471.969267:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.969269:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.969270:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677471.971160:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.971163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.971163:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677471.971165:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.971168:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.971170:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.971171:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.971173:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.971174:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.971175:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.971179:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.971181:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.971188:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.971189:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.971190:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.971191:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.971192:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.971193:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.971194:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.971198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.971204:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.971206:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:17.0:1619677471.971206:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:24.0:1619677471.971207:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.971208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.971209:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.971209:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.971210:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.971210:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.971212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:17.0:1619677471.971212:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.971215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677471.971719:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.971722:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677471.971724:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677471.971727:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677471.971728:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677471.971728:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677471.971729:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677471.971731:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677471.971732:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677471.971733:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677471.971737:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.971738:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677471.971741:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677471.971742:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677471.971742:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677471.971744:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677471.971745:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677471.971746:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677471.971746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677471.971762:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.971765:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.971766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.971767:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.971769:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.971773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.971900:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.971903:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.971905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.971905:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.971908:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.971911:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677471.972144:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677471.972145:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:20.0:1619677471.972146:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:24.0:1619677471.972149:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:20.0:1619677471.972149:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677471.972151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:20.0:1619677471.972152:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.972154:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677471.972155:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.972155:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.972157:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.972158:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.972158:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.972159:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677471.972160:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677471.972160:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:24.0:1619677471.972162:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.972163:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.972163:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677471.972164:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:24.0:1619677471.972166:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677471.972166:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677471.972167:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.972167:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.972169:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677471.972169:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677471.972169:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677471.972170:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677471.972170:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677471.972171:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677471.972172:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:1.0:1619677471.972172:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677471.972173:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.972173:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000400:00000200:1.0:1619677471.972173:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677471.972174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:1.0:1619677471.972174:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.972175:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677471.972180:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677471.972183:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.972183:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677471.972186:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.972187:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.972188:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677471.972188:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:24.0:1619677471.972190:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.972190:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677471.972190:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677471.972191:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:24.0:1619677471.972192:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677471.972192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:24.0:1619677471.972193:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.972195:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.972195:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677471.972197:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677471.972198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677471.972201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677471.972204:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.972207:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.972208:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.972209:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.972210:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.972211:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677471.972212:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677471.972213:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677471.972213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:17.0:1619677471.972214:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.972214:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.972217:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.972218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.972220:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.972224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.972224:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677471.972225:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.972228:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.972228:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677471.972229:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.972229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.972230:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.972230:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:8.0:1619677471.972232:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.972234:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677471.972236:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000400:00000200:8.0:1619677471.972237:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677471.972239:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677471.972239:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677471.972240:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.972240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.972241:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000400:00000200:8.0:1619677471.972241:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677471.972242:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:8.0:1619677471.972242:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.972248:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.972251:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.972254:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.972255:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.972256:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.972258:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.972259:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.972261:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.2:1619677471.972264:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.972269:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.972270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677471.972273:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.972276:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.972277:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.972277:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.972279:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.972282:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.972283:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.972287:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.972288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.972290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677471.972776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.972782:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.972785:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.972786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.972787:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.972790:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.972792:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.972793:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.972795:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.972796:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.972798:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.972801:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.972803:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.972805:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.972806:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.972808:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.972810:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.972810:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.972811:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.972812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.972817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.972836:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677471.972869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.972872:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.972873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.972873:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.972875:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.972877:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.972894:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.972896:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.972897:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.972897:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.972898:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.972899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677471.972958:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.2:1619677471.972960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.972961:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677471.972963:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.972965:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.972967:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677471.972968:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.972969:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.972970:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.972971:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677471.972974:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.972976:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.2:1619677471.972978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677471.972979:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.972980:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677471.972981:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677471.972982:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677471.972983:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677471.972984:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677471.972984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677471.972986:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.972988:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677471.972989:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677471.972990:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677471.972990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677471.972992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677471.973340:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.973344:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.973344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.973346:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.973348:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.973350:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.973351:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.973352:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.973353:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.973354:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.973358:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.973360:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.973363:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.973364:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.973365:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.973366:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.973367:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.973368:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.973368:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.973373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.973387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:18.0:1619677471.973643:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.973645:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.973646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.973647:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.973649:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.973653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.974044:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07de000] (21)++
00000800:00000200:1.0:1619677471.974047:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.1@o2ib3
00000800:00000200:17.0:1619677471.974048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (22)--
00000400:00000200:1.0:1619677471.974050:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.1@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.974053:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.974055:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:1.0:1619677471.974056:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.974057:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.974058:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.974059:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:1.0:1619677471.974063:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.974065:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:1.0:1619677471.974068:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.974069:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:1.0:1619677471.974069:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677471.974071:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:1.0:1619677471.974072:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07de000] (21)++
00000800:00000200:1.0:1619677471.974073:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07de000] (22)--
00000800:00000200:1.0:1619677471.974073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (21)--
00000800:00000200:24.2:1619677471.974077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677471.974081:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677471.974084:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677471.974085:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.974085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677471.974087:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.974090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.2:1619677471.974137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.974146:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677471.974149:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677471.974150:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.974152:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.974155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677471.974447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.974456:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.974458:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677471.974459:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.974460:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.974462:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.974465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677471.974499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.974510:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.974535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.974543:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.974545:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.974546:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.974546:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.974548:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.974550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.974670:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.974673:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.974674:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.974674:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.974676:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.974678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677471.975499:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.975502:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677471.975505:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677471.975508:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.975509:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677471.975511:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.975512:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.975513:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.975514:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677471.975517:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677471.975519:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677471.975522:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.975522:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677471.975523:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677471.975525:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677471.975526:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.975526:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677471.975527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677471.975666:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.975669:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.975670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.975671:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.975674:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.975676:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.975677:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.975678:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.975679:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.975680:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.975683:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.975685:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.975688:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.975689:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.975689:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.975691:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.975692:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.975692:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.975693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.975706:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.975709:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.975711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.975711:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.975713:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.975716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.976074:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.976076:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.976077:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.976078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.976079:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.976082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677471.976825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.976831:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.976834:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.976835:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.976837:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.976840:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.976841:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.976842:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.976844:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.976845:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.976847:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.976850:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.976852:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677471.976852:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.976854:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.976855:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677471.976855:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.976856:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.976856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.976857:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677471.976857:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.976858:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.976859:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:24.0:1619677471.976860:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.976860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:24.0:1619677471.976861:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.976862:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.976864:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.976865:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.976866:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.976868:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.976870:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677471.976871:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677471.976872:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.976872:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.976873:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677471.976873:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677471.976874:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:2.0:1619677471.976874:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.976875:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.976875:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677471.976875:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.976876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677471.976876:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.976878:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.976885:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.976888:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.976888:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.976889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.976890:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.976891:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.976892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677471.976894:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.976895:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.976895:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.976896:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.976896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.977022:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.977024:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.977025:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.977025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.977027:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.977029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.977301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.977306:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.977310:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.977311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.977312:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.977314:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.977316:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.977318:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.977320:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.977321:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.977322:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.977326:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.977328:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.977332:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.977333:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.977334:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.977336:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.977337:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.977338:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.977339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677471.977365:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.977367:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.977368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.977368:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.977370:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.977372:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.977385:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.977387:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677471.977388:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.977389:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.977389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677471.977390:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.978204:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.978206:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.978207:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.978208:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.978210:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.978211:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.978212:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.978214:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.978214:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.978215:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.978218:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.978220:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.978223:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.978223:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.978224:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.978226:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.978226:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.978227:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.978227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.978232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.978240:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677471.978243:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677471.978244:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.978244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.978246:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.978249:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677471.978543:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677471.978620:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.978623:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.978624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677471.978625:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.978627:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.978630:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677471.978645:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.978648:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677471.978649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677471.978650:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.978653:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.978654:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677471.978656:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.978657:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.978658:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.978659:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677471.978663:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.978665:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677471.978668:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.978668:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677471.978669:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.978671:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677471.978671:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677471.978673:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.978674:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.978674:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.978677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.978685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677471.978685:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.978687:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.978687:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677471.978688:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.978689:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.978689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677471.978690:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.978693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677471.978932:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:13.0:1619677471.979300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a4607a4c00] (21)++
00000800:00000200:13.0:1619677471.979303:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.61@o2ib3
00000400:00000200:13.0:1619677471.979306:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.61@o2ib3 : GET - for me
00000800:00000200:12.0:1619677471.979307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a4607a4c00] (22)--
00000400:00000200:13.0:1619677471.979309:0:19942:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.61@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:13.0:1619677471.979311:0:19942:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.61@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:13.0:1619677471.979313:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a4607a4c00] (21)++
00000800:00000200:13.0:1619677471.979314:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a4607a4c00] (22)--
00000400:00000200:13.0:1619677471.979316:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.61@o2ib3 local destination
00000800:00000200:13.0:1619677471.979317:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7d81ca300] -> 10.51.5.61@o2ib3 (2) version: 12
00000800:00000200:13.0:1619677471.979318:0:19942:0:(o2iblnd.c:1251:kiblnd_query()) peer_ni 10.51.5.61@o2ib3 ffff91a7d81ca300, alive 0 secs ago
00000800:00000200:13.0:1619677471.979319:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a4607a4c00] (21)--
00000800:00000200:8.2:1619677471.979905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.979913:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:24.0:1619677471.979915:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677471.979916:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:24.0:1619677471.979917:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:20.0:1619677471.979917:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.979918:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.979918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677471.979919:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:24.0:1619677471.979921:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.979922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677471.979924:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677471.979928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677471.979936:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.979939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.979940:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677471.979941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:8.0:1619677471.979941:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.979942:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.979942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677471.980648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.980651:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677471.980653:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677471.980655:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.980657:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677471.980658:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.980659:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.980660:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.980661:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677471.980665:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677471.980667:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677471.980669:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.980670:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677471.980670:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677471.980672:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677471.980673:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.980674:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677471.980675:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677471.980676:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.2:1619677471.980809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.980823:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.2:1619677471.981098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.981101:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.981104:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677471.981106:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.981108:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.981110:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677471.981111:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.981112:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.981114:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.981114:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677471.981118:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.981119:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677471.981122:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.981123:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677471.981123:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677471.981125:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677471.981125:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.981126:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677471.981126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677471.981131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.981139:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.981142:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.981143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.981143:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.981145:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.981147:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.981348:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.981350:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.981351:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.981351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.981353:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.981355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.981964:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.981966:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.981967:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.981968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.981969:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.981971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.982245:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.982247:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.982249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.982250:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.982252:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.982253:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.982254:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.982256:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.982256:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.982257:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.982260:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.982262:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.2:1619677471.982262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677471.982265:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.982265:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.982266:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.982268:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.982268:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677471.982268:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677471.982269:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.982270:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.982271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677471.982271:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.982273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677471.982274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:2.0:1619677471.982274:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.982277:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.982279:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.982281:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677471.982282:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:2.0:1619677471.982282:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.982284:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677471.982285:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:2.0:1619677471.982285:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677471.982286:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.982286:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677471.982286:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677471.982289:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.982289:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.982290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.982290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677471.982290:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677471.982292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677471.982292:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677471.982294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677471.982301:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.982302:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.982303:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677471.982303:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677471.982305:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.982306:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677471.982306:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.982307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677471.982307:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.982308:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677471.982308:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.982309:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677471.982310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.982311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.982325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677471.982335:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.982338:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677471.982339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677471.982340:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.982342:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677471.982345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.0:1619677471.982934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.982937:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677471.982939:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677471.982941:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.982943:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677471.982944:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.982945:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.982946:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.982947:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677471.982950:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677471.982951:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677471.982954:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.982955:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677471.982956:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677471.982957:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677471.982958:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677471.982959:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677471.982959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677471.982980:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.982983:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.982984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.982984:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.982986:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677471.982989:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.983304:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.983306:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.983307:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.983307:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.983308:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.983310:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677471.983439:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677471.983457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.983463:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.983467:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677471.983468:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677471.983469:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.983472:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.983474:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.983475:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.983477:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.983478:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.983480:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677471.983483:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677471.983485:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.983488:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.983489:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677471.983489:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677471.983491:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.983492:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677471.983493:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.983494:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677471.983505:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.983508:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.983509:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.983509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.983511:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.983514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.983641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.983644:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.983647:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677471.983649:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.983652:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.983653:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677471.983655:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.983656:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.983657:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.983658:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677471.983662:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.983665:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677471.983668:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.983669:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677471.983669:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677471.983670:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677471.983671:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677471.983671:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677471.983673:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677471.983673:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.0:1619677471.983673:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677471.983674:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677471.983675:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677471.983677:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677471.983679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677471.983680:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.983683:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.983684:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677471.983685:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:8.0:1619677471.983686:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.983687:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677471.983688:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:8.0:1619677471.983688:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.983689:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.983689:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677471.983689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.983692:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:8.0:1619677471.983693:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677471.983695:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677471.983699:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.983701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.983703:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677471.983703:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.983703:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.983704:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677471.983704:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.983704:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677471.983705:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.983706:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677471.983706:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.983706:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677471.983707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.983707:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.983708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.983711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.983719:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.983722:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.983723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.983723:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.983725:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.983727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.983765:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.983767:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.983768:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.983768:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.983770:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.983773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677471.984014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677471.984046:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.984049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.984049:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.984050:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.984052:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.984054:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.984078:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.984080:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.984081:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.984081:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.984082:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.984083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.984750:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.984752:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.984754:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.984755:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.984758:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.984760:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677471.984760:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.984762:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.984763:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.984764:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677471.984766:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.984768:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:8.0:1619677471.984768:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677471.984771:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:8.0:1619677471.984771:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.984772:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:1.0:1619677471.984772:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.0:1619677471.984773:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:17.0:1619677471.984774:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677471.984775:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677471.984775:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.984775:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677471.984776:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.984776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677471.984777:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.984778:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.984780:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.984781:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.984782:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.984785:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.984787:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677471.984790:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:17.0:1619677471.984790:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.984790:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.984791:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.984792:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677471.984794:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677471.984794:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.984794:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.984795:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677471.984796:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.984796:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:24.0:1619677471.984797:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677471.984799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677471.984800:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.2:1619677471.984800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677471.984803:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.984807:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.984807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.984809:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677471.984810:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.984812:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.984813:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.984814:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677471.984816:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:24.0:1619677471.984819:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.984819:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677471.984820:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:24.0:1619677471.984822:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677471.984822:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677471.984822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677471.984825:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.984826:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:24.0:1619677471.984827:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:8.0:1619677471.984828:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.984829:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:8.0:1619677471.984829:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.984830:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (23)++
00000800:00000200:24.0:1619677471.984831:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (24)--
00000800:00000200:24.0:1619677471.984832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677471.984832:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677471.984833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.984834:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.984836:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677471.984837:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.984838:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.984839:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.984840:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:12.0:1619677471.984842:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:8.0:1619677471.984842:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677471.984844:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:8.0:1619677471.984844:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:12.0:1619677471.984845:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.984845:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677471.984846:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.984846:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677471.984846:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677471.984847:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:12.0:1619677471.984849:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:8.0:1619677471.984849:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677471.984850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677471.984850:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.984851:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.984852:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.984862:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.984864:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.984865:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.984866:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:24.0:1619677471.984867:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.984867:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.984868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.984869:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.984869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.984870:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.985348:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.985351:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677471.985352:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.985353:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.985355:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.985359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677471.985367:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677471.985370:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677471.985370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.985371:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677471.985372:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677471.985373:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677471.985947:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.985950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677471.985950:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.985951:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.985953:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.985955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677471.985957:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.985960:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.985962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.985962:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.985964:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.985966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.986161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.986170:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.986172:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.986173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.986174:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.986176:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.986179:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677471.986356:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677471.986358:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677471.986359:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.986360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677471.986361:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.986363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677471.986375:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677471.986378:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.986378:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677471.986379:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.986379:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677471.986380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677471.986764:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.986766:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.986767:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.986768:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.986770:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.986771:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677471.986772:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.986773:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.986774:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.986775:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677471.986778:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.986779:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.986781:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.986781:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677471.986782:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.986783:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677471.986784:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.986784:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.986784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677471.986795:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.986796:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.986797:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.986798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.986798:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.986800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677471.987368:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677471.987370:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[2] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677471.987373:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : PUT - routed
00000800:00000200:12.0:1619677471.987375:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.987376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:24.0:1619677471.987850:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.987853:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.987854:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.987854:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.987855:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.987858:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.987941:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.987943:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.987944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.987946:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677471.987948:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.987949:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677471.987950:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.987951:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.987952:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.987953:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677471.987956:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677471.987957:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677471.987959:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.987960:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:8.0:1619677471.987961:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:8.0:1619677471.987962:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:8.0:1619677471.987963:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.987963:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677471.987963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677471.987974:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.987977:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677471.987977:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.987979:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: ACK: OK
00000800:00000200:12.0:1619677471.987981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.987982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677471.988051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677471.988058:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.988061:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677471.988061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677471.988064:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.988066:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.988068:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677471.988070:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.988072:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.988073:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.988074:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677471.988078:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.988080:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677471.988084:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.988085:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677471.988086:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677471.988088:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677471.988089:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677471.988090:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677471.988091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.0:1619677471.988593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.2:1619677471.988602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.988605:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.988607:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677471.988609:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.988612:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.988614:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677471.988616:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.988617:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.988618:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.988619:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677471.988622:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.988624:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677471.988626:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.988627:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677471.988628:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677471.988629:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677471.988630:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.988631:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677471.988631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677471.988636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.988644:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.988647:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677471.988648:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677471.988648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.988650:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.988652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677471.988698:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.988701:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.988702:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.988704:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.988706:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.988708:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.988709:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.988711:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.988711:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.988713:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.988716:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.988717:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.988720:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.988721:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.988722:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.988724:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.988724:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.988725:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.988725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.988730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.988737:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.988740:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.988741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.988741:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.988743:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.988746:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.989315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.989317:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.989318:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.989319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.989320:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.989322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677471.989430:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677471.989471:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.989474:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.989475:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.989475:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.989477:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677471.989480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.989491:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677471.989494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.989494:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677471.989495:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.989496:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.989496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677471.989573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.989576:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.989576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.989578:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.989581:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.989582:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.989583:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.989585:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.989586:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.989586:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.989589:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.989591:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.989594:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.989595:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.989595:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.989597:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.989597:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.989598:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.989599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.989603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.989611:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.989613:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.989614:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.989616:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.989619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677471.989674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.989680:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.989683:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.989684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.989686:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.989688:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.989689:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677471.989691:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.989693:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.989694:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.989695:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677471.989698:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.989700:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677471.989703:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.989704:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677471.989705:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677471.989707:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677471.989708:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.989710:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677471.989711:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677471.989712:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677471.989717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.989723:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677471.989726:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677471.989726:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.989727:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677471.990223:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.990226:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.990227:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.990227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.990229:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.990231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.990576:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.990579:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677471.990580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677471.990581:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677471.990584:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.990585:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677471.990586:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.990587:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.990588:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.990589:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677471.990592:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.990593:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677471.990596:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677471.990597:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677471.990598:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677471.990599:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677471.990600:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.990601:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.990602:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677471.990603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677471.990605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.990613:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677471.990616:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677471.990616:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.990617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.990617:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.990618:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.990618:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.990619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677471.990620:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.990622:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677471.990625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677471.990633:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677471.990636:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677471.990637:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.990637:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.990639:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677471.990639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677471.991491:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07de000] (21)++
00000800:00000200:1.0:1619677471.991493:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.1@o2ib3
00000800:00000200:17.0:1619677471.991495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (22)--
00000400:00000200:1.0:1619677471.991496:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.1@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677471.991498:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.991500:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:1.0:1619677471.991501:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677471.991502:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677471.991503:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677471.991504:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:1.0:1619677471.991507:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677471.991509:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:1.0:1619677471.991511:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677471.991512:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:1.0:1619677471.991513:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677471.991515:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:1.0:1619677471.991515:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07de000] (21)++
00000800:00000200:1.0:1619677471.991516:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07de000] (22)--
00000800:00000200:1.0:1619677471.991516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (21)--
00000800:00000200:24.2:1619677471.991521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677471.991525:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677471.991528:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677471.991529:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.991529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677471.991531:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.991534:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:18.0:1619677471.991656:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.991658:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.991660:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.991660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677471.991662:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.991666:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677471.991969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677471.991975:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.991978:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677471.991978:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:8.0:1619677471.991981:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677471.991981:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677471.991983:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:2.0:1619677471.991983:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.991984:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677471.991985:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677471.991986:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677471.991987:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677471.991988:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.991989:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.991991:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677471.991991:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677471.991992:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.991992:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.991995:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677471.991996:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.991997:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677471.991998:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.992001:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:8.0:1619677471.992002:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677471.992004:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.992005:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.992006:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677471.992007:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.992007:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677471.992008:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.992011:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677471.992012:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (23)++
00000800:00000200:2.0:1619677471.992012:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677471.992014:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.992014:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:8.0:1619677471.992015:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677471.992015:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677471.992015:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:8.0:1619677471.992016:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677471.992016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677471.992021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.992027:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000800:00000200:24.0:1619677471.992028:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:18.0:1619677471.992029:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000800:00000200:2.0:1619677471.992029:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:24.0:1619677471.992030:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:18.0:1619677471.992030:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677471.992030:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677471.992031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:24.0:1619677471.992032:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.992032:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.992032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677471.992033:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000400:00000200:24.0:1619677471.992034:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.992034:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000800:00000200:18.0:1619677471.992035:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (24)--
00000400:00000200:2.0:1619677471.992035:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.992036:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677471.992036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677471.992037:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.992038:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.992040:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.992041:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (23)++
00000800:00000200:2.0:1619677471.992041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677471.992042:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (24)--
00000800:00000200:18.0:1619677471.992042:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677471.992043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677471.992286:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.992289:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.992290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.992292:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.992300:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.992301:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677471.992303:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.992304:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.992305:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.992306:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677471.992310:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.992311:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677471.992314:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.992315:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677471.992316:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677471.992317:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677471.992318:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.992318:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.992319:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677471.992451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.992459:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.992463:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.992464:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.992464:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.992466:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.992469:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677471.992734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.2:1619677471.992863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677471.992865:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677471.992868:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677471.992870:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.992873:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.992874:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677471.992876:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.992877:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.992878:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.992879:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677471.992882:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.992884:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677471.992885:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.992886:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677471.992887:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.992889:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677471.992889:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677471.992890:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677471.992891:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677471.992895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.992908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.992943:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07de000] (21)++
00000800:00000200:17.0:1619677471.992946:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.1@o2ib3
00000800:00000200:1.0:1619677471.992947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (22)--
00000400:00000200:17.0:1619677471.992949:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.1@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.992951:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.992952:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:17.0:1619677471.992954:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.992955:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.992956:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.992957:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:17.0:1619677471.992961:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.992963:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:17.0:1619677471.992966:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.992967:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:17.0:1619677471.992967:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677471.992969:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:17.0:1619677471.992970:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07de000] (21)++
00000800:00000200:17.0:1619677471.992971:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07de000] (22)++
00000800:00000200:17.0:1619677471.992973:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07de000] (23)--
00000800:00000200:17.0:1619677471.992973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (22)--
00000800:00000200:24.2:1619677471.992975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677471.992978:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677471.992981:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677471.992982:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.992982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677471.992984:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.992984:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07de000] (22)++
00000800:00000200:1.0:1619677471.992986:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07de000] (23)--
00000800:00000200:24.0:1619677471.992987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:1.0:1619677471.992987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (22)--
00000800:00000200:1.0:1619677471.992988:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07de000] (21)--
00000800:00000200:24.2:1619677471.992996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677471.992999:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677471.993001:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677471.993002:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677471.993002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677471.993003:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677471.993004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:17.0:1619677471.993475:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.993478:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.993479:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.993482:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.993484:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.993486:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.993487:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.993488:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.993490:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.993490:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.993494:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.993496:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.993498:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.993499:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.993499:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.993501:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.993502:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.993502:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677471.993503:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677471.993508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677471.993516:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.993519:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677471.993520:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.993521:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.993524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677471.993526:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677471.993526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677471.993529:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:12.0:1619677471.993843:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677471.993846:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677471.993846:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677471.993848:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677471.993850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677471.993851:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677471.994108:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.994110:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677471.994111:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677471.994111:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.994113:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.994115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.994293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677471.994300:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.994302:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677471.994303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677471.994305:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.994308:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.994309:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677471.994310:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.994312:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.994313:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.994314:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677471.994317:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.994319:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677471.994323:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.994323:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677471.994324:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677471.994327:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677471.994328:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677471.994329:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677471.994329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677471.994421:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.2:1619677471.994484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677471.994486:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.994489:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677471.994491:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.994493:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.994494:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677471.994496:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.994497:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.994498:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.994499:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677471.994502:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.994504:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677471.994506:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.994507:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677471.994508:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677471.994509:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677471.994510:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677471.994511:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677471.994512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677471.994516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677471.994524:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677471.994527:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677471.994528:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677471.994528:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.994530:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.994532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677471.994605:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.994607:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677471.994608:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677471.994610:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.994612:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.994613:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677471.994615:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.994616:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.994617:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.994618:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677471.994621:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.994622:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677471.994625:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.994626:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677471.994627:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677471.994628:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677471.994629:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677471.994629:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677471.994630:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677471.995072:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.995075:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.995076:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.995077:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.995080:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.995082:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677471.995083:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.995084:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.995085:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.995086:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677471.995089:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.995091:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677471.995093:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.995094:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677471.995095:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677471.995096:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677471.995097:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.995098:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.995098:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677471.995103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677471.995110:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677471.995113:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677471.995114:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677471.995114:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677471.995117:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677471.995120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677471.995182:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.995184:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677471.995185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.995185:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.995187:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.995189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677471.995611:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677471.995614:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677471.995615:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.995616:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677471.995618:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.995621:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677471.995946:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.995949:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.995950:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.995950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.995952:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.995954:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677471.995966:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677471.995968:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677471.995969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.995969:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677471.995970:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.995971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677471.996825:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.996827:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.996828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.996829:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.996832:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.996834:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.996835:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.996836:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.996837:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.996838:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.996841:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.996842:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.996845:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.996846:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.996846:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.996848:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.996848:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.996849:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677471.996849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677471.996863:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.996866:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677471.996867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677471.996868:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.996870:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677471.996874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677471.997024:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677471.997027:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677471.997027:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677471.997028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677471.997029:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.997032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677471.997559:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.997562:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677471.997562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677471.997565:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677471.997567:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.997569:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677471.997570:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677471.997572:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677471.997573:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677471.997574:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677471.997577:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677471.997579:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677471.997582:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677471.997582:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677471.997583:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677471.997585:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677471.997585:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677471.997587:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677471.997588:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677471.997589:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677471.997592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677471.997600:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677471.997602:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677471.997603:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677471.997604:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.0:1619677471.997609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677471.997764:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.997767:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677471.997768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677471.997769:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677471.997772:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677471.997774:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677471.997775:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677471.997776:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677471.997777:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677471.997778:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677471.997781:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677471.997783:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677471.997786:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677471.997787:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677471.997787:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677471.997789:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677471.997790:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677471.997791:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677471.997792:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677471.997793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677471.997804:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677471.997805:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677471.997807:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677471.997807:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677471.997808:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677471.997809:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677471.997809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677471.997809:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.997811:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.997814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677471.998142:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.998145:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.998146:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.998146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.998147:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.998150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677471.998357:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f783000] (21)++
00000800:00000200:20.0:1619677471.998360:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.13.4@o2ib3
00000400:00000200:20.0:1619677471.998362:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.4@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677471.998365:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.998366:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677471.998368:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677471.998369:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677471.998370:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677471.998371:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677471.998375:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677471.998376:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677471.998379:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677471.998380:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677471.998380:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677471.998382:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677471.998382:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946f783000] (21)++
00000800:00000200:20.0:1619677471.998383:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946f783000] (22)--
00000800:00000200:20.0:1619677471.998384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (21)--
00000800:00000200:8.2:1619677471.998388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677471.998397:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677471.998399:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677471.998400:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677471.998401:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677471.998403:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677471.998405:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677471.998612:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.998615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677471.998615:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677471.998616:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677471.998617:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677471.998619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677471.998632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677471.998638:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.998641:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677471.998642:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677471.998644:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677471.998647:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677471.998648:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677471.998650:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677471.998652:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677471.998653:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677471.998654:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677471.998658:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677471.998660:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677471.998662:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677471.998663:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677471.998664:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.998666:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.998667:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677471.998668:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677471.998669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677471.998679:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677471.998681:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677471.998682:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677471.998683:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677471.998684:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677471.998686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677471.998697:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677471.998699:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677471.998702:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677471.998703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.998704:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677471.998705:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677471.998706:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677471.998875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.998883:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677471.998885:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677471.998887:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677471.998888:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677471.998891:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677471.998897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677471.998911:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677471.998942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677471.998948:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.998951:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677471.998952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677471.998954:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677471.998956:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677471.998957:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677471.998959:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677471.998961:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677471.998962:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677471.998963:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677471.998967:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677471.998969:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677471.998971:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677471.998972:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677471.998973:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677471.998975:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677471.998976:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677471.998977:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677471.998977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677471.998981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677471.998994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677471.999915:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677471.999918:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677471.999919:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677471.999919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677471.999921:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677471.999923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.000028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.000036:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.000039:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.000040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.000042:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.113@o2ib7 : GET - for me
00000400:00000200:1.0:1619677472.000045:0:19931:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.113@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1619677472.000047:0:19931:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.113@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:1.0:1619677472.000048:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.000050:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.000051:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677472.000055:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : REPLY try# 0
00000800:00000200:1.0:1619677472.000057:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.000059:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.000059:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.000060:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.000062:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.000062:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.2:1619677472.000067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.000075:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.000077:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.000078:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.000078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.000079:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: REPLY: OK
00000800:00000200:17.0:1619677472.000081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.000107:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.000109:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677472.000110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677472.000111:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.000113:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.000115:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677472.000116:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.000117:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.000118:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.000119:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677472.000122:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.000123:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677472.000125:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.000126:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677472.000126:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677472.000128:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677472.000129:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.000129:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.000130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677472.000142:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.000145:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.000146:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.000146:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.000148:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.000151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677472.000243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.000246:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.000249:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.000251:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.000253:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.000255:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.000257:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.000258:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.000259:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.000260:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.000263:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.000265:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.000267:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.000268:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.000269:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.000270:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.000271:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.000271:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.000272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.000276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.000284:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.000286:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.000287:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.000287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.000289:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.000292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.000296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.000304:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.000307:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.000308:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.000308:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.000309:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.000310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.000433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.000436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.000436:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.000438:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.000441:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.000443:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.000444:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.000446:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.000447:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.000448:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.000451:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.000453:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.000456:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.000457:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.000457:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.000459:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.000460:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.000460:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.000461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.000465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.000473:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.000476:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.000477:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.000477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.000479:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.000482:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.001335:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.001337:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.001338:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.001339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.001340:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.001342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.001355:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.001357:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.001358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.001358:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.001359:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.001359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.001659:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.001692:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:2.0:1619677472.001693:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677472.001694:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.001695:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.001696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677472.001696:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.001697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.001697:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.001699:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.001703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.001971:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.001974:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.001975:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.001975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.001976:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.001979:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.001989:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.001991:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.001992:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.001992:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.001993:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.001994:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.002455:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.002458:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.002459:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.002460:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.002463:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.002464:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.002465:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.002466:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.002467:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.002468:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.002471:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.002472:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.002474:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.002474:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.002475:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.002476:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.002476:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.002477:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.002477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.002489:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.002490:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.002491:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.002492:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.002492:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.002494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.003187:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.003191:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.003191:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.003193:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.003195:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.003197:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.003199:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.003200:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.003201:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.003202:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.003205:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.003207:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.003209:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.003210:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.003211:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.003212:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.003213:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.003213:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.003213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.003218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.003226:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.003228:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.003229:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.003231:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.003234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.003251:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.003254:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.003255:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.003256:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.003259:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.003260:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.003261:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.003263:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.003264:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.003265:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.003268:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.003270:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.003273:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.003274:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.003275:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.003276:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.003277:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.003278:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.003278:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.003282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.003287:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.003289:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:17.0:1619677472.003292:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.003292:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.003298:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.003299:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.003300:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677472.003300:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.003302:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.003302:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677472.003304:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.003305:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.003306:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677472.003309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.003310:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.003310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.003311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.003312:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677472.003315:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.003316:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677472.003317:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677472.003318:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677472.003319:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.003319:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.003320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677472.003334:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.003337:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.003338:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.003339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.003341:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.003343:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.003649:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.003652:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.003653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.003653:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.003655:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.003657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.003998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.004009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.004033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.004036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.004044:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.004045:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.004046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.004048:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677472.004366:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.004369:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.004371:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.004371:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.004373:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.004377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.004386:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.004388:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677472.004389:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.004390:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.004391:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.004391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677472.004455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.004463:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.004466:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.004467:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.004469:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.004471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.005007:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.005010:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.005011:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.005011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.005012:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.005015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.006005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.006013:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.006016:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.006017:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.006018:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.006020:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.2:1619677472.006023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.2:1619677472.006023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.006027:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.006028:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.006029:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677472.006031:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.006032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.006032:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:1.0:1619677472.006032:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.006033:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.006033:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677472.006034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.006035:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.006038:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.006040:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.006042:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.006044:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.006045:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.006046:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.006050:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.006053:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.006056:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.006057:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.006057:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.006059:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.006060:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.006061:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.006062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.006073:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.006076:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.006076:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.006077:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:2.0:1619677472.006077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.006078:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.006079:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.006080:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.006080:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.006081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:8.0:1619677472.006082:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.006084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.006097:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.006100:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.006101:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.006101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.006102:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.006102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.006156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.006166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.006191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.006200:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.006202:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.006203:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.006203:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.006206:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.006208:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.006838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.006844:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.006847:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.006848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.006850:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.006852:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.006854:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.006855:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.006856:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.006857:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.006858:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.006862:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.006864:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.006867:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.006868:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.006869:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.006871:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.006872:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.006873:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.006874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.007201:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.007204:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.007205:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.007205:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.007207:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.007209:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.007315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.007317:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.007319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.007320:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.007323:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.007324:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.007326:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.007327:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.007328:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.007329:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.007333:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.007335:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.007335:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.007336:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.007337:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.007338:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677472.007338:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.007339:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.007339:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677472.007341:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677472.007341:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.007341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)--
00000800:00000200:8.0:1619677472.007342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.007343:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.007345:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.007345:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.007355:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.007357:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.007357:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.007359:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.007359:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.007360:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.007360:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.007361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.007361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.007362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677472.007920:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.007922:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.007923:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.007925:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.007928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.007946:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.008329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.008347:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677472.008352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.008360:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.008362:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.008363:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.008364:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.008365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.2:1619677472.008407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.008414:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.008417:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.008418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.008421:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.008423:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.008425:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677472.008427:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.008429:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.008430:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.008431:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677472.008435:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.008437:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677472.008440:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.008441:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677472.008442:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.008444:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.008445:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.008446:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.008446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.008450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.008458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.008461:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.008462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.008462:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.008464:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.008467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.008879:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.008882:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.008883:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.008885:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.008887:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.008888:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.008890:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.008891:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.008892:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.008893:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.008896:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.008898:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.008901:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.008901:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.008902:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.008903:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.008904:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.008905:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.008905:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.008917:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.008920:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.008921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.008921:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.008924:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.008927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.009259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.009262:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.009264:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.009266:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.009269:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.009270:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.009272:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.009273:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.009274:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.009275:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.009278:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.009280:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.009283:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.009284:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.009284:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.009286:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.009287:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.009287:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.009288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.009292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.009306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677472.009334:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.009337:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.009338:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.009338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.009340:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.009342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.010163:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.010165:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.010166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.010168:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.010170:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.010172:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.010173:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.010174:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.010175:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.010176:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.010179:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.010180:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.010183:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.010184:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.010185:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.010186:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.010187:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.010187:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.010188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.010192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.010201:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.010204:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.010205:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.010205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.010207:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.010210:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.010340:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.010343:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.010343:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.010345:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.010347:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.010349:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.010350:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.010351:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.010352:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.010353:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.010356:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.010357:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.010360:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.010360:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.010361:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.010363:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.010363:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.010364:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.010364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.010369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.010378:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.010380:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.010381:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.010383:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.010385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.011037:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.011039:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.011040:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.011040:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.011042:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.011044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.011058:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.011061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.011061:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.011062:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.011063:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.011063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.011567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.011580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.011680:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.011682:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.011684:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.011687:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.011689:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.011690:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.011691:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.011692:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.011693:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.011697:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.011699:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.011701:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.011702:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.011703:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.011704:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.011705:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.011706:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.011706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677472.011995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.012002:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.012005:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.012006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.012008:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.012011:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.012012:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.012014:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.012016:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.012017:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.012018:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.012022:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.012024:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.012027:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.012028:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.012029:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.012031:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.012032:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.012033:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.012035:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.012035:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.012037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.012040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.012046:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.012048:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.012049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.012050:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677472.012051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677472.012056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.012064:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.012066:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.012067:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.012068:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.012069:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:12.0:1619677472.012225:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.012227:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.012228:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.012230:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.012232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.012234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677472.012245:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.012248:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.012249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.012250:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.012253:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.012254:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.012255:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.012256:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.012257:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.012258:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.012260:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.012262:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.012263:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.012264:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.012265:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.012266:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.012266:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.012267:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.012267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.012278:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.012279:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.012280:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.012281:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.012282:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.012283:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.012578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.012582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.012584:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.012586:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.012589:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.012591:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.012592:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.012594:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.012595:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.012596:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.012600:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.012601:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.012604:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.012605:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.012605:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.012607:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.012607:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.012608:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.012609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.012614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.012622:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.012625:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.012626:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.012626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.012628:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.012631:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.013098:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.013100:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.013101:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.013102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.013103:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.013105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.013464:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.013466:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.013467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.013467:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.013469:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.013474:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677472.013498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.013501:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.013503:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.013503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.013506:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.013509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:13.0:1619677472.013596:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.013599:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.013600:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.013601:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.013603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.013605:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677472.013707:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.013710:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.013711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.013711:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.013714:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.013718:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677472.013722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.013730:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.013732:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:1.0:1619677472.013733:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677472.013734:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:17.0:1619677472.013734:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.013734:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.013735:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.013736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:1.0:1619677472.013737:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.013738:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.013741:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.013742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677472.014121:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.014124:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.014125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.014126:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.014129:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.014132:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.014143:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.014146:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.014147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.014147:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.014149:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.014150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.014213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.014219:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.014222:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.014223:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.014224:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.014227:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.014229:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677472.014231:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.014232:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.014233:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.014235:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677472.014238:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.014240:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.014244:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.014245:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.014246:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677472.014248:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677472.014248:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.014250:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677472.014252:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677472.014252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677472.014257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.014263:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677472.014265:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677472.014266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.014267:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677472.014412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.014423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.014455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.014457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.014460:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.014462:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.014463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.014464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677472.014464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.014466:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.014467:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.014468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.014470:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.014473:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.014474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.014481:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.014484:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.014485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.014488:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.014491:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.014493:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.014494:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.014496:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.014497:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.014498:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.014502:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.014504:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.014506:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.014507:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.014508:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.014510:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.014511:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.014512:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.014513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.014523:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.014526:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.014527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.014527:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.014529:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.014530:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677472.014562:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.014565:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.014566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.014568:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.014570:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.014571:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.014573:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.014574:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.014575:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.014576:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.014580:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.014582:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.014584:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.014585:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.014586:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.014588:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.014588:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.014589:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.014589:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.014594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.014601:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.014603:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.014604:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.014606:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.014608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.014612:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.014615:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.014616:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.014616:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.014618:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.014620:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677472.014767:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.014770:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.014770:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.014772:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.014774:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.014775:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677472.014783:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677472.014786:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.014787:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.014787:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677472.014788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.014789:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677472.014996:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.014999:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.015000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.015001:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.015003:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.015004:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.015005:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.015007:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.015007:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.015008:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.015011:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.015013:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.015015:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.015016:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.015017:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.015018:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.015019:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.015020:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.015022:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.015022:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.015033:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.015035:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.015036:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677472.015037:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677472.015037:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.015038:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.015038:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.015039:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.015041:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.015044:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.015474:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.015476:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.015477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.015477:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.015479:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.015481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.015497:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.015499:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.015500:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.015500:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.015501:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.015502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.015574:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.015576:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.015578:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.015578:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.015581:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.015585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.015886:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.015889:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.015890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.015892:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.015895:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.015896:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677472.015898:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.015899:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.015900:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.015901:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677472.015905:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.015906:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677472.015909:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.015910:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677472.015910:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677472.015912:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677472.015913:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.015914:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.015914:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677472.016058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.016072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.2:1619677472.016455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.016462:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.016465:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.016466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.016468:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.016470:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.016472:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.016473:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.016475:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677472.016475:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677472.016476:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677472.016478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677472.016478:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677472.016478:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.016481:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.016483:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677472.016484:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.016486:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:8.0:1619677472.016486:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.016487:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.016488:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.016489:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:8.0:1619677472.016489:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677472.016490:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:8.0:1619677472.016490:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.016491:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.016492:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:8.0:1619677472.016494:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.016495:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.016496:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677472.016496:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.016497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.016498:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677472.016503:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.016504:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.016505:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.016506:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677472.016506:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.016507:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.016507:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677472.016508:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:8.0:1619677472.016508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677472.016509:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.016511:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.016514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:2.0:1619677472.016521:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.016523:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.016525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.016525:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.016527:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.016530:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.016540:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.016543:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.016544:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.016544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.016545:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.016546:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.016906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.016921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.2:1619677472.017268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677472.017318:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.2:1619677472.017320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:2.0:1619677472.017321:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.017323:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.017323:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.017325:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.017329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.017545:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.017547:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:18.0:1619677472.017547:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000400:00000200:24.0:1619677472.017548:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.017548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.017550:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.017550:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.017550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.017552:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.017553:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:18.0:1619677472.017555:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.017556:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677472.017558:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.017559:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.017560:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.017561:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677472.017565:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.017567:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.017570:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.017571:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.017571:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677472.017573:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677472.017574:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.017575:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.017576:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:8.0:1619677472.017776:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.017779:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.017780:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.017782:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.017785:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.017786:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.017787:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.017789:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.017790:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.017791:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.017794:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.017795:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.017798:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.017799:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.017800:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.017802:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.017802:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.017803:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.017803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.017808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.017817:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.017819:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.017820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.017821:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.017823:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.017826:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.017827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.017835:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677472.017837:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:17.0:1619677472.017838:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.017838:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.017839:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:20.0:1619677472.017840:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:17.0:1619677472.017840:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.017840:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:20.0:1619677472.017842:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.017844:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.017846:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.017847:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.017848:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.017849:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.017850:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.017853:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.017855:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.017858:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.017858:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.017859:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.017861:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.017862:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.017863:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.017864:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.017864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677472.017873:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.017876:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.017877:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.017877:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.017879:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.017882:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.018410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677472.018443:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.018446:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.018447:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.018448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.018449:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.018453:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.018519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.018522:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.018525:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.018527:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.018530:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.018532:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.018533:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.018534:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.018535:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.018536:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.018539:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.018540:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.018543:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.018544:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.018544:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.018546:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.018546:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.018547:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.018548:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.018553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.018561:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.018563:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.018565:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.018565:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.018567:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.018569:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.018571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.018578:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.018580:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.018581:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.018581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.018582:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.018583:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677472.018740:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.018743:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.018743:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.018745:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.018747:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.018749:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.018750:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.018751:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.018752:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.018753:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.018756:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.018757:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.018760:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.018761:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.018761:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.018763:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.018763:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.018765:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.018766:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.018766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.018769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.018777:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.018778:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.018779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.018780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.0:1619677472.018784:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677472.018792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.018800:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.018802:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.018803:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.018804:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.018805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677472.019282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.019289:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.019292:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.019293:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.019297:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.019299:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.019302:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.019337:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.019339:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.019340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.019340:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.019342:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.019344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.019494:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.019496:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.019497:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.019498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.019499:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.019501:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.019517:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.019520:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.019521:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.019521:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.019522:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.019522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.019559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.019561:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.019562:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.019562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.019564:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.019566:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.020343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.020346:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.020349:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.020351:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.020353:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.020355:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.020356:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.020357:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.020359:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.020360:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.020363:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.020365:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.020367:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.020368:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.020368:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.020370:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.020371:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.020371:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.020372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.020377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.020386:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.020388:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.020389:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.020389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.020392:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.020394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.020820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.020827:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.020830:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.020831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.020833:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.020835:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.020837:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.020838:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.020839:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.020840:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.020841:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.020845:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.020846:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.020848:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.020849:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.020850:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.020852:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.020852:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.020854:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.020855:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.020855:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.020861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.020867:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.020869:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.020869:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.020869:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.020870:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.020870:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.020871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.020872:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677472.020872:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.020873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.021172:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.021175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.021175:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.021177:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.021180:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.021182:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.021183:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.021184:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.021185:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.021186:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.021189:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.021191:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.021193:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.021194:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.021195:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.021197:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.021197:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.021198:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.021198:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.021210:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.021213:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.021214:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.021214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.021217:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.021220:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.021231:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.021234:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.021235:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.021235:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.021236:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.021236:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.021715:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.021717:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.021718:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.021718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.021720:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.021722:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.021737:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.021739:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.021740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.021740:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.021741:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.021742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.021756:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677472.021787:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.021790:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.021791:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.021791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.021793:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.021795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.022837:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.022840:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.022841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.022842:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.022845:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.022847:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.022848:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.022849:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.022850:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.022851:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.022854:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.022855:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.022857:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.022857:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.022858:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.022859:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.022860:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.022861:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.022862:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.022863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.022874:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.022875:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.022876:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.022877:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.022878:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.022879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.022879:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.022880:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.022880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.022881:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.023446:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.023448:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.023449:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.023449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.2:1619677472.023450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:24.0:1619677472.023451:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.023453:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677472.023457:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.023460:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.023461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.023463:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.023465:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.023467:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.023468:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.023469:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.023470:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.023471:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.023475:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.023477:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.023479:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.023480:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.023481:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.023483:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.023484:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.023485:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.023485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.023524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.023529:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.023532:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.023533:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.023535:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.023538:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.023539:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677472.023541:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.023542:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.023543:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.023544:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677472.023547:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.023549:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.023551:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.023552:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677472.023552:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.023554:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.023555:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.023556:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.023556:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.023561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.023569:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.023572:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.023573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.023573:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.023575:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.023577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.023585:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.023587:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.023588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.023590:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.023592:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.023593:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.023595:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.023596:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.023597:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.023598:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.023601:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.023603:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.023606:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.023606:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.023607:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.023609:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.023609:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.023610:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.023610:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.023623:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.023626:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.023626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.023627:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.023629:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.023632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.024781:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.024783:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.024784:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.024784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.024785:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.024788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.025173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.025182:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.025184:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.025186:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.025187:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.025191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.025192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.025206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.025501:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.025503:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.025504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.025504:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.025506:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.025508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.025518:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.025520:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.025521:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.025521:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.025522:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.025523:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.025540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.025550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.025575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.025583:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.025585:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.025586:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.025588:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.025590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.025844:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.025847:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.025847:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.025848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.025849:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.025851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.026444:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677472.026620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.026629:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.026632:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.026633:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.026634:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.026635:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.026638:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.026763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.026770:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.026773:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.2:1619677472.026775:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677472.026775:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:20.0:1619677472.026776:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.026779:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.026779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.026779:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:20.0:1619677472.026782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:1.0:1619677472.026782:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.026783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677472.026785:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.026787:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.026788:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.026790:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.026792:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.026793:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.026803:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.026805:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.026808:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.026809:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.026810:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (23)++
00000800:00000200:1.0:1619677472.026812:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677472.026813:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.026814:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.026814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.026815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.2:1619677472.026821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.026826:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:20.0:1619677472.026827:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:20.0:1619677472.026828:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.026830:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.026832:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (24)--
00000800:00000200:20.0:1619677472.026833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677472.026834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677472.027195:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.027197:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.027198:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.027198:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.027200:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.027202:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.027746:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.027748:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.027749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.027751:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.027753:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.027755:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.027756:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.027757:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.027758:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.027759:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.027762:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.027764:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.027766:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.027766:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.027767:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.027768:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.027768:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.027769:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.027769:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.027780:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.027782:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.027782:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.027783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.027784:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.027785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.028653:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.028655:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.028656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.028658:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.028660:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.028661:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.028662:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.028663:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.028664:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.028664:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.028667:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.028669:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.028670:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.028671:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.028671:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677472.028672:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:8.0:1619677472.028672:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.028673:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.028673:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.028673:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677472.028675:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.028676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.028678:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.028680:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.028682:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677472.028683:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.028684:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:17.0:1619677472.028684:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.028685:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677472.028686:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:17.0:1619677472.028686:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677472.028687:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.028688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.028690:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:17.0:1619677472.028690:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.028692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.028692:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677472.028696:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.028697:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677472.028698:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677472.028700:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677472.028701:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.028702:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677472.028703:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677472.028704:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:12.0:1619677472.028714:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:1.0:1619677472.028714:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677472.028716:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677472.028717:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:12.0:1619677472.028718:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:1.0:1619677472.028718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:12.0:1619677472.028719:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.028722:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.028723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.028726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677472.029019:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.029021:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.029022:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.029022:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.029024:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.029026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.029037:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.029039:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.029040:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.029041:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.029041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.029042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.029219:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.029222:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.029223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.029223:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.029224:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.029227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.029238:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.029240:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.029241:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.2:1619677472.029241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677472.029241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.029242:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.029243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.029243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.029249:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.029251:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.029251:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.029252:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677472.029254:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.029255:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.029257:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.029258:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.029259:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677472.029260:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.029262:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.029263:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.029264:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677472.029266:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.029267:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.029269:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.029269:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677472.029270:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.029271:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.029272:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677472.029276:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.029277:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.029280:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.029280:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.029281:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.029281:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677472.029282:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.029282:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677472.029284:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.029285:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.029285:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677472.029285:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.029286:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.029286:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.029287:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.029288:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677472.029288:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.029289:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677472.029290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677472.029297:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677472.029300:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:18.0:1619677472.029301:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:1.0:1619677472.029301:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:18.0:1619677472.029303:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.029304:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.029304:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677472.029305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:17.0:1619677472.029306:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.029307:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677472.029308:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677472.029309:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.029310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:17.0:1619677472.029311:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.029312:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.029313:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677472.029316:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677472.029318:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677472.029321:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.029322:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:17.0:1619677472.029323:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677472.029325:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677472.029325:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677472.029326:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677472.029326:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677472.029327:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:5.2:1619677472.029334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.029340:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677472.029342:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677472.029343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677472.029343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:20.0:1619677472.030664:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.030667:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.030669:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.030671:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.030673:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.030674:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.030676:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.030677:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.030678:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.030681:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.030683:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.030685:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.030686:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.030687:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.030688:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.030689:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.030690:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.030690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.030803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.030812:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.030815:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.030816:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.030816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.030819:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.030821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.030824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.030825:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.030827:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.030828:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.030829:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.030829:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.030830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.030928:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.030930:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.030931:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.030931:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.030933:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.030935:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.031312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.031323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.031350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.031358:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.031360:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.031361:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.031362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.031363:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.031366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.031838:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78b90000] (21)++
00000800:00000200:24.0:1619677472.031841:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.26@o2ib3
00000800:00000200:8.0:1619677472.031842:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000400:00000200:24.0:1619677472.031844:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.26@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.031845:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.031847:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:24.0:1619677472.031849:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.031850:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.031851:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.031852:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:24.0:1619677472.031855:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.031857:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:24.0:1619677472.031860:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.031861:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:24.0:1619677472.031861:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:24.0:1619677472.031863:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:24.0:1619677472.031864:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78b90000] (21)++
00000800:00000200:24.0:1619677472.031865:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78b90000] (22)--
00000800:00000200:24.0:1619677472.031865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (21)--
00000800:00000200:24.2:1619677472.031869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:8.0:1619677472.031956:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.031958:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.031959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.031959:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.031961:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.031962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.032562:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.032564:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.032566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.032567:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.032569:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.032570:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.032572:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.032573:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.032574:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.032574:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.032577:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.032579:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.032582:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.032582:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.032583:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.032585:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.032585:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.032586:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.032588:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.032588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677472.032600:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.032603:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.032604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.032604:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.032606:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.032610:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.032664:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.032666:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.032667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.032668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.032720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.032728:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.032731:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.032732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.2:1619677472.032749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.032756:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.032759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.032759:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[2] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.032764:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.032776:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.032777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.2:1619677472.032788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.032790:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.032793:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.032795:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.032797:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.032799:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.032800:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96515203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.032802:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.032802:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.032804:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.032807:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.032809:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.032812:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.032812:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.032813:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677472.032815:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:20.0:1619677472.032815:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.032816:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.032817:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677472.032829:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.032831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.032832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.2:1619677472.032913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.032922:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:1.0:1619677472.032925:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (24)++
00000800:00000200:17.0:1619677472.032926:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (25)--
00000800:00000200:1.0:1619677472.032926:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (25)--
00000400:00000200:17.0:1619677472.032927:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.032927:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.032929:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: REPLY: OK
00000400:00000200:1.0:1619677472.032929:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.032932:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.032933:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.032934:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.033114:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.033117:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.033118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.033118:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.033120:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.033122:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.033311:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78b90000] (21)++
00000800:00000200:24.0:1619677472.033313:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.26@o2ib3
00000800:00000200:8.0:1619677472.033314:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000400:00000200:24.0:1619677472.033316:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.26@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.033318:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.033319:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:24.0:1619677472.033320:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.033322:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.033323:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.033324:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:24.0:1619677472.033327:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.26@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.033329:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:24.0:1619677472.033331:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.033331:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:24.0:1619677472.033332:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:24.0:1619677472.033333:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:24.0:1619677472.033334:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78b90000] (21)++
00000800:00000200:24.0:1619677472.033334:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c78b90000] (22)++
00000800:00000200:24.0:1619677472.033336:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78b90000] (23)--
00000800:00000200:24.0:1619677472.033336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000800:00000200:24.2:1619677472.033339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:8.0:1619677472.033349:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78b90000] (22)++
00000800:00000200:8.0:1619677472.033351:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c78b90000] (23)--
00000800:00000200:8.0:1619677472.033352:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (22)--
00000800:00000200:8.0:1619677472.033353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78b90000] (21)--
00000800:00000200:24.2:1619677472.033362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:24.0:1619677472.033384:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677472.033386:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:24.0:1619677472.033387:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.033387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.033389:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.033391:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.033658:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.033660:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.033661:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.033663:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.033664:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.033666:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.033667:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.033668:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.033669:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.033670:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.033673:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.033674:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.033677:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.033678:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.033679:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.033680:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.033681:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.033681:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.033682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.033686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.033695:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.033698:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.033698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.033699:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.033701:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.033704:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.033827:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.033830:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.033830:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.033831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.033832:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.033835:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.033888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.033904:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.033905:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.033906:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.033907:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.033908:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.033909:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.033909:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.033925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.033932:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.033935:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.033935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.033938:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.033940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.033941:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.033943:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.033944:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.033945:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.033946:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.033950:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.033951:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.033954:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.033955:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.033955:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.033957:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.033958:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.033959:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.033961:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.033961:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.033964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.033967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.033973:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:20.0:1619677472.033974:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677472.033975:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.033976:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677472.033976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677472.033977:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.033978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.033979:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.033980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.033982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.034299:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.034302:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.034303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.034305:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.034307:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.034308:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.034310:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.034311:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.034312:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.034313:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.034316:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.034318:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.034319:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.034320:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.034321:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.034322:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.034322:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.034323:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.034324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.034335:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677472.034337:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:24.0:1619677472.034337:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.034338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.034339:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.034340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.2:1619677472.034751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.034759:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.034761:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.034763:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.034765:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.034766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.034768:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.034928:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.034931:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.034932:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.034932:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.034934:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.034935:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.034946:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.034949:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.034950:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.034950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.034951:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.034951:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.035127:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:8.0:1619677472.035129:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:8.0:1619677472.035130:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.035131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.035132:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.035134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.035147:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.035149:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.035150:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)++
00000800:00000200:8.0:1619677472.035150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.035151:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.035152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.2:1619677472.035930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.035939:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.035941:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.035942:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.035944:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.035945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.035947:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.035951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.035959:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.035961:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.035962:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.035963:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.035964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.035965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.036212:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.036215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.036215:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.036219:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.036233:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.036233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.036333:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.036337:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.036337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.036338:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.036340:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: REPLY: OK
00000800:00000200:24.0:1619677472.036343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.037333:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.037336:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.037337:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677472.037340:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.037342:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.037343:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677472.037345:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.037346:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.037347:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.037348:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677472.037351:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.037353:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.037354:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.037355:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.037356:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.037357:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.037357:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.037359:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677472.037360:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677472.037360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.2:1619677472.037364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.037373:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.037374:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.037374:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:17.0:1619677472.037375:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.037375:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:17.0:1619677472.037376:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.037376:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.037377:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.037378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.037378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:21.0:1619677472.037616:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.037619:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.037621:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.037621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.037623:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.037624:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.037626:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.037627:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.037628:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.037629:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.037632:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.037634:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.037637:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.037637:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.037638:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.037639:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.037640:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.037642:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.037643:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.037643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.037652:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.037655:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.037655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.037656:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.037710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.037713:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.037715:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.037717:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.037718:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.037720:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.037721:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.037723:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.037724:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.037725:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.037726:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.037729:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.037730:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.037732:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.037733:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.037733:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.037735:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.037736:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.037736:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.037737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.037741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.037749:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.037752:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.037753:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.037754:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.037755:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.037757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.038188:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.038190:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.038191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.038191:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.038193:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.038195:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.038357:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.038360:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.038361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.038362:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.038363:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.038365:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.038366:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.038367:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.038368:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.038369:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.038371:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.038373:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.038374:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.038375:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.038375:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.038376:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.038377:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.038377:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.038378:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.038389:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.038391:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.038391:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.038393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.038393:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.038394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.038533:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.038535:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:24.0:1619677472.038537:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.038537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677472.038539:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.038540:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677472.038541:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.038542:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.038543:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.038544:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677472.038547:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.038548:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677472.038550:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.038551:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677472.038551:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677472.038553:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677472.038553:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.038554:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677472.038554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:20.0:1619677472.038595:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.038599:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.038601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.038602:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.038604:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.038607:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.038608:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.038610:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.038612:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.038613:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.038622:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.038625:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.038628:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.038629:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.038630:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.038631:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677472.038632:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.038633:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.038634:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:17.0:1619677472.038638:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.038640:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.038641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.038643:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.038645:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.038646:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677472.038646:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000400:00000200:17.0:1619677472.038647:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96515181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.038649:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)++
00000400:00000200:17.0:1619677472.038649:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677472.038649:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677472.038650:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:17.0:1619677472.038650:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.038650:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.038651:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.038651:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.038653:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:8.0:1619677472.038653:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:17.0:1619677472.038655:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.038657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.038658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.038658:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.038658:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677472.038660:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.038661:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.038662:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677472.038665:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.038665:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.038666:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.038667:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677472.038681:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.038683:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.038684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.038684:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.038685:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.038686:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.038767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.038774:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677472.038777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.038777:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.038778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.038780:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.038783:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.038784:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.038785:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677472.038785:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:2.0:1619677472.038787:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677472.038787:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.038788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:2.0:1619677472.038788:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.038789:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677472.038789:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.038791:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677472.038793:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.038795:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677472.038795:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677472.038797:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.038799:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.038800:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.038802:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.038803:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.038803:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.038804:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.038816:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.038818:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.038818:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.038819:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.038820:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.038822:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.038839:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.038841:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.038842:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.038843:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.038843:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.038844:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.039061:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.039064:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.039066:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.039066:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.039069:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.039070:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.039071:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.039073:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.039074:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.039074:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.039077:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.039079:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.039081:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.039082:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.039083:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.039085:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.039085:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.039086:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.039087:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.039188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.039190:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.039193:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.039195:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.039196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.039197:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.039198:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.039199:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.039200:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.039201:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.039202:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.039205:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.039207:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.039209:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.039210:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.039210:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.039212:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.039213:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.039214:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.039215:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.039216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.039218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.039221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.039226:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677472.039226:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677472.039229:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677472.039229:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677472.039230:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677472.039230:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.039230:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.039231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677472.039232:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.039235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.039416:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.039418:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.039420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.039421:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.039423:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.039425:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.039426:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.039427:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.039428:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.039429:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.039432:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.039434:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.039435:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.039436:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.039437:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.039438:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.039439:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.039440:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.039441:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.039442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.039453:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.039455:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.039455:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.039457:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.039457:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.039458:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.039459:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.039459:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.039460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.039460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.039704:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.039706:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.039707:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.039707:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.039709:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.039711:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677472.040178:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.040181:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.040182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.040183:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.040185:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.040188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677472.040229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.040237:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.040239:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.040240:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.040242:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.040243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.040245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.040270:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.040273:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.040275:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.040275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.040278:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.040279:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.040280:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.040282:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.040283:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.040284:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.040287:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.040289:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.040291:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.040292:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.040293:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.040299:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.040300:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.040301:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.040302:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.040312:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.040315:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.040316:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.040316:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.040318:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.040321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.040364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.040370:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.040373:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.040374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.040376:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.040378:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.040380:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.040381:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.040383:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.040384:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.040385:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.040389:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.040391:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.040394:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.040395:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.040396:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.040398:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.040399:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.040400:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.040401:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.040531:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.040533:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.040534:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.040534:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.040536:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.040538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.040897:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.040899:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677472.040901:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677472.040902:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.040903:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.040905:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.040906:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.040907:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.040908:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.040908:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.040911:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.040913:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.040916:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.040917:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.040917:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.040919:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.040919:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.040920:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.040920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:18.0:1619677472.040933:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.040935:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.040936:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.040937:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.040939:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.040942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.041104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.041109:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.041113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.041113:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:2.0:1619677472.041115:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.041118:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.041120:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677472.041121:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.041123:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.041124:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.041126:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677472.041129:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.041131:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.041134:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.041135:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677472.041136:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677472.041138:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677472.041139:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.041140:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.041140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.0:1619677472.041330:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.041333:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.041334:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.041336:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.041339:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.041340:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.041341:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.041342:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.041343:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.041344:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.041347:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.041348:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.041351:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.041352:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.041353:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.041354:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.041355:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.041355:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.041356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.041370:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.041372:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.041373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.041374:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.041376:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.041380:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.041437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:8.2:1619677472.042636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677472.042638:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.042639:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.042640:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.042642:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.042642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.042645:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.042645:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.042647:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.042649:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.042651:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.042653:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.042654:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.042656:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.042656:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677472.042656:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.042657:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.042658:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.042659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.042659:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.042660:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.042660:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.042661:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.042662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677472.043116:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.043120:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.043123:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.043125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.043127:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.043129:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677472.043131:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.043134:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.043135:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.043137:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677472.043142:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.043145:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677472.043149:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.043151:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:13.0:1619677472.043152:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:13.0:1619677472.043154:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677472.043155:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.043156:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.043157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677472.043170:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.043172:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.043174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.043174:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.043176:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.043180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.043192:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.043195:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.043196:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.043196:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.043198:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.043198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677472.043615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.043618:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.043621:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.043623:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.043625:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.043626:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.043627:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.043628:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.043629:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.043630:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.043631:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.043634:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.043635:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.043637:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.043638:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.043638:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.043640:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.043641:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.043642:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.043642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.043647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.043656:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.043658:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.043659:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.043661:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.043661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.043663:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677472.043847:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.043850:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.043851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.043853:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.102@o2ib7 : GET - for me
00000400:00000200:2.0:1619677472.043856:0:19932:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.102@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:2.0:1619677472.043859:0:19932:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.102@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:2.0:1619677472.043861:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.043862:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.043864:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677472.043868:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : REPLY try# 0
00000800:00000200:2.0:1619677472.043870:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677472.043872:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.043873:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.043874:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.043876:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.043876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.043889:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.043890:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.043891:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.043892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.043893:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: REPLY: OK
00000800:00000200:18.0:1619677472.043895:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.0:1619677472.043906:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.043909:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.043911:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.043912:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.043913:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.043915:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.043916:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.043918:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.043919:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.043919:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.043922:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.043924:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.043927:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.043927:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.043928:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.043930:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.043930:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.043932:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.043933:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.043933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.043944:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.043946:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.043947:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.043948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677472.044022:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.044025:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.044026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.044027:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.044030:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.044031:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.044032:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.044034:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.044035:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.044035:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.044038:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.044040:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.044044:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.044045:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.044046:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.044047:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.044048:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.044049:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.044049:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.044059:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.044061:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.044061:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.044062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.044063:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.044067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.044069:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.044070:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.044071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.044071:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.044073:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.044074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.044623:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.044625:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.044626:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.044626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.044627:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.044629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.044819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.044828:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.044831:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.044832:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.044832:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.044834:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.044837:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.044838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.044847:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.044849:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.044850:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.044850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.044851:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.044852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.044949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677472.044996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.044999:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.045001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.045001:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.045003:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.045007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.045182:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.045184:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.045185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.045185:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677472.045187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677472.045187:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.045189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.045193:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.045195:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.045197:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.045198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.045200:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.045201:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:21.0:1619677472.045202:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.045203:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.045204:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.045205:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:21.0:1619677472.045205:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.045205:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677472.045205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.045206:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.045206:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:21.0:1619677472.045207:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.045210:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.045211:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.045213:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.045214:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.045214:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.045216:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.045217:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.045218:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.045218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.045222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.045230:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.045232:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.045233:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.045235:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.045236:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.045237:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.045244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.045252:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.045254:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.045255:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.045256:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.045256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.045257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.045451:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.045454:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.045455:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.045457:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.045459:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.045460:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.045462:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.045463:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.045464:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.045465:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.045468:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.045469:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.045472:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.045473:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.045474:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.045475:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.045476:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.045477:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.045477:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.045482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.045491:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.045493:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.045494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.045494:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.045496:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.045499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.046310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.046318:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.046321:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.046322:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.046324:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.046324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.046327:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.046403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.046410:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.046413:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.046413:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.046416:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.046418:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.046419:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677472.046421:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.046422:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.046423:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.046424:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677472.046428:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.046429:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677472.046432:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.046433:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677472.046434:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677472.046436:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677472.046436:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.046437:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.046438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.046602:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.046604:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.046605:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.046606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.046607:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.046610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.047235:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.047238:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.047239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.047240:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.047242:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.047243:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.047245:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.047246:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.047247:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.047248:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.047250:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.047252:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.047255:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.047255:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.047256:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.047258:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.047258:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.047259:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.047261:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.047261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.047263:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.047272:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.047274:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.047275:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677472.047276:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677472.047276:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.047276:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.047277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.047278:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677472.047278:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.047281:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.047654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.047662:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.047665:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.047666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.047666:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.047668:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.047670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.047674:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.047675:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.047677:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.047678:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.047678:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.047679:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.047679:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.047748:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.047752:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677472.047753:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677472.047754:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.047757:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.047759:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677472.047760:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.047761:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.047762:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.047763:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677472.047766:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.047767:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.047770:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.047771:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677472.047771:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677472.047773:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.047773:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.047774:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.047775:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:20.2:1619677472.048337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:24.0:1619677472.048338:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.048340:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.048341:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.048342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.048343:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.048344:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677472.048346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.048346:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.048348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.048350:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.048353:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.048354:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.048356:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.048358:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:2.0:1619677472.048358:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.048359:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.048360:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.048361:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.048362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.048362:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.048363:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.048363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:2.0:1619677472.048364:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.048366:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.048368:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.048369:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.048370:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.048372:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.048372:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.048373:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.048376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.048385:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.048387:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.048388:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.048388:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.048390:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.048391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.048865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.048873:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.048875:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.048876:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.048878:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.048879:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.048881:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677472.048994:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.048997:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.048999:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.048999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.049001:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.049003:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.049004:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.049005:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.049007:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.049008:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.049011:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.049012:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.049015:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.049016:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.049016:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.049018:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.049019:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.049020:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.049021:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.049022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.2:1619677472.049031:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:20.0:1619677472.049034:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.049036:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:2.0:1619677472.049036:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:20.0:1619677472.049037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.049038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:2.0:1619677472.049040:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.049041:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.049043:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.049045:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.049046:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.049048:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.049049:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.049051:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.049051:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.049055:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.049056:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.049059:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.049060:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.049061:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.049063:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.049064:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.049065:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.049065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.049069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.049076:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.049078:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.049079:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.049081:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.049082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.049084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:18.0:1619677472.049309:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.049312:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.049312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.049313:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.049315:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677472.049319:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.049335:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.049338:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.049339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.049340:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.049343:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.049344:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.049345:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.049347:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.049348:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.049348:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.049351:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.049352:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.049354:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.049355:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.049355:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.049356:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.049357:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.049358:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.049358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.049368:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.049370:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.049370:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.049371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.049372:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.049374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.049874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.049880:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.049883:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.049885:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.049885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.049888:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.049889:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.049890:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.049891:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.049893:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.049893:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.049897:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.049898:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.049901:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.049902:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.049902:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.049904:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.049905:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.049905:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.049906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.049921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.049928:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.049931:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:24.0:1619677472.049933:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:20.0:1619677472.049933:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.049934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.049935:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.049936:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.049937:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.049937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.049937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.049939:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.049941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.050036:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.050039:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677472.050039:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:8.0:1619677472.050041:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.050043:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.050045:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677472.050046:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.050047:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.050048:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.050049:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677472.050052:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.050054:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677472.050057:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.050058:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677472.050058:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677472.050060:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677472.050060:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.050061:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677472.050062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677472.050066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.050074:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.050077:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.050078:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.050078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.050080:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.050083:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.050350:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.050352:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.050353:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.050353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.050355:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.050357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.050497:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.050499:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.050500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.050500:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.050502:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.050504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677472.050521:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.050523:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.050526:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.050528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.050529:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.050530:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677472.050531:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.050533:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.050534:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.050534:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677472.050538:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.050539:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677472.050542:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.050542:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677472.050543:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677472.050545:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677472.050546:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.050547:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677472.050548:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677472.050549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.050562:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677472.050564:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677472.050565:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.050566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:21.0:1619677472.050913:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.050915:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.050917:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.050918:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.050919:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.050921:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.050922:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.050924:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.050925:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.050926:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.050929:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.050931:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.050934:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.050935:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.050935:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.050937:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.050938:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.050939:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.050939:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.050962:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.050964:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.050965:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.050965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.050967:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.050970:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.051231:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.051234:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.051234:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.051236:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.051239:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.051240:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.051241:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.051242:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.051243:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.051244:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.051247:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.051248:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.051251:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.051252:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.051253:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.051254:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.051255:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.051255:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.051256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.051261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.051269:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.051272:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.051273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.051274:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.051276:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.051278:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.051427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.051433:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.051436:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.051437:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.051439:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.051442:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.051443:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.051445:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.051446:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.051447:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.051448:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.051451:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.051453:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.051456:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.051457:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.051458:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.051460:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.051460:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.051461:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.051462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.051466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.051475:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.051477:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.051478:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.051480:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.051481:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.051483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.051571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.051576:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.051579:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.051581:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.051582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.051584:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.051585:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.051586:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.051588:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.051589:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.051590:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.051593:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.051595:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.051598:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.051598:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.051599:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.051601:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.051601:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.051602:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.051604:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.051604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.051607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.051609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.051612:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.051614:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.051614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.051615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677472.051615:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.051617:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.051618:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.051619:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.051621:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.051623:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.051918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.051925:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.051928:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.051929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.051929:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.051931:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.051934:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.051937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.051944:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.051947:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.051948:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.051948:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.051949:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.051949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.052214:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.052216:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.052218:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.052219:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.052221:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.052223:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.052224:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.052225:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.052226:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.052228:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.052232:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.052234:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.052238:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.052239:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.052240:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.052241:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.052242:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.052243:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.052244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677472.052406:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.052409:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.052410:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.052410:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.052411:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.052414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.052429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.052432:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.052432:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.052432:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.052433:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.052434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.052909:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.052912:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.052914:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.052914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.052917:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.052918:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.052920:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.052921:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.052922:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.052923:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.052926:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.052928:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.052930:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.052931:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.052931:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.052933:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.052934:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.052935:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.052935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.052989:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.052992:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.052993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.052994:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.052997:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.052999:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.053000:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.053001:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.053002:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.053003:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.053006:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.053008:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.053010:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.053011:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.053012:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.053014:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.053014:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.053015:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.053015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.053020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.053028:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.053031:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.053032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.053032:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.053035:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.053037:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.053087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.053096:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.053098:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.053099:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.053099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.053102:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.053104:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.053797:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.053800:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677472.053801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677472.053803:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677472.053805:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.053806:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.053808:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.053809:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.053810:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.053811:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.053813:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677472.053815:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.053816:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.053817:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.053817:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.053818:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.053819:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.053820:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.053820:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:24.0:1619677472.053831:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.053832:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.053833:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.053834:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.053835:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.053836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.054798:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.054801:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.054802:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.054804:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.054806:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.054807:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.054808:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.054809:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.054810:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.054811:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.054814:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.054815:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.054818:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.054819:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.054820:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677472.054821:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)++
00000800:00000200:24.0:1619677472.054822:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (25)--
00000800:00000200:8.0:1619677472.054822:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (24)--
00000400:00000200:24.0:1619677472.054823:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.054823:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677472.054824:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.054824:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.054825:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.054826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.054826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.054827:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.054829:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.054829:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.054830:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.054831:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.054834:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.054835:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.054837:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.054837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.054839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.055179:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.055181:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.055182:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.055182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.055184:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.055186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.055225:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.055228:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677472.055229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677472.055231:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.055233:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.055235:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677472.055236:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.055237:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.055238:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.055239:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677472.055243:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.055244:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677472.055247:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.055248:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677472.055248:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677472.055250:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677472.055251:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.055251:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.055252:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677472.055264:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.055267:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.055268:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.055268:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.055270:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.055273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677472.055287:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.055289:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.055290:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.055291:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.055292:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.055292:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677472.055555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.055561:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.055563:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.055564:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.055566:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.055568:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.055570:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.055571:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.055572:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.055573:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.055574:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.055577:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.055579:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.055580:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.055581:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.055582:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.055583:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.055584:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.055585:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.055586:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.055586:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.055593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.055599:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.055600:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.055600:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.055601:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.055601:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.055601:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.055602:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.055603:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.055604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.055604:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.055620:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.055622:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.055623:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.055624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.055624:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.055625:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.055960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.055963:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.055965:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.055968:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.055969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.055970:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.055971:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.055973:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.055974:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.055975:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.055976:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.055979:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.055981:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.055983:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.055983:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.055984:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.055985:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.055986:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.055987:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.055987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.055992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.056000:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.056002:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.056003:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.056005:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.056006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.056007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677472.056209:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.056212:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.056212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.056215:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.056217:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.056219:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.056220:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.056221:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.056222:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.056223:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.056227:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.056228:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677472.056228:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.056230:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.056231:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677472.056231:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.056231:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:8.0:1619677472.056231:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.056232:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:8.0:1619677472.056233:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.056234:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.056234:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.056235:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.0:1619677472.056235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.056236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.056240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.056248:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:24.0:1619677472.056249:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677472.056250:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.056251:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.056252:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.056253:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:20.0:1619677472.056253:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.056253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.056254:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.056254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.056254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.056255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.056259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.056267:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.056269:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.056270:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.056270:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.056271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.056272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.056555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.056557:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.056558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.056558:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.056560:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.056563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.056564:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.056566:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.056566:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.056567:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.056568:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.056568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.056805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.056812:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.056815:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.056816:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.056817:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.056819:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.056821:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.056822:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.056824:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.056825:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.056826:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.056829:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.056830:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.056833:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.056834:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.056835:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.056837:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.056837:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.056838:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.056839:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.056840:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.056845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.056851:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.056853:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.056854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.056855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.057223:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.057225:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.057226:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.057228:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.057230:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.057232:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.057233:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.057235:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.057236:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.057237:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.057240:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.057241:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.057244:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.057245:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.057245:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.057247:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.057247:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.057248:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.057249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.057253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.057262:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.057264:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.057265:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.057266:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.057268:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.057271:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.057308:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.057310:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.057312:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.057313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.057314:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.057316:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.057317:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.057319:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.057320:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.057321:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.057324:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.057326:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.057328:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.057329:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.057330:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.057331:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.057332:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.057333:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.057333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.057524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.057533:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.057535:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.057536:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.057537:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.057539:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.057541:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.057559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.057564:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.057566:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.057568:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.057569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.057570:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.057572:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.057573:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.057575:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.057575:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.057577:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.057580:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.057581:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.057584:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.057584:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.057585:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.057587:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.057587:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.057588:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.057589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.057592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.057600:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.057602:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.057602:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.057604:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.057605:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.057606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.057790:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.057792:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.057794:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.057795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.057797:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.057798:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677472.057799:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.057800:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.057801:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.057802:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677472.057805:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.057807:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.057809:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.057809:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677472.057810:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.057811:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.057812:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.057813:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.057813:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.057826:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.057829:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.057830:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.057830:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.057832:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.057835:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.058106:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.058109:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677472.058110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677472.058112:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677472.058114:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.058115:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.058117:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.058118:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.058119:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.058120:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.058122:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677472.058124:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.058125:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.058126:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.058127:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.058128:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.058129:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.058130:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677472.058131:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677472.058132:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.058139:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.058141:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.058142:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.058143:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.058143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.058144:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.058157:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677472.058159:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677472.058159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.058160:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.2:1619677472.058698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.058707:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.058709:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.058711:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.058713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.058713:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.058716:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.058723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.058730:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.058733:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.058734:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.058735:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.058735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677472.058736:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.058737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.058739:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.058740:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.058740:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.058742:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.058744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.059068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.059076:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.059079:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.059080:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.059080:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.059082:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.059085:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.059088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.059096:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.059098:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.059099:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.059099:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.059100:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.059101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.059244:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.059246:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677472.059247:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677472.059249:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.059252:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.059253:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677472.059254:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.059256:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.059257:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.059257:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677472.059260:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.059262:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677472.059264:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.059265:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677472.059265:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677472.059267:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677472.059268:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.059269:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677472.059270:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677472.059270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.059283:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677472.059285:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677472.059285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.059287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.0:1619677472.059541:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.059544:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.059545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.059546:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.059549:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.059550:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.059551:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.059552:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.059553:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.059554:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.059556:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.059557:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.059559:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.059559:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.059560:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.059561:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.059562:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.059562:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.059563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.059573:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.059575:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.059575:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.059577:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.059577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.059579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.059936:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.059939:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.059939:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.059940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.059941:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.059943:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.060266:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.060269:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.060271:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.060272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.060273:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.060275:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.060276:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.060278:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.060279:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.060280:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.060283:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.060285:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.060288:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.060288:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.060289:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.060291:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.060291:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.060293:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.060297:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.060298:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677472.060303:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.060306:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.060307:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.060307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.060308:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:24.0:1619677472.060309:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.060310:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.060311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.060311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.060312:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.060322:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.060325:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.060326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.060326:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.060327:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.060328:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.060903:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.060905:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.060906:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.060906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.060908:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.060910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.061183:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.061186:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677472.061187:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677472.061189:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.061192:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.061193:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677472.061195:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.061197:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.061198:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.061199:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677472.061203:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.061205:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.061209:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.061209:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.061210:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677472.061212:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.061213:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.061214:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.061214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.2:1619677472.061917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.061920:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.061922:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.061925:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.061925:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.061928:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.061929:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.061930:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.061932:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.061933:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.061934:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.061937:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.061939:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.061941:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.061942:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.061942:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.061944:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.061945:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.061945:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.061946:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.061951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.061959:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.061961:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.061962:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.061964:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.061965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.061966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.062180:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.062182:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.062183:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.062184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.062185:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.062187:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.062237:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.062239:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.062240:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.062242:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.062244:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.062245:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.062246:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.062247:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.062248:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.062249:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.062252:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.062253:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.062256:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.062257:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.062257:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.062259:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.062259:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.062260:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.062261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.062265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.062274:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.062277:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.062278:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.062278:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.062280:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.062283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.062291:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.062297:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.062298:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.062300:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.062303:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.062304:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.062306:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.062307:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.062308:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.062309:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.062312:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.062313:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.062316:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.062317:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.062317:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.062319:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.062320:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.062321:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.062321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.062332:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.062335:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.062336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.062336:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.062338:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.062341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.062391:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.062395:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.062395:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.062397:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.062399:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.062400:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.062402:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.062403:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.062404:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.062405:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.062409:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.062410:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.062413:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.062414:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.062414:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.062416:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.062416:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.062417:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.062418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.062422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.062430:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.062432:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.062433:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.062435:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.062435:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.062438:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.062439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.062447:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.062450:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.062451:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.062452:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.062452:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.062453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.062559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677472.062562:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.062565:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:21.0:1619677472.062567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677472.062567:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.062569:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.062571:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677472.062572:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.062573:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.062574:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.062575:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677472.062578:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.062579:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.062582:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.062582:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.062583:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.062585:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.062585:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.062586:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677472.062587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.062591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.062598:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.062601:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.062601:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.062603:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.062604:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.062605:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.062657:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.062660:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.062661:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.062661:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.062663:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.062665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.062887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:18.0:1619677472.062895:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677472.062898:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:2.0:1619677472.062898:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:18.0:1619677472.062900:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.062903:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.062905:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677472.062906:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.062908:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.062909:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.062910:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677472.062913:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.062915:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677472.062917:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.062918:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677472.062919:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677472.062921:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677472.062921:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677472.062922:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:18.0:1619677472.062923:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:20.0:1619677472.062934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.062936:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.062937:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.062939:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.062940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.062942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.063464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.063473:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.063476:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.063477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.063478:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.063480:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.063483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.063568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.063574:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.063577:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.063578:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.063580:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.063583:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.063584:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677472.063586:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.063587:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.063588:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.063590:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677472.063594:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.063595:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677472.063599:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.063600:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677472.063601:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677472.063602:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677472.063603:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.063604:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.063605:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.063851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.063856:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.063859:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.063861:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.063861:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.063863:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.063865:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.063866:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.063867:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.063868:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.063869:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.063872:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.063874:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.063876:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.063877:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.063878:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.063879:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.063880:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.063881:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.063882:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.063883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.063885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.063888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.063890:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.063893:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.063893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677472.063893:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677472.063894:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.063896:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.063897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.063897:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.063899:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.063901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.064025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.064031:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.064034:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.064034:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.064037:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.064040:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.064041:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.064042:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.064044:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.064045:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.064046:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.064048:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.064050:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.064052:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.064053:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.064053:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.064055:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.064055:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.064056:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.064057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.064068:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.064070:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.064070:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.064072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.064072:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.064074:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677472.064608:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.064610:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.064611:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.064613:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.064615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.064616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677472.064700:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.064702:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.064703:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.064703:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.064705:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.064707:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.064719:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.064721:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.064722:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.064722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.064723:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.064724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.064902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.064910:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.064913:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.064914:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.064914:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.064917:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.064920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.065008:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.065010:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.065011:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.065013:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.065016:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.065017:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.065018:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.065019:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.065020:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.065021:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.065023:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.065025:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.065027:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.065027:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.065028:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.065029:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.065029:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.065031:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.065032:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.065033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.065043:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.065044:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.065044:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.065046:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.065047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.065047:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.065047:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.065048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.065048:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.065049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.065211:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.065213:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.065214:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.065214:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.065216:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.065218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:13.0:1619677472.065489:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.065491:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.065492:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.065493:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.065495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.065496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677472.065622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.065625:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.065626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677472.065628:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.065630:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.065632:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677472.065633:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.065634:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.065635:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.065636:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677472.065640:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.065642:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677472.065645:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.065646:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677472.065646:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677472.065648:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677472.065649:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.065649:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.065650:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677472.065661:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.065664:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.065665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.065665:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.065668:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.065670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.066715:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.066717:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.066718:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.066718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.066720:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.066722:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.066792:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.066795:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.066797:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.066799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.066800:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.066801:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.066802:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.066804:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.066805:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.066805:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.066810:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.066813:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.066816:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.066816:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.066817:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.066819:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.066819:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.066820:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.066821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.067291:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.067295:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.067298:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.067299:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.067301:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.067304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.067647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.067653:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.067655:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.067658:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.067658:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.067661:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.067662:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.067664:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.067665:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.067666:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.067667:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.067670:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.067672:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.067674:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.067675:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.067676:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.067677:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.067678:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.067679:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.067680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.067684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.067692:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.067694:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.067695:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.067697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.067697:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.067699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677472.067834:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.067837:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.067838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.067839:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.067841:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.067843:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677472.067845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.067846:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.067847:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.067849:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.067851:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.067852:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.067853:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.067854:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.067855:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.067856:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.067859:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.067860:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.067863:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.067864:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.067865:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.067866:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.067867:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.067867:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.067868:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.067881:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.067883:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.067884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.067885:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.067887:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.067890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.067903:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.067905:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.067907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.067907:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.067908:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.067909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.067947:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.067950:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:17.0:1619677472.067951:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:21.0:1619677472.067952:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.067952:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.067954:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.067954:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.067955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:21.0:1619677472.067956:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.067958:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.067958:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.067960:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.067961:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.067961:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.067962:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.067963:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.067964:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.067966:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677472.067966:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677472.067968:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.067968:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.067969:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.067971:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.067972:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.067972:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:17.0:1619677472.067973:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.067974:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.067975:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:17.0:1619677472.067975:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.067976:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.067977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:17.0:1619677472.067978:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.067979:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.067979:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.067981:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.067982:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.067983:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.067984:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.067985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.067987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.067995:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.067995:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677472.067997:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.067998:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.067998:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.067998:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677472.067999:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.067999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.068001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.068002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.068237:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.068239:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.068240:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.068241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.068242:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.068244:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.068255:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.068258:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.068259:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.068259:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.068260:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.068260:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677472.068501:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.068504:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.068505:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.068507:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.068509:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.068511:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677472.068512:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.068514:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.068515:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.068516:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.068519:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.068521:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677472.068523:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.068524:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677472.068525:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677472.068526:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677472.068527:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.068529:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677472.068530:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677472.068530:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.068542:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677472.068544:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677472.068545:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.068546:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:24.0:1619677472.068927:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.068930:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.068931:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.068931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.068933:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.068935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.068948:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.068950:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.068951:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.068951:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.068952:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.068953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.069025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.069032:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.069034:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.069035:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.069037:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.069041:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.069042:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.069043:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.069044:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.069045:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.069046:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.069049:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.069051:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.069053:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.069054:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.069055:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.069057:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.069057:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.069058:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.069059:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.069063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.069072:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.069074:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.069075:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.069076:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.069077:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.069079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.069321:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.069323:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.069324:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.069325:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677472.069326:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.069326:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.069328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.069329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:8.0:1619677472.069329:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.069330:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.069331:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.069333:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.069334:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.069335:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.069336:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.069337:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.069340:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.069342:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.069344:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.069345:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.069346:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.069347:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.069348:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.069349:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.069349:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.069361:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.069364:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.069365:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.069365:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.069367:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.069370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.069605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.069613:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.069615:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.069616:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.069618:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.069619:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.069621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.069668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.069673:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.069676:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.069678:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.069678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.069680:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.069682:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.069683:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.069685:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.069685:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.069687:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.069690:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.069692:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.069694:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.069695:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.069696:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.069697:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.069698:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.069699:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.069700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.069704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.069712:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.069715:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.069715:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.069716:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.069718:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.069720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.069722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.069729:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.069731:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.069732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.069733:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.069734:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.069735:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677472.069811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.069816:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.069818:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.069819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.069821:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.069823:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.069825:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.069826:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.069827:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.069828:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.069829:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.069832:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.069834:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.069836:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.069837:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.069838:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.069839:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.069840:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.069841:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.069841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677472.069866:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.069869:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.069870:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677472.069872:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.069874:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.069876:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677472.069877:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.069878:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.069880:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.069880:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677472.069883:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.069885:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677472.069887:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.069887:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677472.069888:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677472.069889:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677472.069890:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.069891:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.069891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677472.070437:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.070438:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.070439:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.070440:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:24.0:1619677472.070441:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.070441:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.070441:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.070443:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.070444:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.070445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.070445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.070446:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.070823:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.070826:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.070827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.070829:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.070832:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.070833:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.070834:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.070835:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.070836:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.070837:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.070840:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.070841:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.070842:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.070843:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.070844:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.070845:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.070845:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.070846:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.070846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.070856:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.070858:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.070858:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.070859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.070860:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.070861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677472.071113:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.071116:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.071118:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.071120:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.071121:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677472.071121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.071122:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96515049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.071124:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.071124:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.071125:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677472.071128:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.071130:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677472.071131:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.071132:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677472.071133:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677472.071135:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.071136:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.071136:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.071137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677472.071147:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.071149:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.071150:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.071151:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.071154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.071154:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677472.071191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.071200:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.071202:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.071203:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.071204:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.071206:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.071209:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.071210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.071217:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.071219:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.071220:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.071220:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.071222:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.071222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.072180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.072187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.072190:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.072190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.072193:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.072195:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.072196:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.072197:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.072199:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.072200:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.072201:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.072203:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.072205:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.072207:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.072208:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.072208:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.072210:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.072211:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.072212:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.072213:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.072224:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.072226:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.072227:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.072228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.072229:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.072230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.072244:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.072247:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.072248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.072250:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.072252:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.072254:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.072255:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.072256:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.072257:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.072258:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.072261:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.072263:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.072265:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.072266:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.072267:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.072268:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.072268:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.072269:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.072270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.072276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.072284:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.072287:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.072288:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.072289:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.072290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.072292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.072296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.072303:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.072306:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.072307:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.072308:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.072309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.072310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.072442:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.072445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.072445:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.072446:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.072448:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.072449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.072460:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.072462:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.072463:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.072463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.072464:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.072464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.072484:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.072486:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.072489:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.072489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.072491:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.072493:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.072494:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.072495:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.072496:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.072497:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.072501:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.072502:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.072505:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.072505:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.072506:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.072508:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.072508:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.072509:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.072510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.072973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677472.072976:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.072978:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677472.072980:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.072981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677472.072983:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.072984:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.072986:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.072987:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.072988:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.072989:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.072991:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.072993:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.072995:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.072995:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.072996:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.072998:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.072998:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.072999:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677472.073000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:8.2:1619677472.073004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.073012:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.073015:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.073015:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.073017:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.073017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.073019:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.073383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.073386:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.073388:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.073391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.073391:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.073393:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.073394:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.073395:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.073397:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.073397:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.073398:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.073401:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.073403:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.073404:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.073405:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.073405:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.073407:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.073407:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.073408:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.073409:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.073413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.073421:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.073423:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.073424:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.073425:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.073426:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.073427:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.073455:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.073457:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677472.073458:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677472.073460:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.073463:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.073464:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677472.073466:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.073467:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.073468:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.073469:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677472.073473:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.073475:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677472.073477:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.073478:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677472.073478:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677472.073480:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677472.073481:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.073481:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.073482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677472.073496:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.073498:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.073499:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.073499:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.073501:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.073504:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677472.073930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.073936:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.073939:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.073940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.073942:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.073943:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.073944:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.073946:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.073947:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.073948:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.073949:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.073953:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.073955:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.073957:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.073958:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.073959:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.073960:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.073961:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.073962:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.073962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.073966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.073975:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.073977:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.073978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.073979:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.073980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.073982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.074056:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.074058:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.074059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.074061:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.074063:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.074065:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.074066:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.074068:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.074068:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.074069:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.074072:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.074074:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.074077:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.074078:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.074078:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677472.074079:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677472.074080:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.074080:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677472.074081:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:8.0:1619677472.074082:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:21.0:1619677472.074083:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677472.074083:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.074083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677472.074084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.074085:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677472.074086:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:21.0:1619677472.074087:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.074088:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.074089:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.074090:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.074091:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677472.074094:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:21.0:1619677472.074095:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.074096:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:21.0:1619677472.074097:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677472.074097:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677472.074098:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677472.074098:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.074098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.074099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677472.074100:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.074100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677472.074101:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.074101:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:17.0:1619677472.074101:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.074103:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.074104:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:17.0:1619677472.074104:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.074105:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.074106:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.074107:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677472.074115:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.074115:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677472.074118:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677472.074118:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.074119:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677472.074119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.074120:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.074120:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:24.0:1619677472.074122:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.074125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.074154:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.074156:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.074157:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.074157:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.074159:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.074161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.074292:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.074296:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.074299:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.074300:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.074301:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.074303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.075169:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.075171:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.075172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.075172:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.075174:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.075176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677472.075187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.075190:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.075191:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.075191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.075194:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.075198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677472.075202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.075210:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.075213:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.075214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.075214:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.075216:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.075219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677472.075367:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.075370:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.075371:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.075371:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.075373:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.075377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.075417:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.075420:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.075421:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.075422:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.075425:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.075427:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.075428:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.075430:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.075430:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.075431:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.075434:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.075436:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.075439:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.075440:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.075440:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.075442:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.075442:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.075443:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.075443:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.075448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.075456:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.075459:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.075460:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.075460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.075462:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.075465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677472.075510:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.075513:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.075514:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.075514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.075516:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.075518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677472.075534:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.075537:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.075538:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.075538:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.075539:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.075540:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.0:1619677472.075979:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.075981:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.075984:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.075985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.075986:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.075988:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677472.075989:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.075991:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.075992:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.075993:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.075996:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.075998:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.076001:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.076001:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.076002:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.076004:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.076004:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.076005:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.076007:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.076007:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.076018:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.076020:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.076021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.076021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.076028:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.076031:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.076032:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.076032:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.076034:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677472.076037:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.076039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.076044:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.076046:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.076049:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.076049:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.076051:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.076051:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:21.0:1619677472.076052:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677472.076054:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677472.076055:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.076056:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677472.076056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.076056:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000400:00000200:21.0:1619677472.076057:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677472.076057:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000400:00000200:21.0:1619677472.076058:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677472.076058:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:21.0:1619677472.076062:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.076064:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.076066:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.076067:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.076067:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.076069:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.076070:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.076070:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.076072:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.076072:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.076075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.076077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.076080:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.076082:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.076083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677472.076083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677472.076084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.076086:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.076087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.076087:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.076089:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.076091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.076244:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.076246:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.076247:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.076247:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.076249:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.076251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.076295:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.076301:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.076301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.076303:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.076306:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.076307:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.076308:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.076310:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.076310:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.076311:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.076314:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.076316:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.076319:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.076320:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.076320:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.076322:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.076322:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.076323:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.076323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.076328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.076336:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.076339:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.076340:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.076340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.076342:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.076345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.076350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.076358:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.076361:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.076362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.076362:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.076363:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.076363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.076778:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.076780:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.076782:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.076784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.076784:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.076786:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.076787:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.076788:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.076789:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.076790:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.076793:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.076795:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.076797:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.076798:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.076799:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.076801:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.076801:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.076802:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.076802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.076881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.076893:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.076896:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.076897:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.076899:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.076899:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.076901:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.076902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.076902:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:20.0:1619677472.076903:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.076904:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.076905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.077288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.077298:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.077300:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.077301:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.077303:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.077303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.077306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677472.077887:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.077890:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.077891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.077893:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.077895:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.077897:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.077898:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.077900:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.077901:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.077902:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.077905:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.077907:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.077909:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.077910:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.077911:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.077912:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.077913:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.077914:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.077914:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.077919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.077927:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.077929:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.077930:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.077932:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.077933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.077934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.078618:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.078621:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.078622:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.078622:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.078623:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.078626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.078640:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.078643:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.078643:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.078644:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.078644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.078645:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.079369:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.079371:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.079373:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.079374:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.079376:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.079378:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.079379:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.079380:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.079381:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.079382:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.079386:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.079387:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.079390:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.079391:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.079391:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.079393:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.079394:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.079395:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.079395:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.079407:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.079410:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.079411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.079411:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.079413:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.079416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.079655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.079660:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.079662:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.079664:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.079665:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.079667:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.079668:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.079669:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.079670:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.079671:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.079672:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.079675:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.079676:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.079678:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.079679:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.079679:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.079681:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.079681:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.079682:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.079683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.079687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.079695:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.079697:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.079698:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.079699:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.079700:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.079701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.079754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677472.079757:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.079759:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:8.2:1619677472.079759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:20.0:1619677472.079762:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.079763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677472.079764:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.079765:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.079766:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.079766:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.0:1619677472.079767:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000400:00000200:20.0:1619677472.079767:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.079768:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677472.079768:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.079768:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677472.079769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.079769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:20.0:1619677472.079770:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.079770:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677472.079771:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.079772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677472.079774:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.079775:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.079777:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:24.0:1619677472.079778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.079779:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.079779:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:24.0:1619677472.079780:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677472.079780:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677472.079781:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96515014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.079781:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:24.0:1619677472.079782:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.079782:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000400:00000200:24.0:1619677472.079783:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.079783:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000400:00000200:24.0:1619677472.079784:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677472.079784:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677472.079785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677472.079787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677472.079789:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677472.079790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:24.0:1619677472.079791:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.079794:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.079794:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:24.0:1619677472.079795:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.079795:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677472.079795:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000800:00000200:20.0:1619677472.079795:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.079795:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.079796:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:24.0:1619677472.079797:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.079797:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677472.079797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677472.079797:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.079798:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.079799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677472.079799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.079800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.079801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.079803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.079812:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.079815:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.079816:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.079816:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.079819:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.079821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.080212:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.080214:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.080215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.080215:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.080219:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.080222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.080229:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.080231:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.080232:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.080232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.080233:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.080233:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.080745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.080752:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.080755:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.080755:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.080757:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.080760:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.080761:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677472.080763:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.080765:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.080766:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.080767:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677472.080770:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.080772:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677472.080775:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.080776:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677472.080776:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677472.080778:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677472.080779:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.080780:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.080781:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.080782:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.080787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.080792:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.080794:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.080795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.080796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.081022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.081031:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.081034:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.081035:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.081035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.081036:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677472.081037:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.081039:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.081040:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677472.081040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:8.0:1619677472.081041:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.081044:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.081045:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.081046:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.081048:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.081048:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.081049:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.081052:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.081054:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.081057:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.081058:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.081059:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.081060:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.081061:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.081062:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.081063:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.081064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.081076:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.081078:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.081079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.081080:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.081085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.081087:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.081088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.081089:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.081091:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.081094:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.081233:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.081235:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.081236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.081236:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.081239:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.081241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.081412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.081419:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.081422:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.081423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.081425:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.081427:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.081429:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.081430:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.081432:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.081433:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.081434:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.081438:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.081439:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.081442:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.081443:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.081443:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.081445:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.081446:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.081447:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.081448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.081459:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.081462:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.081462:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.081463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.081464:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.081465:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.081794:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.081797:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.081798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.081799:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.081802:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.081804:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.081805:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96515005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.081806:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.081807:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.081808:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.081811:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.081814:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.081816:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.081817:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.081818:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.081820:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.081820:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.081821:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.081821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.081833:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.081836:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.081837:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.081837:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.081839:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.081843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.082126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.082132:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.082134:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.082137:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.082137:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.082139:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.082141:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.082142:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96515003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.082144:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.082145:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.082146:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.082150:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.082152:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.082155:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.082155:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.082156:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.2:1619677472.082157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:21.0:1619677472.082158:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.082158:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.082159:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677472.082160:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:21.0:1619677472.082160:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677472.082163:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677472.082164:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677472.082164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:24.0:1619677472.082166:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.082169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677472.082170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:1.0:1619677472.082172:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.082175:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.082176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.082176:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.082178:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.082180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.082435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.082443:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.082446:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.082447:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.082447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.082449:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.082452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.083165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.083173:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.083175:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.083177:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.083178:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.083179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.083181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677472.083527:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.083529:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.083531:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.083532:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.083534:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.083535:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.083537:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96515000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.083538:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.083539:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.083540:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.083544:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.083546:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.083549:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.083550:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.083551:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.083552:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.083553:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.083554:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.083555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.2:1619677472.084046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677472.084053:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.084055:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677472.084056:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677472.084058:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677472.084061:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.084063:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677472.084064:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.084065:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.084066:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.084067:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677472.084070:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677472.084072:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.084074:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.084074:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.084075:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.084077:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.084077:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.084078:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677472.084079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.2:1619677472.084083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.084091:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.084093:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.084094:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.084094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.084096:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677472.084097:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.084330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.084338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.084341:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.084341:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.084343:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.084344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.084347:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677472.084503:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.084506:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.084507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.084509:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.084511:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.084513:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.084514:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.084515:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.084516:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.084517:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.084520:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.084522:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.084525:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.084525:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.084526:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.084528:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.084529:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.084529:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.084530:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.084534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.084542:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.084544:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.084545:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.084546:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.084547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.084549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.084770:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.084772:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.084773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.084773:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.084775:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.084777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677472.085485:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.085488:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.085490:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.085492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.085493:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.085494:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677472.085495:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.085497:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.085498:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.085499:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677472.085502:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.085504:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677472.085506:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.085507:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677472.085508:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677472.085510:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677472.085510:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.085511:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.085511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677472.085524:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.085527:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.085528:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.085528:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.085531:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.085534:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.085547:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.085549:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.085550:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.085551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.085552:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.085552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.085617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.085620:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.085623:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.085625:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.085627:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.085627:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.085629:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.085630:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.085631:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.085632:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.085633:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.085636:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.085638:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.085640:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.085641:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.085641:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.085643:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.085644:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.085644:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.085645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.085649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.085657:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.085659:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.085660:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.085662:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.085662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.085664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.085745:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.085748:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.085749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.085750:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.085753:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.085755:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.085756:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.085757:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.085758:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.085759:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.085762:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.085764:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.085768:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.085768:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.085769:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.085771:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.085771:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.085772:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.085773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.085777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.085786:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.085788:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.085789:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.085789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.085791:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.085794:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.085800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.085807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.085809:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.085810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.085810:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.085811:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.085812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.085983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.085990:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.085992:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.085993:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.085995:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.085998:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.085999:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.086001:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.086002:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.086003:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.086004:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.086008:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.086010:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.086013:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.086013:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.086014:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.086016:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.086017:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.086018:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.086018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.086656:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.086658:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.086659:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.086659:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.086661:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.086663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.086666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.086673:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.086676:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.086677:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.086679:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.086680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.086682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.086688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.086695:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.086698:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.086699:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.086699:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.086700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.086701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.086748:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.086750:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.086751:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.086752:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.086753:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.086755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.086976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.086984:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.086987:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.086988:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.086988:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.086990:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.086993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.087134:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.087137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.087137:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.087139:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.087142:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.087143:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.087145:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.087147:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.087148:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.087149:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.087151:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.087153:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.087154:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.087155:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.087155:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.087157:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.087157:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.087158:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.087158:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.087168:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.087170:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.087170:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.087171:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.087172:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.087173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.087189:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.087191:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.087192:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.087192:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.087192:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.087193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.087314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.087320:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.087323:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.087324:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.087326:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.087329:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.087331:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.087332:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.087334:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.087335:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.087336:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.087339:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.087341:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.087343:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.087344:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.087345:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.087346:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.087347:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.087348:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.087350:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.087350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.087356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.087362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.087363:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.087364:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.087364:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.087365:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677472.087366:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.087366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.087367:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.087367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.087368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.0:1619677472.087406:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.087408:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.087409:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.087411:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.087411:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.087414:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.088117:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.088120:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.088121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.088122:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.088124:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.088126:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.088127:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.088129:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.088130:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.088130:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.088134:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.088135:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.088138:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.088139:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.088139:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.088141:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.088141:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.088143:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.088144:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.088144:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.2:1619677472.088149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.088153:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677472.088155:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677472.088156:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677472.088157:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:20.0:1619677472.088158:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.088158:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677472.088159:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677472.088159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.088159:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.088160:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.088160:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.088160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.088161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677472.088162:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.088164:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.088164:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:20.0:1619677472.088166:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.088167:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.088168:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677472.088168:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:20.0:1619677472.088172:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.088175:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.088177:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.088178:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.088179:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.088181:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.088181:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.088182:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.088184:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.088184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.088187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.088189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677472.088194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.088195:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.088197:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.088198:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.088199:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.088201:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.088202:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.088202:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.088203:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677472.088203:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677472.088204:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.088205:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.088205:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.088206:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.088207:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:20.0:1619677472.088207:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.088207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.088210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677472.088214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.2:1619677472.088215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.088217:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.088218:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.088218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.088219:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.088220:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677472.088222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.088225:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.088226:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.088226:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.088227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.088228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.088349:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.088352:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.088352:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.088353:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.088355:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.088357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.088767:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.088769:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.088771:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.088772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.088774:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.088775:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677472.088777:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.088778:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.088779:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.088780:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677472.088784:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.088785:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.088788:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.088789:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677472.088789:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.088791:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.088792:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.088793:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.088793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.088805:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.088808:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.088809:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.088809:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.088811:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677472.088814:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677472.090187:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.090190:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.090191:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.090193:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.090196:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.090197:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.090198:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.090200:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.090201:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.090202:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.090205:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.090206:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.090209:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.090210:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.090210:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.090212:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.090213:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.090214:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.090215:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.090216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.090219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.090226:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677472.090227:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.090228:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.090228:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677472.090229:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677472.090229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677472.090230:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.090232:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.090233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.090234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.090621:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.090623:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.090624:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.090624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.090626:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.090628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.090671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:2.0:1619677472.090677:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.090680:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:18.0:1619677472.090681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:2.0:1619677472.090683:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.090686:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.090687:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.090689:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.090690:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.090691:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.090692:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.090696:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.090698:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.090700:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.090701:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.090702:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.090704:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.090705:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.090706:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907cbe3400] (22)++
00000800:00000200:2.0:1619677472.090707:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (23)--
00000800:00000200:2.0:1619677472.090707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:8.2:1619677472.090710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.090712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677472.090718:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)++
00000800:00000200:21.0:1619677472.090719:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:18.0:1619677472.090720:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907cbe3400] (23)--
00000800:00000200:21.0:1619677472.090721:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:18.0:1619677472.090721:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:21.0:1619677472.090722:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.090722:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000400:00000200:21.0:1619677472.090723:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.090724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.090726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.090915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:2.0:1619677472.090921:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.090923:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:18.0:1619677472.090925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:2.0:1619677472.090927:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.090929:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.090931:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677472.090932:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.090934:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.090935:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.090936:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677472.090940:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.090943:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677472.090946:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.090946:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:2.0:1619677472.090947:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677472.090949:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677472.090950:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.090951:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:2.0:1619677472.090952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:20.2:1619677472.091387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.091390:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.091392:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.091395:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.091396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.091397:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.091398:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.091400:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.091401:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.091402:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.091403:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.091406:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.091407:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.091410:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.091410:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.091411:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.091412:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.091413:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.091414:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.091415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.091419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.091426:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.091428:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.091429:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.091431:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.091431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.091433:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.091660:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.091662:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.091663:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.091665:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.091668:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.091669:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.091671:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.091672:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.091673:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.091674:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.091677:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.091679:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.091681:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.091681:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.091682:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.091683:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.091684:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.091685:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.091685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.091695:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.091697:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.091698:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.091699:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.091700:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.091701:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.092028:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.092030:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.092031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.092031:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.092033:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.092035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677472.092255:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.092257:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.092260:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.092262:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.092262:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.092264:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677472.092265:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.092266:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.092267:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.092268:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677472.092271:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677472.092273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:12.0:1619677472.092273:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677472.092275:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.092276:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677472.092276:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677472.092278:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677472.092279:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677472.092280:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677472.092280:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.092280:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677472.092282:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.092283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.092285:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.092288:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.092290:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.092291:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.092292:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.092301:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.092303:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.092306:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.092308:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.092311:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.092312:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.092313:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.092314:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.092315:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.092316:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.092317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.092322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.092330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.092332:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.092333:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.092335:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.092336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.092338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.092437:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.092439:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.092440:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.092440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.092442:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.092444:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.092456:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.092459:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.092459:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.092459:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.092460:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.092461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.092586:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.092588:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.092589:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.092589:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.092591:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.092593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.092605:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.092608:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.092608:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677472.092608:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.092609:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.092610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.092902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.092911:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.092914:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.092915:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.092915:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.092917:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.092920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.093138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.093143:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.093146:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.093147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.093150:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.093152:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.093154:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.093155:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.093157:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.093158:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.093159:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.093162:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.093164:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.093166:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.093167:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.093168:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.093170:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.093170:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.093171:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.093172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.093183:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.093185:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.093186:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.093187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.093188:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.093189:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.093204:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.093207:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.093208:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.093208:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.093208:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.093209:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.093261:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.093263:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.093264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.093266:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.093267:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.093269:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.093270:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.093271:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.093272:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.093274:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.093276:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.093278:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.093280:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.093281:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.093281:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.093282:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.093283:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.093284:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.093284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.093294:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.093299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.093301:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.093301:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.093303:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.093304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.093368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.093377:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.093379:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.093380:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.093382:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.093382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.093385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.094496:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.094498:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.094499:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.094499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.094501:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.094503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.094722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.094726:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.094729:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.094731:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.094732:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.094734:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.094735:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.094737:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.094739:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.094740:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.094742:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.094748:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.094751:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.094754:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.094756:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.094757:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.094759:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.094761:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.094762:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.094763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.094766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.094774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.094778:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.094780:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.094780:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.094783:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.094787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.095069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.095079:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.095082:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.095084:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.095085:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.095087:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.095091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.095732:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.095735:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.095735:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.095735:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.095737:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.095739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.096395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.096397:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.096398:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.096399:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.096400:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.096402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.096702:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.096705:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.096706:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.096707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.096709:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.096711:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.096712:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.096713:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.096714:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.096715:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.096719:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.096720:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.096723:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.096724:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.096725:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.096727:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.096727:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.096728:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.096729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677472.096809:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.096812:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.096813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.096813:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.096814:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.096816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.096994:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.096997:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.096997:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.097000:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.097002:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.097004:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.097005:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.097006:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.097007:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.097008:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.097012:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.097014:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.097016:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.097017:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.097017:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.097019:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.097020:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.097020:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.097021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.097025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.097033:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.097036:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.097037:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.097038:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.097039:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.097041:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.097067:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.097070:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677472.097071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677472.097073:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.097075:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.097077:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677472.097078:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.097079:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.097081:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.097081:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677472.097084:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.097086:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.097087:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.097088:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.097088:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.097089:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.097090:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.097091:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.097091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.097096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.097103:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.097105:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.097105:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.097106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.097108:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.097109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.097552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.097557:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.097559:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.097562:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.097562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.097564:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.097566:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.097567:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.097568:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.097569:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.097570:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.097573:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.097574:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677472.097575:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677472.097577:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:21.0:1619677472.097578:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.097578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677472.097579:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.097579:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677472.097580:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.097581:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.097581:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677472.097582:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.097582:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677472.097583:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:21.0:1619677472.097583:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:24.0:1619677472.097584:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.097586:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.097587:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677472.097587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677472.097589:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.097592:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.097594:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.097595:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:24.0:1619677472.097596:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.097597:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.097597:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.097597:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.097598:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.097599:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677472.097599:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.097600:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.097600:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677472.097600:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677472.097601:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677472.097602:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.097607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677472.097614:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677472.097615:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677472.097616:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.097617:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.097617:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677472.097617:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.097618:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.097619:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.097619:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:8.0:1619677472.097619:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.097621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.097621:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.097631:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.097634:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.097635:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.097635:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.097635:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.097636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.098577:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.098579:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.098580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.098580:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.098581:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.098583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.098660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.098667:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.098670:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.098671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.098673:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.098675:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.098677:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.098678:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.098680:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.098681:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.098683:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.098685:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.098687:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.098690:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.098691:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.098692:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.098693:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.098694:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.098695:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.098695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.098920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.098923:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.098924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.098926:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.098927:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.098928:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.098930:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.098931:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.098932:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.098933:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.098936:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.098938:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.098939:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.098940:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.098941:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.098942:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.098942:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.098944:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.098945:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.098945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.098956:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.098957:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.098957:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.098959:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.098959:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.098961:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.098961:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.098961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.098962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.098962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.099089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.099098:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.099100:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.099101:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.099103:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.099104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.099106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.100601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.100607:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.100609:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.100612:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.100612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.100614:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.100616:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.100617:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.100618:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.100620:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.100621:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.100624:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.100626:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.100628:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.100629:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.100630:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.100631:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.100632:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.100633:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.100635:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.100636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.100639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.100640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.100643:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.100646:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.100646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.100647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.100650:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.100653:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.100654:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.100654:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.100656:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.100659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:13.0:1619677472.101133:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.101137:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.101141:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.101143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.101145:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.101147:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677472.101149:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.101152:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.101154:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.101155:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677472.101161:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.101164:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677472.101168:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.101169:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677472.101170:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677472.101172:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677472.101173:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.101175:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677472.101177:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677472.101177:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677472.101179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.101190:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:12.0:1619677472.101193:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:17.0:1619677472.101194:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.101196:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.101196:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:1.0:1619677472.101196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:12.0:1619677472.101198:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.101199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:17.0:1619677472.101200:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.101204:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.101205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.101215:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.101219:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.101221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.101221:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.101223:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.101224:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.102039:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.102041:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.102043:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.102043:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.102044:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.102046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.102057:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.102059:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.102060:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.102060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.102061:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.102062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.102117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.102123:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.102126:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.102127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.102130:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.102132:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.102133:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.102135:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.102137:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.102138:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.102139:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.102142:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.102144:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.102146:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.102146:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.102147:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.102149:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.102150:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.102150:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.102151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.102162:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.102164:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.102165:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.102165:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.102167:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.102168:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.102561:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.102564:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.102565:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.102567:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.102568:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.102569:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.102571:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.102572:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.102573:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.102574:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.102577:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.102579:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.102581:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.102582:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.102582:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.102584:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.102584:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.102585:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.102586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.102613:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.102615:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.102616:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.102616:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.102618:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.102619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.102825:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.102828:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.102830:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.102831:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.102832:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.102834:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.102835:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.102836:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.102837:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.102838:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.102842:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.102843:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.102846:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.102847:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.102848:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.102849:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.102850:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.102851:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.102852:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.102853:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.102863:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.102865:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.102867:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.102867:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677472.103213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.103219:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.103221:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.103223:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.103224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.103226:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.103228:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.103229:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.103230:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.103231:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.103232:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.103236:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.103237:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.103240:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.103240:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.103241:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.103243:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.103243:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.103244:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.103245:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.103249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.103257:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.103259:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.103260:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.103261:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.103262:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.103264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.103857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.103868:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.103871:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.103872:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.103874:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.103875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.103877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.104074:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.104077:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.104078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.104078:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.104080:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.104082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.104408:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.104411:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.104412:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.104413:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.104415:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.104417:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.104418:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.104419:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.104420:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.104421:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.104424:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.104425:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.104428:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.104429:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.104429:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.104431:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.104431:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.104432:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.104433:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.104437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.104446:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.104449:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.104450:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.104450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.104452:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.104455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.105041:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.105044:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.105046:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.105047:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.105048:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.105049:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.105050:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.105052:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.105053:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.105054:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.105057:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.105059:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.105061:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.105062:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.105062:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.105064:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.105064:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.105065:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.105066:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.105133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.105142:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.105144:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.105145:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.105147:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.105147:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.105148:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.105149:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)++
00000800:00000200:21.0:1619677472.105150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (24)--
00000800:00000200:20.0:1619677472.105150:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (24)--
00000800:00000200:21.0:1619677472.105151:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.105151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.105277:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.105280:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.105281:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.105283:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.105285:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.105287:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.105288:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.105289:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.105290:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.105291:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.105300:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.105301:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.105304:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.105305:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.105305:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.105307:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.105308:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.105308:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.105309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.105313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.105322:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.105324:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.105325:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.105326:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.105328:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.105329:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.105746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.105753:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.105756:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.105757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.105759:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.105761:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.105762:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.105764:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.105766:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.105767:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.105768:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.105771:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.105773:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.105776:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.105777:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.105778:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.105780:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.105781:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.105783:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.105784:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.105784:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.105786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.105789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677472.105794:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.105796:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:18.0:1619677472.105796:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:21.0:1619677472.105797:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.105798:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.105798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:21.0:1619677472.105799:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.105799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.105799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.0:1619677472.105802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.105807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.105814:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.105817:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.105818:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.105819:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.105820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.105821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.106700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.106703:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.106706:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.106708:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.106709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.106710:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.106712:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.106713:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.106714:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.106716:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.106717:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.106720:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.106722:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.106724:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.106725:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.106726:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.106728:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.106729:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.106730:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.106731:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.106735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.106743:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.106746:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.106747:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.106747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.106749:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.106752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.106754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.106761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.106764:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.106765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.106765:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.106766:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.106766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.106774:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.106777:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.106778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.106778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.106780:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.106782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.107119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.107127:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.107129:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.107130:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.107131:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.107133:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.107134:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677472.107602:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.107605:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.107606:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.107607:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.107610:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.107611:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677472.107612:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.107614:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.107615:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.107616:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677472.107619:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.107621:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.107623:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.107624:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.107624:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.107626:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.107626:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.107628:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677472.107629:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677472.107629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677472.107632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.107641:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.107642:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677472.107642:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.107642:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.107643:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677472.107643:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.107644:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.107644:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.107645:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677472.107646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.107925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677472.107926:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.107928:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677472.107931:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.107931:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.107933:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.107933:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:24.0:1619677472.107935:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.107936:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.107936:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.107937:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.107937:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:24.0:1619677472.107938:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.107939:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.107939:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.107940:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677472.107942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:24.0:1619677472.107944:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.107945:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.107948:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.107949:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.107949:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.107951:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.107952:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.107952:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.107953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.107966:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.107969:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.107970:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.107970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.107972:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.107975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.108295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.108301:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.108304:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.108305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.108307:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.108308:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.108310:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.108311:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.108313:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.108314:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.108315:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.108318:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.108320:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.108322:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.108323:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.108323:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.108325:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.108326:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.108327:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.108327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.108339:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.108341:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.108341:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.108342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.108343:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.108344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.108855:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.108858:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.108859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.108861:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.108864:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.108865:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.108866:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.108867:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.108868:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.108869:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.108872:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.108873:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.108875:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.108876:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.108876:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.108877:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.108878:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.108879:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.108881:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.108881:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.108891:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.108892:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.108893:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.108894:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.108895:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.108896:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.108896:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.108896:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.108897:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.108897:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677472.108927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.108930:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.108933:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.108935:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.108937:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.108937:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.108939:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.108940:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.108941:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.108942:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.108943:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.108946:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.108948:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.108951:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.108952:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.108952:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.108954:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.108954:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.108955:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.108956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.108960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.108968:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.108970:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.108971:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.108972:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.108974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.108975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.109244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.109247:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.109248:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.109250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.109251:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.109252:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.109253:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.109255:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.109256:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.109257:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.109260:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.109261:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.109263:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.109264:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.109265:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.109266:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.109267:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.109268:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.109269:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.109270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.109280:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.109282:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.109283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.109284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.109924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677472.109929:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.109932:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:21.0:1619677472.109933:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677472.109933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.109934:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.109935:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:24.0:1619677472.109936:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.109936:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.109938:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677472.109938:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677472.109939:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.109940:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.109941:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.109942:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677472.109942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:24.0:1619677472.109943:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.109946:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.109948:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.109951:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.109951:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.109952:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.109954:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.109954:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.109955:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.109955:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.109960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.109968:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.109970:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.109971:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.109972:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.109974:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.109977:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.110069:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.110072:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.110072:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.110073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.110074:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.110076:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.111528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.111537:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.111540:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.111540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.111541:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.111543:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.111545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.111548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.111550:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.111552:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.111553:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.111554:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.111554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.111554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677472.112033:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.112035:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.112036:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.112037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.112038:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.112040:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.112051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.112053:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.112054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.112054:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.112055:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.112056:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.112395:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.112398:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.112399:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.112401:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.112403:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.112404:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.112406:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.112407:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.112408:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.112409:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.112412:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.112414:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.112417:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.112418:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.112419:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.112420:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.112421:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.112422:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.112422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.112427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.112435:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.112438:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.112439:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.112440:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.112442:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.112443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.112448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.112456:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.112459:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.112460:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.112461:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.112461:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.112462:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.112861:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.112864:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.112866:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.112866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.112868:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.112870:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.112871:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.112872:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.112873:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.112875:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.112878:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.112880:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.112882:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.112883:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.112884:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.112886:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.112886:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.112887:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.112888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677472.113130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.113133:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.113135:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.113137:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.113138:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.113140:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.113141:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.113143:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.113145:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.113146:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.113147:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.113150:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.113152:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.113154:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.113155:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.113156:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.113157:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.113158:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.113159:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.113160:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.113161:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.113163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.113166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.113171:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677472.113171:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677472.113173:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.113174:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677472.113174:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677472.113175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677472.113175:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.113175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.113177:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.113180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677472.113503:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.113506:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.113508:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.113509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.113511:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.113512:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.113514:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.113515:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.113516:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.113517:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.113520:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.113522:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.113524:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.113525:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.113525:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.113527:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.113527:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.113528:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.113529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.113608:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.113610:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.113611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.113613:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.113616:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.113618:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.113619:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.113620:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.113621:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.113622:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.113626:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.113627:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.113630:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.113631:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.113632:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.113634:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.113634:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.113635:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.113635:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.113649:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.113652:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.113654:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.113654:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.113656:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.113660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.113671:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.113674:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.113674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.113675:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.113676:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.113677:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677472.114138:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.114140:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677472.114141:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677472.114143:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.114145:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.114147:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677472.114148:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.114149:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.114150:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.114151:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677472.114154:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.114155:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.114157:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.114158:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.114158:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.114160:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.114161:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.114161:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.114162:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.114171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.114178:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.114180:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.114180:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.114181:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.114182:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.114183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.114485:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.114488:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.114489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.114490:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.114493:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.114495:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.114496:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.114497:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.114498:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.114499:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.114503:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.114504:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.114506:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.114507:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.114507:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.114509:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.114509:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.114510:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.114510:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.114521:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.114523:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.114524:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.114525:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.114526:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.114527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.114542:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.114545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.114545:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.114545:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.114546:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.114547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.114666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.114669:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.114671:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.114673:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.114675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.114675:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.114677:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.114678:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.114679:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.114680:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.114681:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.114684:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.114686:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.114688:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.114688:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.114689:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.114690:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.114691:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.114692:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.114692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.114696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.114704:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.114706:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.114707:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.114709:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.114709:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.114711:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.114734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.114740:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.114743:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.114744:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.114746:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.114747:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.114749:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.114750:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.114752:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.114753:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.114754:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.114758:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.114759:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.114762:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.114763:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.114764:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.114766:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.114766:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.114767:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.114768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.114772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.114779:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.114781:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.114782:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.114784:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.114785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.114787:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:18.0:1619677472.115483:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:24.2:1619677472.115485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:18.0:1619677472.115486:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.115487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:24.0:1619677472.115488:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000400:00000200:18.0:1619677472.115488:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.115490:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.115491:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677472.115492:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.115492:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.115494:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.115494:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.115496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.2:1619677472.115823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.115835:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.115838:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.115839:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.115840:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.115841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.115844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.115891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.115897:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.115900:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.115901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.115903:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.115905:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.115907:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.115908:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.115910:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.115911:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.115913:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.115918:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.115920:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.115923:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.115924:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.115925:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.115927:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.115928:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.115930:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.115931:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.115931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.115936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.115942:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.115943:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677472.115944:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.115944:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.115945:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677472.115946:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.115946:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:2.0:1619677472.115946:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:24.0:1619677472.115947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.115947:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.115947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.115949:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:8.0:1619677472.115950:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.115951:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.115952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677472.115952:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.116159:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.116161:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.116163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.116164:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.116166:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.116167:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.116168:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.116170:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.116171:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.116172:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.116175:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.116176:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.116179:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.116180:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.116180:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.116182:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.116183:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.116184:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.116185:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.116186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.116188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.116197:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677472.116198:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.116200:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677472.116200:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.116201:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.116201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677472.116201:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.116203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677472.116203:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.116207:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.116769:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.116771:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.116772:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.116772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.116774:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.116777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677472.116786:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.116789:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.116790:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.116791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.116793:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.116797:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.117738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.117747:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.117749:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.117750:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.117751:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.117753:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.117756:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.117766:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.117768:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.117771:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.117772:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.117773:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.117774:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.117776:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.117777:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.2:1619677472.117778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:20.0:1619677472.117778:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.117779:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.117781:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000400:00000200:20.0:1619677472.117782:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.117783:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.117784:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:8.0:1619677472.117785:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.117786:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.117786:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.117787:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:24.0:1619677472.117788:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.117788:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.117790:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.0:1619677472.117790:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.117790:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.117791:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.117792:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.117802:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.117804:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.117805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.117805:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677472.117807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000400:00000200:8.0:1619677472.117807:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.117810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.117812:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.117814:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.117816:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.117817:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.117819:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.117820:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677472.117822:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.117823:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.117824:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.117825:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677472.117828:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.117830:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677472.117833:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.117833:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677472.117834:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677472.117836:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677472.117836:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.117838:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677472.117838:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:1.0:1619677472.118956:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.118960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.118960:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.118963:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.118964:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.118966:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.118967:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.118968:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.118969:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.118970:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.118973:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.118975:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.118978:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.118978:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.118979:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.118981:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.118981:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.118983:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.118984:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.118984:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.118988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.118995:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677472.118996:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.118996:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.118997:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677472.118998:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677472.118998:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677472.118999:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.119001:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.119002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.119003:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.119019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.119024:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.119027:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.119029:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.119029:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.119032:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.119033:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.119035:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.119036:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.119037:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.119038:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.119041:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.119043:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.119045:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.119046:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.119047:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.119048:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.119049:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.119050:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.119050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.119054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.119062:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.119065:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.119066:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.119066:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.119068:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.119070:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.119072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.119080:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.119082:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.119083:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.119083:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.119084:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.119085:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.119229:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.119232:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.119233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.119234:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.119237:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.119239:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.119240:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.119242:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.119243:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.119244:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.119247:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.119248:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.119251:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.119251:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.119252:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.119253:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.119254:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.119255:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.119255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.119266:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.119267:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.119268:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.119269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.119270:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.119271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:23.0:1619677472.119311:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:20.2:1619677472.119805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.119812:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.119815:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.119816:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.119818:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.119820:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.119822:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.119823:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.119825:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.119826:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.119827:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.119830:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.119832:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.119835:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.119836:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.119837:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.119839:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.119840:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.119841:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.119841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.120322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.120330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.120333:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.120334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.120335:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.120337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.120338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.120530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.120535:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.120538:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.120540:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.120540:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.120543:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.120544:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.120545:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.120547:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.120547:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.120549:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.120552:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.120553:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.120556:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.120556:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.120557:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.120559:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.120559:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.120560:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.120561:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.120565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.120572:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.120575:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.120576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.120576:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.120578:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.120581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.120795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:2.0:1619677472.120806:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.120809:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:18.0:1619677472.120811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:2.0:1619677472.120812:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.120815:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.120817:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677472.120818:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.120820:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.120821:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.120822:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677472.120826:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.120827:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677472.120830:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.120831:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677472.120832:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677472.120834:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677472.120835:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.120835:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:2.0:1619677472.120836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:20.0:1619677472.120848:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.120850:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.120851:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.120852:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.120854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.120855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677472.121402:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.121404:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.121405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.121406:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.121408:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.121412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.121534:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.121537:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.121537:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.121538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.121539:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.121541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.121701:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.121703:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.121704:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.121705:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.121706:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.121708:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.121709:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.121710:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.121711:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.121712:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.121715:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.121717:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.121719:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.121720:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.121721:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.121723:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.121723:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.121724:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.121724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.121729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.121737:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.121740:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.121741:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.121742:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.121744:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.121747:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.121761:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.121764:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.121765:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.121765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.121767:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.121769:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.121898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.121906:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.121909:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.121910:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.121912:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.121912:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.121915:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.121918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.121926:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.121929:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.121930:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.121930:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.121931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.121932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677472.122096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba2200] (20)++
00000800:00000200:2.0:1619677472.122102:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba2200] (21)++
00000800:00000200:2.0:1619677472.122105:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.47@o2ib3
00000800:00000200:18.0:1619677472.122107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba2200] (22)--
00000400:00000200:2.0:1619677472.122108:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.47@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.122111:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.122113:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677472.122114:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.122116:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.122117:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.122118:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677472.122122:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.47@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.122123:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677472.122127:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.122128:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677472.122129:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677472.122131:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677472.122131:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba2200] (21)++
00000800:00000200:2.0:1619677472.122133:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba2200] (22)--
00000800:00000200:2.0:1619677472.122133:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba2200] (21)--
00000800:00000200:8.0:1619677472.122143:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.122146:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.122146:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.122147:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.122149:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.122152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.122167:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.122170:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.122171:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.122171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.122172:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.122172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.122185:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.122187:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.122188:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.122188:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.122189:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.122191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.122207:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.122210:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.122211:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.122211:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677472.122211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.122212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677472.122234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba2200] (20)++
00000800:00000200:18.0:1619677472.122240:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba2200] (21)++
00000800:00000200:18.0:1619677472.122243:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.47@o2ib3
00000800:00000200:2.0:1619677472.122243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba2200] (22)--
00000800:00000200:1.0:1619677472.122244:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:18.0:1619677472.122246:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.47@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.122247:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.122247:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:18.0:1619677472.122249:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.122249:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.122250:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677472.122251:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.122252:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.122253:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677472.122254:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.122254:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.122255:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.122256:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677472.122256:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.122257:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.122258:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677472.122261:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.47@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.122263:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.122264:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677472.122266:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677472.122268:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.122268:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.122269:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677472.122269:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.122269:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.122270:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677472.122271:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.122272:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677472.122272:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.122272:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677472.122273:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba2200] (21)++
00000800:00000200:1.0:1619677472.122273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677472.122274:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba2200] (22)--
00000800:00000200:18.0:1619677472.122274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba2200] (21)--
00000800:00000200:8.2:1619677472.122278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.122285:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677472.122286:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.122288:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:8.0:1619677472.122288:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.122289:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:17.0:1619677472.122289:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.122289:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.122290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.122292:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.122300:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.122303:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:8.0:1619677472.122304:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.122305:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.122307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.122308:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.122309:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.122309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.122310:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.122311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.122892:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.122894:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.122895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.122897:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.122900:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.122901:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.122903:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.122904:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.122905:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.122907:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.122910:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.122912:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.122915:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.122916:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.122916:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.122918:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.122919:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.122919:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.122920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.122933:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.122936:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.122937:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.122938:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.122940:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.122944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.122953:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.122955:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.122956:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.122956:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.122958:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.122959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.123248:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.123251:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.123253:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.123254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.123255:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.123257:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.123258:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.123260:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.123261:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.123263:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.123267:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.123268:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.123272:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.123273:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.123273:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.123275:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.123276:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.123276:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.123277:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677472.123318:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.123321:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.123322:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.123322:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.123324:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.123328:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.124108:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.124111:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.124112:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.124114:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.124117:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.124118:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.124120:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.124121:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.124122:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.124123:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.124126:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.124127:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.124129:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.124130:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.124131:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.124132:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.124132:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.124134:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.124135:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.124136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.124146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.124147:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.124148:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.124150:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.124150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.124151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.124151:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.124151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.124152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.124152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.125087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.125092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.125096:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.125098:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.125098:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.125099:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.125101:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.125101:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.125104:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677472.125105:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.125105:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.125107:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.125107:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.125108:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.125108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677472.125109:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.125109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.125110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:21.0:1619677472.125111:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.125113:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.125115:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.125117:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.125119:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.125125:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.125127:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.125131:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.125132:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.125133:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.125135:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.125136:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.125138:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.125139:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.125140:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.125141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.125145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.125148:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677472.125149:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677472.125150:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.125151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.125152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.125152:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.125153:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.125153:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.125155:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.125158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.125461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.125470:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.125472:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.125473:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.125473:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.125475:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.125478:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.125712:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.125715:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.125716:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.125717:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.125719:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.125720:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.125721:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.125722:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.125723:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.125724:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.125727:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.125729:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.125731:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.125732:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.125733:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.125735:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.125735:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.125736:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.125736:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.125741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.125749:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.125751:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.125752:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.125754:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.125756:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.125757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.126255:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.126258:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.126259:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.126259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.126261:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.126263:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.126335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.126338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.126340:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.126343:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.126344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.126345:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.126346:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.126347:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.126349:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.126350:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.126351:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.126354:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.126356:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.126358:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.126359:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.126360:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.126361:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.126362:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.126363:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.126363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.126368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.126376:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.126379:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.126380:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.126380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.126382:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.126384:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677472.126738:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.126740:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.126743:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.126743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.126745:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.126747:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.126748:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.126749:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.126750:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.126751:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.126754:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.126755:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.126758:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.126758:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.126759:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.126760:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.126761:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.126762:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.126763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.126839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.126847:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.126849:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.126850:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.126852:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.126853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.126855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.126865:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.126867:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.126868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.126870:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.126872:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.126874:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.126875:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.126876:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.126877:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.126878:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.126881:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.126883:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.126886:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.126887:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.126887:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.126889:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.126889:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.126890:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.126890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.126895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.126903:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.126905:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.126906:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.126906:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.126909:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.126911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.127020:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.127023:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677472.127024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677472.127026:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.127028:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.127030:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677472.127031:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.127032:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.127034:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.127035:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677472.127039:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.127039:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.127040:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.127041:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:8.0:1619677472.127042:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.127043:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.127044:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.127045:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:8.0:1619677472.127045:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677472.127046:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:8.0:1619677472.127047:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.127048:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:8.0:1619677472.127048:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.127049:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.127049:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677472.127050:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.127052:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677472.127053:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000400:00000200:8.0:1619677472.127053:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.127054:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677472.127055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.127055:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677472.127059:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.127059:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677472.127060:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677472.127061:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677472.127062:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.127063:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.127064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.127067:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677472.127067:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677472.127069:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677472.127071:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.127071:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:8.0:1619677472.127072:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000400:00000200:24.0:1619677472.127073:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.127073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677472.127074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.127075:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:21.0:1619677472.127075:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677472.127075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000400:00000200:24.0:1619677472.127076:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.127078:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677472.127078:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.127078:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.127079:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.127079:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.127080:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.127081:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:24.0:1619677472.127082:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.127083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.127084:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677472.127087:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.127087:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677472.127088:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677472.127089:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677472.127090:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.127091:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.127091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.127105:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.127108:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.127109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.127110:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.127113:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.127114:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.127115:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.127117:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.127117:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.127118:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.127121:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.127122:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.127123:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.127124:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.127124:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.127125:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.127126:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.127127:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.127128:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.127129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.127139:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.127140:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.127141:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.127142:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.127143:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.127143:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.127144:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.127144:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.127145:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.127145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.127174:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.127176:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.127177:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.127178:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.127179:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.127181:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677472.127182:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.127183:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.127184:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.127184:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677472.127187:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.127188:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677472.127191:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.127191:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:24.0:1619677472.127192:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:24.0:1619677472.127194:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:24.0:1619677472.127194:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.127195:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.127195:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677472.127207:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.127209:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.127210:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.127212:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.127214:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.127215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677472.127219:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.127221:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.127222:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:12.0:1619677472.127223:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:8.0:1619677472.127223:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.127226:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677472.127226:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.127227:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.127227:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:8.0:1619677472.127227:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:12.0:1619677472.127228:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:8.0:1619677472.127228:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677472.127229:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:8.0:1619677472.127230:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.127231:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.127231:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677472.127234:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.127236:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 824 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677472.127239:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.127240:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.0:1619677472.127240:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677472.127242:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677472.127243:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.127244:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.127245:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.127246:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.127247:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.127248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.127250:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.127251:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.127252:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677472.127253:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.127254:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.127255:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.127256:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677472.127256:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677472.127258:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.127259:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.127259:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.127261:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677472.127261:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.127261:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 352 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677472.127262:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.127263:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677472.127263:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677472.127264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.127265:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.127265:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.127266:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.127267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.127276:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.127278:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:21.0:1619677472.127278:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:24.0:1619677472.127279:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.127281:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.127281:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.127282:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.127282:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677472.127282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677472.127283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677472.127284:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.127284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.127285:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.127287:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.127288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.127290:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.127291:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.127292:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677472.127298:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.127299:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.127300:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.127301:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677472.127304:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.127305:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 360 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677472.127307:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.127308:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677472.127308:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677472.127310:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677472.127311:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.127313:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (23)++
00000800:00000200:8.0:1619677472.127314:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (24)--
00000800:00000200:8.0:1619677472.127315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.127316:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.127318:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.127319:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (23)++
00000800:00000200:24.0:1619677472.127320:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (24)--
00000400:00000200:8.0:1619677472.127320:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.127321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.127322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.127322:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.127323:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677472.127324:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.127326:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.127326:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.127327:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677472.127330:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.127332:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677472.127335:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.127336:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677472.127336:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677472.127338:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677472.127338:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.127339:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.127339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.127344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677472.127352:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677472.127352:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:24.0:1619677472.127354:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677472.127355:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.127356:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.127356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.127357:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.127358:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.127359:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.127359:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.127361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:24.0:1619677472.127362:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.127363:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.127365:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.127366:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.127366:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.127369:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.127371:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.127372:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.127372:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.127373:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.127374:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.127375:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.127375:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.127376:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.127388:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.127389:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.127390:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.127390:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677472.127391:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.127392:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.127393:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.127394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.127394:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.127396:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.127396:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677472.127397:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:8.0:1619677472.127397:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.127398:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.127399:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.127400:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.127402:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.127404:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.127406:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.127407:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.127408:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.127409:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.127410:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.127410:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.127411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.127415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.127424:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.127426:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.127427:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.127428:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.127430:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.127432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.127470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.127477:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.127480:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.127481:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.127483:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.127486:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.127488:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.127489:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.127490:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.127491:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.127492:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.127495:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.127497:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.127499:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.127499:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.127500:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.127502:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.127503:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.127503:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.127504:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.127515:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.127517:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.127518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.127518:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.127520:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.127521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.127888:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.127892:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.127893:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.127894:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.127897:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.127899:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.127900:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.127901:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.127902:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.127903:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.127906:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.127908:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.127911:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.127911:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.127912:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.127914:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.127914:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.127915:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.127917:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.127917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.127928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.127930:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677472.127930:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:18.0:1619677472.127931:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.127931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.127932:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.127932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.127933:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.127934:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.127937:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.128132:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.128134:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.128135:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.128136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.128137:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.128139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.128152:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.128154:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.128155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.128155:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.128156:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.128156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677472.128188:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.128191:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.128192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.128192:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.128194:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.128198:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677472.128776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.128783:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.128786:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.128787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.128788:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.128791:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.128793:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.128794:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.128796:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.128797:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.128798:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.128801:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.128803:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.128806:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.128807:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.128807:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.128809:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.128810:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.128811:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.128811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677472.128878:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.128881:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.128882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677472.128884:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.128886:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.128888:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677472.128890:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.128891:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.128892:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.128894:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677472.128897:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.128898:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677472.128901:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.128902:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677472.128903:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677472.128905:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677472.128905:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.128907:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677472.128908:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677472.128908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.128920:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677472.128922:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677472.128923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.128924:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677472.129398:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.129401:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.129402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.129403:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.129406:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.129407:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.129408:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.129409:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.129410:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.129411:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.129414:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.129415:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.129417:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.129418:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.129418:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.129420:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.129420:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.129421:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.129421:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.129433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.129434:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.129435:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.129436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.129437:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.129438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.129452:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.129454:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.129455:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.129455:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.129456:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.129456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.129478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.129480:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.129482:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.129483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.129485:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.129486:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677472.129488:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.129489:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.129490:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.129491:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677472.129494:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.129496:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.129499:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.129500:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677472.129500:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.129502:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.129503:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.129503:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.129504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677472.129516:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.129518:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.129520:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.129520:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.129522:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.129525:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.129543:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.129546:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.129546:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.129547:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.129548:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.129548:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.129553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.129555:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.129557:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.129558:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.129560:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.129562:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.129563:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.129564:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.129565:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.129566:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.129570:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.129572:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.129575:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.129575:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.129576:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.129578:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.129579:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.129580:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.129581:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.129582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677472.129590:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677472.129592:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677472.129593:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.129594:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.129594:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:8.0:1619677472.129594:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.129595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.129596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677472.129596:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.129600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.130419:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.130421:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.130423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.130424:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.130426:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.130427:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.130428:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.130429:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.130430:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.130431:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.130434:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.130436:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.130438:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.130438:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.130439:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.130440:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.130441:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.130441:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.130442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.130452:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.130454:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.130455:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.130456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.130457:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.130458:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.131074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.131080:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.131083:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.131085:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.131085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.131087:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.131089:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.131090:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.131092:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.131093:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.131094:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.131097:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.131098:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.131101:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.131101:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.131102:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.131104:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.131104:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.131105:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.131106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.131110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.131118:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.131121:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.131122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.131122:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.131124:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.131126:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.131131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.131138:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.131141:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.131142:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.131142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.131143:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.131144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677472.131211:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.131213:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.131214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.131216:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.131218:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.131220:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.131221:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.131222:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.131223:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.131224:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.131227:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.131228:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.131231:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.131232:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.131232:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.131234:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.131235:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.131236:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.131236:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.131240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.131248:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.131250:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.131251:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.131253:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.131254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.131255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.131259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.131267:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.131269:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.131270:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.131271:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.131271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.131272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.131357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.131365:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.131367:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.131368:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.131370:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.131371:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.131373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.131462:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.131464:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.131465:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.131466:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.131467:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.131469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.131609:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.131612:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.131614:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.131615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.131616:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.131618:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.131619:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.131620:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.131621:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.131622:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.131626:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.131627:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.131630:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.131630:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.131631:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.131633:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.131633:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.131634:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.131635:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.131754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:12.0:1619677472.131802:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.131805:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.131806:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.131808:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.131810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.131811:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.0:1619677472.131880:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:21.0:1619677472.131883:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.5.10@o2ib3
00000800:00000200:20.0:1619677472.131885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:21.0:1619677472.131886:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.131888:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.131890:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.131891:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.131892:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.131894:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.131895:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.131897:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.131899:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.131901:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.131902:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.131903:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.131904:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.131905:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:21.0:1619677472.131906:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:21.0:1619677472.131907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677472.131925:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677472.131927:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677472.131929:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.131930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677472.131931:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.131933:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.131934:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.131935:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.131936:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.131937:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.131940:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.131942:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.131944:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.131945:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.131946:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.131947:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.131948:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677472.131949:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677472.131949:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.2:1619677472.131953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.131960:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.131963:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.131963:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.131965:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.131966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.131967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.132002:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677472.132004:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677472.132006:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.132007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677472.132008:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.132010:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.132011:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.132013:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.132014:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.132014:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.132018:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.132019:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.132022:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.132023:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.132023:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.132025:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.132025:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677472.132026:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677472.132027:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:8.0:1619677472.132037:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.132040:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.132041:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.132041:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.132042:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:8.0:1619677472.132043:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.132044:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:8.0:1619677472.132046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:21.0:1619677472.132047:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.132047:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:21.0:1619677472.132050:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.132052:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677472.132053:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.132054:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.132055:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.132056:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677472.132060:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.132062:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.132063:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.132064:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.132065:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.132066:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.132067:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:21.0:1619677472.132068:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:21.0:1619677472.132070:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:21.0:1619677472.132071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677472.132072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.132081:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.132083:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677472.132083:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.132083:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.132084:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:21.0:1619677472.132084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677472.132085:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.132086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.132087:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677472.132088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677472.132095:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:21.0:1619677472.132097:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:21.0:1619677472.132099:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.132099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:21.0:1619677472.132101:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.132103:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.132104:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.132105:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.132106:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.132107:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.132111:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.132112:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.132114:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.132115:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.132116:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.132117:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.132118:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:21.0:1619677472.132119:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:21.0:1619677472.132119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.0:1619677472.132180:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677472.132182:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:21.0:1619677472.132185:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677472.132185:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.132187:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.132189:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677472.132190:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.132191:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.132192:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.132193:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677472.132196:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.132198:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677472.132201:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.132202:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:20.0:1619677472.132203:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.0:1619677472.132205:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:20.0:1619677472.132206:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677472.132207:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677472.132208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:13.0:1619677472.132218:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.132221:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.132222:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.132223:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.132226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677472.132226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.132246:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677472.132250:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.132251:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.132252:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677472.132253:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.132255:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.0:1619677472.132309:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:21.0:1619677472.132312:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:21.0:1619677472.132314:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.132314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:21.0:1619677472.132317:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.132319:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:21.0:1619677472.132320:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.132322:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.132323:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.132324:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:21.0:1619677472.132327:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.132329:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.0:1619677472.132331:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.132332:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:21.0:1619677472.132333:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677472.132334:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:21.0:1619677472.132335:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:21.0:1619677472.132336:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:21.0:1619677472.132337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:2.0:1619677472.132350:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.132353:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.132354:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.132354:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.132356:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.132359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.0:1619677472.132375:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677472.132378:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677472.132380:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.132381:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677472.132382:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.132383:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.132385:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.132386:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.132387:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.132388:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.132391:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.132393:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.132394:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.132395:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.132395:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.132397:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.132398:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677472.132399:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677472.132400:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677472.132401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:8.2:1619677472.132404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.132411:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.132412:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.132412:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.132413:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:21.0:1619677472.132414:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000400:00000200:20.0:1619677472.132414:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.132415:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:21.0:1619677472.132416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.132416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.132417:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677472.132419:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:20.0:1619677472.132421:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.132422:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.132423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677472.132423:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.132424:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.132425:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.132426:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.132427:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.132429:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.132430:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.132431:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.132432:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.132433:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.132434:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.132435:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677472.132435:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677472.132436:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:17.0:1619677472.132438:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:8.2:1619677472.132440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.132441:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.132442:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.132445:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.132447:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.132448:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:17.0:1619677472.132448:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:21.0:1619677472.132449:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.132450:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.132450:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000400:00000200:17.0:1619677472.132450:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.132451:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.132452:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:17.0:1619677472.132452:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.132453:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:17.0:1619677472.132454:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:21.0:1619677472.132455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.132455:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.132456:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:20.0:1619677472.132457:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.132457:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677472.132458:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.0:1619677472.132459:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.132460:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000400:00000200:20.0:1619677472.132460:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.132460:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677472.132461:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.132461:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677472.132462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.132463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:20.0:1619677472.132463:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677472.132463:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.132464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:20.0:1619677472.132464:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677472.132464:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677472.132464:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677472.132466:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:20.0:1619677472.132467:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.132467:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.132467:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.132468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.0:1619677472.132469:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.132471:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.132472:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.132473:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.132474:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.132475:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677472.132475:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677472.132476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:20.2:1619677472.132477:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677472.132480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.132484:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.132486:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.132488:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.132488:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.132489:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.132490:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.132491:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.132492:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.132492:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.132494:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.132495:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.132496:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.132497:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.132500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677472.132501:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.132502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.132503:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.132503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677472.132503:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:21.0:1619677472.132505:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.132505:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000800:00000200:21.0:1619677472.132506:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.132507:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:20.0:1619677472.132507:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.132509:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:20.0:1619677472.132509:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.132510:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.132511:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.132512:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.132513:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.132514:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.2:1619677472.132515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:20.0:1619677472.132517:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.132518:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.132519:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.132519:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.132520:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677472.132520:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.132521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677472.132521:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.132522:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.132522:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677472.132523:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.132524:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:20.0:1619677472.132525:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677472.132525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:1.0:1619677472.132525:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.132526:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.132526:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.132528:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.132529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.132531:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677472.132537:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.132539:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.132539:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.132541:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.132542:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.132543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.132602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.132610:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.132613:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.132614:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.132615:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.132616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.132619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.132628:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)++
00000800:00000200:21.0:1619677472.132631:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.10@o2ib3
00000400:00000200:21.0:1619677472.132633:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.132633:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000400:00000200:21.0:1619677472.132634:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.132636:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:21.0:1619677472.132637:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.132639:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.132640:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.132641:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:21.0:1619677472.132644:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.132645:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.0:1619677472.132649:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.132650:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:21.0:1619677472.132650:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.0:1619677472.132653:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677472.132654:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70b000] (21)++
00000800:00000200:21.0:1619677472.132655:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70b000] (22)++
00000800:00000200:21.0:1619677472.132656:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70b000] (23)--
00000800:00000200:21.0:1619677472.132657:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:20.0:1619677472.132665:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)++
00000800:00000200:20.0:1619677472.132667:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70b000] (23)--
00000800:00000200:20.0:1619677472.132668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (22)--
00000800:00000200:12.0:1619677472.132668:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:20.0:1619677472.132669:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70b000] (21)--
00000800:00000200:12.0:1619677472.132672:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.132673:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.132675:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.132677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.132679:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677472.132830:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.132833:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.132835:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.132837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.132838:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.132840:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677472.132841:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.132842:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.132843:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.132844:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:13.0:1619677472.132847:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.132849:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677472.132852:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.132853:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:13.0:1619677472.132853:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:13.0:1619677472.132855:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:13.0:1619677472.132855:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.132856:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.132857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677472.133960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.133967:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.133970:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.133970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.133973:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.133975:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.133977:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.133978:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.133979:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.133980:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.133981:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.133985:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.133987:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.133989:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.133990:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.133991:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.133993:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.133994:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.133995:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.133996:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.133997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.134000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.134003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.134009:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:20.0:1619677472.134011:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677472.134011:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.134012:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677472.134013:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:2.0:1619677472.134013:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:20.0:1619677472.134014:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.134016:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.134017:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.134019:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.134019:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.134020:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.134021:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.134021:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (23)++
00000800:00000200:20.0:1619677472.134022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.134023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.134024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:12.0:1619677472.134185:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.134188:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.134191:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.134193:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.134193:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.134195:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:12.0:1619677472.134196:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96514796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.134198:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.134199:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.134200:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:12.0:1619677472.134203:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.134205:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677472.134207:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.134207:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:12.0:1619677472.134208:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:12.0:1619677472.134210:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:12.0:1619677472.134210:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.134211:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.134211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:24.0:1619677472.134220:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:24.0:1619677472.134221:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:24.0:1619677472.134222:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.134223:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677472.134224:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.134225:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.134227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.134227:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.134228:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.134228:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.134229:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.134230:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.134231:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.134233:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.134234:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.134235:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.134236:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.134237:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.134238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.134239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.134240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:8.0:1619677472.134241:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.134243:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.134246:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.134247:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.134247:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.134249:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.134250:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.134251:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.134251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.134264:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.134267:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.134267:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.134268:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.134270:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.134273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.135074:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.135077:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.135078:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.135079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.135081:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.135082:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.135084:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.135085:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.135086:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.135087:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.135090:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.135092:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.135095:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.135096:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.135096:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.135098:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.135099:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.135100:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.135100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.135111:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.135114:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.135114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.135115:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.135117:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.135120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.135129:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.135132:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.135133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.135133:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.135134:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.135134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677472.135643:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.135646:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.135648:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.135650:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.135650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.135652:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677472.135653:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.135654:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.135655:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.135656:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677472.135659:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.135661:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677472.135664:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.135664:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:13.0:1619677472.135665:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:13.0:1619677472.135667:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677472.135668:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.135669:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.135669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677472.135675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.135683:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.135686:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.135687:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.135687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.135689:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.135692:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.136073:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.136075:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.136077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.136078:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.136081:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.136082:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.136084:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.136085:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.136086:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.136087:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.136089:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.136091:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.136093:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.136093:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.136094:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.136095:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.136096:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.136097:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.136098:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.136099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.136109:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.136110:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.136111:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.136112:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.136113:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.136113:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.136113:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.136114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.136115:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.136115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.136155:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.136158:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.136159:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.136159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.136160:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.136162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.136177:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.136179:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.136180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.136180:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.136181:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.136181:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.136332:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.136335:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.136338:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.136339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.136340:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.136341:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.136343:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.136344:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.136345:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.136346:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.136349:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.136351:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.136354:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.136355:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.136356:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.136357:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.136358:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.136359:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.136359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.136370:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.136373:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.136374:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.136374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.136376:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.136378:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.137341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.137347:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.137349:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.137352:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.137352:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.137355:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.137357:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.137358:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.137359:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.137360:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.137361:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.137365:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.137366:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.137369:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.137370:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.137370:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.137372:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.137373:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.137374:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.137375:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.137376:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.137379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.137381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.137384:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.137386:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.137387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677472.137387:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677472.137388:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.137390:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.137391:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.137391:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.137393:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.137395:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.137452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.137461:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.137463:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.137464:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.137464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.137467:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.137470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.137478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.137485:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.137487:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.137488:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.137488:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.137490:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.137490:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.138409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.138415:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.138418:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.138420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.138421:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.138424:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.138426:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.138427:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.138428:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.138429:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.138431:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.138434:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.138436:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.138438:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.138439:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.138440:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.138442:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.138442:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.138443:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.138444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.138456:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.138458:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.138459:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.138459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.138461:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.138462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.138526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.138529:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.138532:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.138533:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.138535:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.138536:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.138537:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.138538:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.138539:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.138540:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.138541:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.138544:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.138546:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.138547:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.138548:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.138549:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.138550:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.138551:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.138552:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.138552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.138557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.138565:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.138567:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.138568:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.138569:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.138570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.138572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.138769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.138777:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.138779:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.138780:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.138782:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.138783:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.138785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.138788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.138795:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.138798:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.138799:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.138800:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.138800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.138801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.139006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.139009:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.139011:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.139011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.139013:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.139015:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.139016:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.139018:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.139018:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:21.0:1619677472.139019:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.139019:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.139020:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.139024:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677472.139025:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677472.139026:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.139027:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677472.139028:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.139028:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.139031:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.139032:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.139033:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.139036:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677472.139036:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.139037:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.139037:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.139038:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.139040:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.139040:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.139040:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.139040:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.139041:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.139042:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.139043:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.139043:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.139043:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.139044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.139045:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.139045:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (23)++
00000800:00000200:20.0:1619677472.139046:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.139047:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.139055:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.139056:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.139056:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.139057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.139058:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.139058:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.139059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677472.139061:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.139586:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.139589:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.139590:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.139592:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.139594:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.139596:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.139597:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.139598:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.139599:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.139600:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.139604:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.139605:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.139608:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.139609:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.139609:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.139611:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.139612:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.139612:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.139613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.139617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.139626:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.139628:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.139629:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.139630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.139631:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:17.0:1619677472.139632:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.139633:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.139636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:1.0:1619677472.139636:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.139637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.139639:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.139641:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.139642:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.139643:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.139644:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.139645:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.139649:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.139650:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.139653:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.139654:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.139655:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.139657:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.139657:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.139658:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.139658:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.139663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.139670:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.139673:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.139674:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.139676:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.139676:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.139679:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677472.139719:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.139722:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.139723:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.139724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.139725:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.139729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.140295:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.140300:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.140302:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.140303:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.140304:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.140306:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.140884:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.140886:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.140887:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.140887:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.140889:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.140890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.141027:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.141029:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.141030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.141032:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.141034:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.141036:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.141037:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.141039:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.141039:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.141041:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.141043:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.141045:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.141046:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.141047:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.141048:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.141049:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.141049:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.141050:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.141051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.141061:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.141063:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.141064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.141064:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.141066:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.141067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.142159:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.142162:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.142165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.142165:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.142166:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.142168:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.142169:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.142171:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.142172:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.142173:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.142176:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.142178:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.142181:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.142182:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.142182:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.142184:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.142185:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.142186:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.142186:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.142198:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.142201:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.142202:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.142202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.142204:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.142206:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.142217:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.142220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.142220:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.142221:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.142222:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.142222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.142813:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.142816:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.142816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.142818:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.142820:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.142822:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677472.142824:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.142826:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.142828:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.142829:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.142836:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.142836:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:24.0:1619677472.142838:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.142840:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.142840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:24.0:1619677472.142841:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.142842:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677472.142842:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000400:00000200:17.0:1619677472.142843:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.142844:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677472.142844:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.142845:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.142845:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:17.0:1619677472.142845:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.142846:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677472.142848:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96514766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.142849:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.142850:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.142851:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677472.142854:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.142856:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.142858:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677472.142859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677472.142859:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.142859:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.142861:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.142862:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.142862:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.142863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.142867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:18.0:1619677472.142869:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.142873:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.142874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677472.142875:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.142878:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.142878:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.142879:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677472.142879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.142881:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.142882:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.142882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:18.0:1619677472.142885:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.2:1619677472.142887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:18.0:1619677472.142888:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.142890:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.142892:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.142893:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677472.142895:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.142898:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.142898:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.142899:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.142899:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:18.0:1619677472.142900:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677472.142900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677472.142903:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677472.142907:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.142908:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677472.142909:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677472.142912:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677472.142913:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.142914:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.142915:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.143034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.143040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.143043:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.143045:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.143045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.143048:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.143049:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.143051:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.143052:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.143053:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.143054:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.143057:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.143058:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.143060:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.143061:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.143062:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.143064:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.143064:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.143065:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.143066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.143070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.143077:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.143079:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.143080:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.143082:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.143083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.143084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.143272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.143280:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.143283:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.143284:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.143284:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.143286:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.143289:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.143722:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.143725:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.143727:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.143727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.143730:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.143732:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.143733:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.143734:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.143735:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.143736:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.143740:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.143742:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.143745:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.143746:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.143747:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.143748:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.143749:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.143750:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.143751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.143762:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.143765:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.143766:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.143766:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.143768:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.143771:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.144240:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.144243:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.144244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.144246:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.144247:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.144249:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.144250:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.144252:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.144253:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.144254:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.144256:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.144258:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.144260:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.144261:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.144262:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.144263:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.144264:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.144266:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.144267:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.144267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.144277:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:8.2:1619677472.144279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.144279:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.144280:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.144281:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.0:1619677472.144287:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.144290:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.144290:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.144292:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.144293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.144300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.144305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.144311:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.144313:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.144315:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.144316:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.144318:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.144319:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.144320:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.144322:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.144323:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.144324:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.144326:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.144328:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.144331:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.144332:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.144332:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.144334:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.144334:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.144335:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.144336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.144350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.144358:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.144361:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.144362:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.144363:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.144364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.144365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.144371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.144379:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.144382:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.144383:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.144384:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.144385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.144386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.144741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.144749:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.144751:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.144752:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.144754:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.144755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.144757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677472.144781:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.144783:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:21.0:1619677472.144784:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:1.0:1619677472.144784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.144786:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.144787:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.144787:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.144789:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:17.0:1619677472.144789:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.144790:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:17.0:1619677472.144790:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677472.144792:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:17.0:1619677472.144792:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.144793:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.144794:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.144795:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.144798:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.144800:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.144803:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.144803:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.144804:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.144806:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.144807:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.144807:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.144808:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677472.144907:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.144910:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.144911:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.144912:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.144915:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.144917:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.144918:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.144919:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.144920:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.144921:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.144924:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.144926:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.144929:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.144930:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.144930:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.144932:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.144933:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.144933:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.144934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.144938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.144946:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.144949:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.144950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.144950:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.144952:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.144955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.145638:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.145640:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.145641:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.145641:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.145643:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.145645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.146199:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.146201:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.146202:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.146203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.146204:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.146206:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.146220:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.146222:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.146223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.146223:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.146224:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.146225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.146330:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.146333:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.146334:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.146335:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.146338:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.146340:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.146341:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.146342:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.146343:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.146343:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.146346:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.146348:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.146351:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.146351:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.146352:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.146354:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.146354:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.146355:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.146356:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.146357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.146360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.146368:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677472.146370:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.146371:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.146372:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.146372:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677472.146372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.146373:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.146374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677472.146374:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.146378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.146379:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.146381:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:8.2:1619677472.146381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.146383:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.146384:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.146385:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.146386:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677472.146387:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.146387:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.146388:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.146389:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677472.146390:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677472.146390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.146392:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677472.146392:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:17.0:1619677472.146393:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.146394:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677472.146398:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.146400:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677472.146402:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.146403:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:17.0:1619677472.146404:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677472.146406:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677472.146406:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.146407:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677472.146408:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677472.146409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.146419:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677472.146421:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677472.146422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.146423:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:20.2:1619677472.146444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.146452:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.146455:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.146456:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.146458:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.146461:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.146462:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.146463:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.146465:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.146466:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.146467:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.146469:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.146471:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.146473:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.146474:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.146474:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.146476:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.146477:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.146477:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.146478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.146490:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.146492:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.146492:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.146493:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.146494:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.146495:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.146517:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.146519:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.146520:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.146521:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.146521:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.146521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677472.146846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (20)++
00000800:00000200:18.0:1619677472.146852:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (21)++
00000800:00000200:18.0:1619677472.146856:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.15.3@o2ib3
00000800:00000200:2.0:1619677472.146856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000400:00000200:18.0:1619677472.146859:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.3@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.146861:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.146862:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677472.146863:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.146865:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.146866:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.146867:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677472.146870:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.146872:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677472.146874:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.146875:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677472.146876:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677472.146878:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677472.146878:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193efc76600] (21)++
00000800:00000200:18.0:1619677472.146879:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193efc76600] (22)--
00000800:00000200:18.0:1619677472.146879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (21)--
00000800:00000200:5.2:1619677472.146881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (20)++
00000800:00000200:2.0:1619677472.146887:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (21)++
00000800:00000200:2.0:1619677472.146889:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.3@o2ib3
00000800:00000200:18.0:1619677472.146890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000800:00000200:20.0:1619677472.146891:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:2.0:1619677472.146891:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.3@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.146894:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:2.0:1619677472.146894:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.146895:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.146895:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677472.146896:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.146897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677472.146897:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:2.0:1619677472.146898:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.146899:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.146900:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677472.146902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:2.0:1619677472.146904:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.146906:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677472.146910:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.146911:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:2.0:1619677472.146911:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:2.0:1619677472.146914:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:2.0:1619677472.146914:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193efc76600] (21)++
00000800:00000200:2.0:1619677472.146915:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193efc76600] (22)--
00000800:00000200:2.0:1619677472.146916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (21)--
00000800:00000200:5.2:1619677472.146924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (20)++
00000800:00000200:12.0:1619677472.146927:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:18.0:1619677472.146930:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (21)++
00000800:00000200:12.0:1619677472.146930:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.146931:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.146933:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.3@o2ib3
00000800:00000200:2.0:1619677472.146933:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000400:00000200:12.0:1619677472.146934:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.146936:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.3@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.146936:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.146937:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:18.0:1619677472.146940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.146941:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677472.146942:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.146944:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.146945:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.146946:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677472.146949:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.146951:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677472.146953:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.146954:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:18.0:1619677472.146954:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677472.146954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (21)++
00000800:00000200:18.0:1619677472.146957:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:18.0:1619677472.146957:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193efc76600] (22)++
00000800:00000200:18.0:1619677472.146958:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193efc76600] (23)--
00000800:00000200:18.0:1619677472.146959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000800:00000200:18.0:1619677472.146960:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (21)++
00000800:00000200:18.0:1619677472.146962:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.3@o2ib3
00000800:00000200:8.2:1619677472.146962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:2.0:1619677472.146963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000400:00000200:18.0:1619677472.146964:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.3@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.146966:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.146967:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677472.146968:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.146969:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.146970:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677472.146970:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:18.0:1619677472.146971:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677472.146973:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:18.0:1619677472.146974:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677472.146974:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.146974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:18.0:1619677472.146976:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677472.146976:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.146978:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677472.146979:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.146980:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677472.146980:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677472.146982:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677472.146983:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193efc76600] (21)++
00000800:00000200:18.0:1619677472.146984:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193efc76600] (22)++
00000800:00000200:18.0:1619677472.146986:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193efc76600] (23)--
00000800:00000200:5.2:1619677472.146986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (23)++
00000800:00000200:18.0:1619677472.146989:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (23)--
00000800:00000200:18.0:1619677472.146990:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (22)++
00000800:00000200:18.0:1619677472.146991:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.3@o2ib3
00000800:00000200:2.0:1619677472.146992:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (23)++
00000800:00000200:2.0:1619677472.146993:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193efc76600] (24)--
00000800:00000200:2.0:1619677472.146993:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (23)--
00000400:00000200:18.0:1619677472.146994:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.3@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.146995:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000400:00000200:18.0:1619677472.146997:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.146998:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.147000:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.147002:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.147003:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.147003:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.147007:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.147008:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.147011:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.147012:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.147013:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.147014:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.147015:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193efc76600] (21)++
00000800:00000200:18.0:1619677472.147015:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193efc76600] (22)--
00000800:00000200:18.0:1619677472.147016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (21)--
00000800:00000200:5.2:1619677472.147019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (20)++
00000800:00000200:2.0:1619677472.147024:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (21)++
00000800:00000200:2.0:1619677472.147027:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.3@o2ib3
00000800:00000200:18.0:1619677472.147028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000400:00000200:2.0:1619677472.147029:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.3@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.147032:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.147034:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.147035:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.147037:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.147037:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.147039:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.147042:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.147044:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.147046:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.147047:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.147048:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.147049:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.147050:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193efc76600] (21)++
00000800:00000200:2.0:1619677472.147051:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193efc76600] (22)--
00000800:00000200:2.0:1619677472.147052:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (21)--
00000800:00000200:8.2:1619677472.147066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.147075:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.147077:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.147078:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.147080:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.147080:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.147083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677472.147192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (20)++
00000800:00000200:18.0:1619677472.147198:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (21)++
00000800:00000200:18.0:1619677472.147201:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.3@o2ib3
00000800:00000200:2.0:1619677472.147202:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000400:00000200:18.0:1619677472.147204:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.3@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.147206:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.147207:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677472.147209:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.147210:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.147211:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.147212:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677472.147215:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.147217:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677472.147220:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.147220:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:18.0:1619677472.147221:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677472.147223:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:18.0:1619677472.147224:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193efc76600] (21)++
00000800:00000200:18.0:1619677472.147225:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193efc76600] (22)--
00000800:00000200:18.0:1619677472.147225:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (21)--
00000800:00000200:8.2:1619677472.147229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.147237:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.147239:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.147240:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.147242:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.147243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.147244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677472.147257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (20)++
00000800:00000200:2.0:1619677472.147263:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (21)++
00000800:00000200:2.0:1619677472.147265:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.3@o2ib3
00000800:00000200:18.0:1619677472.147266:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000400:00000200:2.0:1619677472.147268:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.3@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.147270:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.147271:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.147273:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.147274:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.147275:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.147276:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.147279:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.147280:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.147283:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.147283:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.147284:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.147285:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.147286:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193efc76600] (21)++
00000800:00000200:2.0:1619677472.147287:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193efc76600] (22)++
00000800:00000200:2.0:1619677472.147288:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193efc76600] (23)--
00000800:00000200:2.0:1619677472.147289:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000800:00000200:5.2:1619677472.147294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (21)++
00000800:00000200:18.0:1619677472.147301:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (22)++
00000800:00000200:8.2:1619677472.147302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.147303:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193efc76600] (23)--
00000800:00000200:18.0:1619677472.147304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000800:00000200:18.0:1619677472.147305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (21)--
00000800:00000200:21.0:1619677472.147310:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.147312:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.147313:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.147315:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.147316:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.147318:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677472.147579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (20)++
00000800:00000200:2.0:1619677472.147584:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (21)++
00000800:00000200:18.0:1619677472.147587:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (22)++
00000800:00000200:2.0:1619677472.147587:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.3@o2ib3
00000800:00000200:18.0:1619677472.147589:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.3@o2ib3
00000400:00000200:2.0:1619677472.147589:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.3@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.147591:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.3@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.147592:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.147593:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.147594:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677472.147595:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.147597:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.147598:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.147599:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677472.147602:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677472.147603:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677472.147603:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677472.147604:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.147606:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.147606:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.147607:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.147610:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.147611:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.147612:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.147612:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677472.147613:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.147613:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.147614:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.147615:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.147615:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.147615:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193efc76600] (23)++
00000800:00000200:18.0:1619677472.147616:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.147616:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193efc76600] (24)--
00000800:00000200:18.0:1619677472.147617:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193efc76600] (23)++
00000800:00000200:2.0:1619677472.147617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (23)--
00000800:00000200:18.0:1619677472.147618:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193efc76600] (24)--
00000800:00000200:18.0:1619677472.147618:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000800:00000200:2.0:1619677472.147619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (21)--
00000800:00000200:18.0:1619677472.147631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.147632:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.147633:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.147633:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.147633:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.147634:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.147635:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.147636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.147636:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.147637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677472.147637:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.147638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.147648:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.147651:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.147651:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.147652:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.147652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.147653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.147736:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.147739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.147739:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677472.147742:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.147745:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.147746:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.147747:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.147748:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.147749:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.147750:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.147753:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.147754:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.147756:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.147757:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.147757:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.147759:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.147760:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.147760:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.147761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:24.0:1619677472.147771:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.147772:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.147773:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.147774:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.147775:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.147776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.148545:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.148548:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677472.148549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677472.148551:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.148553:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.148554:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677472.148556:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.148557:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.148558:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.148559:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677472.148562:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.148564:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.148567:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.148568:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.148569:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677472.148571:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.148571:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.148572:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.148572:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.0:1619677472.148955:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.148958:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.148960:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.148961:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.2:1619677472.148962:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:20.0:1619677472.148965:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.148967:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677472.148967:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.148968:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.148969:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.148970:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.148971:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.148972:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.148975:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.148977:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:21.0:1619677472.148978:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.148979:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677472.148979:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.148981:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.148982:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.148983:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.148984:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.148987:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.148989:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.148990:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.148991:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.148991:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.148992:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.148992:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.148993:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.148994:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.148994:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.148995:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.148995:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.148995:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.148996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:21.0:1619677472.148997:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (23)++
00000800:00000200:20.0:1619677472.148997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.148998:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.148999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.149001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.149005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.149012:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.149013:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.149013:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677472.149014:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.149014:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.149015:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.149016:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.149016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.149017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677472.149018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.149097:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.149100:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.149101:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.149103:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.149104:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.149105:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677472.149106:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.149108:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.149109:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.149110:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.149113:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.149114:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.149117:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.149117:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.149118:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.149120:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.149120:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.149121:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.149121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.149134:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.149137:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.149137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.149138:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.149140:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677472.149142:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677472.149594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (20)++
00000800:00000200:2.0:1619677472.149600:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (21)++
00000800:00000200:2.0:1619677472.149603:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.3@o2ib3
00000800:00000200:18.0:1619677472.149604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000400:00000200:2.0:1619677472.149606:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.3@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.149609:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.149611:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677472.149612:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.149613:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.149614:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.149616:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677472.149619:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.149620:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677472.149623:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.149624:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677472.149625:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677472.149626:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677472.149627:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193efc76600] (21)++
00000800:00000200:2.0:1619677472.149628:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193efc76600] (22)--
00000800:00000200:2.0:1619677472.149628:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (21)--
00000800:00000200:8.0:1619677472.149639:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.149642:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.149643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.149643:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.149645:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.149648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.149678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.149687:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.149690:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.149691:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.149692:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.149693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.149695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.149701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.149709:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.149712:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.149713:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.149714:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.149715:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.149716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677472.149809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (20)++
00000800:00000200:18.0:1619677472.149814:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (21)++
00000800:00000200:18.0:1619677472.149817:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.3@o2ib3
00000800:00000200:2.0:1619677472.149818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000400:00000200:18.0:1619677472.149820:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.3@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.149823:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.149824:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.149825:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.149827:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.149828:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.149828:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.149832:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.149833:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.149836:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.149836:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.149837:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.149839:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.149839:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193efc76600] (21)++
00000800:00000200:18.0:1619677472.149841:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193efc76600] (22)++
00000800:00000200:18.0:1619677472.149842:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193efc76600] (23)--
00000800:00000200:18.0:1619677472.149843:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000800:00000200:8.2:1619677472.149845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:5.2:1619677472.149848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193efc76600] (21)++
00000800:00000200:20.0:1619677472.149852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677472.149853:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193efc76600] (22)++
00000800:00000200:20.0:1619677472.149854:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:2.0:1619677472.149855:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193efc76600] (23)--
00000400:00000200:20.0:1619677472.149856:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.149856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (22)--
00000400:00000200:20.0:1619677472.149857:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.149857:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193efc76600] (21)--
00000800:00000200:21.0:1619677472.149859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.149860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.150369:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.150372:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.150373:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.150373:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.150374:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.150377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.150480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.150487:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.150490:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.150490:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.150492:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.150495:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.150496:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.150498:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.150499:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.150500:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.150501:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.150505:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.150506:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.150509:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.150510:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.150511:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.150512:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.150513:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.150514:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.150515:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.150602:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.150604:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.150605:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.150607:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.150608:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.150610:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.150611:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.150612:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.150613:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.150614:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.150617:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.150618:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.150621:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.150622:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.150623:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.150625:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.150625:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.150626:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.150626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.150631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.150639:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.150642:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.150643:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.150643:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.150645:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.150648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.150784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.150790:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.150792:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.150794:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.150795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.150797:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.150798:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.150800:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.150801:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.150802:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.150803:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.150806:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.150808:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.150811:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.150812:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.150812:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.150814:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.150814:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.150816:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.150816:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.150820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.150828:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.150830:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.150831:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.150831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.150833:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.150836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677472.151019:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.151022:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.151023:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.151024:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.151027:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.151028:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.151029:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.151030:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.151032:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.151032:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.151035:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.151037:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.151039:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.151040:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.151041:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.151043:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.151043:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.151044:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.151044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.151049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.151056:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.151058:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.151059:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.151061:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.151063:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.151065:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.151318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.151327:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.151330:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.151330:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.151331:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.151333:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.151336:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.151630:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.151632:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.151634:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:24.0:1619677472.151635:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.151635:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677472.151637:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.151638:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.151640:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.151641:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.151643:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.151644:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.151645:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.151646:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.151647:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.151648:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677472.151648:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.151650:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.151650:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.151651:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.151652:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.151654:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677472.151656:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.151657:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.151658:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.151658:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.151659:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.151659:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.151659:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.151661:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.151661:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.151661:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677472.151662:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.151662:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677472.151663:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.151663:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.151664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.151664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.151665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.2:1619677472.151667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.151675:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.151678:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.151679:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.151679:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.151681:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.151683:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:1.0:1619677472.151684:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.151686:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.151686:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.151687:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.151689:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.151691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.151701:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.151703:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.151704:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.151704:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.151705:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.151706:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.151723:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677472.151726:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.151728:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.151729:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.151729:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.151731:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.151732:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677472.151732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.151734:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.151735:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.151736:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.151739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.151740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.151741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.151749:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.151751:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.151752:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.151753:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.151754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677472.151754:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.2:1619677472.152316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.152319:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.152321:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677472.152322:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.152322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.152324:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.152326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677472.152341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677472.152344:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.152347:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677472.152347:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.152348:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677472.152348:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677472.152349:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677472.152407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.152410:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.152412:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677472.152413:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.152413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.152414:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.152416:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677472.152544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.152547:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.152549:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677472.152550:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.152551:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.152552:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.152554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.0:1619677472.152630:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda10800] (21)++
00000800:00000200:24.0:1619677472.152633:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda10800] (22)--
00000800:00000200:8.0:1619677472.152633:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.17@o2ib3
00000400:00000200:8.0:1619677472.152635:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.17@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.152637:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.2:1619677472.152639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:8.0:1619677472.152640:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677472.152641:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.152643:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000400:00000200:8.0:1619677472.152643:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677472.152644:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677472.152644:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.152645:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.152645:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:24.0:1619677472.152646:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.152648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677472.152648:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.152649:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.0:1619677472.152650:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 968 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677472.152654:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.152654:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.0:1619677472.152655:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677472.152657:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677472.152657:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda10800] (21)++
00000800:00000200:8.0:1619677472.152658:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda10800] (22)++
00000800:00000200:8.0:1619677472.152660:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda10800] (23)--
00000800:00000200:8.0:1619677472.152660:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda10800] (22)--
00000800:00000200:20.0:1619677472.152671:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.152673:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.152674:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.152676:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.152679:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda10800] (22)++
00000800:00000200:21.0:1619677472.152680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.152680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677472.152681:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda10800] (23)--
00000800:00000200:24.0:1619677472.152683:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda10800] (22)--
00000800:00000200:8.0:1619677472.152683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda10800] (21)--
00000800:00000200:24.2:1619677472.152860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.152863:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.152865:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677472.152866:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.152867:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.152868:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.152870:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677472.152883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677472.152886:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.152888:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677472.152889:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.152889:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:8.0:1619677472.152889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677472.152890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.0:1619677472.152939:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.152941:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.152944:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.152944:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.152946:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.152947:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.152948:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.152950:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.152950:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.152951:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.152954:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.152956:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.152958:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.152959:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.152959:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.152961:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.152962:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.152963:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.152964:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.152965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.152976:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.152978:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.152979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.152980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.154517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.154524:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.154527:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.154528:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.154530:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.154533:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.154534:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.154536:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.154537:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.154538:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.154539:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.154542:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.154544:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.154546:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.154547:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.154548:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.154549:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.154550:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.154552:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.154553:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.154553:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.154559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.154566:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.154566:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.154567:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.154567:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.154567:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.154568:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.154569:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.154569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.154570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.154571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.154967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.154973:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.154976:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.154978:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.154978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.154981:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.154982:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.154984:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.154985:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.154986:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.154987:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.154991:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.154993:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.154995:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.154996:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.154996:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.154998:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.154999:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.155000:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.155000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.155005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.155013:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.155015:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.155016:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.155018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.155018:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.155020:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.155146:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.155149:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.155149:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.155150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.155151:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.155153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.155359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.155367:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.155370:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.155371:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.155373:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.155374:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.155376:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.156164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:2.0:1619677472.156169:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.156172:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:18.0:1619677472.156173:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:2.0:1619677472.156175:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.156177:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.156178:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677472.156180:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.156181:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.156182:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.156183:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677472.156186:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.156187:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677472.156190:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.156191:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677472.156191:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677472.156193:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677472.156194:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.156195:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:2.0:1619677472.156195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:20.0:1619677472.156206:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.156209:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.156209:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.156211:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.156212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.156213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.156225:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.156227:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.156228:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.156229:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.156230:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.156230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.156284:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.156287:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.156288:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.156289:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.156292:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.156299:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.156300:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.156301:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.156302:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.156303:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.156307:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.156309:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.156312:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.156312:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.156313:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.156315:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.156315:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.156316:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.156316:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.156321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.156330:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.156333:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.156333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.156334:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.156336:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.156339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.156342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.156350:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.156353:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.156354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.156354:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.156355:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.156355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.156780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.156785:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.156788:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.156790:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.156790:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.156792:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.156794:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.156795:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.156796:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.156797:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.156798:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.156802:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.156804:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.156806:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.156807:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.156807:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.156809:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.156810:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.156810:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.156811:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.156815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.156817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.156823:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677472.156823:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:18.0:1619677472.156826:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677472.156826:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.156827:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.156827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677472.156827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.156829:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.156830:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677472.156831:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:18.0:1619677472.156832:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.156834:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.156835:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.156837:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.156838:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.156839:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.156842:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.156844:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.156846:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.156847:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.156847:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.156849:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.156850:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.156851:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.156852:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.156856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.156866:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.156868:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.156868:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.156868:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.156870:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.156870:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000400:00000200:21.0:1619677472.156872:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.156873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.156874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:21.0:1619677472.156875:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.156876:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677472.156876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.156878:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.156879:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.156880:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.156881:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.156884:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.156886:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.156889:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.156889:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.156890:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.156892:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.156892:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.156893:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.156894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.156911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.156918:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.156920:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.156923:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.107@o2ib7 : GET - for me
00000800:00000200:21.0:1619677472.156924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.156926:0:19934:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.107@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:20.0:1619677472.156928:0:19934:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.107@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:20.0:1619677472.156929:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.156930:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.156931:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.156935:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : REPLY try# 0
00000800:00000200:20.0:1619677472.156937:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.156938:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.156939:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.156940:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.156941:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.156942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677472.156947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.156954:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.156957:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.156957:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.156959:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: REPLY: OK
00000800:00000200:20.0:1619677472.156960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.156961:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.157321:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.157324:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.157325:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.157326:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.157329:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.157331:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.157332:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.157334:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.157335:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.157336:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.157339:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.157340:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.157342:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.157343:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.157343:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.157345:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.157345:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.157347:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.157348:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.157348:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.157359:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.157361:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.157361:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.157363:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.157363:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.157363:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.157364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.157365:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.157365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.157366:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.157588:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.157591:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.157591:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.157592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.157593:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.157595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.157606:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.157609:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.157610:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.157610:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.157611:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.157611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.157762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.157770:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.157773:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.157774:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.157775:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.157777:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.157780:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.158425:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.158427:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677472.158430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.158430:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.158432:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.158433:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677472.158435:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.158436:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.158437:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.158438:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.158441:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.158443:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.158445:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.158446:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.158447:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.158448:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.158449:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.158450:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.158450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677472.158463:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.158466:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.158467:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.158468:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.158469:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.158472:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.158481:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.158484:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.158484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.158485:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.158486:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.158486:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.159462:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.159465:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.159466:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.159468:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.159470:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.159472:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.159473:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.159475:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.159476:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.159477:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.159480:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.159481:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.159484:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.159485:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.159486:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.159488:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.159489:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.159489:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.159490:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.159518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.159526:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.159529:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.159529:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.159531:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.159532:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.159534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.160973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.160981:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.160983:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.160985:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.160986:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.160988:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.160990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.161032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.161038:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.161040:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.161042:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.161042:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.161045:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.161046:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.161048:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.161049:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.161050:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.161051:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.161054:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.161056:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.161059:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.161060:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.161060:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.161062:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.161063:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.161063:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.161065:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.161065:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677472.161070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.161073:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.161075:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.161076:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.161077:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.161080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.161087:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.161089:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.161090:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.161091:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.161092:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.161094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.161102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.161109:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.161112:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.161113:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.161113:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.161114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.161115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.161359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.161366:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.161369:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.161370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.161371:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.161374:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.161375:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.161377:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.161378:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.161379:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.161380:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.161383:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.161385:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.161387:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.161387:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.161388:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.161390:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.161391:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.161392:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.161392:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.161405:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.161407:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.161408:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.161408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.161411:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.161412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.161595:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.161597:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.161598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.161598:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.161600:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.161602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.161606:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.161608:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.161611:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.161611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.161613:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.161615:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.161616:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.161618:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.161619:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.161620:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.161623:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.161625:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.161628:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.161629:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.161629:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.161631:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.161632:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.161633:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.161633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.161643:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.161646:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.161647:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.161647:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.161649:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.161652:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.161833:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.161835:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.161836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.161838:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.161840:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.161841:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.161842:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.161843:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.161844:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.161845:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.161848:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.161849:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.161851:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.161852:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.161852:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.161854:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.161854:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.161855:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.161855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.161865:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.161866:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.161867:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.161868:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.161868:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.161869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.161885:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.161888:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.161888:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.161889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.161889:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.161890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.162749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.162752:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.162755:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.162757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.162757:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.162759:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.162761:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.162762:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.162764:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.162765:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.162766:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.162769:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.162771:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.162774:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.162774:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.162775:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.162776:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.162777:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.162778:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.162778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.162784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.162792:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.162795:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.162796:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.162796:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.162797:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677472.162798:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.162800:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677472.162801:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.162802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:24.0:1619677472.162803:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.162805:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.162806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.162807:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.162808:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.162809:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:24.0:1619677472.162810:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.162811:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.162812:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677472.162812:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.162813:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.162814:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:24.0:1619677472.162815:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.162815:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.162816:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.162817:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.162819:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.162820:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.162820:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.162821:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.162822:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.162823:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.162823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.162834:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.162836:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.162837:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.162837:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.162838:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.162840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.163219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.163225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.163228:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.163229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.163231:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.163233:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.163234:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.163235:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.163237:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.163238:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.163239:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.163242:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.163244:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.163247:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.163248:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.163249:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.163250:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.163251:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.163252:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.163253:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.163254:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.163259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.163265:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.163267:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.163268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.163269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.163447:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.163450:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.163450:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.163451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.163452:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.163454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.163908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.163911:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.163913:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.163914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.163915:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.163917:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.163918:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.163919:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.163920:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.163921:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.163924:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.163926:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.163928:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.163929:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.163929:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.163931:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.163931:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:21.0:1619677472.163932:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.163933:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.163933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.163934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677472.163934:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:8.0:1619677472.163936:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.163938:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.163939:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677472.163940:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.163942:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.163943:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.163944:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677472.163947:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.163949:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677472.163952:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.163952:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677472.163953:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677472.163954:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677472.163955:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.163956:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677472.163956:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:18.0:1619677472.164308:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.164311:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.164311:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.164314:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.164316:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.164318:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677472.164319:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.164321:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.164322:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.164323:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.164326:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.164328:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677472.164330:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.164331:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677472.164332:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677472.164334:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677472.164334:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.164335:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.164336:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:8.2:1619677472.164952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.164962:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.164965:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.164966:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.164967:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.164969:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.164972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.165028:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.165031:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.165031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.165033:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.165035:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.165037:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677472.165038:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.165040:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.165041:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.165042:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677472.165045:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.165047:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677472.165050:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.165051:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677472.165052:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677472.165054:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677472.165054:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.165055:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.165055:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677472.165067:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.165070:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.165071:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.165072:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.165074:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.165075:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677472.165146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.165148:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.165149:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.165150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.165151:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.165153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.2:1619677472.165724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.165728:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.165730:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677472.165731:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.165732:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.165733:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.165735:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.2:1619677472.166828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.166837:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.166839:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.166840:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.166842:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.166843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.166845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.166881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.166886:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.166889:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.166891:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.166891:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.166894:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.166895:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.166896:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.166898:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.166899:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.166900:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.166903:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.166904:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.166907:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.166908:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.166908:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.166910:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.166911:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.166912:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.166912:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.166916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.166923:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.166925:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.166926:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.166928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.166928:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.166930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.166954:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.166957:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.166959:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.166959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.166961:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.166963:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677472.166964:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.166965:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.166966:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.166967:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677472.166970:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.166972:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.166974:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.166974:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677472.166975:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.166976:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.166977:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.166978:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.166978:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677472.166991:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677472.166991:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.166994:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677472.166994:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677472.166995:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.166995:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.166995:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.166997:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.166998:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.167001:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.167001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:8.0:1619677472.167002:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.167003:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.167005:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.167006:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.167007:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.167009:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.167011:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.167012:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.167013:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.167014:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.167015:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.167015:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.167016:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.167017:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.167037:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.167039:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.167040:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.167041:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.167041:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.167042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677472.167082:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.167085:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.167086:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.167086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.167088:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.167091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.0:1619677472.167135:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)++
00000800:00000200:20.0:1619677472.167137:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.57@o2ib3
00000400:00000200:20.0:1619677472.167139:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.57@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.167141:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000400:00000200:20.0:1619677472.167142:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.167143:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677472.167144:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.167145:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.167146:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.167147:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677472.167150:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.167152:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.167153:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.167154:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.167155:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.167156:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.167157:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c400] (21)++
00000800:00000200:20.0:1619677472.167158:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c400] (22)--
00000800:00000200:20.0:1619677472.167158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)--
00000800:00000200:8.2:1619677472.167162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.167171:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.167173:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.167174:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.167176:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.167176:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.167178:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677472.167244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.167247:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.167249:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.167250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.167250:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.167252:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.167253:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.167254:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.167255:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.167256:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.167259:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.167261:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.167263:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.167264:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.167264:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.167266:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.167266:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.167268:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.167269:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.167269:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677472.167278:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.167280:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.167281:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.167281:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.167283:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.167286:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.167290:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.167292:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.167293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.167299:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.0:1619677472.167321:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)++
00000800:00000200:20.0:1619677472.167323:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.57@o2ib3
00000400:00000200:20.0:1619677472.167325:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.57@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.167326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (22)--
00000400:00000200:20.0:1619677472.167328:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.167329:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677472.167330:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.167331:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.167332:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.167333:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.167336:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.167337:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.167340:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.167340:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.167341:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.167342:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.167343:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197eb55c400] (21)++
00000800:00000200:20.0:1619677472.167343:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197eb55c400] (22)--
00000800:00000200:20.0:1619677472.167344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197eb55c400] (21)--
00000800:00000200:2.0:1619677472.167358:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.167361:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.167362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.167362:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.167364:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.167367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677472.167481:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.167484:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.167485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.167487:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.167490:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.167492:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.167493:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.167495:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.167496:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.167497:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.167500:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.167502:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.167505:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.167505:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.167506:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.167508:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.167508:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.167509:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.167509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.167514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.167522:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.167525:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.167526:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.167527:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.167528:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.167530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.167935:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.167938:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.167939:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.167941:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.167942:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.167944:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.167945:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.167947:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.167948:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.167949:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.167952:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.167954:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.167956:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.167956:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.167957:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.167958:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.167959:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.167960:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.167962:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.167962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.167973:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.167974:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.167974:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.167976:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.167976:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.167977:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.167977:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.167978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.167978:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.167979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.167990:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.167992:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.167993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.167993:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.167993:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.167994:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.168450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.168459:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.168462:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.168463:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.168464:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.168466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.168468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.168610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.168616:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.168619:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.168620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.168622:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.168624:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.168626:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.168627:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.168628:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.168629:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.168630:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.168633:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.168635:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.168636:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.168637:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.168638:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.168640:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.168641:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.168642:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.168642:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677472.168650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677472.168654:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677472.168656:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677472.168657:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.168657:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.168658:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677472.168658:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.168660:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.168661:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.168661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677472.168662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:21.0:1619677472.168663:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.168665:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.168666:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.168668:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.168669:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.168670:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.168673:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.168673:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.168674:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677472.168676:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.168677:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.168677:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.168677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.168678:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.168678:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:18.0:1619677472.168678:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.168678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.168680:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.168681:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.168682:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.168682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.168686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.168695:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.168698:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.168699:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.168699:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.168701:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.168704:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677472.168725:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.168727:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.168729:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.168730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.168732:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.168733:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.168734:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.168736:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.168737:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.168738:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.168741:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.168743:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.168745:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.168746:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.168747:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.168748:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.168749:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.168749:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.168750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.169178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.169186:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.169189:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.169190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.169190:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.169192:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.169194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.169958:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.169961:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.169962:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.169962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.169964:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.169966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.170819:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.170822:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.170823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.170825:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.170827:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.170828:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677472.170829:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.170831:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.170832:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.170833:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677472.170836:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.170837:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677472.170841:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.170841:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677472.170842:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677472.170844:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677472.170845:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.170846:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.170846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677472.170860:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.170863:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.170865:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.170867:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.170870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.170872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.2:1619677472.171561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.171569:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.171572:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.171573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.171574:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.171576:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.171578:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.171733:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.171736:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.171736:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.171738:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.171741:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.171742:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.171743:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.171744:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.171745:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.171746:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.171749:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.171751:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.171753:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.171754:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.171755:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.171757:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.171757:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.171759:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.171760:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.171760:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.171763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.171771:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677472.171773:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.171773:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.171773:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.171774:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.171775:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.171776:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.171778:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.171778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.171781:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.171784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.171791:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.171794:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.171795:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.171796:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.171796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.171797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.171828:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.171831:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.171833:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.171834:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.171835:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.171836:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677472.171837:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.171839:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.171840:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.171842:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.171844:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.171846:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.171848:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.171849:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.171850:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.171851:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.171852:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.171853:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.171854:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.171855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.171865:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.171868:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.171868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.171869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677472.171878:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.171881:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.171882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.171882:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.171883:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.171886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.171951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.171957:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.171960:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.171961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.171963:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.171965:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.171966:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677472.171967:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.171969:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.171970:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.171971:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677472.171974:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.171976:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677472.171979:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.171980:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677472.171981:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677472.171982:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677472.171983:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.171984:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.171984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.172104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.172109:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.172112:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.172114:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.172114:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.172117:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.172118:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.172119:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.172121:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.172122:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.172123:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.172126:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.172128:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.172129:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.172130:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.172131:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.172132:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.172133:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.172134:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.172135:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.172136:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.172138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.172142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.172148:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.172150:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.172150:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677472.172151:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.172151:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677472.172152:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.172152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.172153:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.172154:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.172155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677472.172267:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.172270:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.172271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.172272:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.172274:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.172275:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.172277:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.172278:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.172279:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.172280:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.172283:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.172284:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.172287:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.172288:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.172289:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.172290:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.172291:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.172292:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.172292:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.172297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.172305:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.172308:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.172309:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.172310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.172312:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.172315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.172318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.172320:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.172322:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.172322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.172323:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.172324:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.172325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.172337:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.172340:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.172341:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.172343:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.172343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.172345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.173016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.173024:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.173026:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.173027:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.173029:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.173030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677472.173030:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677472.173032:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:20.0:1619677472.173032:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:24.0:1619677472.173033:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.173033:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.173035:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.173037:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.173052:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.173054:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.173055:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.173055:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.173056:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.173057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.173116:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.173118:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677472.173120:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.173120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.173123:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.173124:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.173125:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.173127:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.173128:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.173128:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.173131:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.173133:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.173136:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.173137:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.173137:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.173139:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.173139:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.173140:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.173140:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.173145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.173153:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.173156:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.173157:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.173158:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.173160:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.173163:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.173179:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.173182:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.173184:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.173184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.173186:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.173187:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.173189:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.173190:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.173191:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.173192:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.173195:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.173197:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.173200:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.173201:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.173202:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.173203:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.173204:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.173205:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.173206:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.173216:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.173218:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.173219:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.173220:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.173222:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.173225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.173556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.173566:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.173568:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.173569:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.173571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.173571:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.173574:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.173576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.173585:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.173587:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.173588:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.173589:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.173590:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.173591:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.175017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.175026:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677472.175026:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:21.0:1619677472.175029:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677472.175030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677472.175030:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.175030:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:20.0:1619677472.175031:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677472.175032:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:21.0:1619677472.175032:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.175033:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.175033:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.175034:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.175034:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.175036:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.175036:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677472.175036:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.175038:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.175038:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:24.0:1619677472.175039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:21.0:1619677472.175039:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.175040:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.175040:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.175042:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677472.175042:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.175043:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.175044:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:21.0:1619677472.175050:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677472.175050:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.175053:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.175053:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.0:1619677472.175056:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.175057:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.175057:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.175058:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.175058:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677472.175059:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677472.175060:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.175060:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.175061:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677472.175061:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677472.175061:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:21.0:1619677472.175062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677472.175062:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.175063:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.175066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.175074:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.175076:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.175077:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:2.0:1619677472.175077:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:1.0:1619677472.175077:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.175079:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.175081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677472.175082:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.175084:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.175085:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.175087:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.175091:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677472.176242:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.176245:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.176246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.176248:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.176251:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.176252:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.176254:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.176255:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.176256:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.176257:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.176261:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.176263:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.176265:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.176266:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.176266:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.176268:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.176269:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.176270:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.176270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.176275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.176283:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.176286:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.176287:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.176288:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.176289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.176291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.177401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.177412:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.177415:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.177416:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.177418:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.177418:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.177420:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.177421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.177421:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)++
00000800:00000200:20.0:1619677472.177421:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (24)--
00000800:00000200:20.0:1619677472.177422:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.177423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.177575:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.177577:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.177578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.177580:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.177582:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.177584:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.177585:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.177586:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.177587:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.177588:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.177591:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.177592:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.177594:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.177594:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.177595:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.177597:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.177597:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.177598:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.177598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.177609:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.177611:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.177612:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.177612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.177613:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.177615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.177915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.177922:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.177925:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.177925:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.177927:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.177930:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.177931:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.177933:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.177934:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.177935:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.177936:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.177939:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.177941:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.177942:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.177943:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.177944:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.177945:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.177946:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.177947:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.177947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.177959:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.177961:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.177961:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.177962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.177963:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.177965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.177980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.177983:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.177985:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.177987:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.177988:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.177989:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.177991:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.177992:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.177993:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.177994:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.177995:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.177998:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.178000:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.178003:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.178004:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.178004:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.178006:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.178006:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.178007:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.178008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.178012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.178020:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.178022:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.178023:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.178025:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.178025:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.178027:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.178596:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.178599:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.178600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.178602:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.178603:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.178604:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.178606:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.178607:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.178608:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.178609:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.178613:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.178615:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.178618:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.178619:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.178619:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.178621:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.178621:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.178623:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.178624:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.178624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.178627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.178635:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.178638:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677472.178638:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.178639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.178639:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.178640:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.178641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.178642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677472.178642:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.178646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.178661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.178669:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.178671:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.178673:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.178674:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.178675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.178677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.178966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.178972:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.178975:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.178976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.178978:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.178980:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.178982:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.178983:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.178984:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.178985:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.178986:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.178989:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.178991:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.178993:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.178994:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.178995:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.178996:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.178997:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.178998:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.178998:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.179002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.179010:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.179013:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.179013:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.179015:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.179015:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.179018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.179680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.179686:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.179689:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.179690:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.179691:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.179694:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.179695:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677472.179697:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.179698:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.179699:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.179700:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677472.179702:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.179704:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.179706:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.179707:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677472.179707:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677472.179709:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677472.179709:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.179710:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677472.179711:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677472.179712:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677472.179717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.179724:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677472.179726:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677472.179727:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.179728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.0:1619677472.179758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.179761:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.179762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.179762:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.179764:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.179766:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.180077:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.180080:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677472.180081:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677472.180082:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.180084:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.180086:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.180087:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.180088:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.180089:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.180090:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.180093:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.180095:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.180098:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.180099:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.180099:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.180101:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.180101:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.180102:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677472.180102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677472.180107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.180116:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.180118:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.180119:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.180120:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.180122:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.180124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:13.0:1619677472.180268:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.180273:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.180277:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.180277:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.180280:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.180283:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677472.180285:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.180287:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.180289:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.180290:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677472.180304:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.180307:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677472.180310:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.180312:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677472.180312:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677472.180315:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677472.180316:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.180318:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677472.180320:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677472.180321:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:18.0:1619677472.180330:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.180333:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.180333:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.180334:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.180335:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:18.0:1619677472.180336:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.180337:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677472.180338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.180339:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677472.180340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.180956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.180959:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.180962:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.180964:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.180965:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.180966:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.180967:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.180969:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.180970:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.180971:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.180972:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.180976:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.180978:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.180980:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.180981:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.180982:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.180983:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.180984:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.180984:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.180985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.180990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.180999:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.181001:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.181002:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.181003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.181005:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.181007:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.181009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.181016:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.181019:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.181020:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.181020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.181021:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.181021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677472.181094:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.181097:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.181099:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.181099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.181102:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.181103:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677472.181105:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.181106:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.181107:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.181108:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677472.181112:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.181113:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.181116:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.181117:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677472.181117:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.181119:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.181120:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.181121:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.181122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677472.181133:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.181135:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.181136:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.181137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.181139:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.181142:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.181155:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.181158:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.181158:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.181159:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.181160:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.181160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677472.181326:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.181329:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.181330:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.181331:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.181334:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.181335:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.181336:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.181337:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.181338:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.181339:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.181342:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.181344:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.181346:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.181347:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.181348:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.181350:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.181351:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.181351:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.181352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.181357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.181365:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.181368:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.181369:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.181370:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.181372:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.181372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677472.182367:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.182369:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.182371:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.182371:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.182373:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.182376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677472.182624:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.182626:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.182627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.182629:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.182632:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.182634:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677472.182635:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.182636:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.182637:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.182638:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677472.182641:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.182643:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677472.182646:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.182647:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677472.182648:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677472.182650:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677472.182650:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.182651:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677472.182652:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677472.182653:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.0:1619677472.182663:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:1.0:1619677472.182664:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:8.0:1619677472.182665:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.182666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677472.182666:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:8.0:1619677472.182667:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.182667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.182668:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:8.0:1619677472.182669:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.182673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.182674:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.182676:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.182677:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.182678:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.182678:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.182679:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.182680:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.182682:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.182683:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.182684:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.182685:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.182686:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.182687:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.182689:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.182691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.182691:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.182692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.182693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.182694:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.182694:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.182695:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.182696:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.182697:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.182698:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.182698:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.182708:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.182709:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.182710:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.182711:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.182711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.182712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.182729:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.182732:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.182733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.182733:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.182734:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.182734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.182910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.182917:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.182919:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.182920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.182922:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.182924:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.182926:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677472.182927:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.182929:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.182930:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.182931:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677472.182934:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.182936:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677472.182938:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.182939:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677472.182940:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.182942:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.182942:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.182943:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.182944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.182948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.182957:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.182959:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.182960:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.182960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.182962:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.182965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.182970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.182977:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.182980:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.182981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.182981:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.182981:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.182982:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.183683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.183689:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.183691:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.183694:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.183694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.183701:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.183702:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.183704:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.183705:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.183706:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.183707:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.183710:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.183712:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.2:1619677472.183713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.183715:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.183716:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.183717:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (23)++
00000800:00000200:21.0:1619677472.183719:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (24)--
00000800:00000200:21.0:1619677472.183719:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.183721:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.183721:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:21.0:1619677472.183722:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.183722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.183722:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:20.0:1619677472.183723:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.183724:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:21.0:1619677472.183725:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:20.0:1619677472.183725:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.183726:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.183727:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.2:1619677472.183727:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.183730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.183731:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.183732:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.183733:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.183734:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.183734:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.183735:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.183735:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.183735:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.183736:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.183737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.183738:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677472.184834:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.184837:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.184838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.184839:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.184841:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.184843:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.184844:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.184845:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.184846:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.184847:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.184850:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.184851:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.184854:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.184855:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.184856:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.184858:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.184858:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.184859:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.184859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.184864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.184873:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.184876:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.184877:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.184877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.184879:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.184882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.185260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.185268:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.185270:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.185271:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.185273:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.185274:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.185275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.185398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.185405:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.185407:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.185409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.185410:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.185414:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.185415:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.185417:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.185418:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.185419:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.185420:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.185424:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.185425:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.185428:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.185429:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.185429:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.185431:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.185432:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.185432:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.185433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677472.185659:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.185662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.185662:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.185664:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.185667:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.185668:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.185670:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.185671:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.185672:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.185673:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.185676:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.185678:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.185680:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.185681:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.185681:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.185683:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.185684:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.185685:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.185685:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.185690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.185697:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.185699:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.185700:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.185702:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.185703:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.185704:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.186242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677472.186245:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.186247:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677472.186250:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.186251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677472.186252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.186254:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677472.186255:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.186256:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.186257:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.186258:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677472.186262:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.186263:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.186266:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.186266:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.186267:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.186269:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.186269:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.186270:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677472.186271:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.186275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.186283:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.186285:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.186286:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.186287:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.186288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.186289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.187495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.187498:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.187501:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.187503:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.187504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.187505:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.187506:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.187508:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.187509:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.187510:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.187511:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.187514:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.187516:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.187518:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.187519:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.187520:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.187521:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.187522:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.187523:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.187523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.187528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.187536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.187539:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.187540:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.187540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.187542:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.187545:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.187563:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.187566:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.187567:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.187569:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.187571:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.187573:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.187574:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.187575:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.187576:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.187577:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.187580:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.187582:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.187584:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.187584:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.187585:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.187586:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.187587:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.187587:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.187588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.187599:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.187600:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.187601:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.187602:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.187603:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.187604:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677472.187930:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.187933:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.187934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.187936:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.187938:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.187940:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.187941:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.187943:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.187944:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.187945:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.187947:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.187949:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.187951:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.187952:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.187953:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.187954:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.187955:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.187956:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.187956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677472.187967:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.187969:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.187970:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.187971:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.187972:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.187973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.188468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677472.188474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.188477:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.188479:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.188480:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:18.0:1619677472.188481:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.188482:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:18.0:1619677472.188483:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.188484:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.188484:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.188485:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.188487:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.188487:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.188488:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.188491:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.188492:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.188494:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677472.188494:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.188495:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:2.0:1619677472.188495:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:18.0:1619677472.188496:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.188497:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.188497:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.188498:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.188499:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.188500:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677472.188500:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.188502:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (23)++
00000800:00000200:18.0:1619677472.188503:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (24)--
00000400:00000200:2.0:1619677472.188504:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.188505:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.188505:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.188506:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.188507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677472.188509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.188509:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.188509:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.188510:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.188510:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.188510:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.188510:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:18.0:1619677472.188512:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.188513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.188513:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.188513:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.188514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.188514:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.188515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:18.0:1619677472.188516:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.188517:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.188517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.188518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677472.189030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.189042:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.189044:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.189045:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.189047:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.189047:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.189050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.189051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.189059:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.189061:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.189062:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.189063:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.189064:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.189064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.2:1619677472.189356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.189359:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.189361:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.189363:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.189364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.189365:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.189367:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.189368:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.189369:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.189370:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.189371:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.189374:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.189376:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.189378:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.189379:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.189379:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.189381:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.189382:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.189382:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.189383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.189387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.189395:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.189397:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.189398:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.189400:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.189400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.189402:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.189945:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.189948:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.189949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.189949:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.189950:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.189953:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.189968:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.189971:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.189971:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.189972:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677472.189972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.189973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.189991:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.189994:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.189995:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.189997:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.189999:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.190000:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.190002:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.190003:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.190004:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.190005:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.190008:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.190010:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.190013:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.190013:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.190014:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.190016:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.190016:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.190018:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.190019:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.190019:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.190022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.190030:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.190030:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.190031:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677472.190032:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677472.190032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677472.190033:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.190033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677472.190035:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.190036:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.190037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.190327:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.190330:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.190331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.190332:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.190335:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.190336:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.190337:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.190338:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.190340:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.190341:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.190344:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.190346:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.190349:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.190350:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.190350:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.190352:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.190352:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.190354:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.190354:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.190355:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.190358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.190366:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.190368:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677472.190368:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.190369:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.190370:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677472.190370:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.190371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.190372:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677472.190372:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.190375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.190519:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.190521:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.190523:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.190524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.190525:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.190526:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677472.190528:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.190529:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.190530:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.190531:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677472.190534:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.190536:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.190538:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.190539:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677472.190540:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.190542:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.190543:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.190544:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.190544:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677472.190557:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.190559:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.190560:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.190561:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.190562:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.190565:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.191366:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.191368:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.191369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.191369:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.191371:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.191374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.191418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677472.191421:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.191423:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677472.191426:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.191427:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677472.191428:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.191429:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677472.191431:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.191432:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.191433:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.191434:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677472.191437:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.191439:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677472.191442:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.191443:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677472.191444:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677472.191446:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677472.191446:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.191447:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677472.191448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:12.0:1619677472.191459:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.191462:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.191463:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.191464:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.191467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.191467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677472.191477:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677472.191480:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677472.191481:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.191482:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677472.191482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.191484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.2:1619677472.191663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.191672:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.191675:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.191676:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.191676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.191678:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.191682:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.192418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.192425:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.192428:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.192429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.192431:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.192433:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.192435:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.192436:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.192437:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.192438:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.192439:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.192442:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.192444:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.192446:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.192447:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.192448:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.192449:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.192450:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.192451:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.192452:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.192452:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.192456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.192458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677472.192464:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677472.192464:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.192466:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.192466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677472.192467:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.192467:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.192467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:21.0:1619677472.192469:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.192470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.192472:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:18.0:1619677472.192560:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.192562:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.192563:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.192564:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.192566:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.192573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.192574:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.192576:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.192577:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.192578:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.192578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.192579:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.192984:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.192986:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.192987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.192990:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.192992:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.192993:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.192994:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.192996:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.192997:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.192998:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.193000:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.193002:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.193004:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.193004:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.193005:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.193006:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.193007:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.193007:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.193008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.193018:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.193020:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.193021:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.193022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.193023:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.193025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.193385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.193388:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.193390:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.193392:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.193393:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.193395:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.193396:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.193398:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.193399:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.193400:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.193401:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.193404:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.193406:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.193409:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.193409:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.193410:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.193412:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.193412:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.193413:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.193414:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.193419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.193427:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.193430:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.193431:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.193431:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.193433:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.193435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.194841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.194851:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.194853:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.194854:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.194856:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.194856:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.194859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677472.194999:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.195003:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.195004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.195006:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.195009:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.195011:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.195013:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.195016:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.195018:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.195019:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.195026:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.195031:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.195035:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.195038:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677472.195039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677472.195039:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.195042:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.195043:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677472.195044:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677472.195044:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.195046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.195048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.195049:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.195053:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.195055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.195056:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.195057:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.195059:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.195059:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.195060:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.195061:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96514561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.195063:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677472.195064:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.195067:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.195071:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677472.195075:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.195078:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:20.0:1619677472.195078:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.195080:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.195081:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677472.195082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.195082:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:24.0:1619677472.195083:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.195084:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.195086:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.195087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.195087:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.195087:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.195089:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)++
00000800:00000200:20.0:1619677472.195089:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (23)++
00000800:00000200:21.0:1619677472.195091:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (25)--
00000800:00000200:21.0:1619677472.195092:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (24)--
00000800:00000200:20.0:1619677472.195092:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (25)--
00000800:00000200:21.0:1619677472.195093:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.195093:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.195095:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.195097:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:8.2:1619677472.195098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.195102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677472.195105:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.195108:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.195110:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.195111:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677472.195111:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.195113:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.195114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.195115:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.195116:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.195118:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677472.195119:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677472.195137:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.195141:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.195143:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.195144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.195147:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.195153:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.195155:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.195158:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.195158:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677472.195160:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.195162:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.195163:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677472.195574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677472.195583:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.195587:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677472.195589:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677472.195592:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677472.195596:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.195599:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677472.195601:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.195603:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.195605:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.195607:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677472.195612:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677472.195615:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677472.195619:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.195620:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677472.195621:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677472.195624:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677472.195625:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.195626:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677472.195627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.0:1619677472.195641:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.195645:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.195646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.195647:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.195650:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.195654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677472.195698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.195702:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.195706:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.195708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.195709:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.195711:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677472.195714:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.195716:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.195718:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.195720:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677472.195727:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.195731:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677472.195735:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.195736:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677472.195738:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677472.195740:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677472.195742:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.195743:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.195744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677472.195828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.195838:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.195842:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.195843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.195847:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.195850:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.195853:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.195855:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.195857:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.195859:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.195861:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.195867:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.195871:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.195875:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.195876:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.195877:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.195879:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.195880:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.195882:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.195883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.195895:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.195898:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.195900:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.195901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.195903:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.195905:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.196031:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.196034:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.196035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.196036:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.196039:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.196040:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.196042:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.196043:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.196044:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.196045:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.196048:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.196049:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.196052:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.196053:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.196054:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.196055:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.196056:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.196056:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.196057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.196061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.196072:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.196075:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.196076:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.196076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.196079:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.196082:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.196084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.196092:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.196095:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.196096:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.196096:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.196097:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.196098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.197865:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.197869:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.197871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.197874:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.197879:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.197881:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.197884:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.197886:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.197888:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.197890:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.197895:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.197898:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.197901:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.197902:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.197903:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.197904:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.197906:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.197907:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.197908:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.197922:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.197926:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.197927:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.197929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.197932:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.197934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.198277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.198289:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.198293:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.198295:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.198303:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.198307:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.198311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.199476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.199482:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.199485:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.199487:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.199488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.199489:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.199491:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.199492:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.199493:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.199494:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.199496:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.199499:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.199501:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.199503:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.199504:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.199505:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.199507:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.199507:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.199508:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.199509:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.199513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.199522:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.199525:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.199526:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.199526:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.199528:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.199530:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677472.199707:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.199710:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.199712:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.199713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.199714:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.199715:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677472.199717:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.199718:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.199719:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.199720:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.199723:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.199725:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.199728:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.199728:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.199729:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.199731:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.199731:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.199732:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.199733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.199747:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.199750:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.199751:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.199751:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.199753:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677472.199756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.199950:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.199952:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.199954:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.199955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.199957:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.199958:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.199959:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.199961:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.199962:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.199963:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677472.199963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:21.0:1619677472.199966:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.199968:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.199968:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677472.199970:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.199971:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677472.199971:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:21.0:1619677472.199972:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677472.199972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677472.199973:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.199974:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:2.0:1619677472.199974:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.199975:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.199975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:2.0:1619677472.199975:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.199977:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677472.199978:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.199979:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.199980:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.199982:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677472.199985:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.199987:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677472.199989:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.199990:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677472.199991:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677472.199992:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677472.199993:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.199994:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.199995:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677472.200006:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.200008:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.200009:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.200011:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.200012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.200013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677472.200051:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.200054:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.200054:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.200057:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.200059:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.200060:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.200062:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.200063:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.200064:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.200065:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.200068:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.200070:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.200073:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.200073:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.200074:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.200076:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.200077:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.200077:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.200078:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.200082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.200090:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.200092:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.200093:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.200095:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.200096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.200097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.200162:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.200165:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.200166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.200166:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.200168:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.200170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.200182:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.200185:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.200185:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.200185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.200186:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.200187:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.200615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.200623:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.200626:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.200627:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.200629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.200629:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.200632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.200766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.200771:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.200773:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.200775:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.200776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.200778:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.200780:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.200781:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.200782:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.200783:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.200784:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.200787:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.200789:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.200791:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.200792:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.200793:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.200794:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.200795:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.200796:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.200796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.200801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.200808:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.200811:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.200812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.200812:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.200814:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.200816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.200818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.200825:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.200827:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.200828:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.200828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.200829:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.200829:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.200892:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:24.0:1619677472.200895:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:8.0:1619677472.200895:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:8.0:1619677472.200895:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.200896:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:24.0:1619677472.200897:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.200897:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.200898:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.200899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.200900:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.200901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.201662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.201664:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.201666:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.201667:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.201669:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.201670:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.201671:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.201673:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.201674:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.201675:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.201678:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.201679:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.201682:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.201683:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.201683:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.201685:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.201685:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.201687:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.201688:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.201688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.201691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.201699:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677472.201702:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677472.201702:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.201703:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.201703:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.201705:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677472.201705:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.201706:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.201707:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677472.201708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.202106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.202112:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.202115:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.202116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.202118:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.202121:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.202122:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677472.202124:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.202125:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.202126:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.202127:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677472.202130:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.202131:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677472.202135:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.202136:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677472.202137:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677472.202138:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677472.202139:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.202140:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.202140:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:13.0:1619677472.202152:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.202155:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.202156:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.202158:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.202161:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.202162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677472.202441:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.202443:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.202445:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.202446:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.202449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.202450:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677472.202462:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677472.202466:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.202468:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.202469:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677472.202470:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.202471:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677472.202971:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.202973:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.202974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.202976:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.202979:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.202981:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.202982:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.202983:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.202984:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.202985:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.202988:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.202989:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.202991:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.202992:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.202992:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.202994:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.202994:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.202995:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.202995:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.203006:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.203008:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.203008:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.203009:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.203010:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.203011:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.203025:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.203027:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.203028:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.203028:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.203028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.203029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.204389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.204395:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.204398:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.204399:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.204401:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.204404:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.204405:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.204407:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.204408:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.204409:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.204410:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.204413:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.204414:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.204416:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.204417:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.204418:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.204419:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.204420:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.204421:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.204421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.204444:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.204446:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:20.0:1619677472.204447:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:18.0:1619677472.204447:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.204447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.204449:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.204450:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:18.0:1619677472.204450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:20.0:1619677472.204452:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.204453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.204454:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.204455:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.204457:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.204458:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.204459:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.204460:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.204463:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.204464:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.204467:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.204467:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.204468:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.204470:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.204471:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.204472:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.204473:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.204474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.204485:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.204487:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.204488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.204489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.205032:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.205035:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.205038:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.205039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.205039:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.205040:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:8.0:1619677472.205041:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.205043:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:8.0:1619677472.205043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:17.0:1619677472.205044:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.205044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.205046:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.205049:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.205431:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.205434:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.205435:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.205437:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.205438:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.205440:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.205441:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.205442:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.205443:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.205444:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.205447:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.205449:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.205452:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.205453:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.205454:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.205456:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.205456:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.205457:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.205457:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.205463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.205471:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.205473:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.205474:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.205476:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.205476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.205478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.205480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.205488:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.205491:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.205491:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.205492:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.205493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.205494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.205548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.205553:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.205556:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.205558:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.205558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.205560:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.205561:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.205562:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.205564:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.205565:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.205566:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.205569:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.205570:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.205573:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.205574:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.205574:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.205576:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.205576:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.205577:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.205578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.205582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.205590:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.205593:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.205594:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.205594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.205596:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.205598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.205758:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.205760:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.205761:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.205761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.205763:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.205765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.205779:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.205782:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.205783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.205783:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.205784:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.205785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.206748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.206757:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.206760:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.206761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.206761:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.206763:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.206766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.206767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.206774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.206777:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.206778:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.206778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.206779:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.206780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.206787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.206791:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.206793:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.206795:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.206796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.206797:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.206799:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.206800:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.206802:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.206803:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.206804:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.206807:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.206809:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.206812:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.206812:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.206813:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.206815:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.206815:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.206817:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.206818:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.206818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.206821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.206823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677472.206828:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677472.206829:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677472.206830:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677472.206831:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.206831:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677472.206831:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.206832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677472.206832:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.206833:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.206835:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.207107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.207113:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.207116:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.207117:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.207119:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.207121:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.207122:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.207123:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.207125:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.207126:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.207127:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.207130:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.207131:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.207134:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.207136:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.207136:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.207138:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.207139:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.207139:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.207140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.207565:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.207568:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.207569:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.207570:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.207572:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.207574:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.207575:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.207576:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.207577:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.207578:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.207581:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.207583:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.207585:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.207585:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.207586:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.207587:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.207587:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.207588:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.207589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.207599:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.207601:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.207601:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.207603:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.207603:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.207605:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677472.208255:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.208259:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.208260:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.208260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.208262:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.208265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.208552:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.208555:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.208556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.208557:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.208559:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.208561:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.208562:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.208563:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.208564:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.208565:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.208568:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.208570:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.208573:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.208573:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.208574:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.208575:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.208576:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.208577:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.208577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.208581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.208590:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.208593:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.208594:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.208594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.208596:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.208599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.208786:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.208789:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.208791:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.208791:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.208793:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.208795:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.208796:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.208797:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.208798:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.208800:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.208803:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.208805:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.208808:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.208808:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.208809:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.208811:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.208811:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.208812:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.208813:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.208824:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.208827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.208827:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.208828:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.208830:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.208833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.208837:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.208839:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.208841:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.208842:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.208844:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.208845:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677472.208847:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.208848:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.208849:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.208850:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.208853:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.208855:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.208857:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.208858:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.208858:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.208860:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.208861:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.208862:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.208863:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.208864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.208874:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:2.0:1619677472.208874:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.208876:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677472.208877:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.208877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.208878:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.208878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.208878:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.208880:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.208883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.208894:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.208896:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.208897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.208898:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.208898:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.208899:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.210153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.210159:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.210162:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.210163:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.210164:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.210167:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.210169:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.210170:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.210171:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.210172:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.210173:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.210176:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.210177:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.210179:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.210180:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.210181:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.210182:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.210183:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.210184:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.210184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.210195:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.210198:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.210198:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.210199:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.210200:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.210201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.210409:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.210411:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.210412:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.210413:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.210414:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.210416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677472.210587:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.210591:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.210595:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.210596:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.210598:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.210601:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677472.210603:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.210605:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.210606:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.210608:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677472.210612:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:12.0:1619677472.210613:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.210615:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.210615:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:12.0:1619677472.210616:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:1.0:1619677472.210618:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.210619:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.210620:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.210621:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:1.0:1619677472.210621:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:12.0:1619677472.210622:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:1.0:1619677472.210622:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677472.210624:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:1.0:1619677472.210624:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.210624:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677472.210625:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.210626:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000400:00000200:1.0:1619677472.210626:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677472.210627:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:1.0:1619677472.210629:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.210631:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.210633:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.210634:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.210635:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.210636:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.210637:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.210638:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.210639:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.210640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.210643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.210651:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677472.210653:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.210653:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.210654:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677472.210655:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677472.210655:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:21.0:1619677472.210656:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.210658:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.210659:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.210660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.210679:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.210681:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.210682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.210682:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.210684:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.210686:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.211228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.211236:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.211239:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.211240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.211240:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.211243:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.211245:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.211354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.211357:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.211359:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.211361:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.211362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.211363:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.211365:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.211366:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.211368:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.211369:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.211370:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.211373:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.211374:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.211377:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.211377:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.211378:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.211380:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.211380:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.211381:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.211382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.211386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.211394:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.211396:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.211397:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.211399:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.211399:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.211401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.211518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.211526:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.211529:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.211530:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.211530:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.211532:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.211535:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.212426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.212434:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.212437:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.212438:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.212440:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.212441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.212443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.212537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.212542:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.212545:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.212547:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.212548:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.212549:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.212551:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.212552:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.212553:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.212554:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.212555:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.212558:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.212560:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.212562:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.212563:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.212564:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.212566:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.212566:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.212567:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.212568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.212571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.212578:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.212581:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.212581:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.212583:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.212584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.212585:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.212591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.212598:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.212601:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.212602:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.212602:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.212603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.212604:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.213214:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.213217:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.213218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.213220:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.213223:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.213224:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.213225:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.213227:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.213228:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.213228:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.213232:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.213233:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.213235:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.213235:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.213236:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.213237:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.213237:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.213239:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.213240:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.213241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.213252:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.213253:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.213254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.213254:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.213255:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677472.213255:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.213256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.213256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.213257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.213257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677472.213976:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.213979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.213979:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677472.213982:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.213984:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.213985:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677472.213987:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.213988:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.213989:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.213990:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677472.213993:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.213994:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677472.213997:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.213997:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677472.213998:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677472.214000:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677472.214000:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.214001:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.214001:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677472.214015:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.214017:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.214018:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.214019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.214020:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.214023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.214461:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.214464:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.214465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.214467:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.214469:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.214471:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.214472:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.214473:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.214474:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.214475:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.214479:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.214480:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.214483:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.214484:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.214485:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.214486:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.214487:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.214488:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.214488:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.214492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.214501:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.214504:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.214504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.214505:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.214507:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.214510:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.215267:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.215270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.215270:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.215273:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.215275:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.215276:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.215278:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.215279:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.215280:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.215281:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.215284:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.215286:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.215290:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.215290:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.215291:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.215298:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.215298:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:8.2:1619677472.215299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.215299:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.215300:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.215308:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.215310:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.215311:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.215313:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.215313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.215316:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.215588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.215595:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.215598:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.215599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.215601:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.215603:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.215604:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.215606:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.215607:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.215608:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.215609:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.215613:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.215615:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.215618:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.215618:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.215619:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.215621:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.215621:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.215622:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.215622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677472.215822:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.215825:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.215827:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.215829:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.215830:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.215831:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677472.215832:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.215834:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.215835:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.215836:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677472.215839:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.215841:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677472.215844:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.215845:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:13.0:1619677472.215845:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:13.0:1619677472.215847:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677472.215848:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.215848:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.215849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677472.215860:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.215863:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.215864:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.215864:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.215867:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.215870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677472.216388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.216396:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.216399:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.216400:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.216402:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.216403:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.216405:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.216829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.216834:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.216836:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.216839:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.216839:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.216841:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.216842:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.216843:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.216845:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.216846:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.216847:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.216850:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.216852:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.216854:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.216855:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.216856:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.216857:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.216858:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.216858:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.216859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.216863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.216871:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.216873:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.216874:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.216875:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.216876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.216877:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.217981:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.217983:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.217985:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.217986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.217987:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.217989:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.217990:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.217991:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677472.217991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:21.0:1619677472.217992:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.217993:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.217996:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.217998:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.218000:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.218000:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:21.0:1619677472.218001:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.218001:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.218003:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.218004:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.218004:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.218005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.218005:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.218007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.218010:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.218014:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.218016:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677472.218019:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.218021:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.218023:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.218026:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.2:1619677472.218031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:18.0:1619677472.218031:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.218034:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677472.218034:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677472.218036:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677472.218039:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677472.218040:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.218040:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:21.0:1619677472.218041:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677472.218041:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:20.0:1619677472.218042:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.218044:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:18.0:1619677472.218044:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:20.0:1619677472.218045:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.218045:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:20.0:1619677472.218047:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677472.218047:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000400:00000200:20.0:1619677472.218048:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.218049:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677472.218049:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677472.218050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677472.218054:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:20.0:1619677472.218057:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.218057:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.218059:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.218060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677472.218060:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:8.0:1619677472.218061:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.218061:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:20.0:1619677472.218063:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.218064:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:8.0:1619677472.218064:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.218065:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677472.218065:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:20.0:1619677472.218066:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:2.0:1619677472.218066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.0:1619677472.218067:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677472.218067:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.218068:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:2.0:1619677472.218068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.0:1619677472.218069:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.218070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.218072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.218075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.218078:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.2:1619677472.218078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677472.218079:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.218080:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677472.218080:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.218081:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.218082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.218082:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.218084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.218085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.218086:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677472.218086:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.218089:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.218090:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.218090:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.2:1619677472.218092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:17.0:1619677472.218093:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.218096:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.218098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:21.0:1619677472.218100:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.218102:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.218103:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.218104:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.218104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.218105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.218106:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.218108:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.218109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.218110:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.218111:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.218111:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.218131:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.218134:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.218135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.218137:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.218139:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.218140:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.218142:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.218143:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.218144:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.218145:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.218149:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.218150:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.218152:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.218153:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.218153:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.218154:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.218155:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.218156:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.218156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.218166:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.218168:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.218169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.218169:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.218171:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.218172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.218185:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.218187:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.218188:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.218188:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.218188:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.218189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.218384:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.218387:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.218389:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.218390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.218392:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.218393:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.218395:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.218396:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.218397:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.218398:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.218401:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.218403:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.218405:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.218406:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.218406:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.218408:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.218408:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.218409:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.218410:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.218594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.218602:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.218607:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.218608:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.218612:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.218616:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.218618:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.218620:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.218622:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.218624:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.218626:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.218631:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.218634:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.218637:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.218638:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.218639:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.218641:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.218642:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.218644:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.218646:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.218647:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.218652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.218659:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.218660:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.218661:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.218662:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677472.218662:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.218663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.218665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.218665:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.218666:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.218667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.220180:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.220182:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.220183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.220183:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.220185:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.220187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.220199:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.220201:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.220202:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.220202:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.220203:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.220203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.220424:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.220428:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.220429:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.220433:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.220436:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.220439:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.220441:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.220444:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.220445:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.220447:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.220452:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.220456:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.220460:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.220461:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.220462:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.220464:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.220465:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.220467:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.220468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.220471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.220480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.220483:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.220484:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.220485:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.220488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.220489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.220827:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.220829:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.220830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.220832:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.220834:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.220835:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.220837:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.220838:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.220839:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.220840:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.220843:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.220845:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.220847:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.220848:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.220849:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.220851:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.220851:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.220852:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.220852:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.220857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.220867:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.220871:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.220873:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.220873:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.220876:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.220880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.220919:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.220923:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:20.0:1619677472.220925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.220926:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677472.220929:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.220932:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.220934:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.220937:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.220939:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.220941:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.220947:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.220951:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.220954:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.220956:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.220957:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.220959:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.220960:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.220962:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.220963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.221048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.221056:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.221060:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.221061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.221063:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.221065:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.221067:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.221069:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.221070:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.221071:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.221072:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.221076:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.221078:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.221081:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.221082:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.221083:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.221085:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.221086:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.221087:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.221087:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.221091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.221099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.221102:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.221102:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.221105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.221105:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.221108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.221118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677472.221126:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677472.221127:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677472.221128:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.221129:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.221129:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.221130:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677472.221130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.221131:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.221131:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.221132:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.221133:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.221134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.221391:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.221393:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.221394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.221394:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.221396:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.221399:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.221508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677472.221514:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.221516:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.221519:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.221519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.221521:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.221523:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:21.0:1619677472.221525:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.221526:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.221527:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.221528:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:21.0:1619677472.221531:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.221533:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677472.221536:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.221537:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.0:1619677472.221537:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677472.221540:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.0:1619677472.221540:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.221542:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677472.221543:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677472.221544:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677472.221549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.221554:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677472.221556:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677472.221557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677472.221558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:2.0:1619677472.221846:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.221849:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.221850:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.221850:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.221852:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.221856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.221868:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.221871:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.221871:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.221871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.221873:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.221873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.222969:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.222971:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.222973:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.222974:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.222976:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.222978:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.222979:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.222980:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.222981:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.222982:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.222985:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.222986:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.222987:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.222988:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.222989:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.222990:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.222990:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.222991:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.222993:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.222993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.223004:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.223005:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.223005:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.223007:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.223007:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.223008:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.223008:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.223009:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.223009:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.223010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677472.223097:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.223100:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.223101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.223103:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.223105:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.223107:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677472.223108:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.223110:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.223110:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.223111:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.223115:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.223116:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.223119:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.223120:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.223120:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.223122:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.223123:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.223123:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.223124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677472.223303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.223305:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.223308:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.223311:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.223311:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.223313:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.223315:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.223316:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.223318:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.223319:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.223320:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.223323:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.223325:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.223327:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.223328:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.223329:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.223330:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.223331:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.223332:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.223333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.223337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.223346:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.223348:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.223349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.223349:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.223351:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.223353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.223361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.223368:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.223371:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.223372:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.223372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.223373:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.223373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.223423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.223435:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.223438:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.223439:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.223441:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.223441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.223444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.223889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.223892:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677472.223894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.223894:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.223896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.223898:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.223899:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.223900:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.223901:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.223902:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.223906:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.223907:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.223911:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.223911:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.223912:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.223914:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.223915:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.223916:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.223916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677472.224055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.224061:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.224064:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.224066:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.224067:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.224070:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.224072:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.224073:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.224075:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.224076:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.224077:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.224080:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.224082:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.224085:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.224086:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.224086:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.224088:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.224089:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.224090:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.224091:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.224091:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.224097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.224103:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.224104:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.224105:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.224106:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.0:1619677472.224815:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.224817:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.224819:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.224820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.224822:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.224824:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.224825:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.224827:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.224827:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.224829:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.224832:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.224834:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.224836:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.224837:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.224838:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.224839:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.224840:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.224841:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.224842:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.224843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.224863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.224866:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.224866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.224867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.225172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.225180:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.225183:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.225184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.225184:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.225186:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.225189:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.225531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677472.225535:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:24.0:1619677472.225538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677472.225538:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:2.0:1619677472.225538:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677472.225539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:24.0:1619677472.225540:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.225541:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.225541:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.225541:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.225542:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677472.225542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.225544:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:17.0:1619677472.225544:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.225545:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.225546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:17.0:1619677472.225546:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.225547:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.225548:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.225548:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.225549:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677472.225550:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.225550:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.225551:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677472.225552:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.225553:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.225555:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677472.225555:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677472.225558:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:2.0:1619677472.225559:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.225561:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.225561:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:2.0:1619677472.225561:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677472.225562:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677472.225563:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.225564:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.225564:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:2.0:1619677472.225564:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.225564:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677472.225565:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.225566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677472.225566:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.225566:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.225567:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.225568:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677472.225570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.225578:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:18.0:1619677472.225579:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677472.225580:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.225581:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.225581:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.225582:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.225582:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.225583:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.225583:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.225585:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.225586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.225587:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.225897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677472.225940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:2.0:1619677472.225944:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.225947:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.225948:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.225948:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.225950:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.225954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677472.226174:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.226176:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.226178:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.226179:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.226181:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.226182:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.226183:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.226185:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.226186:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.226187:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.226190:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.226192:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.226195:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.226196:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.226196:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.226198:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.226198:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.226199:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.226200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:18.0:1619677472.226231:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.226233:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.226234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.226235:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.226237:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.226241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.226257:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.226259:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.226260:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.226260:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.226262:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.226264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.226278:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.226280:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.226281:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.226281:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.226282:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.226282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.226746:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.226749:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.226750:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.226752:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.226753:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.226755:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.226756:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.226757:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.226758:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.226759:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.226762:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.226764:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.226767:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.226767:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.226768:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.226770:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.226770:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.226771:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.226771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.226776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.226784:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.226787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.226787:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.226788:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.226790:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.226794:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.227438:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.227441:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677472.227442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677472.227444:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.227446:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.227448:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.227449:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.227451:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.227452:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.227452:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.227455:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.227456:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.227459:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.227460:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.227460:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.227462:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.227462:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.227463:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.227464:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:2.0:1619677472.227478:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.227481:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.227482:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.227482:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.227484:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.227487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.227497:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.227499:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.227500:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.227501:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.227502:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.227502:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.228039:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.228042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.228042:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.228043:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.228044:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.228047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.228410:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.228412:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.228415:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.228415:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.228416:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.228418:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.228420:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.228421:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.228422:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.228423:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.228426:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.228428:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.228431:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.228432:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.228432:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.228434:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.228435:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.228436:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.228436:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.228459:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.228462:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.228463:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.228463:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.228466:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.228468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.228930:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.228933:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.228933:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.228935:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.228938:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.228940:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.228941:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.228942:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.228943:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.228944:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.228946:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.228948:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.228950:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.228951:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.228952:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.228953:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.228953:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.228954:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.228955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.228964:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.228965:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.228966:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.228967:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.228967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.228969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.228982:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.228984:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.228987:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.228987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.228989:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.228991:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.228992:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.228993:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.228994:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.228995:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.228998:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.229005:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.229007:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.229008:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.229009:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.229010:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.229011:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.229012:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.229012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.229022:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.229025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.229025:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.229026:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.229028:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.229030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.229047:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.229050:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.229051:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.229051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.229052:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.229053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.229891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.229895:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.229897:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.229899:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.229900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.229902:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.229903:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.229904:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.229906:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.229907:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.229908:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.229911:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.229913:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.229915:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.229916:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.229917:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.229918:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.229919:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.229920:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.229920:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.229925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.229934:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.229937:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.229938:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.229938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.229940:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.229942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.230074:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.230076:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.230077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.230077:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.230079:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.230081:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.230420:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.230423:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.230424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.230426:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.230428:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.230430:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.230431:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.230433:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.230434:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.230435:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.230438:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.230440:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.230442:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.230443:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.230444:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.230445:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.230446:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.230447:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.230448:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.230449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.230452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.230459:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.230461:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677472.230462:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.230462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.230463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.230464:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.230465:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.230467:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.230468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.230469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.230472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.230480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.230482:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.230483:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.230484:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.230485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.230486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.231555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.231557:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.231558:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.231559:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.231560:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.231562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.231565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:8.2:1619677472.232008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.232016:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.232019:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.232020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.232020:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.232023:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.232025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.232027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.232034:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.232037:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.232038:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.232038:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.232041:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.232041:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.232104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.232109:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.232112:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.232114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.232115:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.232117:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.232119:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.232120:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.232121:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.232122:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.232123:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.232126:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.232128:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.232130:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.232131:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.232131:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.232133:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.232134:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.232135:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.232135:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.232146:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.232149:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.232149:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.232150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.232151:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.232152:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.232207:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.232210:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677472.232212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.232212:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.232214:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.232215:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.232217:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.232218:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.232219:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.232220:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.232224:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.232225:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.232229:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.232229:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.232230:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.232232:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.232233:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.232233:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.232234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:21.0:1619677472.232641:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.232644:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.232646:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.232646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.232649:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.232650:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.232652:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.232653:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.232654:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.232655:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.232657:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.232659:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.232661:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.232662:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.232662:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.232664:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.232665:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.232666:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.232667:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.232668:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.232676:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.232679:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.232680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.232681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.232734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.232743:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.232746:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.232747:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.232747:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.232749:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.232752:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.233001:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.233004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.233004:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.233006:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.233009:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.233010:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.233011:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.233013:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.233014:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.233014:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.233017:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.233019:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.233022:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.233022:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.233023:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.233025:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.233025:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.233026:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.233027:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.233031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.233039:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.233042:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.233042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.233043:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.233045:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.233048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.233402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.233405:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.233408:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.233410:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.233411:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.233412:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.233414:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.233415:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.233416:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.233417:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.233418:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.233422:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.233423:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.233425:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.233426:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.233427:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.233429:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.233429:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.233430:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.233431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.233435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.233443:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.233445:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.233446:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.233448:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.233449:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.233450:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.234031:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.234033:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.234034:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.234036:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.234039:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.234040:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.234042:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.234043:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.234044:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.234045:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.234049:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.234050:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.234053:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.234054:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.234055:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.234056:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.234057:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.234058:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.234060:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.234060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.234062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.234071:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677472.234073:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677472.234073:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.234074:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.234075:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.234075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677472.234075:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.234077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677472.234077:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.234079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.234609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.234616:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.234619:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.234621:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.234622:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.234624:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.234626:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.234627:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.234628:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.234629:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.234630:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.234634:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.234636:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.234638:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.234639:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.234640:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.234641:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.234642:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.234643:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.234643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.234648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.234656:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.234659:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.234659:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.234661:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.234663:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.234664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677472.235168:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.235171:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.235172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.235174:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.235176:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.235178:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677472.235179:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.235181:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.235182:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.235183:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.235185:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.235187:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.235190:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.235191:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.235191:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.235193:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.235194:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.235194:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.235195:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677472.235266:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.235269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.235269:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.235272:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.235274:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.235275:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.235277:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.235278:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.235279:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.235280:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.235283:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.235284:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.235286:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.235287:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.235287:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.235289:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.235289:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.235290:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.235291:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.235296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.235305:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.235307:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.235308:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.235309:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.235310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.235311:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.235774:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.235776:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.235777:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.235778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.235779:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.235781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.235791:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.235794:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.235795:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.235795:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.235796:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.235796:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.236115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.236118:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.236121:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.236123:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.236123:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.236125:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.236126:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.236128:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.236129:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.236130:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.236131:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.236134:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.236135:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.236138:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.236139:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.236139:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.236141:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.236141:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.236142:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.236144:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.236144:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.236147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.236149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.236154:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677472.236155:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677472.236156:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.236157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.236158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677472.236158:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.236159:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.236159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.236161:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.236163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.236515:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.236517:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.236519:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.236520:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.236523:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.236525:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677472.236526:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.236527:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.236528:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.236529:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.236532:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.236534:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677472.236537:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.236538:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677472.236538:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677472.236540:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677472.236540:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.236542:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677472.236543:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677472.236544:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.236558:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677472.236559:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677472.236560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.236561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:21.0:1619677472.237144:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.237147:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.237149:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.237150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.237151:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.237153:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.237154:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.237155:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.237156:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.237157:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.237160:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.237162:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.237164:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.237165:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.237165:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.237167:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.237167:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.237168:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.237169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.237851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.237859:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.237862:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.237863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.237863:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.237866:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.237869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.237872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.237880:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.237882:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.237883:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.237883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.237884:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.237885:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.238315:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.238318:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.238318:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.238320:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.238323:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.238325:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.238326:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.238327:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.238328:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.238329:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.238332:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.238334:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.238337:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.238337:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.238338:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.238339:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.238340:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.238341:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.238341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.238345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.238353:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.238357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.238357:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.238358:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.238360:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.238362:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.238579:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.238582:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.238583:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.238583:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.238585:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.238587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.238844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.238847:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.238849:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.238851:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.238853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.238853:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.238855:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.238856:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.238858:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.238858:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.238860:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.238863:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.238865:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.238867:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.238868:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.238868:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.238870:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.238870:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.238871:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.238872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.238876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.238884:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.238886:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.238887:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.238889:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.238889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.238891:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.239954:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.239956:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.239958:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.239959:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.239962:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.239964:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.239965:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.239966:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.239967:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.239968:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.239971:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.239973:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.239975:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.239975:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.239976:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.239977:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.239978:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.239979:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.239979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.239990:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.239992:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.239992:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.239993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.239994:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.239996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.240170:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.240174:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.240174:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.240176:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.240179:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.240180:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.240181:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.240183:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.240184:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.240185:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.240188:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.240190:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.240193:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.240193:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.240194:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.240196:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.240196:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.240197:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.240197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.240202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.240210:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.240212:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.240213:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.240214:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.240216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.240217:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.240221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.240229:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.240232:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.240233:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.240233:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.240234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.240235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.240334:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.240336:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.240337:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.240337:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.240339:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.240342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.240413:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.240416:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.240418:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.240418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.240420:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.240421:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.240423:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.240424:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.240425:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.240426:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.240429:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.240431:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.240434:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.240434:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.240435:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.240437:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.240437:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.240438:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.240439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.240476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.240483:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.240487:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.240489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.240490:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.240493:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.240494:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.240496:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.240497:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.240498:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.240499:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.240502:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.240504:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.240505:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.240506:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.240507:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.240508:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.240509:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.240510:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.240510:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.240521:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.240524:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.240524:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.240525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.240526:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.240527:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.241634:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.241637:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.241639:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.241640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.241641:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.241643:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.241644:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.241645:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.241646:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.241647:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.241650:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.241652:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.241655:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.241656:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.241657:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.241659:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.241659:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.241661:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.241662:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.241662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.241673:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.241675:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.241676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.241677:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677472.241700:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.241703:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.241704:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.241704:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.241705:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.241708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.241915:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.241918:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677472.241919:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677472.241920:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.241923:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.241924:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677472.241926:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.241927:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.241928:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.241929:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677472.241932:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.241933:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677472.241937:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.241937:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677472.241938:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677472.241940:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677472.241940:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.241941:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.241941:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:20.2:1619677472.242073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.242079:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.242081:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.242083:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.242084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.242086:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.242088:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.242089:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.242090:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.242091:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.242092:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.242095:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.242097:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.242099:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.242100:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.242101:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.242103:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.242103:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.242104:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.242105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.242109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.242117:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.242120:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.242121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.242121:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.242123:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.242125:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.242236:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.242239:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677472.242240:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677472.242242:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677472.242244:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.242246:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.242247:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.242248:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.242249:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.242250:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.242253:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677472.242254:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.242256:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.242256:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.242257:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.242258:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.242259:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.242260:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.242260:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:24.0:1619677472.242281:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.242283:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.242284:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.242284:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.242285:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.242287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.242633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.242640:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.242643:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.242643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.242645:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.242647:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.242648:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.242649:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.242651:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.242652:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.242653:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.242656:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.242658:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.242661:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.242662:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.242662:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.242664:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.242664:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.242665:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.242666:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.242681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.242690:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.242692:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.242693:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.242694:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.242695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.242696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677472.243153:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.243155:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.243157:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.243158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.243160:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.243161:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.243162:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.243164:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.243165:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.243166:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.243169:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.243171:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.243173:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.243174:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.243175:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.243176:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.243177:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.243178:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.243178:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.243900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.243909:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.243912:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.243913:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.243913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.243915:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.243918:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.244133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.244140:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.244143:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.244144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.244144:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.244146:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.244149:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.244153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.244160:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.244163:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.244164:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.244165:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.244166:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.244166:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677472.244422:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.244425:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.244427:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.244428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.244430:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.244431:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.244432:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.244434:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.244435:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.244436:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.244439:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.244441:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.244443:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.244444:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.244444:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.244446:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.244447:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.244448:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.244449:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.244450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.244460:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.244462:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.244463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.244464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.244498:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.244501:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.244502:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.244504:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.244506:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.244507:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.244509:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.244510:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.244511:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.244512:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.244515:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.244516:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.244519:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.244520:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.244521:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.244522:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.244523:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.244524:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.244524:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.244539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.244548:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.244550:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.244551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.244551:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.244554:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.244556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.245214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.245219:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.245222:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.245224:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.245224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.245226:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.245228:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.245229:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.245230:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.245231:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.245232:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.245236:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.245237:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.245240:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.245240:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.245241:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.245243:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.245244:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.245245:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.245245:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.245249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.245257:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.245259:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.245260:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.245262:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.245263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.245264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677472.245587:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.245590:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.245590:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.245592:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.245594:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.245596:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.245597:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.245598:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.245600:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.245601:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.245604:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.245605:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.245608:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.245608:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.245609:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.245610:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.245611:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.245612:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.245612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.245617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.245626:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.245628:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.245629:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.245631:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.245632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.245633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.245784:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.245787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.245787:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677472.245789:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.245792:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.245793:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677472.245794:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.245796:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.245797:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.245797:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677472.245800:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.245803:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.245804:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.245805:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.245805:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.245806:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.245807:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.245807:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.245808:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677472.245812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.245820:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.245822:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.245822:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.245823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.245824:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.245825:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.245833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.245840:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.245843:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.245843:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.245844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.245844:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.245844:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677472.246171:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.246174:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.246176:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.246178:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.246179:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.246181:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677472.246182:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.246183:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.246184:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.246185:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677472.246189:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.246190:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677472.246193:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.246193:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677472.246194:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677472.246195:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677472.246196:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.246197:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677472.246199:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677472.246199:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.246213:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677472.246215:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677472.246216:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.246217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:24.0:1619677472.246393:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.246395:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.246396:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.246398:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.246401:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.246403:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.246404:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.246405:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.246406:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.246407:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.246410:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.246411:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.246412:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.246413:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.246414:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.246415:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.246416:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.246417:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.246418:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.246419:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.246430:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.246432:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.246432:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.246434:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.246434:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677472.246434:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.246435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.246436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.246436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.246437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.246448:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.246450:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.246451:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.246451:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.246451:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.246452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677472.246775:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.246778:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.246778:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.246780:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.246782:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.246783:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:17.0:1619677472.247069:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.247071:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.247072:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.247074:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.247076:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.247078:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677472.247079:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.247081:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.247082:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.247083:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677472.247086:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.247087:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677472.247090:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.247091:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677472.247092:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677472.247094:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677472.247094:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.247096:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677472.247097:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:8.0:1619677472.247097:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677472.247098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.0:1619677472.247100:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.247101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.247101:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.247104:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.247106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.247109:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677472.247111:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677472.247112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.247113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677472.247188:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.247190:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.247191:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.247191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.247192:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.247194:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.247208:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.247211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.247211:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677472.247211:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.247212:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.247213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.247521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.247530:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.247532:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.247533:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.247535:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.247536:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.247538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.247988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.247996:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.247999:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.248000:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.248000:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.248002:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.248005:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.248574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.248580:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.248583:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.248585:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.248586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.248588:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.248590:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.248591:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.248593:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.248594:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.248595:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.248600:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.248603:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.248606:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.248606:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.248607:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.248609:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.248610:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.248611:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.248612:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.248613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.248616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.248618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.248620:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.248623:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.248623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.248624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.248624:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.248627:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.248628:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.248628:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.248630:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.248632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.248635:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.248638:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.248639:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.248639:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.248641:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.248643:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.249012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.249020:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.249023:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.249023:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.249025:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.249026:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.249028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.249028:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.249030:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.249031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.249032:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.249034:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.249038:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677472.249042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.249046:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.249048:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.249049:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.249050:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.249051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.249051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.249563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.249571:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.249574:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.249575:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.249575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.249578:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.249580:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.249765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.249772:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.249774:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.249776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.249778:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.249780:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.249782:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.249783:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.249784:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.249785:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.249786:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.249790:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.249792:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.249793:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.249794:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.249795:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.249796:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.249797:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.249798:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.249799:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.249800:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.249808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.249815:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.249816:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.249816:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677472.249817:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.249817:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.249817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.249818:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.249818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.249819:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.249820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.249980:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.249982:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.249983:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.249985:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.249987:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.249988:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.249989:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.249990:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.249992:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.249993:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677472.249994:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:24.0:1619677472.249995:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.249997:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677472.249997:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.249997:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:24.0:1619677472.250000:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677472.250000:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.250001:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.250001:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:1.0:1619677472.250002:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.250003:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.250003:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.250004:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.250004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677472.250004:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.250005:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.250006:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.250007:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.250008:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.250011:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.250013:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677472.250014:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:1.0:1619677472.250015:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.250016:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677472.250016:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:24.0:1619677472.250017:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.250017:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.250017:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.250018:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:8.0:1619677472.250019:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.250019:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:8.0:1619677472.250020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.250020:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.250021:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.250022:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.250024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.250033:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677472.250035:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.250035:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.250035:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.250036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.250037:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.250038:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.250040:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.250040:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.250042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.250905:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.250907:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.250908:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.250910:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.250912:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.250913:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.250914:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.250916:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.250916:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.250917:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.250920:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.250922:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.250923:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.250924:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.250925:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.250926:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.250926:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.250927:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.250927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677472.250936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677472.250938:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.250939:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677472.250940:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.250940:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.250941:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.250942:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:8.0:1619677472.250943:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.250944:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.250945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.250945:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.250948:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.250950:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.250951:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.250952:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.250953:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.250953:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.250954:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.250955:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.250958:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.250961:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.250962:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.250963:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677472.250964:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.250964:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.250964:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:21.0:1619677472.250966:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677472.250966:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677472.250967:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.250968:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.250969:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.250970:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.250971:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.250972:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.250972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:21.0:1619677472.250973:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.250973:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677472.250975:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.250975:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.250976:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.250976:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.250977:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.250978:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.250979:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.250979:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677472.250979:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.250980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.250981:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.250981:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.250982:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.250982:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.250983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.250998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.251006:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.251009:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.251010:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.251010:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.251011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.251012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.251316:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.251319:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.251319:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.251320:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.251321:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.251323:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.251425:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.251427:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.251429:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.251430:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.251431:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.251433:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.251434:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.251436:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.251437:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.251438:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.251441:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.251443:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.251446:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.251447:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.251447:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.251449:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.251450:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.251451:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.251451:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677472.251709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.251715:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.251718:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.251719:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.251721:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.251723:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.251724:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.251726:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.251727:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.251728:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.251729:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.251732:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.251734:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.251736:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.251737:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.251738:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.251740:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.251741:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.251742:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.251742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.251746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.251754:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.251756:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.251757:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.251759:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.251760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.251762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.251848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.251856:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.251859:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.251860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.251860:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.251862:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.251865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.252493:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.252495:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.252496:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.252497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.252498:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.252500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.252509:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.252512:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.252512:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.252513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.252514:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.252514:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.254383:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.254386:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.254387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.254388:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.254390:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.254391:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.254392:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.254394:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.254395:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.254396:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.254398:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.254400:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.254401:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.254402:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.254402:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.254404:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.254404:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.254405:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.254405:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.254416:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.254418:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.254418:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.254419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.254420:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.254421:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.254566:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.254568:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.254569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.254569:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.254571:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.254573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.254577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677472.254583:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.254586:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677472.254587:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.254588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677472.254590:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.254592:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.254593:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.254594:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.254595:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.254596:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.254599:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.254601:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.254603:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.254604:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.254605:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.254607:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.254607:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.254608:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677472.254609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.254613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.254620:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.254622:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.254623:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.254625:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.254626:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.254627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677472.254731:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.254734:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.254734:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.254736:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.254739:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.254740:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.254741:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.254743:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.254744:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.254745:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.254748:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.254750:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.254752:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.254753:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.254754:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.254755:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.254756:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.254756:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.254757:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.254761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.254769:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.254772:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.254772:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.254774:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.254775:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.254776:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.255056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.255063:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.255066:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.255067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.255069:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.255071:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.255073:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.255074:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.255076:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.255077:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.255078:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.255082:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.255083:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.255087:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.255088:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.255089:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.255091:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.255091:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.255092:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.255093:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677472.255143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.255145:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.255148:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.255150:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.255151:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.255152:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.255153:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.255155:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.255156:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.255157:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.255158:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.255161:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.255163:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.255165:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.255166:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.255167:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.255169:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.255169:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.255170:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.255171:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.255175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.255183:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.255186:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.255187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.255187:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.255189:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.255191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677472.256002:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.256004:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.256007:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.256007:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.256009:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.256011:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.256012:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.256013:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.256014:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.256016:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.256019:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.256020:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.256023:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.256024:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.256024:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.256026:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.256026:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.256027:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.256028:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.256040:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.256042:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.256043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.256044:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.256045:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.256048:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.256058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.256060:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.256061:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.256061:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.256062:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.256063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.256105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.256114:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.256116:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.256118:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.256119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.256120:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.256124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.256125:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.256128:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.256129:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.256130:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.256132:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.256133:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.256135:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.256136:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.256137:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.256138:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.256141:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.256143:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.256146:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.256146:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.256147:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.256149:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.256149:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.256151:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.256152:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.256152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.256154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.256163:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677472.256165:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.256166:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.256167:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.256167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.256168:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.256168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677472.256170:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.256170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677472.256172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.256428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.256434:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.256437:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.256438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.256440:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.256444:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.256445:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.256446:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.256447:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.256449:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.256449:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.256452:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.256454:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.256456:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.256456:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.256457:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.256459:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.256459:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.256460:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.256461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.256472:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.256475:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.256475:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.256476:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.256477:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.256478:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.256493:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.256495:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.256496:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.256496:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.256496:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.256497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.256808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.256814:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.256816:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.256817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.256819:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.256820:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.256822:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677472.256823:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.256824:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.256825:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.256826:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677472.256829:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.256831:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677472.256834:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.256834:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677472.256835:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677472.256837:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677472.256837:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.256839:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.256840:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.256840:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.256845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677472.256849:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:2.0:1619677472.256851:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:24.0:1619677472.256852:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.256853:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.256853:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677472.256853:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.256854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:24.0:1619677472.256855:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.256855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.256859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.257299:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.257301:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.257302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.257302:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.257304:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.257306:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.258043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.258050:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.258053:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.258054:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.258056:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.258059:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.258060:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677472.258062:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.258063:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.258064:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.258065:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677472.258068:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.258069:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677472.258073:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.258074:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:18.0:1619677472.258074:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:18.0:1619677472.258076:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:18.0:1619677472.258077:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.258078:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.258078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677472.258091:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.258094:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.258094:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.258096:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.258099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.258100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:24.0:1619677472.258260:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.258262:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.258263:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.258264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.258265:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.258267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.258726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.258734:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.258737:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.258738:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.258738:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.258741:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.258744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.258748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.258756:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.258758:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.258759:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.258759:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.258760:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.258761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.258850:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.258853:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.258855:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.258856:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.258858:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.258859:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.258861:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.258862:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.258863:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.258864:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.258867:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.258869:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.258872:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.258872:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.258873:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.258875:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.258875:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.258876:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.258876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.259259:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.259262:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.259262:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.259264:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.259267:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.259268:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.259269:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.259271:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.259272:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.259273:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.259276:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.259277:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.259279:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.259280:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.259280:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.259282:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.259282:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.259283:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.259284:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.259293:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.259298:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.259301:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.259301:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.259302:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.259303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.259791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.259798:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.259801:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.259802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.259804:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.259806:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.259808:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677472.259809:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.259810:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.259812:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.259813:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677472.259816:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.259817:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677472.259820:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.259821:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677472.259822:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677472.259823:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677472.259824:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.259825:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.259825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677472.259836:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.259839:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.259840:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.259842:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677472.259842:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.259844:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.260094:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.260097:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.260098:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.260100:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.260101:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.260103:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.260104:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.260105:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.260106:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.260107:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.260110:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.260112:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.260115:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.260116:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.260117:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.260118:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.260119:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.260119:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.260120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.260132:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.260135:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.260136:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.260136:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.260138:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.260142:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.260198:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.260201:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.260203:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.260203:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.260206:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.260207:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.260208:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.260209:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.260210:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.260211:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.260214:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.260216:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.260219:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.260220:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.260221:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.260222:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.260223:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.260224:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.260224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677472.260630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.260636:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.260639:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.260640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.260642:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.260644:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.260645:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.260647:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.260648:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.260649:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.260650:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.260653:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.260654:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.260657:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.260658:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.260659:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677472.260659:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:2.0:1619677472.260661:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.260661:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677472.260662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:2.0:1619677472.260662:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677472.260662:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:2.0:1619677472.260663:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677472.260665:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.260667:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.260668:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.260670:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.260671:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.260672:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.260673:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.260676:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.260678:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.260680:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.260681:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.260682:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.260684:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.260684:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.260685:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.260686:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.260691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.260699:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.260701:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.260702:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.260704:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.260705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.260706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.260709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.260717:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.260720:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.260721:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.260721:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.260722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.260723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.260753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.260758:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.260760:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.260763:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.260763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.260765:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.260767:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.260768:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.260769:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.260770:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.260771:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.260774:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.260776:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.260778:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.260779:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.260779:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.260781:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.260782:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.260782:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.260783:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.260787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.260796:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.260798:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.260799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.260800:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.260802:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.260804:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.260808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.260815:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.260818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.260818:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.260819:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.260820:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.260820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.261209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.261217:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.261219:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.261220:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.261222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.261222:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.261225:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.262141:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.262143:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.262144:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.262145:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.262146:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.262148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.262160:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.262162:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.262163:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.262164:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.262165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.262165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.262282:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.262284:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.262286:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.262287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.262289:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.262290:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.262292:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677472.262297:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:20.0:1619677472.262301:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.262302:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.262303:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677472.262304:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.262306:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:20.0:1619677472.262307:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.262307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677472.262309:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:18.0:1619677472.262310:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.262312:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.262313:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.262313:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:18.0:1619677472.262313:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.262314:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.0:1619677472.262315:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.262316:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:18.0:1619677472.262316:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.262317:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000400:00000200:18.0:1619677472.262317:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.262318:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.262319:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.262319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:18.0:1619677472.262319:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677472.262322:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.262325:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677472.262327:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.262329:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:18.0:1619677472.262329:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677472.262329:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.262330:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:18.0:1619677472.262331:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677472.262332:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677472.262332:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:18.0:1619677472.262332:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:24.0:1619677472.262333:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.262333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:18.0:1619677472.262333:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:8.0:1619677472.262333:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.262334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:18.0:1619677472.262334:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:24.0:1619677472.262335:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.262335:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677472.262338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.262340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677472.262345:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677472.262345:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:21.0:1619677472.262347:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.262347:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.262347:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.262348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:21.0:1619677472.262349:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.262349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677472.262350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.262352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.262593:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.262596:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.262597:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.262597:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.262599:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.262601:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.262612:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.262614:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.262615:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.262615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.262616:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.262617:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.262867:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.262869:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677472.262870:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677472.262872:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.262874:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.262875:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677472.262876:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.262878:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.262878:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.262879:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677472.262882:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.262884:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677472.262887:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.262887:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677472.262888:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677472.262889:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677472.262890:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.262891:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677472.262892:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677472.262893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.2:1619677472.262895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.262904:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:24.0:1619677472.262906:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:17.0:1619677472.262906:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.262907:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677472.262908:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000400:00000200:17.0:1619677472.262908:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.262909:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.262910:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:17.0:1619677472.262910:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.262913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.263141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.263148:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.263151:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.263151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.263154:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.263156:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.263158:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.263159:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.263160:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.263161:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.263162:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.263166:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.263167:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.263169:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.263170:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.263171:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.263172:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.263173:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.263174:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.263174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.263185:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.263187:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.263188:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.263189:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.263190:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.263191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.264786:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.264789:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.264791:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.264792:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.264793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.264795:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.264796:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.264798:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.264798:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.264800:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.264803:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.264805:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.264807:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.264808:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.264808:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.264810:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.264810:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.264811:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.264812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:21.0:1619677472.264868:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.264870:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.264871:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.264873:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.264873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.264875:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.265294:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.265301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.265304:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.265308:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.265312:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.265314:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.265316:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.265319:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.265320:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.265322:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.265327:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.265330:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.265332:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.265333:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.265334:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.265336:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.265337:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.265338:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.265339:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.265395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.265398:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.265399:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.265401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.265402:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.265404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.265445:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.265449:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.265450:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.265451:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.265452:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.265453:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.265924:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.265928:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.265932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.265932:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.265935:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.265937:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.265939:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.265942:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.265943:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.265945:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.265951:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.265954:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.265957:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.265959:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.265960:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.265962:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.265963:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.265964:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.265965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.265975:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.265978:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.265979:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.265979:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.265981:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.265984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.266931:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.266934:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.266934:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.266935:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.266936:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.266938:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.267397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.267405:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.267407:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.267408:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.267410:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.267411:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.267413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.267524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.267527:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.267531:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.267533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.267534:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.267537:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677472.267537:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.267538:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:24.0:1619677472.267540:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:20.0:1619677472.267540:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.267541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677472.267542:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.267543:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.267544:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677472.267545:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.267547:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.267548:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.267549:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.267551:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.267551:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677472.267552:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.267553:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677472.267554:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.267554:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:24.0:1619677472.267555:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.267555:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.267556:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.267557:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677472.267558:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.267558:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:24.0:1619677472.267560:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.267560:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.267560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.267563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677472.267564:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.267565:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.2:1619677472.267565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677472.267566:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.267568:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.267569:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.267570:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:21.0:1619677472.267570:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:24.0:1619677472.267571:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.267572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677472.267572:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677472.267573:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.267573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.267574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.267575:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.267576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.267576:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.267578:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.267580:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677472.267583:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.267585:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.267586:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.267587:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.267587:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.267587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677472.267587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.267588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.267589:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.267593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.267866:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:24.0:1619677472.267869:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:24.0:1619677472.267870:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.267870:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:8.0:1619677472.267870:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:8.0:1619677472.267871:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.267872:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.267873:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.267874:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677472.267874:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.267875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.267971:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.267974:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.267974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.267976:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.267978:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.267980:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.267981:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.267982:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.267984:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.267985:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.267988:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.267989:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.267992:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.267992:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.267993:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.267995:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.267995:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.267996:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.267996:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.268012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.268020:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.268022:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.268023:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.268025:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.268025:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.268027:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.268916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.268922:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.268924:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.268926:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.268927:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.268929:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.268930:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.268931:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.268933:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.268934:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.268935:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.268938:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.268940:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.268942:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.268943:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.268944:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.268945:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.268946:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.268947:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.268947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.268952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.268960:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.268962:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.268963:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.268965:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.268966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.268968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.269391:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000800:00000200:24.0:1619677472.269395:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:8.0:1619677472.269395:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:8.0:1619677472.269397:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.269400:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.269402:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000400:00000200:8.0:1619677472.269402:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677472.269403:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.269404:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677472.269405:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000400:00000200:8.0:1619677472.269406:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677472.269406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:8.0:1619677472.269408:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677472.269409:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.269411:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.269414:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:18.0:1619677472.269415:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.269417:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000400:00000200:18.0:1619677472.269417:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:8.0:1619677472.269418:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.269419:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:18.0:1619677472.269419:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.269419:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:8.0:1619677472.269420:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:18.0:1619677472.269421:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.269422:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677472.269422:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.269423:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:8.0:1619677472.269423:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000400:00000200:24.0:1619677472.269424:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.269425:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8667200] (24)++
00000400:00000200:24.0:1619677472.269426:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.269426:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000400:00000200:24.0:1619677472.269427:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677472.269427:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.269427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677472.269428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000400:00000200:24.0:1619677472.269429:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96514317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.269429:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:24.0:1619677472.269431:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.269432:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.269434:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677472.269434:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.269435:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.269436:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:24.0:1619677472.269438:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.269438:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.269438:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.269439:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.269439:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:24.0:1619677472.269440:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677472.269440:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:8.0:1619677472.269440:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:18.0:1619677472.269441:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000400:00000200:2.0:1619677472.269441:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.269442:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:8.0:1619677472.269442:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677472.269442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:24.0:1619677472.269443:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677472.269443:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.269444:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677472.269444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:8.0:1619677472.269444:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:24.0:1619677472.269445:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.269447:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677472.269447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.269448:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (21)++
00000800:00000200:24.0:1619677472.269449:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (22)--
00000800:00000200:24.0:1619677472.269452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:24.0:1619677472.269454:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000800:00000200:18.0:1619677472.269454:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:24.0:1619677472.269456:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:18.0:1619677472.269456:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677472.269457:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:8.0:1619677472.269457:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.269458:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:2.0:1619677472.269458:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:24.0:1619677472.269459:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.269459:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.269461:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.269462:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.269463:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.269464:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677472.269464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:8.0:1619677472.269465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:24.0:1619677472.269466:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.269467:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.269468:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.269469:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677472.269472:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.269474:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677472.269477:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.269478:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677472.269478:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677472.269480:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677472.269480:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (21)++
00000800:00000200:24.0:1619677472.269481:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (22)--
00000800:00000200:24.0:1619677472.269481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (21)--
00000800:00000200:8.2:1619677472.269486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677472.269494:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000800:00000200:1.0:1619677472.269495:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:24.0:1619677472.269497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:8.0:1619677472.269497:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:1.0:1619677472.269498:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.269499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:8.0:1619677472.269499:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.269499:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.269501:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.269501:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.269503:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677472.269504:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.269505:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.269506:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.269507:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.269509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:8.0:1619677472.269512:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.269513:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000800:00000200:24.0:1619677472.269514:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:8.0:1619677472.269514:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:24.0:1619677472.269517:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.269518:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677472.269519:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.269519:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677472.269519:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677472.269521:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.0:1619677472.269521:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:24.0:1619677472.269522:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.269523:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.269524:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.269525:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.269525:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000800:00000200:8.2:1619677472.269527:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677472.269529:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.269531:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.269532:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677472.269533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:8.0:1619677472.269533:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000800:00000200:24.0:1619677472.269534:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.269535:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.269535:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.269535:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677472.269535:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:8.0:1619677472.269537:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.269538:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.269538:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000800:00000200:21.0:1619677472.269538:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:24.0:1619677472.269539:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8667200] (24)++
00000400:00000200:21.0:1619677472.269539:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.269539:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.269541:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000400:00000200:21.0:1619677472.269541:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.269541:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677472.269542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000400:00000200:8.0:1619677472.269542:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677472.269543:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000400:00000200:8.0:1619677472.269543:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.269544:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:8.0:1619677472.269545:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677472.269547:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:8.0:1619677472.269551:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.269552:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:8.0:1619677472.269552:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677472.269554:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:18.0:1619677472.269554:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.269555:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.269555:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:24.0:1619677472.269556:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.269556:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:18.0:1619677472.269557:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.269557:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.269559:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.269559:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.269559:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677472.269559:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.0:1619677472.269560:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000400:00000200:24.0:1619677472.269561:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:21.0:1619677472.269561:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:24.0:1619677472.269562:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.269562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:24.0:1619677472.269563:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.269564:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.269565:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677472.269567:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:24.0:1619677472.269569:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.269569:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.269570:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:24.0:1619677472.269571:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677472.269571:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677472.269573:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677472.269574:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.269574:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (24)++
00000800:00000200:24.0:1619677472.269575:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:24.0:1619677472.269575:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677472.269575:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000800:00000200:8.0:1619677472.269576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677472.269576:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:2.0:1619677472.269576:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.269577:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:24.0:1619677472.269578:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (24)++
00000800:00000200:8.0:1619677472.269578:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:24.0:1619677472.269579:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000800:00000200:24.0:1619677472.269579:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000800:00000200:2.0:1619677472.269579:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.269580:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:18.0:1619677472.269580:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.269581:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8667200] (24)--
00000400:00000200:8.0:1619677472.269581:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.269581:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.269582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:2.0:1619677472.269582:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.269582:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.269583:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000800:00000200:24.0:1619677472.269584:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:8.0:1619677472.269584:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.269585:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677472.269587:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96514311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.269588:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.269589:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677472.269590:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:8.0:1619677472.269590:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677472.269593:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.269593:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.269593:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.269594:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.269595:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.269595:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.269596:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:13.0:1619677472.269597:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:24.0:1619677472.269598:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96514310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.269598:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:24.0:1619677472.269599:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.269601:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.269602:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677472.269604:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.269605:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677472.269606:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677472.269607:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.269608:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.269608:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677472.269608:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677472.269609:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:24.0:1619677472.269610:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.0:1619677472.269610:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677472.269611:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000800:00000200:24.0:1619677472.269612:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:24.0:1619677472.269612:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (24)++
00000800:00000200:8.0:1619677472.269612:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (24)--
00000800:00000200:24.0:1619677472.269613:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8667200] (24)++
00000800:00000200:8.0:1619677472.269613:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:24.0:1619677472.269614:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677472.269614:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:24.0:1619677472.269615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677472.269615:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:24.0:1619677472.269616:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:24.0:1619677472.269617:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:8.0:1619677472.269617:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.269619:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.269620:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.269620:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.269622:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677472.269623:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96514309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.269624:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.269625:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.269626:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677472.269628:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.269629:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677472.269629:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96514308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.269630:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677472.269631:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:8.0:1619677472.269631:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.269632:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.269634:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.269635:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677472.269636:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:8.0:1619677472.269636:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.269637:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.269638:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677472.269638:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (24)++
00000800:00000200:24.0:1619677472.269639:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000800:00000200:24.0:1619677472.269640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677472.269640:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.269641:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:8.0:1619677472.269641:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.269642:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:8.0:1619677472.269642:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:24.0:1619677472.269644:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.269644:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.269645:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (24)++
00000800:00000200:8.0:1619677472.269645:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000400:00000200:24.0:1619677472.269646:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.269646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000800:00000200:8.0:1619677472.269647:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:8.0:1619677472.269647:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8667200] (24)--
00000400:00000200:24.0:1619677472.269648:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:8.0:1619677472.269648:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000400:00000200:24.0:1619677472.269649:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96514307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677472.269650:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:24.0:1619677472.269651:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.269653:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.269654:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677472.269654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:17.0:1619677472.269658:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:24.0:1619677472.269659:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.269661:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677472.269661:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.269662:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.269662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.269664:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.269664:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:17.0:1619677472.269664:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.269665:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677472.269667:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677472.269668:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (21)++
00000800:00000200:17.0:1619677472.269668:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.269669:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (22)--
00000800:00000200:24.0:1619677472.269670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (21)--
00000800:00000200:8.2:1619677472.269673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.269680:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.269683:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.269684:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.269686:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.269686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.269689:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:18.0:1619677472.269911:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)++
00000800:00000200:18.0:1619677472.269914:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.1@o2ib3
00000800:00000200:2.0:1619677472.269915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000400:00000200:18.0:1619677472.269918:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.1@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.269921:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.269922:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:8.0:1619677472.269923:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000400:00000200:18.0:1619677472.269924:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.269925:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.269926:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677472.269927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000400:00000200:18.0:1619677472.269927:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.269927:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:18.0:1619677472.269932:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677472.269932:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.269934:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.269936:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.269938:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677472.269939:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677472.269940:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677472.269940:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:8.0:1619677472.269940:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.269941:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677472.269941:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)++
00000400:00000200:8.0:1619677472.269942:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.269943:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677472.269943:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.1@o2ib3
00000800:00000200:18.0:1619677472.269944:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907e3cf600] (23)++
00000400:00000200:8.0:1619677472.269944:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677472.269945:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907e3cf600] (24)--
00000800:00000200:18.0:1619677472.269946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (23)--
00000800:00000200:18.0:1619677472.269947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000400:00000200:2.0:1619677472.269947:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.1@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.269948:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000800:00000200:24.0:1619677472.269949:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000400:00000200:8.0:1619677472.269949:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677472.269950:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.269951:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677472.269952:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677472.269953:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96514304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.269954:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.269955:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.269956:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.269956:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677472.269956:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677472.269957:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:2.0:1619677472.269957:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.269958:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.0:1619677472.269958:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:24.0:1619677472.269960:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96514303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.269960:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:2.0:1619677472.269961:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.269962:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.269963:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677472.269963:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.269964:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.269965:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000800:00000200:2.0:1619677472.269966:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.269967:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8667200] (24)++
00000800:00000200:2.0:1619677472.269967:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.269968:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000800:00000200:2.0:1619677472.269968:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:24.0:1619677472.269969:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.269969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000800:00000200:2.0:1619677472.269969:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.269970:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:2.0:1619677472.269970:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907e3cf600] (21)++
00000800:00000200:24.0:1619677472.269971:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.269971:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:2.0:1619677472.269971:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907e3cf600] (22)++
00000800:00000200:2.0:1619677472.269972:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907e3cf600] (23)--
00000400:00000200:8.0:1619677472.269973:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.269973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000800:00000200:24.0:1619677472.269974:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.269975:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.2:1619677472.269975:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.269976:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.269978:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.269979:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (24)++
00000800:00000200:24.0:1619677472.269980:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000800:00000200:24.0:1619677472.269981:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000400:00000200:8.0:1619677472.269981:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.269983:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:18.0:1619677472.269984:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)++
00000400:00000200:8.0:1619677472.269984:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:1.0:1619677472.269984:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.269985:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.269985:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96514302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.269986:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.269987:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907e3cf600] (23)--
00000400:00000200:8.0:1619677472.269987:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677472.269987:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:24.0:1619677472.269988:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.269988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000800:00000200:17.0:1619677472.269988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:8.0:1619677472.269988:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.269989:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)--
00000400:00000200:8.0:1619677472.269989:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677472.269989:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.269991:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:1.0:1619677472.269992:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.269993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.269993:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:8.0:1619677472.269994:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677472.269994:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.269994:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:2.0:1619677472.269996:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.269997:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.269999:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.269999:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.270000:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:8.0:1619677472.270000:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677472.270001:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8667200] (24)--
00000800:00000200:24.0:1619677472.270001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:8.0:1619677472.270001:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677472.270002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.270003:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.270003:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.270003:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (22)++
00000800:00000200:24.0:1619677472.270004:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000800:00000200:8.0:1619677472.270004:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (24)--
00000800:00000200:24.0:1619677472.270005:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:20.2:1619677472.270005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:8.0:1619677472.270005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:8.0:1619677472.270006:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000400:00000200:24.0:1619677472.270007:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.270007:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:20.0:1619677472.270008:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000400:00000200:8.0:1619677472.270009:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.270010:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.270010:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:24.0:1619677472.270012:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:21.0:1619677472.270013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677472.270013:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.270014:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.270015:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.270015:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.270016:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.270017:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677472.270017:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677472.270018:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.270019:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.270020:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.270021:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.270021:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.270022:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677472.270023:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677472.270023:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96514299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.270025:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677472.270026:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677472.270027:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677472.270027:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.270028:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.270030:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677472.270031:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.270032:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:24.0:1619677472.270033:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677472.270033:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.270034:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:8.0:1619677472.270034:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.270035:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677472.270035:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677472.270036:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000800:00000200:20.0:1619677472.270036:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677472.270036:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677472.270037:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (24)--
00000800:00000200:20.0:1619677472.270037:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:24.0:1619677472.270038:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:20.0:1619677472.270038:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677472.270038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:24.0:1619677472.270039:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (22)++
00000800:00000200:8.0:1619677472.270039:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.270040:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:8.0:1619677472.270040:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677472.270040:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.2:1619677472.270042:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:24.0:1619677472.270043:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.270045:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.270047:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.270048:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:508, d:10, s:-96514298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.270048:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:24.0:1619677472.270049:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.270050:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.270051:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.270051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677472.270052:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (23)++
00000800:00000200:21.0:1619677472.270053:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:8.0:1619677472.270053:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8667200] (24)++
00000400:00000200:21.0:1619677472.270054:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.270054:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (25)--
00000800:00000200:8.0:1619677472.270055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (24)--
00000400:00000200:21.0:1619677472.270056:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.270056:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.0:1619677472.270057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:24.0:1619677472.270058:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.270058:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.270058:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.270059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.270061:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.270061:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.270063:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677472.270068:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.270070:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (23)++
00000800:00000200:24.0:1619677472.270071:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.270071:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677472.270071:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8667200] (24)--
00000800:00000200:24.0:1619677472.270072:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.270072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (23)--
00000800:00000200:24.0:1619677472.270073:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.270073:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677472.270073:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.270074:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.270075:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (22)++
00000800:00000200:20.0:1619677472.270075:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.270075:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:8.0:1619677472.270075:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677472.270075:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.270076:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (23)--
00000800:00000200:20.0:1619677472.270076:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677472.270076:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.270077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000800:00000200:21.0:1619677472.270077:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.270078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.270079:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (21)--
00000800:00000200:8.0:1619677472.270079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677472.270089:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.270092:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.270093:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.270093:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.270095:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.270099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.270160:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.270162:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.270164:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.270164:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.270166:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.270168:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.270176:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.270179:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677472.270179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.270180:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:24.0:1619677472.270181:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667200] (21)++
00000800:00000200:18.0:1619677472.270181:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.270182:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.270184:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.35@o2ib3
00000800:00000200:8.0:1619677472.270185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (22)--
00000400:00000200:24.0:1619677472.270187:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.35@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.270189:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.270191:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.270192:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.270194:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.270194:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.270196:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.270199:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.35@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.270201:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.270203:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.270204:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.270205:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.270206:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.270207:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667200] (21)++
00000800:00000200:24.0:1619677472.270208:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667200] (22)--
00000800:00000200:24.0:1619677472.270209:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667200] (21)--
00000800:00000200:8.2:1619677472.270212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.270220:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.270223:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.270224:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.270224:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.270227:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.270231:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.270532:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.270535:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677472.270536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677472.270538:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.270540:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.270541:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677472.270543:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.270544:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.270545:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.270546:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677472.270548:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.270550:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677472.270553:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.270553:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677472.270554:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677472.270556:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677472.270557:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.270558:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677472.270559:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677472.270560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677472.270563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677472.270570:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:20.0:1619677472.270571:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:17.0:1619677472.270571:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677472.270572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.270573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:20.0:1619677472.270574:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.270575:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.270576:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.270578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.270579:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.270813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.270821:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.270823:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.270824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.270825:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.270827:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.270829:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.270834:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677472.270838:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.270840:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.270841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677472.270841:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:8.0:1619677472.270843:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.270844:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.270845:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:8.0:1619677472.270845:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.270845:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.270846:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.270847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:8.0:1619677472.270847:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.270848:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.270849:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.270850:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.270851:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.270854:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.270855:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.270857:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.270858:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.270858:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.270860:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.270860:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.270861:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.270862:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.270872:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.270874:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.270875:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.270876:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.270876:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.270878:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.270899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.270902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.270902:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.270902:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.270903:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.270904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.271382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.271387:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.271390:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.271392:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.271393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.271394:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.271396:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.271397:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.271399:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.271399:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.271401:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.271404:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.271406:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.271408:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.271409:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.271410:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.271411:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.271411:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.271412:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.271413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.271418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.271425:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.271427:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.271428:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.271430:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.271431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.271432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.271437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.271444:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.271447:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.271448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.271449:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.271452:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.271454:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.271455:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.271456:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.271457:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.271458:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.271461:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.271463:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.271464:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.271465:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.271466:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.271467:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.271468:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.271469:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.271469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.271481:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.271484:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.271484:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.271485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.271486:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.271487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.272015:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.272017:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.272019:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.272020:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.272022:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.272024:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.272025:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.272027:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.272028:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.272029:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.272032:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.272033:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.272036:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.272037:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.272038:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.272039:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.272040:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.272041:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.272041:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677472.273048:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.273050:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.273052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.273053:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.273055:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.273056:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.273057:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.273059:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.273059:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.273061:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.273064:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.273066:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.273069:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.273069:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.273070:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.273072:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.273072:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.273073:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.273073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.273086:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.273089:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.273090:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.273090:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.273092:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.273095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.273107:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.273109:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.273110:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.273110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.273112:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.273112:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.274145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.274151:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.274154:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.274155:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.274157:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.274159:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.274160:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.274162:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.274163:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.274164:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.274165:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.274169:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.274170:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.274173:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.274174:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.274175:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.274177:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.274177:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.274178:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.274179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.274459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.274462:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.274465:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.274467:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.274468:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.274470:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.274471:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.274473:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.274474:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.274475:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.274476:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.274479:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.274481:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.274483:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.274484:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.274485:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.274487:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.274487:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.274488:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.274489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.274494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.274502:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.274505:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.274506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.274506:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.274508:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.274511:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.275737:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.275739:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.275740:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.275743:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.275746:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.275747:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.275748:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.275749:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.275750:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.275751:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.275754:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.275756:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.275757:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.275759:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.275759:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.275760:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.275761:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.275761:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.275762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.275773:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.275774:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.275775:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.275776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.275777:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.275778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.275839:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.275842:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.275843:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.275844:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.275847:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.275848:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.275849:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.275851:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.275852:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.275853:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.275857:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.275860:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.275862:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.275863:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.275864:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.275867:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.275868:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.275869:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.275870:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.275871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.275874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.275882:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677472.275883:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.275883:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.275884:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.275885:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.275886:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.275886:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.275888:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.275889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.275891:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.276335:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.276337:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.276339:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.276339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.276341:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.276344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677472.277587:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.277591:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.277595:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.277595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.277598:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.277601:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677472.277603:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.277605:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.277607:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.277608:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677472.277614:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.277617:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677472.277620:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.277621:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677472.277622:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677472.277625:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677472.277625:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.277627:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.277628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677472.277631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.277640:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.277643:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.277644:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.277644:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.277646:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.277650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.277652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.277659:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.277661:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.277662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.277662:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.277664:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.277664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677472.277984:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.277987:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.277988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.277988:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.277990:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.277994:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.278417:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.278420:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.278421:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.278421:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.278422:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.278425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.278595:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.278598:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.278598:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.278601:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.278603:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.278605:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677472.278606:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.278607:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.278608:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.278609:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677472.278613:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.278615:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677472.278618:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.278619:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677472.278619:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677472.278621:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677472.278622:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.278622:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.278622:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677472.279386:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.279389:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.279390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.279391:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.279393:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.279395:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.279396:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.279397:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.279398:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.279399:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.279402:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.279404:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.279407:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.279408:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.279408:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.279410:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.279410:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.279412:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.279413:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.279413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.279425:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.279427:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.279428:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.279428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.279429:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.279429:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.279430:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677472.279431:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.279431:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.279435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.280582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.280592:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.280594:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.280595:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.280597:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.280598:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:21.0:1619677472.280599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.280601:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.280602:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.280603:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.280605:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.280608:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.280609:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.280610:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.280612:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.280613:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.280614:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.280617:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.280619:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.280621:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.280622:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.280623:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.280624:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.280625:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.280626:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.280626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.280630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.280638:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.280640:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.280641:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.280643:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.280644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.280645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.280649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.280656:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.280659:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.280660:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.280661:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.280661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.280662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.280757:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.280759:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.280761:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.280762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.280763:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.280765:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.280766:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.280768:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.280768:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.280770:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.280773:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.280775:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.280778:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.280779:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.280779:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.280781:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.280782:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.280783:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.280784:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.280785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.280796:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.280798:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.280799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.280799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677472.280879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.280885:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.280888:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.280889:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.280891:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.280894:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.280895:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.280897:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.280898:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.280899:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.280900:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.280904:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.280905:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.280907:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.280908:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.280909:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.280911:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.280911:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.280913:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.280914:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.280914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.280920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.280926:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.280927:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.280927:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.280927:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677472.280928:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.280928:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.280929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.280930:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.280930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.280932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:12.0:1619677472.281016:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.281018:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.281020:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.281022:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.281024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.281025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677472.281142:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.281144:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.281146:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.281147:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.281150:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.281151:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.281152:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.281153:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.281154:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.281155:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.281158:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.281160:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.281161:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.281162:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.281162:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.281163:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.281164:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.281165:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.281165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.281175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.281177:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.281177:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.281178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.281179:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.281181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.281580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.281589:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.281591:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.281592:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.281594:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.281595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.281597:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.281622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.281628:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.281631:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.281632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.281633:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.281636:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.281638:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.281639:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.281640:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.281641:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.281643:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.281646:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.281648:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.281652:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.281653:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.281653:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.281655:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.281656:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.281657:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.281657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.282223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.282226:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.282229:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.282231:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.282232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.282233:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.282235:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.282236:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.282237:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.282238:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.282239:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.282243:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.282244:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.282247:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.282248:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.282248:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.282250:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.282251:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.282251:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.282252:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.282256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.282264:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.282267:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.282268:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.282268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.282270:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.282273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.282940:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.282942:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.282943:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.282943:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.282945:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.282946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.283076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.283084:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.283087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.283087:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.283088:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.283091:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.283093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.283705:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.283707:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.283708:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.283708:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.283710:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.283712:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.285196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:18.0:1619677472.285246:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.285248:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.285249:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.285250:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.285252:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.285258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.285260:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.285262:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.285263:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.285263:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.285264:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.285265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.285855:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.285858:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.285859:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.285860:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.285862:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.285863:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.285864:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.285865:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.285866:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.285867:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.285869:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.285871:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.285873:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.285873:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.285874:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.285875:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.285876:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.285876:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.285877:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.285888:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.285889:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.285890:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.285891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.285892:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.285893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.286634:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.286636:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.286637:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.286639:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.286640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.286642:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:13.0:1619677472.286970:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.286973:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.286975:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.286978:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.286979:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677472.286979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.286980:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.286981:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.286982:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.286983:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677472.286986:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.286988:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677472.286991:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.286991:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677472.286992:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677472.286993:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.286994:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.286995:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.286995:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677472.287008:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.287011:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.287012:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.287014:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.287015:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.287017:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677472.287036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.287043:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.287046:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.287047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.287049:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.287052:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.287053:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.287055:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.287056:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.287057:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.287058:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.287062:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.287063:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.287066:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.287067:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.287068:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.287069:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.287070:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.287071:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.287073:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.287073:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.287079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.287085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.287087:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.287088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.287089:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.287168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.287174:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.287177:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.287178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.287180:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.287184:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.287185:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.287187:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.287189:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.287190:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.287191:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.287194:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.287196:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.287198:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.287199:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.287199:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.287201:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.287201:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.287202:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.287203:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.287214:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.287217:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.287217:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.287218:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.287219:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.287221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.287370:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.287373:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.287373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.287375:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.287378:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.287379:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.287380:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.287382:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.287383:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.287384:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.287387:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.287388:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.287391:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.287392:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.287392:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.287394:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.287394:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.287395:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.287396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.287401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.287410:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.287412:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.287413:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.287414:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.287416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.287417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.287682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.287687:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.287689:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.287692:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.287692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.287694:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.287695:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.287697:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.287698:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.287699:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.287700:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.287703:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.287705:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.287708:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.287708:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.287709:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.287711:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.287711:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.287712:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.287714:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.287714:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.287717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.287719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.287722:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.287724:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.287725:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677472.287725:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677472.287726:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.287727:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.287729:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.287729:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.287731:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.287733:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.288068:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.288070:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.288071:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.288071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.288073:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.288075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.288087:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.288089:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.288090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.288090:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.288091:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.288092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.288729:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.288732:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.288734:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.288735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.288737:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.288738:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.288740:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.288741:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.288742:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.288743:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.288747:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.288748:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.288751:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.288752:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.288753:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.288755:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.288756:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.288757:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.288757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677472.289003:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.289005:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.289006:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.289007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.289008:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.289010:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677472.289413:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.289415:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.289417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.289417:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.289419:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.289423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.289843:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.289846:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.289846:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.289847:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.289848:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.289851:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.289865:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.289868:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.289869:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.289869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.289870:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.289871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.290380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.290389:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.290392:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.290392:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.290393:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.290395:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.290398:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.290400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.290407:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.290410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.290410:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.290411:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.290412:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.290412:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.291259:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.291262:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.291263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.291265:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.291266:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.291267:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.291269:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.291270:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.291271:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.291272:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.291274:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.291276:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.291278:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.291278:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.291279:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.291280:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.291281:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.291281:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.2:1619677472.291282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677472.291282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677472.291288:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.291291:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:24.0:1619677472.291292:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.291294:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677472.291299:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.291300:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.291300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677472.291301:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.291302:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.291303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:21.0:1619677472.291304:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.291305:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.291307:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.291308:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.291309:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.291310:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.291313:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.291314:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.291316:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.291317:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.291318:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.291319:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.291320:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.291321:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.291321:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.291326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.291334:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.291336:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.291337:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.291339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.291339:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.291341:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.291366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.291373:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.291376:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.291377:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.291379:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.291379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.291381:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:18.0:1619677472.292011:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.292014:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.292014:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.292017:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.292019:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.292021:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677472.292022:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.292023:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.292024:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.292025:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.292028:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.292031:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677472.292033:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.292034:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677472.292035:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677472.292037:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677472.292037:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.292038:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.292039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:17.0:1619677472.293130:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.293133:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.293134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.293136:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.293138:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.293140:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677472.293142:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.293143:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.293144:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.293145:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677472.293148:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.293150:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.293152:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.293153:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.293153:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.293155:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.293156:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.293156:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.293157:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.293162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.293169:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.293171:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.293171:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.293173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.293173:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.293174:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677472.293731:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.293734:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.293736:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.293737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.293738:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.293739:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.293741:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.293742:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.293743:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.293744:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.293747:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.293749:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.293752:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.293753:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.293754:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.293755:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.293756:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.293757:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.293758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.293769:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.293772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.293772:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.293773:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.293775:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.293778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.293934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.293940:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.293943:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.293945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.293946:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.293949:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.293950:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.293951:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.293953:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.293954:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.293954:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.293957:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.293959:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.293961:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677472.293962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:2.0:1619677472.293962:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.293963:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.293964:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.293965:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.293966:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.293966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677472.293968:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677472.293971:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677472.293972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677472.293974:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677472.293976:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.293977:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677472.293978:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.293979:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:17.0:1619677472.293980:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.293980:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.293981:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:17.0:1619677472.293981:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677472.293982:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.293982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.293985:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.293986:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:17.0:1619677472.293986:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677472.293988:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.293989:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.293990:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.293991:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.293992:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.293993:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677472.293993:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677472.293995:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.2:1619677472.293999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.294006:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.294008:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.294009:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.294009:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.294010:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677472.294012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.294406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.294412:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.294415:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.294416:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.294418:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.294420:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.294421:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.294423:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.294424:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.294425:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.294426:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.294430:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.294431:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.294434:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.294434:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.294435:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.294437:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.294438:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.294439:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.294439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.294444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.294453:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.294455:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.294456:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.294458:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.294458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.294460:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677472.294745:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.294747:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.294748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.294750:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.294751:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.294753:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.294754:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.294755:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.294756:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.294757:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.294760:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.294762:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.294764:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.294765:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.294766:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.294767:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.294767:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.294768:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.294768:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.294773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.294781:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.294783:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.294784:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.294786:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.294786:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.294788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.294792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.294799:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.294802:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.294803:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.294804:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.294805:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.294806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.295074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.295080:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.295082:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.295085:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.295085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.295087:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.295089:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.295090:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.295092:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.295093:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.295094:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.295097:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.295099:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.295101:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.295102:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.295102:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.295104:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.295105:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.295105:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.295106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.295110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.295118:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.295120:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.295121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.295122:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.295124:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.295126:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677472.295653:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.295656:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.295657:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.295658:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.295661:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.295663:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677472.295664:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.295665:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.295666:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.295667:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677472.295670:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.295672:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677472.295675:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.295676:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677472.295676:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677472.295678:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677472.295678:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.295679:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.295680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677472.295692:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.295695:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.295696:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.295696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.295698:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.295700:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:13.0:1619677472.295911:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.295914:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.295916:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.295919:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.295919:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.295920:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677472.295921:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.295922:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.295923:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.295924:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:13.0:1619677472.295927:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.295929:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677472.295931:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.295932:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:13.0:1619677472.295932:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:13.0:1619677472.295934:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:13.0:1619677472.295935:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.295936:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.295936:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:24.0:1619677472.296146:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.296148:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.296150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.296151:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.296154:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.296155:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.296156:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.296157:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.296158:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.296159:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.296162:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.296163:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.296165:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.296166:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.296166:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.296168:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.296168:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.296170:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.296171:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.296171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.296182:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.296183:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.296184:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.296185:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.296186:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.296186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.296187:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.296187:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.296188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.296188:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.298234:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.298238:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.298241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.298242:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.298245:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.298247:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.298249:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.298252:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.298253:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.298256:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.298263:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.298267:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.298270:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.298272:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.298273:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.298275:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.298276:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.298278:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.298280:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.298281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677472.298289:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677472.298292:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677472.298292:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.298293:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.2:1619677472.298299:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.298300:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.298302:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:8.0:1619677472.298302:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.298303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677472.298304:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.298305:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:24.0:1619677472.298307:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677472.298307:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:8.0:1619677472.298307:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.298308:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.298308:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.298309:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.298309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.298310:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677472.298310:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.298310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677472.298311:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.298313:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.298319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.298329:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.298333:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.298334:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.298336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.298336:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.298337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.298792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.298800:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.298803:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.298804:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.298805:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.298808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.298809:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.299266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.299272:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.299275:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.299276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.299277:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.299279:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.299281:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.299283:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.299286:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.299287:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.299288:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.299292:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.299301:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.299304:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.299305:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.299305:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.299307:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.299308:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.299309:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.299310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.299329:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.299331:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.299332:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.299333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.299334:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.299336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.300099:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.300103:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.300104:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.300105:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.300108:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.300110:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.300111:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.300112:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.300113:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.300114:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.300117:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.300119:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.300121:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.300122:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.300123:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.300124:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.300125:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.300127:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.300128:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.300128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.300131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.300139:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.300139:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677472.300141:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677472.300141:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000400:00000200:20.0:1619677472.300142:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.300142:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.300142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677472.300144:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.300145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.300146:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.301599:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.301601:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.301602:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.301604:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.301606:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.301608:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.301609:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.301610:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.301611:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.301612:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.301614:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.301616:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.301617:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.301618:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.301618:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677472.301619:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.301620:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.301620:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.301621:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.301621:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.301622:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.301623:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.301623:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.301625:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.301629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.301631:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.301633:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.301634:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.301635:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.301636:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.301638:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.301639:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.301642:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.301642:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.301643:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.301644:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.301644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.302634:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.302637:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.302639:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.302640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.302641:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.302643:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.302644:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.302645:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.302646:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.302647:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.302650:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.302652:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.302655:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.302656:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.302657:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.302659:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.302659:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.302660:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.302661:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677472.302959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.302962:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.302964:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.302966:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.302967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.302969:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.302971:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.302972:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.302973:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.302974:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.302975:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.302979:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.302981:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.302982:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.302983:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.302984:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.302985:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.302986:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.302987:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.302987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.302992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.303000:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.303002:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.303003:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.303005:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.303005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.303007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.303334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.303341:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.303344:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.303345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.303347:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.303349:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.303351:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.303352:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.303353:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.303354:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.303355:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.303358:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.303360:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.303362:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.303363:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.303364:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.303365:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.303366:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.303367:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.303367:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.303378:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.303381:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.303381:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.303382:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.303383:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.303384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.303506:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:17.0:1619677472.303509:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.45@o2ib3
00000800:00000200:1.0:1619677472.303510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000400:00000200:17.0:1619677472.303512:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.45@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.303515:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.303516:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:17.0:1619677472.303517:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.303519:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.303520:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.303521:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:17.0:1619677472.303524:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.303526:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:17.0:1619677472.303529:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.303530:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:17.0:1619677472.303531:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677472.303532:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:17.0:1619677472.303533:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:17.0:1619677472.303534:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce0bf7800] (22)++
00000800:00000200:17.0:1619677472.303535:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf7800] (23)--
00000800:00000200:17.0:1619677472.303535:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:24.2:1619677472.303539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.303543:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.303546:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:1.0:1619677472.303546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)++
00000400:00000200:24.0:1619677472.303547:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.303547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:1.0:1619677472.303548:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce0bf7800] (23)--
00000400:00000200:24.0:1619677472.303549:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.303549:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:1.0:1619677472.303549:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)--
00000800:00000200:24.0:1619677472.303553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677472.303737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.303744:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.303746:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.303747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.303749:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.303751:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.303753:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677472.303754:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.303755:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.303756:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.303758:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.303761:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.303762:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677472.303765:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.303766:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677472.303767:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677472.303768:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677472.303769:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.303770:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.303770:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677472.304139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.304145:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.304148:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.304149:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.304151:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.304154:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.304155:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.304156:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.304158:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.304159:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.304160:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.304163:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.304164:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.304167:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.304168:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.304168:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.304170:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.304171:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.304171:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.304172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677472.304352:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.304354:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.304355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.304358:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.304361:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.304362:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677472.304364:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.304365:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.304366:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.304367:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.304370:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.304372:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.304374:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.304375:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.304376:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.304378:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.304378:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.304379:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.304379:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677472.304982:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:1.0:1619677472.304985:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.45@o2ib3
00000800:00000200:17.0:1619677472.304986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000400:00000200:1.0:1619677472.304988:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.45@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.304990:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.304992:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:1.0:1619677472.304993:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.304994:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.304995:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.304997:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:1.0:1619677472.305000:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.305002:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:1.0:1619677472.305005:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.305005:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:1.0:1619677472.305006:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677472.305008:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:1.0:1619677472.305008:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:1.0:1619677472.305009:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:1.0:1619677472.305009:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)--
00000800:00000200:24.2:1619677472.305014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.305018:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.305021:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677472.305022:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.305022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.305024:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.305027:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.0:1619677472.305696:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.305699:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.305700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.305701:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.305703:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.305705:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.305706:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.305708:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.305709:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.305710:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.305713:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.305714:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.305717:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.305718:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.305718:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.305720:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.305720:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.305721:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.305721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.305738:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.305741:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.305742:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.305742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.305744:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.305748:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.306506:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.306509:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.306510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.306512:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.306515:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.306516:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677472.306517:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.306519:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.306519:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.306521:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677472.306524:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.306526:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677472.306528:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.306529:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677472.306529:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677472.306531:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677472.306532:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.306533:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.306533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677472.306546:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.306548:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.306550:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.306550:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.306551:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.306554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.306848:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.306851:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.306852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.306854:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.306857:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.306858:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.306859:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.306860:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.306861:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.306862:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.306865:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.306867:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.306868:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.306869:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.306869:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.306871:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.306871:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.306872:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.306873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.306883:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.306885:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.306885:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.306886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.306887:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.306888:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677472.308156:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.308159:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.308159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.308160:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.308162:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.308166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677472.308177:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.308179:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.308180:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.308180:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.308181:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.308182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677472.308219:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.308222:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.308223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.308225:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.308227:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.308229:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.308230:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.308231:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.308232:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.308233:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.308236:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.308238:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.308240:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.308241:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.308242:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.308243:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.308244:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.308244:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.308245:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.308250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.308258:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.308261:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.308261:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.308263:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.308264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.308266:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.309658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.309665:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.309668:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.309669:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.309671:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.309674:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.309675:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.309676:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.309678:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.309678:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.309680:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.309683:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.309685:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.309686:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.309687:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.309688:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.309690:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.309690:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.309691:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.309692:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.309693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.309698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.309704:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.309706:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.309706:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.309706:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.309707:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.309707:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.309708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.309708:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.309709:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.309710:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.309888:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.309890:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.309893:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.309893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.309895:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.309896:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.309898:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.309900:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.309901:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.309902:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.309904:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.309906:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.309908:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.309909:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.309910:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.309912:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.309912:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.309913:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.309914:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.310013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.310022:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.310024:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.310025:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.310025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.310028:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.310031:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.310032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.310039:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.310042:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.310043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.310044:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.310045:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.310045:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.310746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.310754:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.310756:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.310757:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.310759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.310759:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.310762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.310767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.310774:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.310777:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.310778:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.310779:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.310779:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.310781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.310994:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.310996:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.310998:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.310999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.311000:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.311002:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.311003:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.311004:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.311005:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.311006:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.311010:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.311012:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.311014:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.311015:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.311016:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.311018:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.311019:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.311019:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.311020:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.311029:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.311033:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.311033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.311036:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.311038:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.311040:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677472.311041:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.311042:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.311043:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.311044:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677472.311047:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.311049:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.311053:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.311053:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.311054:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677472.311056:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677472.311056:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.311057:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.311058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.2:1619677472.311316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.311322:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.311324:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.311325:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.311327:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.311330:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.311331:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.311333:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.311334:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.311335:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.311336:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.311339:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.311340:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.311343:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.311344:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.311344:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.311346:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.311346:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.311347:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.311348:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.311348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.311354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.311360:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.311362:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.311362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.311363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.311375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.311382:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.311384:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.311385:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.311387:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.311389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.311390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.311691:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.311694:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.311695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.311697:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.311699:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.311700:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.311701:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.311703:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.311704:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.311705:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.311707:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.311709:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.311710:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.311711:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.311712:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.311713:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.311714:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.311715:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.311716:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.311716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.311728:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.311729:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.311729:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.311731:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.311731:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677472.311731:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.311732:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.311733:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.311733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.311734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.311954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.311962:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.311964:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.311965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.311966:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.311968:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.311970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.312604:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.312607:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.312608:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.312610:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.312612:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.312613:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.312615:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.312616:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.312617:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.312618:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.312620:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.312622:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.312623:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.312624:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.312624:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.312626:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.312626:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.312627:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.312627:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.312639:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.312641:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.312641:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.312642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.312643:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.312645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.312659:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.312662:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.312663:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.312663:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.312663:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.312664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.312843:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.312846:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.312848:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.312849:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.312850:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.312852:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.312853:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.312855:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.312856:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.312857:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.312860:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.312861:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.312863:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.312864:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.312865:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.312866:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.312867:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.312867:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.312868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:2.0:1619677472.314034:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.314036:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.314038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.314038:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.314040:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.314043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.314624:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.314626:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.314627:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.314627:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.314629:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.314631:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.315344:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.315346:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.315347:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.315347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.315349:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.315351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.316046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.316053:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.316056:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.316057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.316059:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.316061:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.316062:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.316064:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.316065:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.316066:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.316067:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.316071:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.316072:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.316076:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.316077:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.316077:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.316079:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.316080:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.316081:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.316081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.316546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.316549:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.316552:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.316554:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.316556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.316556:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.316558:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.316559:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.316561:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.316562:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.316563:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.316566:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.316568:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.316570:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.316571:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.316572:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.316573:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.316574:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.316575:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.316576:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.316577:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.316580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.316582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.316589:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677472.316590:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677472.316592:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.316593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.316594:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.316594:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.316595:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.316596:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.316599:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.316602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.316946:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.316949:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.316949:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.316951:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.316953:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.316954:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.316956:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.316957:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.316958:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.316959:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.316961:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.316963:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.316964:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.316965:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.316966:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.316967:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.316967:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.316968:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.316969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.316979:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.316980:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.316981:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.316982:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.316983:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.316984:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.317006:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.317010:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.317012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.317015:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.317018:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.317020:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.317022:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.317025:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.317026:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.317028:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.317033:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.317036:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.317039:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.317041:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.317042:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.317044:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.317045:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.317046:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.317048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.317050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.317058:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.317060:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.317061:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.317063:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.317065:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.317066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.317074:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.317076:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.317079:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.317080:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.317081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.317082:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.317083:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.317084:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.317085:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.317086:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.317090:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.317092:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.317095:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.317096:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.317096:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.317098:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.317099:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.317100:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.317100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.317110:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.317113:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.317114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.317114:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.317116:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.317119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.317769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.317775:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.317778:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.317779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.317781:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.317784:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.317785:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.317786:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.317788:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.317789:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.317790:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.317793:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.317795:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.317797:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.317798:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.317799:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.317800:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.317801:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.317802:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.317802:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.317814:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.317816:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.317817:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.317818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.317819:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.317820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.317839:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.317841:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.317842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.317842:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.317843:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.317843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.318661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.318670:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.318672:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.318673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.318674:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.318676:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.318679:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.319009:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.319012:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.319012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.319014:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.319016:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.319017:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677472.319019:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.319020:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.319021:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.319022:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677472.319025:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.319027:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.319029:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.319030:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.319030:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.319032:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.319033:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.319034:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677472.319035:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677472.319036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677472.319038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.319047:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.319048:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677472.319048:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677472.319048:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.319048:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.319049:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.319050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.319050:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.319051:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677472.319051:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677472.319146:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.319149:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.319151:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.319151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.319153:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.319155:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.319156:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.319158:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.319159:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.319160:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.319163:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.319165:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.319167:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.319168:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.319168:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.319170:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.319171:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.319172:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.319173:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.319174:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.319184:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.319186:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.319187:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.319188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.319532:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.319535:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.319536:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.319536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.319538:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.319540:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.319553:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.319555:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.319556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.319556:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.319557:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677472.319558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.319686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677472.319690:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.319693:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677472.319695:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.319697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677472.319698:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.319700:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677472.319702:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.319703:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.319704:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.319705:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677472.319710:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.319712:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677472.319716:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.319717:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:20.0:1619677472.319717:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.0:1619677472.319720:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:20.0:1619677472.319720:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.319721:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677472.319722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:18.0:1619677472.319737:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.319740:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.319741:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.319741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.319743:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.319747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.320249:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.320251:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.320252:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.320253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.320254:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.320256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.320265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:8.0:1619677472.320271:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.320274:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.320275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.320275:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.320275:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.320276:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.320299:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.320301:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.320302:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.320302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.320303:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.320305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677472.320319:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.320321:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.320322:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.320323:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.320326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.320327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677472.320528:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.320531:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.320532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.320533:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.320535:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.320537:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.320538:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.320539:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.320540:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.320541:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.320544:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.320546:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.320549:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.320550:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.320550:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.320552:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.320552:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.320553:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.320553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.320567:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.320569:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.320570:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.320571:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.320573:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.320576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.321176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.321183:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.321186:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.321186:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.321188:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.321190:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.321192:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677472.321193:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.321194:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.321195:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.321196:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.321199:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.321201:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677472.321204:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.321204:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677472.321205:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677472.321207:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677472.321207:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.321208:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677472.321209:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677472.321210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677472.321215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.321221:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677472.321222:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677472.321223:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.321225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677472.321985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.321991:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.321994:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.321995:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.321996:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.321999:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.322001:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.322002:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.322003:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.322004:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.322005:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.322008:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.322010:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.322012:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.322013:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.322014:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.322016:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.322016:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.322017:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.322017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.322068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.322074:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.322076:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.322078:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.322079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.322082:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.322083:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.322085:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.322086:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.322087:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.322088:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.322091:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.322093:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.322095:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.322096:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.322097:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.322098:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.322099:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.322100:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.322100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.322105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.322113:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.322116:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.322117:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.322117:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.322119:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.322121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.322124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.322131:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.322134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.322134:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.322135:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.322136:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.322136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677472.322241:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.322244:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.322247:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.322248:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.322251:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.322253:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.322254:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.322255:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677472.322255:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.322256:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677472.322257:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:20.0:1619677472.322258:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.322259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.322261:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.322262:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.322264:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.322264:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.322266:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.322267:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.322267:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677472.322268:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.322268:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.322268:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:24.0:1619677472.322270:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.322270:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.322270:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.322271:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.322271:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.322272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:24.0:1619677472.322274:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.322275:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.322277:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.322277:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.322278:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.322279:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.322280:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.322281:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.322282:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.322283:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.322298:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.322299:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.322299:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.322300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.322300:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.322301:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:8.0:1619677472.322301:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.322302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.322303:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.322304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.324695:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.324699:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.324700:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.324701:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.324703:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.324703:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:24.0:1619677472.324706:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677472.324707:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.324709:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.324710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.324713:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.324717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677472.324736:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.324738:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.324741:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.324743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.324744:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.324746:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.324747:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.324749:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.324750:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.324751:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.324755:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.324757:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.324759:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.324760:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.324761:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.324763:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.324763:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.324765:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.324765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.324779:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.324783:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.324784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.324784:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.324788:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.324792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.325203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677472.325210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677472.325215:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.325219:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677472.325220:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677472.325221:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.325222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.325224:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:17.0:1619677472.325225:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.325225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.325228:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.325229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:18.0:1619677472.325231:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.325233:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.325234:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.325236:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.325237:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.325239:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.325243:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.325245:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.325247:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.325248:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.325249:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.325250:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.325251:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.325252:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.325253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.325264:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.325266:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.325267:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.325268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.325269:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.325271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.325429:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.325431:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.325432:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.325432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.325434:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.325436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.325613:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.325617:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.325619:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.325620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.325621:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.325623:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.325625:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.325626:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.325628:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.325629:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.325632:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.325636:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.325638:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.325640:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.325640:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.325642:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.325642:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.325643:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677472.325643:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.325643:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.325644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:21.0:1619677472.325645:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.325645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.325648:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.325649:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.325651:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96514122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.325652:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.325653:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.325654:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.2:1619677472.325656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:21.0:1619677472.325658:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.325660:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.2:1619677472.325660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:21.0:1619677472.325663:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.325664:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677472.325664:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677472.325665:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.325666:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:1.0:1619677472.325666:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677472.325667:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.325667:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677472.325667:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.325668:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.325669:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:18.0:1619677472.325669:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:21.0:1619677472.325670:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:2.0:1619677472.325670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:1.0:1619677472.325670:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.325671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:18.0:1619677472.325673:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.325674:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:18.0:1619677472.325675:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.325676:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677472.325677:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.325679:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:18.0:1619677472.325679:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.325680:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.325681:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.325682:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.325683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.325684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677472.325684:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.325687:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677472.325689:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.325690:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677472.325691:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677472.325693:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677472.325693:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677472.325694:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907cbe3400] (22)++
00000800:00000200:18.0:1619677472.325696:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (23)--
00000800:00000200:18.0:1619677472.325696:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:20.2:1619677472.325702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.325707:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)++
00000800:00000200:20.0:1619677472.325709:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677472.325710:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907cbe3400] (23)--
00000800:00000200:2.0:1619677472.325710:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:20.0:1619677472.325711:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:2.0:1619677472.325711:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000400:00000200:20.0:1619677472.325712:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.325714:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.325715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.325717:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.325728:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.325731:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.325732:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.325732:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.325733:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.325734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677472.326022:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.326025:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.326026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.326028:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.326030:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.326031:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.326033:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.326034:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.326035:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.326036:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.326039:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.326041:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.326043:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.326044:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.326045:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.326047:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.326047:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.326048:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.326048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.326053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.326061:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.326063:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.326064:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.326066:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.326067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.326069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.326280:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.326283:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.326284:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.326284:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.326287:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.326291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677472.326301:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.326303:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.326304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.326305:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.326306:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.326306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.326334:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.326337:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677472.326338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677472.326339:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.326341:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.326342:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677472.326344:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.326345:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.326346:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.326347:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677472.326350:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.326352:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.326354:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.326355:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677472.326356:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677472.326357:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.326358:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.326359:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.326359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:24.0:1619677472.326447:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.326450:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.326451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.326453:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.326456:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.326457:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.326459:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.326460:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.326461:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.326462:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.326465:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.326466:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.326469:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.326469:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.326470:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.326472:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.326473:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.326474:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.326474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.326486:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.326489:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.326490:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.326490:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.326493:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.326496:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.326616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677472.326659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:2.0:1619677472.326662:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.326665:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.326666:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.326666:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.326669:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.326672:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.326681:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.326683:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.326684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.326685:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.326686:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.326686:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.327755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.327758:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.327760:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.327763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.327763:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.327766:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.327767:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.327769:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.327770:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.327771:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.327772:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.327776:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.327777:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.327780:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.327781:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.327781:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.327783:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.327783:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.327784:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.327785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.327790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.327798:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.327801:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.327802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.327802:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.327804:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.327806:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.328073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.328079:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.328082:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.328083:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.328084:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.328087:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.328089:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.328090:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.328092:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.328093:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.328094:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.328097:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.328098:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.328101:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.328102:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.328103:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.328104:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.328105:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.328106:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.328106:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.328111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.328120:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.328122:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.328123:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.328124:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.328125:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.328127:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.329139:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.329142:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.329143:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.329144:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.329148:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.329149:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.329151:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.329152:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.329153:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.329154:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.329157:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.329159:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.329160:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.329161:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.329161:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.329162:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.329163:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.329164:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.329164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.329174:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.329176:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.329176:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.329177:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.329178:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.329179:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.329245:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.329247:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.329249:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.329250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.329252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.329254:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.329255:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.329256:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.329257:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.329258:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.329262:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.329263:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.329266:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.329267:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.329267:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.329269:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.329270:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.329271:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.329272:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.329272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677472.329281:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677472.329283:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677472.329284:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.329285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.329285:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:8.0:1619677472.329285:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.329286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.329286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677472.329287:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.329290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.329303:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.329306:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.329306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.329307:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.329308:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.329309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.329969:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.329971:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.329972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.329972:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.329974:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.329976:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.329992:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.329995:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.329996:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.329996:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.329996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.329997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.331198:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.331201:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.331201:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.331203:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.331206:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.331208:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677472.331209:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.331210:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.331211:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.331213:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677472.331216:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.331218:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677472.331220:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.331221:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677472.331222:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677472.331224:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677472.331224:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.331225:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.331225:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677472.331237:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.331240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.331240:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.331241:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.331243:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.331246:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.331345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.331354:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.331357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.331357:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.331358:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.331360:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.331363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.331364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.331372:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.331374:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.331375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.331376:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.331377:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.331377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.331445:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.331447:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.331449:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.331450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.331453:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.331454:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.331456:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.331457:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.331458:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.331459:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.331462:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.331464:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.331466:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.331467:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.331468:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.331470:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.331470:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.331471:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.331472:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.331473:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.331475:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.331477:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.331478:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.331480:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.331482:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677472.331483:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96514102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.331485:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.331486:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.331487:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.331491:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.331494:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677472.331497:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.331498:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677472.331499:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677472.331501:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:24.0:1619677472.331502:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.331503:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.331503:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:20.2:1619677472.331986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.331993:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.331996:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.331996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.331999:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.332002:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.332003:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.332005:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.332006:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.332007:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.332008:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.332010:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.332012:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.332014:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.332015:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.332015:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.332017:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.332018:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.332019:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.332019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.332030:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.332032:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.332033:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.332034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.332035:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.332036:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.332167:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677472.332170:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677472.332171:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677472.332173:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.332175:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.332177:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677472.332178:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.332180:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.332181:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.332182:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677472.332185:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.332186:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677472.332189:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.332190:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677472.332191:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677472.332193:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677472.332193:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677472.332194:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677472.332194:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677472.332198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.332207:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.332209:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.332210:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.332211:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.332213:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.332216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677472.332321:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.332323:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.332324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.332326:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.332328:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.332330:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.332331:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.332332:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.332333:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.332334:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.332337:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.332339:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.332342:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.332342:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.332343:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.332345:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.332345:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.332347:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.332348:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.332348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.332351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.332360:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.332360:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677472.332362:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677472.332362:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000400:00000200:20.0:1619677472.332363:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.332363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.332364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677472.332365:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.332366:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.332367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.332774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:18.0:1619677472.332822:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.332825:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.332826:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.332826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.332828:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.332831:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.333004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.333009:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.333012:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.333012:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677472.333014:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.333016:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.333017:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.333018:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.333020:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.333021:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.333022:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.333024:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.333026:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.333028:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.333029:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.333030:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.333032:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.333032:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.333034:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.333035:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.333035:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.333038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.333040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677472.333046:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:18.0:1619677472.333046:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.333048:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:21.0:1619677472.333049:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:18.0:1619677472.333049:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:21.0:1619677472.333050:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.333050:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:21.0:1619677472.333051:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.333052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.333054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.333058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.333066:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.333069:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.333070:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.333070:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.333071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.333072:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.333828:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.333830:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.333831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.333833:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.333836:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.333837:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.333838:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.333840:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.333841:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.333841:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.333844:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.333846:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.333847:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.333848:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.333848:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.333850:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.333850:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.333851:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.333851:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.333862:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.333863:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.333864:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.333865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.333866:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.333867:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.334277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.334280:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.334282:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.334284:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.334285:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.334286:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.334289:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.334290:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.334292:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.334298:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.334299:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.334303:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.334306:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.334308:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.334309:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.334310:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.334312:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.334312:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.334313:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.334314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.334318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.334327:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.334329:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.334330:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.334330:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.334332:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.334335:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.334337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.334344:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.334347:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.334348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.334348:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.334350:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.334350:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.335101:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.335103:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.335104:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.335104:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.335106:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.335108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677472.335135:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.335138:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.335139:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.335139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.335142:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.335145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.335197:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.335199:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.335201:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.335202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.335203:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.335205:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.335206:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.335208:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.335209:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.335210:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.335213:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.335215:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.335217:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.335218:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.335219:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.335221:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.335221:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.335222:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.335223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.335243:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.335246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.335246:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.335247:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.335249:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.335251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.335316:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.335318:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.335319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.335321:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.335323:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.335324:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.335325:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.335326:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.335327:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.335328:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.335331:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.335333:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.335334:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.335335:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.335335:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.335337:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.335337:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.335338:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677472.335339:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677472.335340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.335350:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.335351:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.335352:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.335353:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677472.335354:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677472.335354:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677472.335354:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.335355:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.335356:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677472.335356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.335366:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.335369:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.335370:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.335370:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.335371:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.335371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.335380:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.335383:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.335384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.335384:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.335387:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.335390:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.336071:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.336074:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.336074:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.336075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.336077:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.336079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.336551:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.336553:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.336554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.336556:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.336558:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.336560:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.336561:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.336563:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.336563:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.336564:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.336567:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.336569:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.336571:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.336572:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.336573:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.336575:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.336575:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.336575:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.336576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.336581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.336590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.336592:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.336593:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.336595:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.336596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.336597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.336602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.336610:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.336612:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.336614:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.336614:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.336615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.336615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.336963:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.336965:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.336967:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.336968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.336970:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.336971:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.336972:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.336974:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.336974:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.336976:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.336979:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.336981:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.336983:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.336983:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.336984:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.336986:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.336986:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.336987:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.336988:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.2:1619677472.337583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.337592:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.337595:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.337596:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.337596:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.337598:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.337601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677472.338000:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.338003:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.338004:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.338004:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.338006:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.338009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.339785:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.339788:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.339789:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.339791:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.339792:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.339794:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.339795:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.339796:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.339797:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.339798:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.339801:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.339803:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.339804:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.339805:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.339806:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.339807:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.339807:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.339808:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.339809:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.339810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.339821:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.339822:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.339823:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.339825:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.339825:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.339826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.339826:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.339826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.339827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.339827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677472.339899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677472.339902:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.339905:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677472.339907:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.339908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677472.339909:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.339911:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677472.339912:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677472.339914:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:20.0:1619677472.339917:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.339918:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.339919:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:2.0:1619677472.339920:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677472.339923:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.339923:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.339924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677472.339926:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677472.339927:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.339929:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677472.339929:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.339930:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677472.339931:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:2.0:1619677472.339931:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.339932:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.339933:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:2.0:1619677472.339933:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.339934:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.339934:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000400:00000200:2.0:1619677472.339934:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.339935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:2.0:1619677472.339935:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.339939:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.339940:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.339943:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.339944:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.339945:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677472.339947:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:2.0:1619677472.339947:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.339948:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.339948:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.339949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677472.339950:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.339951:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.339953:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.339955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.339957:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677472.339970:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677472.339974:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677472.339976:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.339977:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677472.339977:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.339979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677472.340216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.340222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.340224:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.340227:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.340227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.340230:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.340231:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.340233:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.340234:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.340235:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677472.340236:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:21.0:1619677472.340236:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677472.340239:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:21.0:1619677472.340239:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.340241:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.340241:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677472.340242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677472.340243:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.340244:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:24.0:1619677472.340245:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.340245:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.340246:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.340247:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677472.340248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.340248:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.340249:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.340250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.340253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.340255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.340258:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.340260:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.340261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.340262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677472.340262:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.340265:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.340266:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.340266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.340268:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.340271:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:13.0:1619677472.340724:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.340727:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.340729:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.340732:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.340732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.340733:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677472.340735:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.340736:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.340737:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.340738:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677472.340741:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.340743:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677472.340745:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.340746:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677472.340747:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677472.340749:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677472.340749:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.340751:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677472.340752:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677472.340753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.340768:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677472.340771:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677472.340772:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.340774:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.0:1619677472.341291:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.341295:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677472.341298:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.341299:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.341300:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.341302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.341309:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.341312:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.341313:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.341313:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.341314:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.341314:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.341353:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.341356:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.341357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.341359:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.341361:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.341362:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.341363:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.341364:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.341366:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.341366:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.341370:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.341372:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.2:1619677472.341373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677472.341374:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.341375:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.341376:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.341377:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.341378:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.341378:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.341379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677472.341380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.341382:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:8.2:1619677472.341383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677472.341383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.341385:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.341388:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.341389:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677472.341391:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:18.0:1619677472.341391:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.341392:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.341393:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.341394:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.341394:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.341394:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.341397:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.341398:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:18.0:1619677472.341398:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.341399:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.341401:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.341403:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.341404:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.341404:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.341406:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.341407:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.341408:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.341409:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.341410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.341415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.341421:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.341422:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.341423:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677472.341423:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.341424:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.341424:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.341425:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.341425:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.341426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.341427:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.341441:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.341443:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.341444:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.341445:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.341445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.341445:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.341500:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.341502:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.341504:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.341505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.341507:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.341509:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.341510:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.341511:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.341512:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.341513:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.341516:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.341518:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.341521:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.341522:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.341523:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.341525:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.341525:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.341526:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.341526:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677472.341585:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.341587:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.341590:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.341590:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.341592:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.341594:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.341595:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.341596:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.341597:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.341598:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.341601:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.341603:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.341605:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.341606:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.341606:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.341608:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.341609:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.341610:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.341611:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.341612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.341622:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.341625:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.341625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.341626:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.343426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:12.0:1619677472.343469:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.343472:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.343472:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.343474:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.343476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.343477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.2:1619677472.343635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.343643:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.343646:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.343647:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.343647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.343650:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.343653:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.344200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677472.344242:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:13.0:1619677472.344245:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:2.0:1619677472.344245:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.344246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.344246:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.344248:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:2.0:1619677472.344248:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:13.0:1619677472.344249:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.344250:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.344251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:2.0:1619677472.344252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677472.344253:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677472.344318:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.344321:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.344322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.344323:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.344326:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.344327:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.344328:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.344330:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.344331:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.344332:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.344335:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.344336:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.344339:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.344340:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.344341:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.344343:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.344343:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.344344:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.344344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.344357:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.344360:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.344361:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.344361:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.344364:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.344367:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677472.344527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a46d7f8000] (20)++
00000800:00000200:12.0:1619677472.344537:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a46d7f8000] (21)++
00000800:00000200:12.0:1619677472.344540:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.8@o2ib3
00000400:00000200:12.0:1619677472.344542:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.4.8@o2ib3 : GET - for me
00000800:00000200:13.0:1619677472.344545:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a46d7f8000] (22)--
00000400:00000200:12.0:1619677472.344545:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.4.8@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677472.344547:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.4.8@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677472.344548:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a46d7f8000] (21)++
00000800:00000200:12.0:1619677472.344549:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a46d7f8000] (22)--
00000400:00000200:12.0:1619677472.344551:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.4.8@o2ib3 local destination
00000800:00000200:12.0:1619677472.344551:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a46d7f8000] (21)--
00000800:00000200:20.2:1619677472.344645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.344650:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.344653:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.344654:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.344656:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.344658:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.344660:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.344661:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.344662:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.344663:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.344664:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.344667:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.344669:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.344672:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.344673:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.344673:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.344675:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.344676:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.344677:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.344677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.345124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.345130:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.345132:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.345134:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.345135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.345137:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.345139:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.345140:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.345141:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.345142:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.345143:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.345147:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.345149:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.345151:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.345152:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.345152:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.345154:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.345155:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.345155:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.345156:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.345160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.345168:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.345170:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.345171:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.345173:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.345173:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.345175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.345427:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.345430:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.345431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.345432:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.345436:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.345437:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.345438:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.345439:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.345440:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.345441:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.345444:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.345445:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.345447:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.345448:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.345448:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.345450:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.345450:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.345451:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.345451:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.345462:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.345464:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.345464:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.345465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.345466:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.345467:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.345600:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.345603:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.345604:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.345604:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.345606:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.345608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677472.346680:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.346682:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.346684:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.346684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.346686:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.346688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677472.346719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.346728:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.346730:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.346731:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.346733:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.346734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.346740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.346742:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.346745:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677472.346746:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677472.346748:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.346751:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.346752:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677472.346753:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.346755:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.346756:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.346757:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677472.346760:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.346761:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677472.346764:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.346765:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677472.346766:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677472.346768:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677472.346768:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.346769:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.346770:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677472.346781:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.346783:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.346784:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.346785:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.346786:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.346789:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.346799:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.346801:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677472.346802:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.346803:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.346804:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.346804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677472.346944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.346951:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.346954:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.346955:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.346956:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.346958:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.346959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.347124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.347129:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.347132:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.347134:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.347134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.347136:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.347138:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.347139:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.347141:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.347142:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.347143:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.347146:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.347148:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.347150:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.347151:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.347152:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.347153:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.347154:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.347155:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.347155:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.347160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.347167:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.347170:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.347171:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.347171:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.347173:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.347175:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677472.347751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677472.347757:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.347760:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677472.347761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677472.347762:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677472.347764:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.347766:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677472.347767:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.347768:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.347769:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.347770:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677472.347773:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677472.347775:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677472.347778:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.347779:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677472.347779:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677472.347781:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677472.347781:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.347783:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677472.347784:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677472.347784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:5.2:1619677472.347791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:8.0:1619677472.347794:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677472.347797:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:8.0:1619677472.347797:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.347798:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.347798:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.347799:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677472.347799:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677472.347800:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000400:00000200:8.0:1619677472.347800:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.347803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677472.348201:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.348203:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.348205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.348205:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.348207:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.348211:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677472.348441:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.348444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.348444:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.348447:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.348449:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.348451:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.348452:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.348454:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.348455:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.348456:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.348459:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.348460:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.348463:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.348464:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.348464:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.348466:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.348466:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.348467:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.348467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.348473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.348481:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.348483:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.348484:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.348487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.2:1619677472.348487:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:20.0:1619677472.348490:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.348491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.348494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.348496:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.348498:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.348499:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.348499:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.348500:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.348500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.348546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:18.0:1619677472.348550:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.348553:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.348554:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.348554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.348556:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.348560:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677472.348869:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.348872:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.348874:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.348874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.348877:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.348878:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.348880:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.348882:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.348883:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.348884:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.348887:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.348889:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.348892:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.348893:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.348893:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.348895:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.348896:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.348896:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.348897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.348907:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.348910:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.348911:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.348911:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.348913:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.348916:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.349312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677472.349369:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.349371:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.349371:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.349372:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.349372:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.349373:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.349374:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.349375:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.349378:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.349379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.349379:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.349380:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.350112:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.350115:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.350116:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.350118:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.350120:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.350121:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.350123:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.350124:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.350125:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.350126:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.350129:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.350130:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.350132:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.350133:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.350133:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.350134:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.350135:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.350136:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.350136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.350146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.350148:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.350148:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.350149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.350150:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.350151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.350167:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.350170:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.350171:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.350171:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.350172:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.350172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.350302:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.350305:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.350306:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.350306:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.350308:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.350311:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.350327:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.350330:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677472.350330:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.350331:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.350332:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.350332:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677472.350378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.350387:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.350389:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.350390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.350391:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.350393:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.350396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.350414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.350420:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.350422:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.350423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.350425:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.350428:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.350429:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.350431:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.350432:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.350433:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.350434:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.350437:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.350439:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.350442:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.350443:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.350444:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.350445:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.350446:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.350447:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.350447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.350451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.350459:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.350462:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.350463:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.350464:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.350465:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.350467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:18.0:1619677472.350584:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.350586:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.350587:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.350587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.350589:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.350592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.350748:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.350750:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.350751:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.350751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.350753:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.350755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.351656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.351663:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.351665:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.351667:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.351669:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.351671:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.351672:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.351674:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.351675:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.351676:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.351678:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.351681:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.351683:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.351685:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.351686:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.351687:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.351688:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.351689:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.351690:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.351690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.351702:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.351704:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.351705:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.351706:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.351707:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.351709:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.351722:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.351724:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.351725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.351726:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.351726:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.351727:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.352062:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.352065:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.352067:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.352067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.352069:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.352071:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.352072:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96514024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.352073:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.352074:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.352075:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.352078:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.352079:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.352082:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.352083:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.352084:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.352085:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.352086:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.352087:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.352087:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.352099:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.352102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.352102:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.352103:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.352105:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.352107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.352724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.352727:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.352729:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.352732:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.352732:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.352734:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.352736:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.352737:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.352738:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.352739:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.352740:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.352744:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.352745:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.352748:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.352749:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.352749:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.352751:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.352752:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.352752:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.352753:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.352767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.352776:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.352779:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.352780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.352780:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.352782:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.352784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.353100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.353112:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.353114:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.353116:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.353117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677472.353118:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.353121:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.353331:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.353334:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.353335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.353337:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.353338:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.353340:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.353341:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.353342:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.353344:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.353345:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.353347:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.353349:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.353351:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.353351:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.353352:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.353353:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.353353:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.353355:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.353356:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.353356:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.353368:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.353369:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.353370:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.353371:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.353371:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.353372:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.353372:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.353373:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.353373:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.353374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677472.353515:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.353518:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.353520:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.353522:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.353522:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.353524:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677472.353525:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.353526:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.353527:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.353528:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677472.353531:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.353533:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677472.353536:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.353536:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:13.0:1619677472.353537:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:13.0:1619677472.353539:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677472.353539:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.353540:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.353540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677472.353553:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.353556:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.353557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.353558:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.353560:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.353563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.0:1619677472.354312:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.354315:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.354317:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.354317:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.354319:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.354321:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.354322:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.354324:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.354325:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.354326:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.354329:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.354330:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.354333:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.354334:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.354335:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.354336:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.354337:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.354338:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.354339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.354350:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.354353:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.354354:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.354354:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.354356:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.354359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.354714:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.354717:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.354717:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.354720:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.354722:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.354723:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.354725:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.354726:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.354727:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.354728:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.354732:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.354734:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.354736:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.354737:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.354738:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.354739:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.354740:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.354741:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.354742:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.354743:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.354745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.354754:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.354756:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.354756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.354757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.354758:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.354760:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.354761:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.354763:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.354763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.354765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.354945:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.354948:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.354949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.354951:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.354953:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.354955:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.354956:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.354957:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.354958:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.354959:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.354963:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.354965:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.354968:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.354968:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.354969:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.354971:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.354971:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.354972:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.354973:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.354977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.354985:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.354988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.354988:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.354989:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.354992:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.354994:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.354996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.355003:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.355006:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.355007:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.355007:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.355008:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.355009:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677472.355228:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.355231:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.355233:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.355235:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.355236:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.355238:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677472.355239:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.355241:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.355242:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.355243:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677472.355246:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.355248:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677472.355250:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.355251:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677472.355252:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677472.355254:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677472.355254:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.355255:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.355256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677472.355268:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.355271:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.355272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.355272:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.355274:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.355277:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.355845:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.355847:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.355848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.355848:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.355850:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.355852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.355861:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.355864:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.355865:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.355865:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.355866:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.355866:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.356469:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.356472:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.356473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.356474:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.356477:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.356478:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.356480:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96514012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.356481:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.356482:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.356483:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.356486:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.356488:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.356490:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.356491:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.356492:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.356494:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.356494:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.356495:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.356495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.356509:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.356512:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.356513:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.356513:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.356515:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.356519:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.356737:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.356740:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.356741:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.356741:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.356743:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.356745:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.356761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.356767:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.356770:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.356771:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.356773:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.356774:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.356776:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.356778:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.356779:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.356780:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.356781:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.356785:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.356787:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.356790:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.356791:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.356792:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.356793:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.356794:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.356796:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.356797:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.356797:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.356803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.2:1619677472.356809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:18.0:1619677472.356809:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.356811:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.356812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.356812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677472.356870:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.356873:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.356873:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.356875:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.356877:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.356878:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.0:1619677472.357488:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.357491:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.357493:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.357494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.357496:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.357497:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.357498:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.357500:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.357501:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.357502:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.357505:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.357507:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.357509:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.357510:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.357511:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.357512:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.357513:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.357515:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.357516:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.357516:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.357526:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.357529:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.357529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.357530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.357536:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.357539:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.357540:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.357540:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.357543:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.357545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.357556:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.357559:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.357560:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.357560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.357561:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.357561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.357572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677472.357573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:1.0:1619677472.357581:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.357584:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.357585:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.357586:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.357588:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.357596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677472.357616:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.357618:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.357619:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.357621:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.357623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.357624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677472.357636:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677472.357639:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677472.357639:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.357640:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677472.357640:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.357641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:18.0:1619677472.358155:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.358158:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.358159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.358160:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.358162:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.358165:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.358189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677472.358230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:2.0:1619677472.358235:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.358237:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.358238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.358239:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.358241:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.358244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.358285:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.358288:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.358290:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.358290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.358292:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.358300:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.358301:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96514001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.358302:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.358303:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.358304:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.358307:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.358309:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.358312:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.358313:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.358313:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.358315:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.358316:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.358316:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.358317:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.358339:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.358341:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.358342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.358343:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.358344:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.358347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677472.358460:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.358463:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.358465:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677472.358467:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.358468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.358469:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677472.358470:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96514000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.358471:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.358472:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.358473:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677472.358477:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.358479:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677472.358482:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.358482:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677472.358483:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677472.358485:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677472.358485:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.358486:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.358487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677472.358816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.358819:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.358821:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.358823:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.358825:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.358826:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.358827:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.358828:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.358829:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.358830:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.358832:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.358834:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.358836:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.358838:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.358839:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.358840:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.358842:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.358842:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.358844:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.358845:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.358846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.358848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.358850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677472.358857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677472.358859:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677472.358859:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.358860:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.358860:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:21.0:1619677472.358861:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.358862:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677472.358862:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.358862:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:21.0:1619677472.358863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677472.358864:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.358865:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.358866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677472.358867:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677472.358867:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.358869:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.2:1619677472.358870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:1.0:1619677472.358870:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.358872:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.358873:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.358874:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677472.358878:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:1.0:1619677472.358879:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.358880:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.358881:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.358881:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677472.358882:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.358883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.358884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677472.358885:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.358886:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677472.358887:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677472.358889:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677472.358889:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.358890:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.358890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677472.358902:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.358904:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.358905:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.358907:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.358908:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.358910:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:18.0:1619677472.359076:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.359079:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.359080:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.359080:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.359083:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.359086:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.359095:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.359098:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.359098:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.359099:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.359100:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.359100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.359114:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.359117:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.359118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.359119:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.359122:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.359124:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.359125:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.359127:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.359128:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.359128:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.359131:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.359133:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.359135:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.359135:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.359136:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.359137:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.359138:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.359139:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.359139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.359149:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.359151:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.359151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.359152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.359153:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.359155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.359674:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.359677:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.359678:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.359680:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.359683:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.359685:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677472.359687:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.359688:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.359689:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.359690:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677472.359693:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.359695:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677472.359698:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.359699:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677472.359700:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677472.359701:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677472.359702:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.359703:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677472.359704:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677472.359705:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.359946:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677472.359949:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677472.359950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677472.359950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:20.2:1619677472.360045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.360051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.360053:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.360055:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.360056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.360058:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.360059:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.360060:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.360062:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.360062:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.360064:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.360066:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.360068:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.360070:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.360071:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.360071:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.360073:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.360073:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.360074:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.360075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.360080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.360088:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.360090:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.360091:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.360092:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.360095:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.360098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.360669:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.360672:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.360673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.360675:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.360678:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.360679:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.360681:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.360682:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.360683:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.360684:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.360687:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.360689:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.360692:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.360692:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.360693:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.360695:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.360695:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.360696:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.360696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.360702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.360710:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.360712:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.360713:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.360715:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.360716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.360717:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.361061:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.361063:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.361064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.361066:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.361068:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.361069:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.361071:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.361072:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.361073:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.361074:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.361077:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.361079:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.361082:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.361082:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.361083:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.361085:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.361085:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.361087:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.361088:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.361089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.361099:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.361101:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.361102:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.361103:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.361103:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677472.361103:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.361104:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.361105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.361105:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.361109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.361115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.361121:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.361124:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.361124:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677472.361127:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.361129:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.361131:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.361132:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.361134:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.361135:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.361136:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.361139:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.361141:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.361143:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.361144:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.361144:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.361146:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.361147:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.361148:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.361148:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.361159:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.361161:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.361161:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.361162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.361163:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.361164:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.361468:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.361470:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.361472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.361472:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.361474:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.361477:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.361489:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.361492:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677472.361492:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.361493:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.361494:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.361494:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.361934:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.361937:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.361937:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.361938:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.361939:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.361942:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.362216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.362222:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.362224:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.362225:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.362227:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.362229:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.362231:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.362232:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.362234:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.362235:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.362236:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.362239:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.362241:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.362244:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.362245:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.362246:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.362248:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.362248:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.362249:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.362250:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.362620:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.362623:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.362624:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.362625:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.362628:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.362630:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.362631:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.362633:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.362634:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.362635:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.362638:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.362640:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.362642:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.362643:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.362644:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.362645:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.362645:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.362646:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.362647:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.362659:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.362662:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.362663:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.362663:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.362665:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.362669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.362764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677472.362809:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.362812:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.362813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.362813:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.362815:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.362818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677472.363562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a478b7c800] (20)++
00000800:00000200:12.0:1619677472.363570:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a478b7c800] (21)++
00000800:00000200:12.0:1619677472.363573:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.48@o2ib3
00000400:00000200:12.0:1619677472.363575:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.48@o2ib3 : GET - for me
00000800:00000200:13.0:1619677472.363577:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a478b7c800] (22)--
00000400:00000200:12.0:1619677472.363577:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.48@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677472.363578:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.48@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677472.363579:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a478b7c800] (21)++
00000800:00000200:12.0:1619677472.363580:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a478b7c800] (22)--
00000400:00000200:12.0:1619677472.363582:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.48@o2ib3 local destination
00000800:00000200:12.0:1619677472.363583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a478b7c800] (21)--
00000800:00000200:8.2:1619677472.363725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.363734:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.363737:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.363738:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.363738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.363740:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.363743:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.364186:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (21)++
00000800:00000200:1.0:1619677472.364189:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.1.16@o2ib3
00000800:00000200:17.0:1619677472.364190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)--
00000400:00000200:1.0:1619677472.364192:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.364194:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364195:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:1.0:1619677472.364196:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.364198:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.364199:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.364200:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677472.364203:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.364205:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677472.364207:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.364208:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:1.0:1619677472.364209:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677472.364211:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:1.0:1619677472.364211:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (21)++
00000800:00000200:1.0:1619677472.364213:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (22)--
00000800:00000200:1.0:1619677472.364213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (21)--
00000800:00000200:21.0:1619677472.364226:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.364228:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.364229:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.364230:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.364232:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.364237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677472.364239:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (21)++
00000800:00000200:17.0:1619677472.364242:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000800:00000200:1.0:1619677472.364242:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)++
00000800:00000200:1.0:1619677472.364243:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000400:00000200:17.0:1619677472.364245:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.364246:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.364247:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364248:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.364249:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677472.364251:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.364252:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.364253:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.364254:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677472.364256:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677472.364258:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.364258:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.364259:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677472.364260:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677472.364260:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.364261:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.364264:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.364265:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677472.364265:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:1.0:1619677472.364265:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.364267:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677472.364267:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.364268:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:17.0:1619677472.364269:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:1.0:1619677472.364269:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.364270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)--
00000800:00000200:1.0:1619677472.364270:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.364271:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)--
00000800:00000200:1.0:1619677472.364271:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677472.364273:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677472.364274:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (21)++
00000800:00000200:1.0:1619677472.364274:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (22)--
00000800:00000200:1.0:1619677472.364276:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)--
00000800:00000200:1.0:1619677472.364277:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (21)++
00000800:00000200:1.0:1619677472.364279:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000800:00000200:17.0:1619677472.364280:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)++
00000800:00000200:18.0:1619677472.364281:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:17.0:1619677472.364281:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000400:00000200:1.0:1619677472.364281:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.364282:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.364283:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:17.0:1619677472.364283:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.364284:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.364284:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:1.0:1619677472.364284:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677472.364285:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.364286:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677472.364287:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.364288:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.364289:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.364291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:17.0:1619677472.364292:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.364300:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677472.364301:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677472.364302:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.364303:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677472.364303:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:17.0:1619677472.364304:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.364305:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677472.364308:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.364309:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.364310:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677472.364310:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677472.364311:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677472.364313:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.364313:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677472.364314:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677472.364314:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:1.0:1619677472.364314:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:17.0:1619677472.364315:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:17.0:1619677472.364316:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:1.0:1619677472.364316:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:17.0:1619677472.364317:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (26)--
00000800:00000200:1.0:1619677472.364317:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (26)--
00000800:00000200:17.0:1619677472.364318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (25)--
00000800:00000200:17.0:1619677472.364319:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)++
00000800:00000200:1.0:1619677472.364319:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)--
00000800:00000200:17.0:1619677472.364322:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000800:00000200:1.0:1619677472.364323:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000400:00000200:17.0:1619677472.364324:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.364325:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000400:00000200:17.0:1619677472.364327:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364327:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.364328:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677472.364328:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.364330:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.364331:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.364332:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.364334:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677472.364336:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677472.364337:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96513973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.364338:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.364339:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677472.364340:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:1.0:1619677472.364340:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.364341:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677472.364344:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.364346:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677472.364347:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.364347:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:17.0:1619677472.364348:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (23)++
00000800:00000200:1.0:1619677472.364348:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.364349:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:1.0:1619677472.364349:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677472.364351:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677472.364352:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:17.0:1619677472.364352:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:1.0:1619677472.364352:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:17.0:1619677472.364353:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (26)--
00000800:00000200:1.0:1619677472.364353:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)--
00000800:00000200:1.0:1619677472.364353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (25)--
00000800:00000200:17.0:1619677472.364354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)--
00000800:00000200:1.0:1619677472.364355:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)++
00000800:00000200:1.0:1619677472.364356:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1400] (23)--
00000800:00000200:17.0:1619677472.364357:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:1.0:1619677472.364357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)--
00000800:00000200:1.0:1619677472.364358:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)++
00000800:00000200:17.0:1619677472.364359:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000800:00000200:1.0:1619677472.364359:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000400:00000200:17.0:1619677472.364361:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.364362:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.364364:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364364:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.364366:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677472.364367:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.364369:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677472.364369:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:17.0:1619677472.364371:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.364372:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677472.364372:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.364373:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.364374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.364375:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677472.364376:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677472.364377:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.364378:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.364379:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677472.364380:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677472.364380:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.364381:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677472.364384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:1.0:1619677472.364384:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.364386:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.364386:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677472.364387:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:17.0:1619677472.364388:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677472.364388:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.364389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677472.364389:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677472.364390:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677472.364390:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:17.0:1619677472.364391:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:2.0:1619677472.364391:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.364391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.364392:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:1.0:1619677472.364392:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.364393:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)--
00000800:00000200:1.0:1619677472.364393:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677472.364393:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677472.364394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:17.0:1619677472.364395:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:17.0:1619677472.364396:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000800:00000200:1.0:1619677472.364396:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677472.364396:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:1.0:1619677472.364397:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)--
00000800:00000200:1.0:1619677472.364397:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:1.0:1619677472.364398:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000400:00000200:17.0:1619677472.364399:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.364399:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000400:00000200:17.0:1619677472.364401:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364401:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.364403:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677472.364404:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.364405:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.364406:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.364406:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677472.364407:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.364409:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:1.0:1619677472.364409:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677472.364410:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.364411:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96513969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.364412:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677472.364413:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:1.0:1619677472.364413:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677472.364414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.364414:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364414:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.364415:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677472.364416:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.364418:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.364419:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677472.364420:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.364420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:1.0:1619677472.364420:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.364422:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677472.364422:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677472.364423:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:17.0:1619677472.364424:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)--
00000800:00000200:17.0:1619677472.364425:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:17.0:1619677472.364426:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:1.0:1619677472.364426:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.364427:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000800:00000200:1.0:1619677472.364427:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:1.0:1619677472.364428:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:17.0:1619677472.364429:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.364430:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:17.0:1619677472.364432:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.364432:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:1.0:1619677472.364432:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)--
00000400:00000200:17.0:1619677472.364433:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:1.0:1619677472.364433:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:24.0:1619677472.364434:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677472.364434:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:1.0:1619677472.364434:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1400] (24)--
00000400:00000200:17.0:1619677472.364435:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677472.364435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)--
00000800:00000200:1.0:1619677472.364436:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)++
00000800:00000200:24.0:1619677472.364437:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:17.0:1619677472.364437:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.364438:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.364438:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677472.364438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677472.364438:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000400:00000200:17.0:1619677472.364439:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677472.364441:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:17.0:1619677472.364442:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.364444:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:24.0:1619677472.364445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.364445:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677472.364446:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:1.0:1619677472.364446:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.364447:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677472.364447:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.364448:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.364448:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:13.0:1619677472.364449:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.364450:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:12.0:1619677472.364450:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:1.0:1619677472.364450:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364451:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677472.364453:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.364454:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.364455:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677472.364456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:1.0:1619677472.364456:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677472.364458:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:17.0:1619677472.364460:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:8.0:1619677472.364460:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:1.0:1619677472.364460:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.364461:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)--
00000800:00000200:17.0:1619677472.364462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:1.0:1619677472.364462:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677472.364463:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.0:1619677472.364463:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.364464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677472.364464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:13.0:1619677472.364464:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677472.364464:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.364465:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677472.364465:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.364466:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000800:00000200:8.0:1619677472.364466:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.364466:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.364467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:1.0:1619677472.364467:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (23)++
00000800:00000200:13.0:1619677472.364468:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677472.364469:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.364470:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:17.0:1619677472.364471:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.364471:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:1.0:1619677472.364471:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:13.0:1619677472.364472:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:1.0:1619677472.364472:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)--
00000400:00000200:17.0:1619677472.364473:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:13.0:1619677472.364473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677472.364473:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (24)--
00000400:00000200:17.0:1619677472.364475:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677472.364475:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:17.0:1619677472.364476:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677472.364476:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.364476:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.364478:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.364478:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677472.364479:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677472.364481:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.364482:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:17.0:1619677472.364483:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.364483:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.364484:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.364484:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:1.0:1619677472.364484:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:17.0:1619677472.364485:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000400:00000200:12.0:1619677472.364485:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.364486:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.364486:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000400:00000200:12.0:1619677472.364487:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.364488:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677472.364488:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677472.364489:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)++
00000400:00000200:1.0:1619677472.364489:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.364490:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)--
00000800:00000200:17.0:1619677472.364490:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:12.0:1619677472.364491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677472.364492:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677472.364493:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000400:00000200:1.0:1619677472.364493:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.364494:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000800:00000200:8.2:1619677472.364494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:1.0:1619677472.364494:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.364496:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.364497:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.364498:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.364498:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.364501:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.364502:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364502:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677472.364504:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:1.0:1619677472.364504:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.364505:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.364507:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.364507:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.364508:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677472.364511:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677472.364512:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.364512:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.364512:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.364514:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677472.364514:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.364515:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:1.0:1619677472.364516:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)--
00000800:00000200:17.0:1619677472.364517:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.364517:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:17.0:1619677472.364518:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677472.364518:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677472.364518:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:1.0:1619677472.364519:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:1.0:1619677472.364520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)--
00000800:00000200:17.0:1619677472.364521:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677472.364521:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (22)++
00000800:00000200:1.0:1619677472.364521:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.364521:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.364522:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1400] (23)++
00000400:00000200:1.0:1619677472.364522:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.364524:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:17.0:1619677472.364524:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)--
00000400:00000200:1.0:1619677472.364524:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.364525:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)++
00000800:00000200:1.0:1619677472.364525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.364526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.364527:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000800:00000200:1.0:1619677472.364527:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:1.0:1619677472.364528:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000400:00000200:17.0:1619677472.364529:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.364530:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364530:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.364531:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677472.364534:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.364535:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.364536:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.364537:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677472.364540:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364541:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677472.364542:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.364543:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.364544:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677472.364544:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.364545:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.364546:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677472.364549:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.364551:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.364551:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677472.364552:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:17.0:1619677472.364553:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677472.364553:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.364554:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)++
00000800:00000200:17.0:1619677472.364555:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677472.364555:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (23)++
00000800:00000200:17.0:1619677472.364556:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:1.0:1619677472.364556:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (23)--
00000800:00000200:17.0:1619677472.364557:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)--
00000800:00000200:17.0:1619677472.364557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:1.0:1619677472.364557:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)++
00000800:00000200:1.0:1619677472.364557:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)--
00000800:00000200:17.0:1619677472.364558:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:24.0:1619677472.364559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677472.364559:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:17.0:1619677472.364560:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000800:00000200:1.0:1619677472.364560:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:8.2:1619677472.364561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (22)++
00000800:00000200:1.0:1619677472.364561:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:1.0:1619677472.364561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)--
00000800:00000200:24.0:1619677472.364562:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677472.364562:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)++
00000400:00000200:17.0:1619677472.364563:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.364564:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000400:00000200:24.0:1619677472.364566:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.364566:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364567:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.364568:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677472.364568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.364569:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.364569:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.364570:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (23)++
00000400:00000200:24.0:1619677472.364571:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677472.364571:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.364572:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (24)--
00000400:00000200:24.0:1619677472.364573:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96513960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.364573:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.364574:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.364574:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677472.364575:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.364575:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.364576:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677472.364576:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (23)++
00000400:00000200:1.0:1619677472.364576:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.364577:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (24)--
00000400:00000200:24.0:1619677472.364578:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.364578:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364578:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677472.364579:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677472.364581:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.364581:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677472.364581:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96513959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.364583:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.364584:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.364584:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.364586:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.364587:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.364587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.364588:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:24.0:1619677472.364589:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.364589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:24.0:1619677472.364590:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677472.364590:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677472.364590:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.364591:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.364591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.364591:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677472.364591:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:24.0:1619677472.364592:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677472.364592:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677472.364593:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.2:1619677472.364593:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:1.0:1619677472.364593:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.364594:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.364594:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677472.364594:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:20.0:1619677472.364595:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677472.364595:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:20.0:1619677472.364596:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000800:00000200:17.0:1619677472.364596:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (24)--
00000800:00000200:1.0:1619677472.364596:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.364597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677472.364597:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)--
00000800:00000200:1.0:1619677472.364597:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:1.0:1619677472.364597:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.364599:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677472.364599:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)++
00000800:00000200:1.0:1619677472.364599:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677472.364600:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.16@o2ib3
00000800:00000200:1.0:1619677472.364600:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:1.0:1619677472.364601:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1400] (24)++
00000800:00000200:20.0:1619677472.364602:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677472.364602:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (25)--
00000800:00000200:1.0:1619677472.364603:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (24)--
00000400:00000200:17.0:1619677472.364604:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.16@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.364604:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:1.0:1619677472.364604:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)--
00000800:00000200:21.0:1619677472.364605:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.364605:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.364607:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.364607:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.364607:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.364608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677472.364608:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.364608:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:20.0:1619677472.364610:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677472.364610:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:13.0:1619677472.364610:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.364611:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.364611:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677472.364612:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96513958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.364613:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:17.0:1619677472.364613:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96513958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.364613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:20.0:1619677472.364614:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.364614:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.364614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677472.364614:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)++
00000800:00000200:1.0:1619677472.364615:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1400] (24)--
00000400:00000200:20.0:1619677472.364616:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.364616:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:17.0:1619677472.364616:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677472.364616:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (23)--
00000400:00000200:20.0:1619677472.364617:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677472.364617:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.364617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677472.364617:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (22)--
00000400:00000200:17.0:1619677472.364618:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.364619:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.364620:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.364622:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:17.0:1619677472.364623:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677472.364626:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.364626:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677472.364628:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677472.364629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.364631:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.364631:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.364632:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.364632:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677472.364632:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677472.364633:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.364635:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.364635:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677472.364636:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677472.364636:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1400] (21)++
00000800:00000200:17.0:1619677472.364637:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1400] (22)--
00000800:00000200:17.0:1619677472.364637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1400] (21)--
00000800:00000200:20.0:1619677472.364638:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.364638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.364641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.364649:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.364652:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.364653:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.364653:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.364656:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.364659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:12.0:1619677472.364865:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677472.364868:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677472.364870:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : GET - for me
00000800:00000200:13.0:1619677472.364871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677472.364873:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.103@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677472.364874:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.103@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677472.364875:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677472.364876:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677472.364878:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:12.0:1619677472.364882:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677472.364883:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677472.364885:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.364886:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677472.364886:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.364888:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677472.364888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.364900:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.364902:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.364903:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.364904:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:13.0:1619677472.364906:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.364907:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677472.365504:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.365507:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.365509:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.365511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.365512:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.365513:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677472.365515:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.365516:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.365517:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.365518:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677472.365521:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.365522:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677472.365525:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.365526:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677472.365526:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677472.365528:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677472.365528:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.365530:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677472.365532:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677472.365532:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.365545:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677472.365547:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677472.365548:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.365549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677472.365698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.365704:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.365707:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.365709:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.365710:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.365711:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.365713:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.365714:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.365716:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.365716:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.365718:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.365720:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.365722:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.365724:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.365725:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.365725:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.365727:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.365727:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.365728:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.365729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.365734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.365741:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.365744:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.365744:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.365746:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.365747:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.365748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.365754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.365762:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.365765:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.365766:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.365767:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.365767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.365769:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.365797:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.365799:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.365800:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.365801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.365802:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.365804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.366701:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.366703:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.366704:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.366704:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.366706:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.366707:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.367428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.367435:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.367437:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.367439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.367440:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.367444:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.367445:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.367447:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.367448:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.367449:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.367450:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.367454:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.367455:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.367458:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.367459:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.367459:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.367461:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.367462:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.367463:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.367463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.367468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.367476:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.367478:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.367479:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.367481:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.367482:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.367484:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.367985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.367991:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.367994:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.367995:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.367997:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.367999:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.368000:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.368002:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.368003:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.368004:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.368005:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.368008:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.368009:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.368011:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.368011:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.368012:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.368013:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.368013:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.368014:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.368015:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.368027:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.368029:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.368030:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.368031:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.368032:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.368033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.368055:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.368058:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.368059:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.368061:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.368064:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.368065:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.368067:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.368068:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.368069:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.368070:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.368072:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.368074:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.368077:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.368077:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.368078:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.368080:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.368080:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.368081:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.368081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.368086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.368094:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.368096:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.368097:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.368099:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.368100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.368101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.369437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.369445:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.369448:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.369449:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.369449:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.369452:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.369455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.369456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.369464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.369466:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.369467:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.369467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.369468:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.369469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.369602:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.369605:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.369606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.369608:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.369609:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.369611:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.369612:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.369613:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.369614:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.369614:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.369617:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.369619:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.369620:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.369621:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.369621:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.369622:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.369623:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.369623:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.369624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.369634:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.369636:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.369636:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.369637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.369638:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.369639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.369658:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.369660:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.369661:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.369661:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.369662:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.369662:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.370372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.370378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.370380:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.370383:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.370383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.370385:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.370387:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.370388:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.370389:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.370390:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.370391:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.370394:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.370396:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.370398:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.370399:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.370399:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.370405:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.370406:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.370407:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.2:1619677472.370407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.370408:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.370410:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.370411:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.370413:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.370413:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677472.370414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:21.0:1619677472.370415:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.370416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.370418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.370419:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.370421:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.370422:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.370423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677472.370866:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.370868:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.370869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.370869:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.370871:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.370873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677472.371012:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.371015:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.371016:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.371016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.371018:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.371022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.371806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.371812:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.371814:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.371817:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.371817:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.371818:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.371819:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.371821:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.371823:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.371824:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.371825:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.371828:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.371830:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.371833:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.371833:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.371834:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.371836:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.371836:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.371837:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.371838:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.371843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.371851:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.371853:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.371855:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.371855:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.371857:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.371860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:12.0:1619677472.371882:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.371884:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.371885:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.371887:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.371889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.371890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677472.371962:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.371965:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.371966:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.371966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.371968:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.371969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.371987:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.371989:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.371990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.371990:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.371991:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.371992:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.372411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:12.0:1619677472.372458:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.372461:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.372462:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.372464:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.372467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.372468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.2:1619677472.372969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.372977:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.372980:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.372980:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.372981:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.372983:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.372985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.372991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.372998:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.373001:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.373001:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.373002:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.373003:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.373004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.373089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.373096:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.373098:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.373099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.373101:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.373103:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.373105:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.373106:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.373108:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.373109:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.373110:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.373113:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.373115:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.373118:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.373119:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.373120:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.373121:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.373122:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.373123:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.373124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.374193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.374201:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.374203:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.374204:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.374206:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.374207:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.374209:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677472.374280:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.374283:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.374284:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.374286:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.374288:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.374289:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.374291:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.374292:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.374297:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.374298:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.374301:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.374302:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.374305:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.374306:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.374307:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.374308:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.374309:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.374309:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.374310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.374315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.374323:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.374325:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.374326:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.374327:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.374328:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.374330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.374491:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.374494:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.374495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.374497:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.374498:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.374499:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.374501:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.374502:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.374503:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.374504:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.374507:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.374509:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.374511:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.374512:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.374512:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.374514:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.374514:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.374515:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.374516:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.374517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.374528:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.374529:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.374529:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.374530:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.374531:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.374532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.374532:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.374533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.374533:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.374534:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677472.374571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:18.0:1619677472.374623:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.374626:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.374627:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.374627:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.374629:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.374633:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677472.375113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.375119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.375121:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.375122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.375125:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.375127:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.375128:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.375130:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.375131:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.375132:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.375133:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.375136:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.375138:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.375140:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.375141:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.375141:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.375143:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.375144:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.375145:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.375146:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.375147:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.375152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.375158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.375159:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.375159:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677472.375160:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.375160:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.375160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.375162:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.375162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.375163:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.375163:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.375176:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.375178:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.375179:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.375179:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.375179:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.375180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.375694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.375697:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.375700:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.375702:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.375703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.375705:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.375706:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.375707:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.375709:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.375710:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.375710:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.375714:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.375715:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.375718:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.375718:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.375719:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.375721:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.375721:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.375722:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.375723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.375727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.375735:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.375737:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.375738:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.375739:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.375740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.375741:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.375746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.375754:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.375756:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.375757:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.375758:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.375759:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.375760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.375956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.375964:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.375967:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.375968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.375968:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.375970:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.375973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677472.375975:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.375978:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.375979:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.375980:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.375982:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.375983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677472.376127:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.376129:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.376130:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.376131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.376133:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.376135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.376389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.376396:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.376398:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.376399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.376401:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.376403:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.376405:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677472.376406:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.376407:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.376408:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.376409:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677472.376412:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.376414:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677472.376416:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.376417:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677472.376418:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.376420:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.376420:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.376421:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.376422:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.376427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.376435:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.376437:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.376438:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.376438:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.376440:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.376442:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.377269:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.377271:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.377272:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.377273:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.377274:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.377276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.378207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.378209:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.378212:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.378214:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.378215:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.378216:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.378218:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.378219:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.378220:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.378221:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.378222:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.378225:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.378227:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.378229:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.378230:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.378231:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.378232:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.378233:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.378234:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.378234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.378239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.378248:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.378250:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.378251:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.378251:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.378254:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.378256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677472.378280:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.378283:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.378283:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.378286:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.378288:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.378289:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677472.378291:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.378292:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.378296:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.378298:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.378301:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.378303:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.378306:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.378306:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.378307:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.378309:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.378309:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.378310:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.378310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677472.378362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.378368:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.378372:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.378373:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.378374:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.378377:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.378379:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677472.378380:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.378381:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.378382:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.378383:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677472.378386:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.378388:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.378391:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.378391:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677472.378392:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.378394:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.378394:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.378395:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.378396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.378400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.378408:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.378411:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.378412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.378412:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.378414:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.378417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.379490:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.379493:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.379494:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.379495:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.379497:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.379498:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.379500:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.379501:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.379502:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.379503:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.379506:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.379507:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.379509:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.379510:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.379510:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.379511:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.379512:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.379512:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.379513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.379523:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.379525:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.379526:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.379527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.379528:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.379529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.380203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.380209:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.380212:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.380213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.380215:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.380217:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.380219:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.380220:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.380221:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.380222:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.380223:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.380227:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.380229:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.380231:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.380232:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.380233:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.380235:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.380235:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.380237:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.380238:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.380238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.380241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.380243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677472.380249:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677472.380249:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.380251:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:21.0:1619677472.380252:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:2.0:1619677472.380252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:21.0:1619677472.380253:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.380253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:21.0:1619677472.380254:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.380255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.380257:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.380260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.380268:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.380270:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.380271:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.380272:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.380273:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.380274:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.380304:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.380306:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.380307:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.380309:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.380311:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.380312:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.380314:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.380315:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.380316:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.380317:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.380321:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.380322:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.380326:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.380326:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.380327:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.380329:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.380329:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.380330:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.380330:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.380344:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.380347:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.380347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.380348:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.380350:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.380353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.381524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.381533:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.381536:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.381537:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.381537:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.381540:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.381543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.381544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.381551:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.381554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.381554:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.381555:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.381556:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.381556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677472.382388:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.382390:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.382391:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.382393:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.382395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.382396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677472.382405:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677472.382407:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677472.382408:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.382409:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677472.382409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.382410:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:24.0:1619677472.382507:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.382509:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.382510:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.382511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.382512:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.382515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677472.382675:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.382677:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.382679:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677472.382681:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.382682:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.382682:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:12.0:1619677472.382684:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.382685:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.382686:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.382687:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:12.0:1619677472.382689:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.382691:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677472.382693:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.382694:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:12.0:1619677472.382694:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:12.0:1619677472.382696:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:12.0:1619677472.382696:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.382697:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.382698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.0:1619677472.382707:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.382710:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.382711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.382711:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.382713:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.382715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.382726:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.382728:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.382729:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.382729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.382730:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.382731:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.382956:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.382959:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.382960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.382962:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.382964:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.382966:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.382967:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.382969:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.382970:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.382971:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.382974:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.382976:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.382978:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.382979:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.382980:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.382982:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.382982:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.382984:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.382985:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.382985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.382988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.382996:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.382996:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.382998:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677472.382999:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677472.382999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677472.383000:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.383000:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.383002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.383002:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.383004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.383017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.383023:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.383026:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.383028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.383029:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.383032:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.383034:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677472.383035:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.383036:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.383037:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.383038:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677472.383042:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.383044:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.383047:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.383048:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.383049:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677472.383051:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.383052:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.383053:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.383053:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677472.383723:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.383725:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.383726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.383728:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.383730:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.383732:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677472.383733:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.383735:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.383736:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.383737:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677472.383740:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.383742:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677472.383745:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.383746:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677472.383746:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677472.383748:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677472.383748:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.383749:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.383750:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677472.383763:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.383765:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.383766:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.383768:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.383770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.383771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677472.383781:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677472.383784:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.383785:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.383786:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677472.383787:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.383788:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677472.383789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.383795:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.383797:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.383800:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.383800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.383802:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.383804:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.383805:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.383807:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.383808:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.383809:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.383812:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.383814:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.383816:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.383817:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.383818:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.383819:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.383820:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.383821:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.383822:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.383823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.383826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.383828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.383831:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.383833:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.383834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.383834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.383834:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.383837:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.383838:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.383838:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.383840:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.383842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.383881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (20)++
00000800:00000200:20.0:1619677472.383884:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.383887:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.15@o2ib3
00000400:00000200:20.0:1619677472.383889:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.15@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.383890:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:20.0:1619677472.383892:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.383894:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:20.0:1619677472.383896:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.383897:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.383898:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.383899:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:20.0:1619677472.383903:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.383904:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:20.0:1619677472.383908:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.383909:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:20.0:1619677472.383909:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:20.0:1619677472.383912:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:20.0:1619677472.383913:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.383914:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b84c00] (22)++
00000800:00000200:20.0:1619677472.383915:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84c00] (23)--
00000800:00000200:20.0:1619677472.383916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:24.2:1619677472.383919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:20.2:1619677472.383920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:21.0:1619677472.383926:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)++
00000800:00000200:21.0:1619677472.383929:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b84c00] (23)--
00000800:00000200:21.0:1619677472.383929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:21.0:1619677472.383931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)--
00000800:00000200:20.2:1619677472.383944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (20)++
00000800:00000200:21.0:1619677472.383949:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:21.0:1619677472.383952:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.15@o2ib3
00000400:00000200:21.0:1619677472.383954:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.15@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.383954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:21.0:1619677472.383958:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.383959:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.383961:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.383962:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.383963:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.383964:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.383967:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.383969:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.383971:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.383972:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.383973:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.383975:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.383975:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:21.0:1619677472.383976:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:21.0:1619677472.383977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)--
00000800:00000200:8.0:1619677472.383987:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.383990:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.383991:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.383991:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.383994:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.383997:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.384057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (20)++
00000800:00000200:20.0:1619677472.384060:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.384063:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.15@o2ib3
00000400:00000200:20.0:1619677472.384065:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.15@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.384066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:20.0:1619677472.384068:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.384070:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.384071:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.384072:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.384073:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.384074:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.384078:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.384079:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.384082:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.384082:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.384083:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.384085:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.384085:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.384086:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:20.0:1619677472.384087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)--
00000800:00000200:8.2:1619677472.384091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.384099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:17.0:1619677472.384099:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.384101:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.384102:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.384102:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.384104:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000400:00000200:17.0:1619677472.384104:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.384106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677472.384107:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.384109:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.384109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.384112:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.384113:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677472.384115:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.384116:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.384117:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.384118:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677472.384122:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.384123:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677472.384126:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.384127:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677472.384128:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677472.384129:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677472.384130:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.384131:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677472.384131:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:20.2:1619677472.384144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (20)++
00000800:00000200:20.0:1619677472.384147:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.384149:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.15@o2ib3
00000800:00000200:21.0:1619677472.384152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:20.0:1619677472.384152:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.15@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.384154:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.384156:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677472.384157:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.384158:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.384160:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.384160:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677472.384164:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.384165:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677472.384168:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.384169:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:20.0:1619677472.384170:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.0:1619677472.384171:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:20.0:1619677472.384172:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.384173:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:20.0:1619677472.384173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)--
00000800:00000200:20.2:1619677472.384181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.384187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.384190:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.384191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.384193:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.384196:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.384197:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.384199:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.384200:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.384201:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.384202:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.384204:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677472.384205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (20)++
00000800:00000200:18.0:1619677472.384206:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.384208:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.384209:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.384209:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.384210:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:18.0:1619677472.384211:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.384212:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.15@o2ib3
00000800:00000200:18.0:1619677472.384212:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.384213:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:21.0:1619677472.384214:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.15@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.384214:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677472.384215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:21.0:1619677472.384217:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.384219:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:21.0:1619677472.384220:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.384222:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.384223:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.384224:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677472.384225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:21.0:1619677472.384227:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.384227:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677472.384228:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:2.0:1619677472.384228:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.384229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.384230:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.384231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.384232:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.384232:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:21.0:1619677472.384233:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677472.384236:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677472.384236:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:21.0:1619677472.384238:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b84c00] (22)++
00000800:00000200:21.0:1619677472.384239:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84c00] (23)--
00000800:00000200:21.0:1619677472.384239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:20.2:1619677472.384244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:24.0:1619677472.384247:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.0:1619677472.384247:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)++
00000800:00000200:20.0:1619677472.384249:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b84c00] (23)--
00000800:00000200:24.0:1619677472.384250:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:20.0:1619677472.384250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:24.0:1619677472.384251:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.384251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)--
00000800:00000200:8.0:1619677472.384251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.384253:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.384256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.384261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (20)++
00000800:00000200:20.0:1619677472.384264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.384266:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.15@o2ib3
00000400:00000200:20.0:1619677472.384268:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.15@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.384269:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:20.0:1619677472.384271:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.384273:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677472.384274:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.384275:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.384276:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.384277:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.384281:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.384283:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.384285:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.384285:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.384286:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.384288:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.384288:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.384289:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:20.0:1619677472.384290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)--
00000800:00000200:18.0:1619677472.384302:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.384304:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.384305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.384306:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.384308:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.384311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.384333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (20)++
00000800:00000200:21.0:1619677472.384338:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:21.0:1619677472.384341:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.15@o2ib3
00000400:00000200:21.0:1619677472.384343:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.15@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.384344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:21.0:1619677472.384346:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.384348:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:21.0:1619677472.384349:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.384350:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.384351:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.384352:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:21.0:1619677472.384355:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.384357:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.0:1619677472.384359:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.384360:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:21.0:1619677472.384361:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677472.384363:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.0:1619677472.384363:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:21.0:1619677472.384364:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:21.0:1619677472.384365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)--
00000800:00000200:8.2:1619677472.384369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.384377:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.384380:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.384381:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.384381:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.384383:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.384386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.384389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (20)++
00000800:00000200:20.0:1619677472.384392:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.384394:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.15@o2ib3
00000800:00000200:21.0:1619677472.384397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:20.0:1619677472.384397:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.15@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.384399:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.384401:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677472.384402:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.384404:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.384405:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.384406:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677472.384409:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.384411:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677472.384413:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.384414:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:20.0:1619677472.384415:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677472.384416:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677472.384417:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.384418:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:20.0:1619677472.384418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)--
00000800:00000200:20.2:1619677472.384514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (20)++
00000800:00000200:21.0:1619677472.384520:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:21.0:1619677472.384522:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.15@o2ib3
00000400:00000200:21.0:1619677472.384524:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.15@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.384525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:21.0:1619677472.384527:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.384529:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.384530:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.384532:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.384533:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.384534:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.384537:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.384539:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.384542:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.384543:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.384543:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.384545:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.384546:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:21.0:1619677472.384547:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b84c00] (22)++
00000800:00000200:21.0:1619677472.384548:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84c00] (23)--
00000800:00000200:21.0:1619677472.384549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:20.2:1619677472.384554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.384556:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)++
00000800:00000200:20.0:1619677472.384558:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b84c00] (23)--
00000800:00000200:20.0:1619677472.384559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:20.0:1619677472.384560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)--
00000800:00000200:20.2:1619677472.384673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (20)++
00000800:00000200:20.0:1619677472.384676:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.384678:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.15@o2ib3
00000800:00000200:21.0:1619677472.384681:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:20.0:1619677472.384681:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.15@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.384685:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.384686:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677472.384688:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.384690:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.384692:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.384693:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677472.384699:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.384702:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677472.384705:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.384706:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:20.0:1619677472.384707:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677472.384709:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677472.384710:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.384711:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:20.0:1619677472.384711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)--
00000800:00000200:21.0:1619677472.384985:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.384987:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.384990:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.384990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.384992:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.384994:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.384995:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.384996:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.384998:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.384999:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.385002:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.385004:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.385007:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.385007:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.385008:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.385010:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.385010:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.385011:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.385012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.385023:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.385025:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.385026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.385027:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.385029:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.385030:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.385032:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:8.0:1619677472.385032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.385035:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.385035:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.385037:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.385039:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.385040:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.385042:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677472.385042:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.385043:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.385044:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677472.385046:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.385046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.385047:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.385048:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677472.385048:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.385049:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.385049:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.385053:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.385055:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.385056:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677472.385057:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.385058:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.385058:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:21.0:1619677472.385059:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.385060:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:21.0:1619677472.385061:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.385061:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.385062:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.385063:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.0:1619677472.385063:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:21.0:1619677472.385064:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.385064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.385065:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.385066:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.385067:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.385071:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.385072:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.385075:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.385075:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.385076:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.385077:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.385078:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.385079:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.385080:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.385084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.385093:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.385094:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.385094:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:17.0:1619677472.385096:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.385096:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:17.0:1619677472.385098:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677472.385099:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.385101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.385102:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:1.0:1619677472.385102:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.385103:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.385104:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:1.0:1619677472.385104:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:17.0:1619677472.385105:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.385105:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.385106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.385107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677472.385107:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677472.385108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:1.0:1619677472.385108:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.385109:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677472.385112:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.385114:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677472.385117:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.385117:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677472.385118:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677472.385120:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677472.385120:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.385121:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.385121:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677472.385134:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.385137:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.385138:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.385138:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.385140:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.385142:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.385153:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.385155:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.385156:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.385156:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.385157:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.385158:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677472.385207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (20)++
00000800:00000200:20.0:1619677472.385209:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.385212:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.15@o2ib3
00000400:00000200:20.0:1619677472.385214:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.15@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.385215:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:20.0:1619677472.385216:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.385218:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677472.385219:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.385220:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.385221:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.385222:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677472.385225:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.385227:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677472.385229:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.385230:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:20.0:1619677472.385231:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677472.385233:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.0:1619677472.385233:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:20.0:1619677472.385234:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:20.0:1619677472.385235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)--
00000800:00000200:8.2:1619677472.385239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.385246:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.385249:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.385250:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.385250:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.385252:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.385254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.385316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b84c00] (20)++
00000800:00000200:21.0:1619677472.385321:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:21.0:1619677472.385324:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.15@o2ib3
00000400:00000200:21.0:1619677472.385325:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.15@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.385326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (22)--
00000400:00000200:21.0:1619677472.385328:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.385329:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:21.0:1619677472.385331:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.385332:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.385333:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.385334:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:21.0:1619677472.385336:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.385338:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.0:1619677472.385341:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.385341:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:21.0:1619677472.385342:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677472.385343:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677472.385344:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84c00] (21)++
00000800:00000200:21.0:1619677472.385345:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84c00] (22)--
00000800:00000200:21.0:1619677472.385346:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84c00] (21)--
00000800:00000200:8.0:1619677472.385415:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.385418:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.385419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.385421:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677472.385422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:8.0:1619677472.385424:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.385425:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.385426:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.385427:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.385428:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.385429:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.385429:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.385431:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.385433:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677472.385433:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.385435:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.385436:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677472.385438:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677472.385438:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.385439:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.385439:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:2.0:1619677472.385439:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:8.0:1619677472.385441:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.385441:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.385442:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677472.385442:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677472.385443:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.385443:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677472.385443:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.385444:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.385447:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.385449:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.385453:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.385454:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.385455:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.385456:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.385456:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.385457:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.385457:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:18.0:1619677472.385458:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.385458:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.385459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:18.0:1619677472.385460:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.385460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.385463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.385463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.385471:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.385473:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:2.0:1619677472.385473:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:20.0:1619677472.385474:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.385475:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.385476:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.385476:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677472.385477:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.385477:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.385477:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:20.0:1619677472.385478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:2.0:1619677472.385478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.386236:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.386238:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677472.386239:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677472.386241:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.386244:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.386246:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.386247:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.386248:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.386249:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.386250:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.386254:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.386256:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.386258:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.386259:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.386260:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.386261:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.386262:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.386262:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.386263:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:18.0:1619677472.386276:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.386279:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.386279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.386280:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.386282:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.386282:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.386284:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.386285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.386285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:8.0:1619677472.386287:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.386290:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.386292:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.386298:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.386300:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.386301:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.386301:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.386304:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.386305:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.386307:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.386307:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.386308:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.386309:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.386310:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.386311:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.386312:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.386313:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.386324:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.386326:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.386326:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.386328:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.386328:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.386329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.386329:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.386330:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.386330:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.386331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.386679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.386688:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.386690:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.386692:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.386693:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.386694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.386696:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.386981:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677472.386984:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:17.0:1619677472.386985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:1.0:1619677472.386987:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.386989:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.386990:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:1.0:1619677472.386992:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.386993:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.386994:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.386995:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677472.386999:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.387001:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677472.387003:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.387004:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:1.0:1619677472.387004:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677472.387006:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:1.0:1619677472.387007:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677472.387007:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:1.0:1619677472.387008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:20.0:1619677472.387019:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.387022:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.387022:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.387024:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.387025:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.387026:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.387775:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.387777:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.387778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.387778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.387780:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.387782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677472.388202:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.388205:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.388206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.388206:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.388208:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.388212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677472.389354:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.389357:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.389357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.389360:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.389362:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.389363:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.389364:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.389366:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.389367:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.389368:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.389371:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.389373:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.389375:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.389376:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.389376:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.389378:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.389378:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.389379:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.389380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.389385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.389393:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.389395:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.389396:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.389398:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.389399:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.389400:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.389535:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.389538:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.389540:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.389541:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.389542:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.389544:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.389545:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.389547:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.389548:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.389549:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.389552:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.389553:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.389556:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.389557:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.389558:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.389560:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.389560:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.389561:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.389561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.389572:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.389575:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.389576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.389576:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.389579:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.389581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.389806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.389811:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.389814:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677472.389816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.389817:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.389820:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.389822:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.389823:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.389825:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.389827:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.389828:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.389836:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.389838:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.389841:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.389842:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.389842:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.389844:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.389845:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.389845:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.389846:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.389846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.2:1619677472.389849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.389855:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.389857:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.389858:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:1.0:1619677472.389858:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.389859:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.389860:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.389861:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677472.389861:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.389865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.389866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.389866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.389867:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.389946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.389953:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.389956:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.389956:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.389959:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.389961:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.389962:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.389964:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.389965:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.389966:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.389967:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.389971:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.389973:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.389976:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.389977:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.389978:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.389980:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.389980:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.389981:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.389982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.390316:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.390318:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677472.390319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677472.390321:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.390323:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.390325:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677472.390326:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.390328:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.390329:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.390329:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677472.390332:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.390334:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677472.390337:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.390337:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677472.390338:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677472.390340:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677472.390340:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.390341:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.390341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.0:1619677472.390833:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.390836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.390836:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.390839:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.390841:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.390842:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.390844:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.390845:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.390846:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.390847:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.390850:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.390851:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.390854:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.390855:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.390855:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.390858:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.390858:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.390859:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.390859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677472.390864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.390867:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.390870:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.390872:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.390872:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677472.390873:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.390874:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.390874:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:20.0:1619677472.390876:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:18.0:1619677472.390876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.390876:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.390878:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.390878:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:20.0:1619677472.390879:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.390881:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.390881:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677472.390882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:20.0:1619677472.390885:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.390887:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.390889:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.390890:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.390890:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.390892:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.390893:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.390894:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.390894:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.390898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.390906:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.390908:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.390909:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.390911:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.390911:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.390913:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.390919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.390927:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.390929:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.390930:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.390931:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.390931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.390933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.391240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.391246:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.391249:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.391250:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.391252:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.391254:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.391255:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.391257:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.391258:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.391259:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.391260:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.391263:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.391265:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.391267:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.391268:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.391269:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.391270:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.391271:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.391272:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.391272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.391283:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.391285:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.391285:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.391286:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.391287:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.391289:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677472.391737:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.391739:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.391740:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.391741:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.391744:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.391745:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677472.392056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:24.0:1619677472.392094:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.392097:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.2:1619677472.392098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:8.0:1619677472.392098:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.392099:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.392102:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:24.0:1619677472.392103:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.392104:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.392105:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96513851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.392105:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.392106:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.392106:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:24.0:1619677472.392107:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.392107:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.392109:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677472.392109:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.392112:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.392113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.392114:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.392115:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.392116:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.392117:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.392118:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.392119:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.392119:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.392120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.392130:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.392132:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.392133:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.392134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.392135:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.392137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.392999:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.393002:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.393002:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.393003:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.393004:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.393007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677472.395057:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.395060:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.395061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.395061:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.395064:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.395067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.395078:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.395080:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.395081:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.395081:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.395083:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.395083:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677472.395440:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.395443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.395443:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.395446:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.395448:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.395450:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.395451:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.395452:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.395453:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.395455:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.395458:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.395459:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.395462:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.395462:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.395463:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.395465:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.395465:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.395466:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.395466:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.395472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.395475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.395479:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.395480:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.395480:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.395481:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.395482:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.395484:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.395484:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.395486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677472.395487:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.395487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:21.0:1619677472.395488:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677472.395488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.395489:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.395491:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.395492:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.395493:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.395496:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.395498:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.395501:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.395501:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.395502:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.395504:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.395504:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.395505:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.395507:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.395507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.395511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.395514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.395521:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.395522:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.395522:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.395522:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.395523:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677472.395524:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.395524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.395525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.395526:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.395527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.395738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.395745:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.395748:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.395749:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.395749:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.395752:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.395754:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.395758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.395765:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.395768:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.395768:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.395769:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.395770:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.395770:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.395782:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.395785:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.395786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.395788:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.395789:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.395791:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.395792:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.395794:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.395795:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.395795:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.395799:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.395800:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.395803:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.395804:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.395804:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.395806:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.395807:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.395808:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.395808:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.395812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.395819:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.395822:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.395823:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.395823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.395825:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.395828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.396439:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.396442:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.396442:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677472.396445:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.396447:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.396448:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677472.396449:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.396451:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.396452:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.396453:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677472.396456:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.396458:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.396460:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.396461:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.396461:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.396463:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.396463:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.396464:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677472.396465:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677472.396466:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677472.396470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.396478:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.396479:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.396479:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000400:00000200:17.0:1619677472.396480:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.396480:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677472.396480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.396481:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.396482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.396483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.396483:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:20.2:1619677472.396567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.396573:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.396575:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.396577:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.396578:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.396578:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.396580:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.396581:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.396582:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.396583:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.396584:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.396588:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.396590:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.396592:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.396593:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.396593:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.396595:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.396596:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.396596:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.396597:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.396601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.396609:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.396611:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.396612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.396612:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.396614:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.396617:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.397108:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.397111:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.397112:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.397114:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.397117:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.397118:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.397120:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.397121:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.397122:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.397123:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.397126:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.397127:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.397129:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.397129:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.397130:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.397131:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.397131:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.397133:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.397134:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.397135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.397145:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.397146:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.397147:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.397149:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.397149:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.397150:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.397151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.397151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.397152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.397153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.397163:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.397165:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.397166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.397166:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.397167:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.397168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.397177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677472.397225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.397228:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.397229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.397229:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.397231:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.397234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.397498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.397501:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.397501:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.397503:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.397505:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.397507:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677472.397508:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.397509:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.397510:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.397511:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.397515:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.397517:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677472.397520:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.397521:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677472.397522:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677472.397523:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677472.397524:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.397525:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.397526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.2:1619677472.397625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.397631:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.397634:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.397635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.397637:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.397640:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.397641:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.397642:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.397644:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.397645:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.397646:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.397649:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.397651:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.397654:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.397655:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.397656:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.397658:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.397658:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.397659:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.397660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.398249:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.398251:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.398252:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.398253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.398254:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.398257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.398267:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.398269:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.398270:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.398270:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.398271:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.398271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.398933:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.398935:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.398936:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.398936:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.398938:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.398941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.398971:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.398973:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.398974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.398976:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.398978:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.398979:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.398980:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.398982:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.398983:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.398984:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.398987:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.398989:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.398992:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.398993:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.398994:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.398996:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.398996:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.398996:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.398997:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.399010:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.399013:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.399014:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.399014:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.399016:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.399020:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.399174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.399179:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.399182:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.399183:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.399185:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.399188:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.399189:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.399190:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.399191:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.399192:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.399193:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.399196:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.399198:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.399200:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.399200:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.399201:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.399203:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.399203:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.399204:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.399205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.399216:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.399218:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.399219:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.399219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.399221:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.399222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.401123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.401130:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.401133:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.401134:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.401135:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.401138:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.401139:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677472.401141:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.401142:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.401143:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.401144:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677472.401147:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.401149:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677472.401152:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.401153:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677472.401154:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677472.401156:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677472.401156:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.401158:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677472.401159:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677472.401160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677472.401165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:24.0:1619677472.401169:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677472.401171:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:24.0:1619677472.401172:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.401173:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.401173:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:8.0:1619677472.401173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677472.401174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:24.0:1619677472.401175:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.401175:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:24.0:1619677472.401178:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.401446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.401449:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.401452:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.401454:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.401455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.401456:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.401458:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.401459:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.401460:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.401462:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.401463:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.401466:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.401467:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.401470:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.401470:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.401471:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.401472:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.401473:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.401474:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.401474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.401479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.401494:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.401496:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.401497:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.401499:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.401500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.401502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.401527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.401536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.401539:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.401540:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.401540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.401542:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.401545:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.401710:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.401712:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.401713:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.401715:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.401718:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.401719:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.401720:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.401722:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.401723:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.401723:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.401726:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.401728:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.401731:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.401731:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.401732:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.401734:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.401734:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.401735:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.401735:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.401739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.401747:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.401749:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.401750:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.401751:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.401753:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.401755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.401758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.401765:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.401768:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.401768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.401769:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.401770:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.401771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.401913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.401919:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.401922:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.401923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.401925:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.401928:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.401929:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.401931:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.401932:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.401933:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.401934:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.401937:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.401939:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.401942:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.401943:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.401944:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.401945:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.401946:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.401948:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.401949:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.401949:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.401954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.401960:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.401962:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.401963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.401964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677472.402441:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.402444:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.402446:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.402447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.402448:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.402450:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.402451:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.402453:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.402453:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.402455:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.402458:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.402459:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.402461:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.402462:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.402463:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.402464:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.402465:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.402466:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.402467:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.402468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.402480:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:1.0:1619677472.402480:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:21.0:1619677472.402482:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.402483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.402484:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:17.0:1619677472.402484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.402484:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.402487:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.402489:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.402490:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.402492:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.402493:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.402494:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.402495:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.402499:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.402501:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.402503:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.402504:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.402505:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.402507:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.402507:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.402508:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.402509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.402514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.402521:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.402523:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.402524:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.402526:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.402526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.402529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.402533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.402541:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.402543:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.402544:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.402545:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.402545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.402546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.402698:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.402701:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.402702:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.402704:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.402707:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.402708:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.402709:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.402711:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.402711:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.402713:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.402716:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.402717:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.402719:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.402720:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.402721:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.402722:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.402723:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.402723:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.402724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.402734:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.402736:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.402736:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.402737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.402738:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.402739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.402974:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.2:1619677472.402975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677472.402976:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.402977:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.402977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.402978:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.402980:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677472.402981:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.402982:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.402984:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.402985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.402987:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.402988:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.402989:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.402991:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.402992:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.402993:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.402996:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.402998:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.403000:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.403001:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.403002:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.403003:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.403004:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.403004:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.403005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.403010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.403017:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.403020:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.403020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.403021:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.403023:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.403025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677472.403210:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.403213:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.403214:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.403214:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.403216:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.403220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.404480:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677472.404483:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677472.404484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677472.404486:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.404487:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.404489:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677472.404490:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.404492:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.404493:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.404494:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677472.404497:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.404498:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677472.404501:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.404501:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:8.0:1619677472.404502:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677472.404504:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.0:1619677472.404504:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677472.404505:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677472.404505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:13.0:1619677472.404518:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.404522:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.404523:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.404525:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.404528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.404529:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677472.404861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:13.0:1619677472.404906:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.404909:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.404910:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.404912:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.404914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.404916:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.0:1619677472.405547:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.405550:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.405552:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.405552:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.405554:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.405555:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.405557:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.405558:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.405559:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.405560:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.405563:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.405565:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.405567:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.405568:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.405569:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.405571:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.405571:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.405572:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.405572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:1.0:1619677472.405768:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.405771:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.405772:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677472.405774:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.405776:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.405778:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677472.405779:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.405780:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.405781:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.405782:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677472.405785:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.405787:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677472.405790:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.405790:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677472.405791:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677472.405793:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677472.405793:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.405794:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.405795:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677472.405851:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.405854:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.405855:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.405855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.405857:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.405861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677472.405869:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.405872:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.405873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.405873:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.405874:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.405875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677472.405964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677472.405969:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.405972:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.405974:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.405974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.405976:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.405978:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.405979:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.405981:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.405982:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.405983:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.405986:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.405988:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.405990:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.405991:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.405992:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.405994:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.405994:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.405995:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677472.405996:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677472.405997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677472.406002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.406005:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677472.406007:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677472.406008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677472.406009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:20.2:1619677472.406309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.406315:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.406318:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.406319:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.406322:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.406325:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.406327:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.406328:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.406329:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.406330:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.406331:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.406335:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.406336:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.406339:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.406340:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.406341:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.406342:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.406343:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.406344:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.406345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.406477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.406483:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.406486:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.406487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.406489:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.406492:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.406493:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.406494:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.406495:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.406496:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.406497:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.406499:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.406501:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.406502:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.406503:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.406503:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.406505:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.406505:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.406507:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.406508:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.406508:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.406513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.406519:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.406520:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.406520:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.406521:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.406521:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.406521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.406522:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:18.0:1619677472.406523:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.406523:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.406525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.406539:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.406542:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.406542:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.406543:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.406543:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.406544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.406950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.406958:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.406960:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.406961:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.406963:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.406964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.406966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.407489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677472.407494:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.407497:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.407499:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.407499:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.407501:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.407503:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677472.407504:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.407505:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.407506:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.407507:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677472.407510:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.407512:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.407515:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.407516:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.407516:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.407518:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.407519:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.407519:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677472.407520:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:8.2:1619677472.407525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.407533:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.407536:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.407537:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.407538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.407538:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.407540:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.407541:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.407544:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.407545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.407547:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.407549:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.407550:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.407551:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.407553:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.407554:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.407555:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.407558:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.407559:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.407561:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.407562:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.407562:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.407563:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.407564:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.407565:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.407566:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.407567:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.407578:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.407578:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.407579:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.407580:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.407581:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677472.407581:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.407582:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.407582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.407583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.407583:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677472.408059:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.408061:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.408063:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.408064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.408065:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.408067:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.408068:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.408070:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.408071:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.408072:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.408075:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.408077:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.408080:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.408081:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.408081:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.408083:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.408084:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.408085:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.408086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677472.408219:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.408221:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.408222:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.408222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.408225:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.408228:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.408319:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.408322:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.408323:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.408326:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.408329:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.408330:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.408331:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.408333:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.408334:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.408335:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.408338:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.408340:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.408342:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.408343:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.408344:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.408345:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.408346:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.408347:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.408348:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.408349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.408352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.408360:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.408360:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.408362:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677472.408363:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.408363:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.408363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.408364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677472.408365:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.408366:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.408368:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.408685:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.408688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.408688:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.408690:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.408692:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.408693:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.408695:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.408696:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.408697:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.408698:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.408701:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.408703:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.408706:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.408706:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.408707:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.408709:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.408709:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.408710:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.408710:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.408715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.408722:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.408725:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.408726:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.408726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.408729:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.408732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.409109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.409114:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.409116:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.409119:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.409119:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.409121:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.409123:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.409124:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.409126:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.409127:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.409128:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.409131:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.409133:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.409135:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.409136:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.409136:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.409138:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.409138:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.409140:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.409141:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.409141:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.409144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.409146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.409149:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.409151:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677472.409151:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677472.409152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.409153:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.409154:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.409155:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.409155:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.409157:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.409160:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677472.409244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.409246:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.409249:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.409249:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.409251:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.409253:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.409254:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.409255:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.409256:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.409257:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.409260:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.409262:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.409265:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.409266:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.409266:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.409268:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.409269:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.409270:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.409270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.409281:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.409284:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.409285:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.409285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.409287:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.409290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.409303:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.409306:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.409306:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.409307:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.409308:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.409308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.411323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.411335:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.411338:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.411339:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.411341:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.411341:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.411344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.411450:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.411453:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.411455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.411455:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.411457:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.411458:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.411460:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.411461:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.411462:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.411463:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.411466:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.411468:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.411471:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.411471:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.411472:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.411474:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.411474:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.411475:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.411476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.411486:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.411489:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.411490:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.411490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.411493:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.411495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.2:1619677472.411749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:8.0:1619677472.411863:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.411866:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.411867:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.411867:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.411869:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.411872:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.411883:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.411885:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.411886:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.411886:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.411887:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.411888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.411923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.2:1619677472.411930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677472.411931:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.411933:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.411934:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.411935:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.411938:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:17.0:1619677472.411938:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.411940:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.411941:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.411943:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.411944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:17.0:1619677472.411945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.411950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:12.0:1619677472.411962:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:8.2:1619677472.411963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:12.0:1619677472.411965:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.411965:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.411967:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.411969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.411970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.0:1619677472.411971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.411974:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.411975:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.411977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.411977:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.411981:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.411983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.411990:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.411993:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.411994:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.411995:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.411996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.411997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.411998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.412000:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.412003:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.412005:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.412005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677472.412006:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:18.0:1619677472.412007:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.412008:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.412009:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.412009:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.412012:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.412018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677472.412020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.412030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.412037:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:8.0:1619677472.412038:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677472.412040:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.412041:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.412041:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677472.412041:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677472.412042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.412042:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.412044:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.412045:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.412047:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.412048:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.412091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.412099:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.412102:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.412103:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.412104:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.412105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.412107:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.412110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.412120:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.412123:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.412124:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.412124:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.412125:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.412126:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.412154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.412161:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.412164:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.412164:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.412167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.412167:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.412174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.412185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.412193:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.412195:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.412196:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.412198:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.412198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.412201:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.412577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.412583:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.412586:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.412587:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.412589:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.412591:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.412592:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.412594:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.412595:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.412596:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.412597:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.412601:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.412602:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.412605:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.412606:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.412607:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.412609:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.412609:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.412610:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.412611:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.413132:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.413135:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.413136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.413137:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.413139:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.413141:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.413142:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.413143:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.413144:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.413145:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.413148:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.413150:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.413151:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.413152:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.413153:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.413154:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.413155:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.413156:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.413156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.413166:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.413168:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.413169:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.413170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.413170:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.413172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.413204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.413212:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.413215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.413215:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.413216:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.413218:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.413221:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.413223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.413231:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.413233:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.413234:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.413234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.413235:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.413236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.413271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.413277:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.413279:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.413280:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.413282:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.413285:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.413286:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.413288:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.413289:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.413290:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.413291:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.413298:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.413300:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.413302:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.413303:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.413303:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.413305:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.413306:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.413307:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.413307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.413320:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.413322:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.413322:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.413323:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.413324:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.413325:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.413437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677472.413444:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677472.413445:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677472.413446:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.413447:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.413448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677472.413448:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677472.413449:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.413449:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.413450:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.413451:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677472.413452:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.413452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677472.414241:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.414243:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.414245:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.414247:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.414248:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.414250:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.414251:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.414252:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.414253:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.414254:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.414258:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.414259:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.414262:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.414262:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.414263:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.414265:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.414265:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.414266:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.414268:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.414268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.414279:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.414281:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.414282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.414283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.414940:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.414942:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.414943:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.414945:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.414946:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.414948:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.414949:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96513757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.414951:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.414951:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.414952:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.414955:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.414957:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.414960:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.414960:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677472.414961:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:8.0:1619677472.414961:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.414962:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.414963:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677472.414964:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:8.0:1619677472.414964:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.414964:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:13.0:1619677472.414966:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.414968:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.414969:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.414971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677472.414971:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677472.414971:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)++
00000800:00000200:12.0:1619677472.414972:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677472.414973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.414974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:18.0:1619677472.414977:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.414980:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.414980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.414981:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.414983:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.414986:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.414988:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.414991:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.414991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.414994:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.414996:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.414998:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.414999:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.415000:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.415002:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.415003:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.415006:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.415008:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.415010:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.415011:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.415012:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.415014:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.415015:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.415016:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.415016:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.415021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.415028:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.415031:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.415031:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.415033:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.415034:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.415036:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.415080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.415083:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.415086:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.415088:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.415088:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.415090:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.415092:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.415093:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.415095:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.415096:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.415097:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.415100:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.415101:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.415104:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.415105:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.415105:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.415107:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.415107:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.415108:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.415109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.415113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.415120:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.415123:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.415124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.415124:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.415126:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.415129:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.415580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.415586:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.415589:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.415590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.415592:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.415594:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.415596:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.415597:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.415598:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.415599:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.415600:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.415604:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.415605:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.415608:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.415609:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.415609:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.415611:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.415612:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.415612:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.415613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.415618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.415626:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.415629:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.415629:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.415631:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.415632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.415634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677472.417115:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.417118:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.417120:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.417121:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.417122:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.417124:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.417125:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.417126:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.417127:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.417128:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.417132:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.417133:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.417136:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.417137:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.417138:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.417140:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.417140:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.417142:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.417144:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.417144:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.417155:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.417157:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.417158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.417158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:21.0:1619677472.417403:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.417406:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.417408:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.417410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.417411:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.417413:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.417415:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.417417:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.417418:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.417419:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.417424:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.417426:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.417429:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.417431:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.417432:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.417434:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.417435:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.417436:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.417437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.417451:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.417455:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.417457:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.417457:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.417460:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.417464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.417694:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.417697:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.417702:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.417705:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.417707:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.417708:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.417710:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.417713:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677472.417714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:20.0:1619677472.417715:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.417717:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.417723:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.417724:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.417726:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.417726:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.417727:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.417730:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.417731:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.417732:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.417733:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.417735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.417735:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.417736:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.417736:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.417738:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.417739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.417751:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.417755:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.417756:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.417759:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:8.0:1619677472.417760:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.417762:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.417763:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677472.417764:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (24)--
00000800:00000200:8.0:1619677472.417765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677472.417766:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.417767:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.417768:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.417769:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.417770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.417775:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.417778:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.417781:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.417783:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.417786:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.417787:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.417789:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.417794:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.417797:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.417801:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.417802:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.417803:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.417805:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.417806:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.417808:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.417809:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.417811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.417823:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.417825:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.417826:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.417829:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:24.0:1619677472.417830:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.417830:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.417831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.417832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.417833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.417834:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.418653:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.418655:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.418656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.418656:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.418658:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.418660:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.418834:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.418837:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.418838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.418839:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.418840:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.418842:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.418842:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.418844:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.418845:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.418845:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.418848:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.418850:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.418852:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677472.418852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677472.418853:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.418854:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.418855:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.418856:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.418856:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.418857:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.418859:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677472.418861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.418862:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.418862:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.418865:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.418867:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.418868:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.418870:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.418870:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:18.0:1619677472.418871:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.418872:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677472.418872:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:18.0:1619677472.418873:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677472.418873:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.418874:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.418876:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.418877:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.418879:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677472.418879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677472.418883:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.418884:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.418885:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.418887:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.418887:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.418888:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.418889:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.419626:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.419628:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677472.419629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677472.419631:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677472.419633:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.419635:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677472.419636:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.419637:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.419638:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.419639:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677472.419642:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677472.419644:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.419647:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.419648:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677472.419648:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677472.419650:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.419651:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.419651:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.419652:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:20.2:1619677472.419720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.419726:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.419728:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.419731:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.419731:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.419734:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.419735:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.419736:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.419737:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.419739:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.419739:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.2:1619677472.419741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:21.0:1619677472.419749:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.419751:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.419754:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.419754:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.419755:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.419757:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.419757:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.419758:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.419759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.419763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.419771:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.419773:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.419774:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.419774:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.419776:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.419778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677472.419912:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.419915:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.419916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.419916:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.419918:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.419924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.419926:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.419928:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.419929:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.419929:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.419930:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.419930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.419968:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.419971:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.419972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.419974:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.419976:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.419978:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677472.419979:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.419981:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.419982:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.419983:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677472.419986:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.419987:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677472.419990:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.419990:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677472.419991:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677472.419993:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677472.419994:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.419994:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.419995:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677472.420007:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.420010:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.420010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.420011:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.420012:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.420015:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.420331:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.420334:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677472.420335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677472.420337:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.420339:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.420341:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.420342:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.420343:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.420344:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.420346:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.420349:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.420351:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.420354:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.420354:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.420355:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.420357:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.420357:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.420359:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677472.420360:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677472.420360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.2:1619677472.420363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.420371:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.420374:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.420374:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:1.0:1619677472.420374:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.420375:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.420376:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677472.420377:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:1.0:1619677472.420377:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.420378:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:1.0:1619677472.420380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.420704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.420710:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.420712:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.420713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.420715:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.420718:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.420720:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677472.420721:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.420722:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.420723:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.420724:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677472.420728:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.420729:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.420732:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.420733:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677472.420733:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677472.420735:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677472.420736:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.420737:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.420737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677472.420786:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.420789:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.420789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.420792:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.420794:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.420796:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.420797:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.420799:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.420800:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.420801:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.420804:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.420806:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.420808:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.420809:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.420809:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.420811:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.420811:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.420812:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.420813:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.420817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.420826:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:24.0:1619677472.420829:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677472.420829:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.420830:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.420832:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:20.0:1619677472.420832:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.420833:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677472.420833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.420835:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.420835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:24.0:1619677472.420838:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677472.420838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:24.0:1619677472.420842:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.420843:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677472.420843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000400:00000200:24.0:1619677472.420845:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.420846:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677472.420846:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677472.420847:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.420848:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.420848:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.420849:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.420849:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:21.0:1619677472.420850:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.420850:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.420851:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:24.0:1619677472.420852:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.420852:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677472.420852:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.420854:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.420855:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.420856:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677472.420857:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.420858:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:20.0:1619677472.420858:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.420859:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:20.0:1619677472.420859:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677472.420860:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:20.0:1619677472.420860:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677472.420861:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.420861:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677472.420861:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677472.420862:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677472.420864:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.420866:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.420868:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.420869:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.420869:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.420871:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.420871:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.420873:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:18.0:1619677472.420873:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677472.420874:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677472.420874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:18.0:1619677472.420876:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.420877:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677472.420877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.420877:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.420879:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.2:1619677472.420880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.420883:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:18.0:1619677472.420883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.420885:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000800:00000200:20.0:1619677472.420885:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.420886:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000400:00000200:20.0:1619677472.420886:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.420887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677472.420887:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.420889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.420890:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677472.420891:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:20.2:1619677472.421012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.421017:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.421020:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.421022:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.421022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.421024:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.421026:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.421027:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.421028:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.421030:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.421030:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.421034:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.421035:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.421037:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.421038:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.421039:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.421040:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.421041:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.421042:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.421042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.421046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.421054:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.421056:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.421057:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.421058:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.421059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.421061:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.421680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.421689:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.421691:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.421692:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.421693:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.421695:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.421698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677472.421890:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.421892:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677472.421893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677472.421895:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.421898:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.421900:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677472.421901:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.421902:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.421903:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.421905:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677472.421916:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.421918:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.421922:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.421923:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.421924:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677472.421925:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.421926:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.421928:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677472.421929:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677472.421929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.421940:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677472.421942:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677472.421943:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.421944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:21.0:1619677472.422380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.422383:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.422385:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.422385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.422387:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.422388:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.422390:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.422391:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.422392:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.422393:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.422396:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.422398:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.422400:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.422400:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.422401:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.422403:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.422403:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.422404:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.422405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.422789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.422794:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.422797:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.422798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.422800:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.422803:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.422805:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.422806:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.422807:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.422808:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.422809:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.422812:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.422814:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.422816:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.422817:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.422817:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.422819:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.422820:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.422821:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.422821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.422833:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.422835:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.422836:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.422836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.422837:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.422839:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.423088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.423096:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.423098:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.423099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.423100:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.423102:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.423105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.423106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.423113:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.423116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.423116:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.423117:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.423118:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.423118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.423227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.423233:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.423236:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.423237:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.423239:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.423241:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.423243:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.423244:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.423245:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.423246:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.423248:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.423251:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.423253:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.423255:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.423256:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.423256:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.423258:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.423259:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.423260:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.423261:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.423261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.423264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.423267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.423273:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:20.0:1619677472.423274:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:18.0:1619677472.423275:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:20.0:1619677472.423276:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:18.0:1619677472.423276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.423277:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.423278:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677472.423280:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.423280:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.423281:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000400:00000200:20.0:1619677472.423281:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.423283:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:20.0:1619677472.423284:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:21.0:1619677472.423285:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677472.423285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.423286:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677472.423287:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000400:00000200:21.0:1619677472.423288:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.423289:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677472.423289:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.423290:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000400:00000200:21.0:1619677472.423291:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.423291:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.423292:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.423292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.423292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:21.0:1619677472.423300:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.423301:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.423305:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.423307:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.423310:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.423310:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.423311:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.423312:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.423313:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.423314:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677472.423314:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.423318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.423326:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.423328:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.423328:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.423330:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.423331:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.423332:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.423628:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.423631:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.423633:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.423634:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.423637:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.423638:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.423640:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.423641:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.423642:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.423643:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.423646:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.423648:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.423650:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.423651:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.423651:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.423653:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.423653:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.423654:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.423655:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.423665:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.423667:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.423668:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.423669:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.423670:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.423672:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.423884:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.423886:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.423887:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.423887:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.423889:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.423891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.423903:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.423905:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.423906:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.423907:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677472.423907:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.423908:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.424204:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.424207:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.424207:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677472.424209:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677472.424211:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.424212:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677472.424213:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.424215:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.424216:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.424216:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677472.424219:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677472.424221:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.424223:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.424224:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677472.424225:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677472.424226:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.424227:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.424227:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.424228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:17.0:1619677472.425207:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.425210:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.425211:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.425213:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.425215:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.425217:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.425218:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.425219:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.425220:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.425221:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.425224:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.425227:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.425229:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.425230:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.425230:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.425232:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.425232:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.425233:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.425233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.425239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.425247:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.425249:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.425250:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.425251:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.425253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.425254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.425412:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.425415:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.425417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.425418:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.425420:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.425422:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.425423:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.425424:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.425425:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.425426:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.425429:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.425431:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.425434:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.425434:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.425435:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.425437:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.425437:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.425438:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.425440:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.425440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677472.425451:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.425453:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677472.425453:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.425455:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.425455:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.425455:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677472.425455:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.425456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677472.425457:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.425461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.425548:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.425551:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.425553:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.425554:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.425555:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.425556:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.425558:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.425559:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.425560:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.425561:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.425564:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.425566:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.425570:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.425571:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.425571:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.425573:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.425573:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.425574:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677472.425574:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.425575:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.425576:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000400:00000200:21.0:1619677472.425577:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.425577:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.425578:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.425579:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.425581:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.425582:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.425583:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.425584:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.425585:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.425585:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.425585:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.425586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.425587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:21.0:1619677472.425589:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.425590:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.425593:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.425594:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.425595:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.425596:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.425597:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.425598:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.425599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677472.425618:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.425621:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.425621:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.425622:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.425624:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.425627:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.426013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.426021:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.426024:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.426025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.426025:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.426027:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.426031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.426041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.426044:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.426046:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.426048:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.426049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.426051:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.426052:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.426054:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.426055:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.426056:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.426057:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.426060:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.426061:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.426064:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.426064:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.426065:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.426067:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.426067:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.426068:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.426069:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.426070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.426073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.426075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.426080:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677472.426080:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677472.426083:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677472.426083:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677472.426084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677472.426084:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.426084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.426085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677472.426086:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.426089:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.426097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.426105:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.426107:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.426108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.426108:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.426110:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.426110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.426449:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.426452:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.426453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.426454:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.426457:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.426459:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.426460:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.426461:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.426462:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.426463:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.426466:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.426468:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.426469:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.426470:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.426471:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.426472:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.426472:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.426473:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.426473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677472.426485:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.426487:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.426487:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.426488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.426489:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.426491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.426507:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.426510:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.426510:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.426511:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.426511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.426512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.426758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.426766:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677472.426767:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677472.426769:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:8.0:1619677472.426769:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.426770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:21.0:1619677472.426770:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.426771:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.426772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:8.0:1619677472.426772:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.426774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:8.0:1619677472.426775:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.426777:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.426778:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96513703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.426779:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.426780:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.426781:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.426784:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.426786:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.426789:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.426790:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.426790:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.426792:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.426792:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.426793:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.426793:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.426806:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.426809:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.426810:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.426810:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.426812:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.426818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.426820:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.426822:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.426823:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.426823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.426824:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.426824:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.427206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.427214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.427217:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.427218:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.427218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.427220:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.427223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.427373:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.427376:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.427378:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.427379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.427380:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.427382:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.427383:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.427385:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677472.427385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:20.0:1619677472.427386:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.427387:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.427390:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.427391:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677472.427393:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:20.0:1619677472.427395:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.427395:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.427396:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677472.427396:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.427397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.427397:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.427398:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.427398:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.427399:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000400:00000200:1.0:1619677472.427399:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.427400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:1.0:1619677472.427402:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.427411:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.427414:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.427415:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.427415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.427417:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.427420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.427641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.427649:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.427651:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.427652:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.427654:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.427655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.427657:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.428671:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.428673:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.428674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.428674:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.428676:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.428678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.428735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.428744:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.428746:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.428748:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.428748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.428750:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.428752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.428755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.428762:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.428764:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.428765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.428766:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.428767:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.428767:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.429596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.429602:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.429605:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.429606:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.429608:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.429611:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.429612:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.429613:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.429614:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.429616:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.429616:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.429620:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.429621:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.429623:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.429624:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.429625:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.429626:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.429627:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.429628:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.429628:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.429640:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.429643:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.429643:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.429644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.429645:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.429646:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.429731:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.429734:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.429736:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.429737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.429739:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.429740:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.429742:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.429743:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.429744:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.429745:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.429748:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.429750:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.429751:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.429752:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.429753:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.429755:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.429755:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.429756:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.429757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.430300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.430307:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677472.430309:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677472.430310:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:24.0:1619677472.430311:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677472.430311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677472.430313:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677472.430313:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.430315:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.430315:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.430317:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677472.430318:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.430318:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.430320:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677472.430320:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.430321:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.430322:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.430323:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677472.430324:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.430325:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.430327:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677472.430327:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:24.0:1619677472.430331:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.430331:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677472.430334:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.430334:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.430334:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.430335:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677472.430337:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.430337:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.430337:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677472.430338:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.430338:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.430338:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.430339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.430340:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.430341:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.430342:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.430343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.430347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.430355:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.430358:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.430359:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.430359:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.430361:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.430364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.430509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.430514:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.430517:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.430519:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.430520:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.430522:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.430523:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.430524:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.430526:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.430527:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.430528:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.430531:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.430533:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.430535:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.430536:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.430536:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.430538:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.430539:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.430539:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.430540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.430544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.430552:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.430554:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.430555:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.430557:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.430557:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.430559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.430563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.430570:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.430573:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.430573:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.430574:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.430575:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.430576:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.430919:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.430922:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.430922:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.430925:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.430927:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.430929:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.430930:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.430932:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.430933:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.430934:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.430937:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.430939:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.430941:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.430942:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.430943:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.430945:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.430945:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.430947:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.430948:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.430949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.430951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.430959:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.430959:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677472.430961:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.430962:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.430963:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.430963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677472.430964:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.430964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.430965:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.430966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.431299:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.431302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.431302:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.431305:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.431307:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.431309:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.431310:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.431312:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.431313:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.431314:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.431317:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.431318:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.431320:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.431320:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.431321:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.431322:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.431323:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.431323:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.431324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.431334:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.431337:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.431337:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.431338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.431339:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.431340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.432035:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.432037:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.432038:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.432040:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.432041:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.432043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.432136:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.432138:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.432139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.432141:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.432143:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.432144:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.432145:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.432147:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.432148:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.432149:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.432152:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.432153:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.432156:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.432157:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.432157:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.432159:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.432160:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.432161:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.432162:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.432163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.432174:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.432175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.432177:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:18.0:1619677472.432177:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.432177:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677472.432178:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677472.432178:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.432180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.432180:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.432184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.433004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.433013:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.433016:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.433016:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.433017:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.433019:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.433022:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.433920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.433922:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.433923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.433923:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.433925:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.433928:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.434367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.434376:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.434379:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.434380:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.434380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.434382:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.434385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.434424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677472.434466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:2.0:1619677472.434472:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.434475:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.434476:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.434477:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.434479:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.434482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.434973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.434976:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.434977:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.434977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.434979:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.434983:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.435365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.435367:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.435370:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.435372:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.435373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.435374:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.435375:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.435377:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.435378:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.435379:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.435380:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.435383:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.435385:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.435388:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.435389:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.435389:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.435391:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.435392:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.435392:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.435393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.435398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.435406:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.435409:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.435410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.435410:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.435412:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.435414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.435419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.435425:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.435428:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.435429:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.435433:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.435435:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.435436:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.435438:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.435439:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.435441:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.435442:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.435447:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.435450:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.435454:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.435454:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.435455:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.435455:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:18.0:1619677472.435456:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.435456:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:18.0:1619677472.435458:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.435458:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.435459:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.435460:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.435461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677472.435462:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.435463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.435463:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.435464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.435464:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.435465:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.435466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.436152:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.436154:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.436156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.436157:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.436160:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.436161:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.436162:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.436163:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.436164:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.436165:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.436167:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.436169:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.436170:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.436171:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.436172:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.436173:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.436173:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.436174:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.436174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.436185:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.436186:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.436187:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.436188:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.436189:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.436190:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.436603:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.436605:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.436607:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.436608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.436610:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.436612:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.436613:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.436614:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.436615:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.436616:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.436620:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.436622:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.436624:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.436625:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.436625:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.436627:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.436628:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.436629:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.436630:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.436811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.436818:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.436822:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.436822:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.436825:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.436828:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.436829:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.436830:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.436831:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.436832:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.436833:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.436836:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.436838:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.436839:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.436840:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.436841:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.436842:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.436843:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.436844:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.436845:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.436846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.436851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.436857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.436859:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.436859:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.436859:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.436860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.436860:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.436861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.436861:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.436862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.436863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.437025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.437027:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.437030:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.437032:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.437033:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.437034:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.437036:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.437037:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.437038:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.437040:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.437041:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.437044:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.437046:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.437048:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.437049:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.437050:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.437051:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.437052:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.437052:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.437053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.437058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.437066:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.437068:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.437069:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.437071:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.437073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.437074:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.437077:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.437080:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.437081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.437083:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.437084:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.437086:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.437087:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.437088:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.437089:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.437090:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.437093:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.437095:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.437096:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.437097:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.437098:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.437099:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.437100:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.437101:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.437101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.437111:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.437113:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.437113:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.437114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.437115:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.437116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.437132:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.437134:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.437135:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.437135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.437136:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.437136:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.437759:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.437763:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.437763:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.437765:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.437768:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.437769:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.437770:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.437772:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.437772:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.437774:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.437777:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.437778:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.437781:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.437782:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.437782:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.437784:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.437785:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.437786:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.437786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.437791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.437799:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.437802:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.437802:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.437804:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.437805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.437806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.438214:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.438217:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.438217:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.438219:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.438221:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.438223:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677472.438224:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.438226:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.438227:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.438228:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677472.438232:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.438233:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.438237:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.438237:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.438238:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677472.438240:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677472.438240:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.438241:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.438242:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:8.2:1619677472.438484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.438492:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.438495:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.438496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.438496:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.438499:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.438501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.438504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.438511:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.438514:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.438515:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.438515:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.438516:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.438517:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.438662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.438664:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.438665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.438665:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.438667:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.438669:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.438910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.438917:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.438920:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.438921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.438921:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.438923:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.438926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.439756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:2.0:1619677472.439763:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.439766:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:18.0:1619677472.439767:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:2.0:1619677472.439769:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.439772:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.439774:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677472.439775:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.439776:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.439777:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.439778:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677472.439782:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.439784:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 600 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677472.439786:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.439787:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677472.439787:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677472.439789:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677472.439790:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.439791:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:2.0:1619677472.439792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:20.0:1619677472.439803:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.439805:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.439806:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.439808:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.439809:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.439811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.439822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.439824:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.439825:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.439826:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.439827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.439827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.440015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.440022:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.440024:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.440025:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.440027:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.440028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.440030:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.440034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.440043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.440046:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.440047:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.440048:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.440049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.440050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.440095:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.440097:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:13.0:1619677472.440098:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:2.0:1619677472.440098:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.440099:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.440100:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:18.0:1619677472.440101:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:13.0:1619677472.440101:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.440103:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.440105:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677472.440105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.440107:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:24.0:1619677472.440253:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.440256:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.440257:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.440257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.440260:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.440262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.440270:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.440273:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.440273:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677472.440274:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.440275:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.440275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677472.440290:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.440292:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.440294:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.440301:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.440304:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.440307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.440399:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.440402:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.440403:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.440403:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.440406:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.440409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677472.440418:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.440420:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.440421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.440421:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.440423:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.440423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677472.440425:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.440429:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.440430:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.440433:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.440435:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.440437:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677472.440461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.440470:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.440472:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.440473:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.440475:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.440476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.440478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.440748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.440754:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.440757:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.440758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.440760:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.440763:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.440765:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.440766:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.440767:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.440769:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.440770:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.440773:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.440774:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.440777:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.440778:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.440778:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.440780:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.440780:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.440781:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.440782:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.440786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.440793:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.440795:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.440796:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.440798:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.440799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.440801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.440805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.440812:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.440815:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.440816:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.440817:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.440817:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.440819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.441639:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.441643:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.441644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.441645:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.441647:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.441649:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.441650:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.441651:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.441652:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.441653:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.441656:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.441658:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.441660:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.441660:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.441661:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.441662:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.441662:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.441663:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.441664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677472.441669:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.441672:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.441674:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.441674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677472.441675:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:21.0:1619677472.441676:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.441677:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.441678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677472.441678:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677472.441678:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.441679:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.441680:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.441681:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.441682:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677472.441682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:21.0:1619677472.441683:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.441686:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.441688:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.441695:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.441696:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.441697:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.441698:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.441699:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.441700:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.441701:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.441702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.441711:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.441713:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.441714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.441715:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.442026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.442028:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.442031:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.442033:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.442034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.442035:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.442037:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.442038:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.442039:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.442040:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.442041:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.442044:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.442046:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.442048:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.442049:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.442049:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.442051:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.442052:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.442052:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.442054:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.442054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.442057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.442060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.442063:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.442065:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.442065:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.442065:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.442066:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677472.442066:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.442067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.442068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.442069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.442069:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677472.442569:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.442572:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:5.2:1619677472.442572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb0a00] (20)++
00000800:00000200:8.0:1619677472.442573:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.442574:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.442576:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.442577:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.442578:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.442579:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb0a00] (21)++
00000400:00000200:24.0:1619677472.442580:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.442581:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.442582:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677472.442582:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.19@o2ib3
00000800:00000200:1.0:1619677472.442582:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb0a00] (22)++
00000800:00000200:1.0:1619677472.442584:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.19@o2ib3
00000400:00000200:17.0:1619677472.442586:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.442588:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.442588:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.19@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.442589:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.442590:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.442592:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677472.442592:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000800:00000200:24.0:1619677472.442593:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:17.0:1619677472.442593:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.442594:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.442595:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.442595:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677472.442595:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.442595:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677472.442597:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677472.442597:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000800:00000200:24.0:1619677472.442598:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.442598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677472.442598:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.442600:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677472.442601:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.442601:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.442602:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.442603:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677472.442604:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000400:00000200:1.0:1619677472.442604:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:17.0:1619677472.442608:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.442609:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000400:00000200:1.0:1619677472.442609:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.442610:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:8.0:1619677472.442610:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.442611:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677472.442611:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:8.0:1619677472.442612:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.442613:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:17.0:1619677472.442613:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb0a00] (23)++
00000800:00000200:24.0:1619677472.442614:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.442614:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.442615:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677472.442615:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb0a00] (24)--
00000800:00000200:17.0:1619677472.442615:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb0a00] (23)--
00000800:00000200:1.0:1619677472.442615:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.442616:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677472.442616:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677472.442616:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677472.442617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb0a00] (22)--
00000800:00000200:8.0:1619677472.442617:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.2:1619677472.442619:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:8.0:1619677472.442619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677472.442619:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677472.442619:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb0a00] (21)++
00000800:00000200:1.0:1619677472.442620:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb0a00] (22)--
00000800:00000200:1.0:1619677472.442621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb0a00] (21)--
00000800:00000200:24.0:1619677472.442623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.442631:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.442633:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.442634:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.442634:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.442637:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.442640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:13.0:1619677472.443197:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.443200:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.443202:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.443205:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.443205:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.443207:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677472.443208:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.443209:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.443210:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.443211:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677472.443214:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.443216:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677472.443219:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.443219:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677472.443220:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677472.443222:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677472.443222:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.443223:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.443224:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677472.443228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.443236:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.443239:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.443240:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.443240:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.443242:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.443245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.443286:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.443289:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.443289:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.443292:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.443297:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.443299:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.443300:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.443302:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.443302:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677472.443304:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000400:00000200:1.0:1619677472.443304:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.443306:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.443307:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677472.443308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:1.0:1619677472.443309:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:24.0:1619677472.443310:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.443312:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.443312:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.443312:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.443313:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677472.443314:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677472.443315:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96513633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677472.443315:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.443315:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:24.0:1619677472.443316:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677472.443316:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.443316:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:24.0:1619677472.443317:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.443318:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:8.2:1619677472.443321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677472.443324:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.443326:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677472.443329:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.443329:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677472.443330:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.0:1619677472.443330:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:24.0:1619677472.443332:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677472.443332:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:21.0:1619677472.443332:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:24.0:1619677472.443333:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000400:00000200:21.0:1619677472.443333:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.443334:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:21.0:1619677472.443335:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.443336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.443337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.443790:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.443793:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.443795:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.443796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.443797:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.443798:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.443799:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.443801:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.443802:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.443803:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.443806:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.443808:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.443811:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.443811:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.443812:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.443814:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.443814:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.443815:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.443816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:2.0:1619677472.443819:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.443822:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677472.443824:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677472.443825:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.443826:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:2.0:1619677472.443828:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.443829:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.443829:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.443830:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.443830:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677472.443832:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.443833:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:2.0:1619677472.443834:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.443835:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677472.443836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:2.0:1619677472.443836:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677472.443839:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.443841:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.443845:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.443846:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.443846:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677472.443848:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.443849:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.443850:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.443850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:24.0:1619677472.443939:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.443941:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.443942:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.443943:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.443944:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.443946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.444650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.444659:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.444662:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.444662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.444663:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.444665:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.444668:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.445142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.445150:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.445152:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.445153:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.445155:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.445158:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.445159:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.445161:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.445162:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.445164:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.445164:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.445167:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.445169:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.445171:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.445172:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.445172:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.445174:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.445175:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.445176:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.445176:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.445189:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.445191:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.445191:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.445192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.445193:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.445195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.445290:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.445293:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:20.0:1619677472.445297:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.445299:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677472.445301:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.445303:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.445304:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.445305:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.445306:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.445307:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.445310:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.445312:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.445314:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.445315:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.445316:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.445318:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.445318:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.445319:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.445320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.445341:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.445344:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.445345:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.445345:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.445347:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.445350:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.445361:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.445363:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.445364:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.445364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.445365:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.445365:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.446022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.446030:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.446033:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.446034:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.446034:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.446036:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.446040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.447099:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.447101:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.447102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.447104:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.447106:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.447108:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.447109:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.447111:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.447112:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.447112:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.447115:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.447117:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.447119:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.447119:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.447120:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.447121:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.447122:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.447123:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.447123:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.447134:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.447135:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.447136:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.447137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.447138:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.447139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.447621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.447624:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.447627:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.447629:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.447630:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.447631:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.447633:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.447634:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.447636:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.447637:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.447638:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.447641:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.447642:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.447645:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.447645:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.447646:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.447648:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.447648:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.447649:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.447650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.447655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.447663:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.447666:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.447667:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.447667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.447669:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.447671:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677472.447883:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.447885:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.447887:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.447888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.447890:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.447892:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.447893:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.447894:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.447895:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.447896:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.447900:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.447902:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.447905:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.447905:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.447906:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.447908:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.447909:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.447910:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.447910:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.447920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.447923:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.447924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.447924:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.447926:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.447929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.447957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.447967:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.447972:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.447973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.447976:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.447980:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.447982:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.447984:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.447987:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.447988:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.447990:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.447995:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.447998:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.448002:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.448004:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.448005:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.448007:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.448008:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.448010:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.448012:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.448013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.448017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.448025:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.448028:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.448029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.448031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.448113:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.448116:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.448117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.448118:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.448120:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.448121:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.448122:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.448123:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.448124:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.448125:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.448128:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.448130:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.448132:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.448133:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.448133:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.448135:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.448135:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.448136:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.448136:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.448146:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.448148:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.448148:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.448149:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.448150:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.448151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.448680:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.448682:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.448683:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.448683:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.448685:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.448687:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.448894:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.448897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.448897:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.448900:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.448902:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.448904:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.448905:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.448907:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.448908:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.448909:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.448912:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.448913:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.448915:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.448916:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.448917:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.448919:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.448919:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.448920:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.448920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.448936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.448944:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.448946:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.448947:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.448949:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.448950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.448952:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.449116:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.449119:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.449120:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.449122:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.449125:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.449126:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677472.449128:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.449129:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.449130:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.449131:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677472.449135:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.449137:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.449140:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.449141:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.449141:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677472.449143:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677472.449144:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.449145:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.449145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:21.0:1619677472.449868:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.449871:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.449873:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.449873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.449875:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.449877:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.449878:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.449879:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.449880:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.449882:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.449884:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.449886:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.449889:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.449890:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.449890:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.449892:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.449893:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.449894:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.449895:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.449895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.449904:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677472.449905:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.449906:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.449907:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677472.449908:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.449908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.449908:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.449909:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.449911:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.449913:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.449926:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.449929:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.449930:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.449930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.449931:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.449931:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.450570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.450579:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.450582:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.450583:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.450583:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.450585:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.450589:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.450590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.450597:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.450600:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.450601:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.450601:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.450602:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.450603:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.450897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.450900:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.450902:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.450904:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.450905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.450907:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.450908:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.450909:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.450911:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.450912:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.450913:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.450915:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.450917:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.450919:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.450919:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.450920:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.450922:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.450922:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.450923:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.450923:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.450928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.450936:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.450938:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.450938:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.450940:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.450941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.450942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.450948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.450956:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.450959:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.450960:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.450960:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.450961:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.450962:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677472.451369:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.451372:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.451374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.451374:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.451376:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.451380:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.451389:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.451391:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.451392:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.451392:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.451393:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.451394:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.451790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.451796:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.451799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.451799:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677472.451801:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.451803:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.451805:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.451806:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.451807:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.451808:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.451809:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.451812:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.451814:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.451816:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.451816:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.451817:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.451819:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.451819:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.451820:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.451820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.451832:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.451834:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.451835:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.451836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.451837:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.451839:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.451850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.451858:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.451860:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.451861:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.451863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.451864:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.451870:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.452400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.452406:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.452410:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.452412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.452413:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.452415:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.452417:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.452418:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.452419:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.452420:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677472.452421:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677472.452422:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677472.452424:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.452425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677472.452427:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677472.452428:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.452430:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:8.0:1619677472.452431:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.452432:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.452434:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96513600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677472.452434:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.452435:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:8.0:1619677472.452436:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677472.452436:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:8.0:1619677472.452437:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.452438:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677472.452438:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.452438:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.452439:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.452440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:8.0:1619677472.452442:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.452443:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.452445:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.452446:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.452447:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.452448:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.452449:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.452449:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.452450:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.452459:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.452461:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.452462:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.452463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.452464:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.452465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.453250:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.453252:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.453254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.453255:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.453257:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.453258:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.453260:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.453261:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.453262:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.453263:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.453267:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.453268:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.453271:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.453272:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.453272:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.453274:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.453275:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.453276:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.453277:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.453278:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.453288:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.453291:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.453291:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.453292:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.453292:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.453293:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.453295:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.453299:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.453299:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.453302:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.453957:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.453960:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.453962:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.453963:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.453964:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.453966:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.453967:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.453968:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.453969:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.453970:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.453973:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.453975:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.453978:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.453979:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.453979:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.453981:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.453982:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.453982:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.453983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677472.454524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.454528:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.454530:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.454533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.454533:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.454535:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.454537:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.454538:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.454539:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.454540:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.454541:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.454545:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.454547:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.454549:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.454550:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.454551:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.454552:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.454553:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.454553:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.454554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.454559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.454568:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.454571:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.454572:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.454572:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.454574:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.454576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.454578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.454585:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.454587:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.454588:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.454589:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.454589:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.454590:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677472.454597:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.454600:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.454600:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.454602:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.454604:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.454606:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.454607:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.454608:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.454609:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.454610:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.454613:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.454615:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.454617:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.454618:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.454619:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.454620:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.454621:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.454622:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.454623:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.454624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.454626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.454635:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677472.454637:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.454637:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.454637:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.454638:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.454639:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.454640:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.454642:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.454642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.454644:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.454800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:12.0:1619677472.454848:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.454850:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.454851:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677472.454852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:12.0:1619677472.454853:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.454855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.454856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.0:1619677472.454860:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.454862:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.454864:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.454865:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.454866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.454868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677472.454939:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.454942:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.454942:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.454944:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.454947:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.454948:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677472.454949:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.454951:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.454952:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.454954:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.454957:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.454960:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.454962:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.454963:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.454964:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.454965:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.454966:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.454967:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677472.454968:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677472.454968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.454981:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677472.454983:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677472.454984:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.454985:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:21.0:1619677472.455452:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.455454:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.455455:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.455456:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.455458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.455459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.455630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.455632:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.455635:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.455637:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.455638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.455640:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.455641:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.455642:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.455644:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.455645:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.455646:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.455649:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.455651:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.455653:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.455654:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.455655:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.455656:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.455657:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.455658:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.455659:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.455660:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.455663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.455666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.455669:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.455670:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.455670:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.455671:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.455672:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677472.455672:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.455673:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.455674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.455675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.455676:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677472.455692:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.455695:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.455696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.455698:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.455701:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.455703:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677472.455705:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.455707:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.455708:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.455709:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677472.455713:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.455715:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677472.455718:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.455719:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:17.0:1619677472.455719:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677472.455721:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:17.0:1619677472.455721:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.455722:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.455723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.455729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.455736:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.455738:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.455739:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.455741:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.455742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.455743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.456149:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.456152:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.456153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.456156:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.456158:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.456160:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.456161:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.456162:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.456163:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.456164:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.456168:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.456170:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.456173:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.456174:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.456174:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.456176:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.456176:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.456177:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.456178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.456191:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.456194:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.456195:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.456196:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.456197:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.456201:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.456501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.456510:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.456512:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.456513:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.456515:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.456516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.456518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.456520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.456527:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.456530:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.456531:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.456531:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.456532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.456533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.457798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.457809:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.457812:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.457813:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.457815:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.457815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.457818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.457971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.457978:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.457981:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.457981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.457983:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.457986:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.457987:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.457989:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.457990:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.457991:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.457992:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.457995:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.457997:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.457999:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.458000:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.458001:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.458003:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.458004:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.458005:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.458005:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.458009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.458016:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.458019:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.458019:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.458021:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.458022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.458024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.458223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677472.458271:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.458274:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.458275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.458275:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.458277:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.458281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.458291:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.458297:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.458298:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.458299:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.458300:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.458300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.458354:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.458356:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.458357:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.458358:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.458360:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.458363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.458518:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.458521:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.458522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.458524:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.458527:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.458528:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.458529:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.458531:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.458532:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.458533:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.458536:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.458538:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.458540:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.458541:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.458542:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.458543:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.458544:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.458545:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.458545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.458549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.458558:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.458560:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.458561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.458562:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.458564:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.458567:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.459513:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.459516:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.459516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.459518:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.459520:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.459522:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677472.459523:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.459524:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.459526:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.459526:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677472.459530:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.459531:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.459533:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.459534:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.459534:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.459536:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.459536:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.459537:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.459537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.459543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.459550:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.459552:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.459552:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.459553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.459554:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.459555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677472.459796:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.459798:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.459799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.459800:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.459802:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.459804:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.459805:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.459806:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.459807:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.459807:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.459810:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.459812:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.459814:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.459814:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.459815:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.459817:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.459817:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.459818:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.459818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.459823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.459830:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.459832:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.459833:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.459835:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.459837:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.459838:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:12.0:1619677472.459848:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.459851:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.459853:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.459856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.459856:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.459857:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677472.459859:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.459860:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.459861:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.459862:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677472.459865:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.459867:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.459869:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.459870:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677472.459870:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677472.459872:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.459872:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.459873:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.459874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:13.0:1619677472.459884:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.459886:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.459886:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.459888:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.459890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.459891:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677472.459997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.460000:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.460002:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.460004:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.460005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.460006:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.460007:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.460009:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.460010:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.460011:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.460012:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.460015:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.460017:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.460019:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.460020:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.460021:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.460022:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.460023:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.460024:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.460024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.460028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.460036:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.460038:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.460039:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.460041:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.460041:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.460043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.460352:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.460355:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.460356:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.460358:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.460361:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.460362:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.460364:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.460365:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.460366:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.460367:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.460370:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.460372:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.460375:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.460375:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.460376:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.460378:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.460378:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.460379:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.460379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.460384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.460392:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.460394:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.460395:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.460396:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.460398:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.460401:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.461054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.461061:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.461064:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.461065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.461068:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.461070:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.461071:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.461073:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.461074:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.461075:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.461076:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.461079:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.461081:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.461083:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.461084:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.461085:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.461087:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.461087:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.461088:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.461089:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.461100:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.461102:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.461103:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.461104:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.461105:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.461107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.461199:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.461201:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.461202:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.461204:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.461207:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677472.461208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677472.461208:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677472.461209:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.461211:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677472.461211:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.461212:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.461213:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677472.461214:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.461217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.461217:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.461218:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677472.461219:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.461220:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:20.0:1619677472.461221:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.461222:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.461223:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677472.461224:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677472.461224:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:20.0:1619677472.461225:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677472.461225:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:20.0:1619677472.461226:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.461227:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677472.461227:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.461228:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:20.0:1619677472.461229:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.461229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.0:1619677472.461231:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.461233:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.461234:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.461235:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.461236:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.461237:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.461238:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.461239:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.461242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.461250:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.461252:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.461253:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.461254:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.461255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.461256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.461263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.461270:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.461273:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.461274:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.461274:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.461275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.461276:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.461502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.461510:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.461512:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.461513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.461514:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.461516:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.461518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.462224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.462232:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.462235:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.462236:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.462237:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.462238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.462240:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.463216:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.463219:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.463220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.463222:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.463225:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.463226:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.463227:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.463228:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.463229:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.463230:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.463233:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.463234:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.463236:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.463236:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.463237:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.463238:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.463239:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.463240:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.463241:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.463242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.463252:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.463253:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.463254:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.463255:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.463256:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.463257:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.463257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.463257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.463258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.463259:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.463510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.463519:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.463521:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.463522:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.463524:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.463525:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.463527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.463957:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.463959:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.463960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.463960:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.463962:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.463964:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.463983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.463985:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.463986:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.463986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.463988:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.463988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.464168:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.464170:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.464171:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.464173:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.464174:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.464175:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.464176:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.464178:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.464178:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.464179:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.464182:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.464183:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.464186:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.464187:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.464188:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.464189:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.464190:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.464190:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.464191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.464205:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.464207:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.464209:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.464209:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.464211:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.464215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.464231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.464239:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.464241:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.464242:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.464244:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.464244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.464246:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.464249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.464257:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.464259:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.464260:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.464261:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.464262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.464263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.464662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.464668:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.464670:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.464672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.464673:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.464676:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.464677:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.464679:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.464680:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.464681:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.464682:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.464685:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.464687:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.464690:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.464691:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.464692:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.464693:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.464694:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.464695:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.464695:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677472.465022:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.465025:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.465026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.465029:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.465031:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.465033:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.465034:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.465035:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.465036:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.465037:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.465040:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.465042:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.465045:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.465046:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.465047:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.465048:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.465049:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.465049:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.465050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.465055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.465063:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.465065:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.465066:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.465068:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.465069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.465070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.465161:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.465164:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.465165:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.465167:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.465170:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.465171:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677472.465173:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.465174:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.465175:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.465176:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677472.465179:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.465181:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.465184:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.465185:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.465185:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677472.465187:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677472.465188:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.465189:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677472.465190:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677472.465191:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.465201:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677472.465203:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677472.465204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.465205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.2:1619677472.465211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.465216:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.465219:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.465221:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.465221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.465223:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.465225:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.465226:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.465227:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.465228:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.465229:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.465233:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.465234:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.465237:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.465238:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.465238:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.465240:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.465241:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.465241:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.465242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.465246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.465253:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.465256:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.465256:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.465258:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.465259:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.465260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.465266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.465273:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.465276:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.465277:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.465277:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.465278:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.465279:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.465922:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.465925:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.465926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.465928:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.465931:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.465932:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.465933:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.465934:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.465935:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.465936:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.465939:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.465941:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.465944:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.465944:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.465945:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.465947:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.465947:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.465948:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.465948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.465953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.465961:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.465964:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.465965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.465966:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.465968:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.465971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.466982:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.466985:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677472.466986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677472.466988:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.466991:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.466992:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677472.466994:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.466995:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.466996:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.466997:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677472.466999:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.467001:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677472.467004:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.467005:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677472.467005:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677472.467007:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677472.467008:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.467008:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677472.467010:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677472.467010:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.2:1619677472.467013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.467021:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:24.0:1619677472.467023:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:17.0:1619677472.467023:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.467024:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677472.467025:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677472.467025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:17.0:1619677472.467025:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.467027:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:17.0:1619677472.467027:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.467030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.468361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.468370:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.468373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.468373:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.468374:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.468376:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.468379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.468383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.468390:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.468393:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.468393:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.468394:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.468395:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.468396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.468401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.468407:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.468410:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.468412:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.468413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.468414:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.468416:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.468417:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.468418:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.468419:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.468421:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.468424:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.468426:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.468428:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.468429:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.468430:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.468432:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.468432:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.468434:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.468435:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.468436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.468438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.468441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.468445:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677472.468446:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677472.468447:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.468449:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.468450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.468450:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.468451:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.468452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.2:1619677472.468454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:1.0:1619677472.468454:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.468460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.468462:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.468465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.468466:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.468468:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.468470:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.468471:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.468801:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.468804:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.468805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.468807:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.468810:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.468812:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.468813:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.468814:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.468816:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.468817:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.468820:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.468821:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.468823:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.468823:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.468824:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.468825:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.468825:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.468826:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.468826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.468838:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.468840:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.468840:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.468841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.468842:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.468844:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.469098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.469108:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.469112:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.469113:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.469113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.469117:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.469121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.469229:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.469232:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.469233:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.469233:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.469235:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.469237:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677472.469482:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.469486:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.469488:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.469489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.469492:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.469496:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677472.469597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.469606:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.469609:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.469610:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.469611:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.469613:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.469617:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.470072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.470079:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.470082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.470082:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677472.470084:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.470087:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.470089:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.470090:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.470091:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.470093:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.470094:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.470097:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.470099:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.470102:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.470103:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.470103:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.470105:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.470106:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.470107:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.470108:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.470108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.470114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.470120:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.470122:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.470122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.470123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.470204:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.470206:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.470208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.470209:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.470211:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.470212:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.470214:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.470215:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.470216:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.470217:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.470220:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.470221:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.470224:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.470225:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.470225:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.470227:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.470228:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.470229:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.470230:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.470231:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677472.470241:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.470243:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677472.470244:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.470245:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.470245:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:2.0:1619677472.470245:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.470246:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.470247:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677472.470247:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.470251:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.470262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.470265:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.470265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.470266:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.470267:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.470267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.470354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.470360:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.470363:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.470364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.470365:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.470369:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.470370:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.470371:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.470373:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.470374:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.470375:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.470378:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.470380:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.470382:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.470383:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.470383:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.470385:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.470385:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.470386:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.470388:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.470388:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.470393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.470399:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.470400:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.470401:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.470401:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.470401:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.470402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.470403:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.470403:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.470404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.470404:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677472.470877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.470885:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.470888:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.470889:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.470889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.470891:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.470894:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.471692:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.471694:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.471695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.471697:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.471700:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.471701:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.471702:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.471704:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.471705:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.471706:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.471709:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.471710:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.471713:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.471714:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.471714:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.471716:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.471716:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.471717:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.471717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.471731:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.471734:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.471735:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.471735:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.471737:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.471740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.472803:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.472806:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.472808:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.472809:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.472810:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.472812:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.472813:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.472815:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.472816:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.472817:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.472820:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.472822:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.472825:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.472826:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.472826:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.472828:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.472829:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.472830:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.472830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677472.473341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.473349:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.473351:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.473352:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.473354:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.473355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.473357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.473781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.473789:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.473791:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.473793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.473795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.473795:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.473798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.473935:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.473937:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.473938:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.473939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.473940:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.473943:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677472.474323:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.474326:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.474327:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.474327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.474329:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.474333:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.474471:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.474474:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.474475:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.474476:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.474479:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.474480:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.474482:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96513515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.474483:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.474484:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.474485:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.474487:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.474489:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.2:1619677472.474491:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.474495:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.474496:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.474496:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.474497:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.474498:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.474499:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.474499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677472.474499:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.474502:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.474503:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.474503:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.474505:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.474509:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:1.0:1619677472.474509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.474511:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.474512:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.474513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.474514:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.474516:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.474528:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.474531:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.474531:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.474532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.474532:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.474533:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.474563:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.474566:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.474567:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.474567:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.474569:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.474572:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.474702:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.474705:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.474706:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.474707:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.474709:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.474717:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.474788:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.474790:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.474791:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.474791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.474793:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.474795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.475072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.475080:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.475083:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.475084:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.475086:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.475086:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.475089:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.475233:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.475235:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.475238:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.475238:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.475240:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.475241:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.475243:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.475244:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.475245:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.475246:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.475249:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.475250:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.475253:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.475253:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.475254:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.475256:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.475256:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.475257:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.475257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.475562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.475569:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.475572:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.475573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.475574:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.475577:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.475579:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.475580:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.475582:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.475583:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.475584:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.475587:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.475589:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.475591:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.475592:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.475593:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.475594:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.475595:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.475596:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.475596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.475601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.475609:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.475612:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.475612:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.475614:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.475615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.475617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:18.0:1619677472.476411:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.476414:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.476415:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.476417:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.476419:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.476421:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.476422:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.476424:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.476425:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.476426:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.476429:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.476431:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.476433:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.476434:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.476435:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.476436:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.476437:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.476438:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.476438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677472.476450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.476453:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.476453:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.476454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.476455:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.476457:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.476471:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.476474:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.476474:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.476474:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.476475:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.476475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.476489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.476497:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.476500:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.476501:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.476501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.476503:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.476506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677472.476581:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.476584:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.476585:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.476585:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.476587:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.476591:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.476859:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.476862:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.476863:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.476864:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.476866:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.476867:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.476869:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.476870:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.476871:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.476872:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.476875:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.476876:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.476879:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.476880:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.476880:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.476882:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.476883:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.476883:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.476884:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.476896:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.476898:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.476899:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.476900:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.476902:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.476905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.478428:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.478431:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.478432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.478434:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.478436:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.478438:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.478439:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.478440:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.478441:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.478442:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.478446:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.478447:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.478450:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.478451:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.478451:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.478453:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.478453:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.478454:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.478455:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.478456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.2:1619677472.478456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677472.478468:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.478469:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.478471:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677472.478471:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.478472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.478472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677472.478472:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.478473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677472.478476:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.478479:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.478486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.478492:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.478495:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.478496:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.478498:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.478501:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.478502:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.478504:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.478505:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.478506:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677472.478506:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:18.0:1619677472.478508:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.478508:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.478509:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.478511:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.478512:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.478513:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.478515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.478516:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.478516:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.478517:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.478517:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.478518:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.478518:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.478520:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.478520:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.478520:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.478521:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.478521:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.478521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.478522:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.478533:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.478534:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.478535:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.478536:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.478536:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.478537:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.478557:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.478559:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.478560:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.478560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.478561:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.478561:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.479044:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.479047:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.479050:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.479051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.479053:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.479054:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.479056:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.479057:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.479058:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.479059:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.479063:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.479065:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.479068:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.479069:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.479069:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677472.479072:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677472.479072:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.479073:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.479074:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.479280:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.479283:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.479284:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.479286:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.479289:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.479290:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.479292:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.479297:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.479298:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.479299:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.479302:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.479304:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.479306:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.479306:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.479307:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.479308:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.479309:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.479309:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.479310:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.479321:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.479322:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.479323:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.479324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.479325:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.479327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.479678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.479686:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.479688:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.479689:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.479691:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.479692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.479694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.479855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.479863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.479866:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.479867:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.479868:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.479869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.479872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:13.0:1619677472.480228:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.480233:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.480236:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.480236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.480238:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.480241:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677472.480243:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.480245:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.480250:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.480252:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677472.480257:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.480260:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677472.480264:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.480265:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677472.480266:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677472.480268:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677472.480269:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.480271:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677472.480273:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677472.480274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.480287:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677472.480290:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677472.480291:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.480292:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677472.480310:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.480314:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.480315:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.480315:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.480317:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.480321:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.481036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.481045:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.481048:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.481049:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.481050:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.481052:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.481055:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677472.481060:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.481063:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.481064:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.481067:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677472.481067:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.481070:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677472.481147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.481153:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.481156:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.481157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.481159:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.481161:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.481162:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.481164:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.481165:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.481166:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.481168:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.481171:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.481173:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.481175:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.481176:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.481177:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.481179:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.481179:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.481180:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.481181:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677472.481637:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.481640:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.481640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.481642:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.481645:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.481646:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677472.481648:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.481649:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.481650:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.481651:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.481654:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.481656:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.481658:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.481660:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.481660:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.481662:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.481663:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.481663:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.481664:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677472.482089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.482097:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.482099:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.482100:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.482102:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.482102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.482105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.482109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.482116:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.482118:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.482119:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.482120:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.482121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.482122:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.483057:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.483060:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.483061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.483062:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.483064:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.483066:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.483067:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.483068:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.483069:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.483070:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.483074:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.483075:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.483078:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.483079:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.483080:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.483081:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.483082:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.483083:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.483083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.483088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.483096:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.483099:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.483100:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.483100:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.483102:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.483105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.483107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.483115:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.483118:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.483118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.483119:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.483120:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.483120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.483243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.483249:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.483251:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.483254:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.483254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.483256:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.483258:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.483259:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.483260:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.483261:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.483262:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.483265:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.483267:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.483269:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.483270:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.483270:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.483272:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.483273:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.483274:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.483275:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.483278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.483286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.483288:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.483289:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.483291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.483291:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.483293:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.483299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.483307:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.483309:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.483310:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.483311:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.483311:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.483312:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.484336:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.484338:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.484339:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.484340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.484341:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.484344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.484440:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.484443:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.484444:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.484445:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.484448:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.484449:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.484450:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.484451:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.484452:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.484453:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.484456:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.484457:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.484459:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.484459:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.484460:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.484461:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.484461:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.484462:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.484462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.484472:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.484474:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.484475:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.484476:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.484476:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.484477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.485669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.485678:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.485681:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.485682:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.485683:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.485684:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.485687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.485981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.485987:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.485990:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.485991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.485993:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.485996:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.485998:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.485999:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.486001:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.486002:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.486003:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.486006:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.486008:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.486010:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.486010:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.486011:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.486013:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.486013:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.486014:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.486015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.486038:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.486040:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.486040:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.486041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.486042:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.486043:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.486357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.486363:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.486365:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.486368:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.486368:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.486370:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.486372:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.486373:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.486375:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.486376:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.486377:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.486380:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.486381:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.486384:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.486384:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.486385:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.486387:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.486387:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.486388:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.486389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.486393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.486400:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.486402:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.486403:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.486405:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.486406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.486407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.486917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.486924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.486927:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.486928:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.486930:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.486930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.486932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.487709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677472.487713:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.487716:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:21.0:1619677472.487719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677472.487719:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.487721:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.487723:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.487724:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.487725:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.487726:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.487727:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.487730:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.487732:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.487735:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.487735:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.487736:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.487738:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.487739:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.487739:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677472.487740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:8.2:1619677472.488097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.488106:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.488108:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.488109:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.488110:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.488111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.488113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.488340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.488347:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.488350:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.488351:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.488353:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.488356:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.488357:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.488358:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.488360:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.488361:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.488362:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.488365:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.488366:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.488369:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.488370:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.488371:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.488372:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.488373:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.488374:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.488374:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.488519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.488521:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.488522:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.488524:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.488526:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.488527:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.488528:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.488529:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.488530:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.488531:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.488534:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.488535:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.488537:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.488538:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.488538:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.488539:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.488540:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.488541:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.488542:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.488542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.488553:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.488554:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.488555:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.488556:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.488556:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.488557:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.488558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.488558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.488559:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.488559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.488574:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.488577:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.488577:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.488578:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.488578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.488579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677472.488782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677472.488789:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.488791:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677472.488792:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677472.488795:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.488797:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.488798:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677472.488800:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.488801:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.488802:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.488803:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677472.488806:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.488808:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677472.488812:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.488813:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677472.488813:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677472.488815:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677472.488816:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.488816:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677472.488817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.0:1619677472.489074:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.489076:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.489077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.489077:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.489079:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.489081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.489096:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.489098:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.489099:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.489100:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.489100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.489101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.489453:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.489455:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.489456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.489457:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.489459:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.489460:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.489461:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.489462:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.489463:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.489464:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.489467:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.489468:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.489471:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.489472:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.489472:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.489474:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.489474:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.489475:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.489475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.489480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.489488:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.489491:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.489492:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.489492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.489494:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.489497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.489600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.489603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.489605:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.489607:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.489609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.489610:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.489611:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677472.489612:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.489614:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.489615:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.489616:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677472.489619:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.489620:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.489623:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.489624:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677472.489624:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.489626:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.489627:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.489627:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.489628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677472.489639:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.489641:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.489642:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.489643:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.489644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.489645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:13.0:1619677472.489964:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.489967:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.489969:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.489972:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.489972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.489974:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677472.489975:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.489977:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.489977:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.489978:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677472.489981:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.489983:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677472.489985:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.489986:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677472.489987:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677472.489988:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677472.489989:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.489990:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.489990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677472.490888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.490897:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.490899:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.490900:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.490902:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.490903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.490905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.491052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.491060:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.491061:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.491061:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.491062:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.491062:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.491065:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.491065:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.491067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677472.491068:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.491068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.491069:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.491070:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.491071:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677472.491071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:21.0:1619677472.491072:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.491073:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.491074:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.491078:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.491080:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.491081:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.491081:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.491082:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677472.491083:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.491083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.491085:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.491085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:21.0:1619677472.491088:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.491088:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:21.0:1619677472.491089:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.491089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:21.0:1619677472.491090:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:18.0:1619677472.491091:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.491092:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.491092:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.491094:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:18.0:1619677472.491094:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.491095:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.491095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:18.0:1619677472.491096:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677472.491097:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.491099:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.491100:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.491101:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677472.491104:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.491105:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.491106:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:18.0:1619677472.491106:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677472.491107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.491108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.491109:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.491110:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.491111:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677472.491112:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677472.491113:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.491114:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.491115:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.2:1619677472.492307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.492313:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.492316:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.492317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.492319:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.492322:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.492324:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.492325:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.492326:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.492327:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.492328:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.492331:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.492333:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.492334:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.492335:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.492335:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.492337:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.492337:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.492338:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.492339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.492352:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.492354:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.492354:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.492355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.492356:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.492357:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.492551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.492554:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.492557:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.492559:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.492560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.492562:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.492563:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.492565:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.492566:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.492567:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.492568:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.492571:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.492573:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.492575:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.492576:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.492576:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.492578:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.492578:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.492579:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.492580:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.492581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.492584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.492587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.492590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.492592:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.492592:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.492593:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677472.492593:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.492594:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.492594:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.492595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.492596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.492597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.492670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.492682:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.492684:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.492685:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.492687:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.492688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.492690:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.493212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.493218:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.493221:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.493222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.493224:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.493226:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.493228:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.493229:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.493231:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.493231:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.493232:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.493235:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.493237:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.493239:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.493240:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.493241:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.493243:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.493244:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.493245:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.493246:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.493246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.493249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.493252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.493257:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:20.0:1619677472.493260:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:18.0:1619677472.493260:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.493260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.493262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677472.493263:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.493264:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.493265:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.493267:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.493268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677472.493527:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.493530:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.493531:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677472.493533:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.493536:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.493538:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677472.493539:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.493541:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.493542:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.493543:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677472.493546:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.493548:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677472.493550:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.493551:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677472.493552:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677472.493554:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677472.493554:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.493555:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.493555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677472.493567:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.493569:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.493570:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.493571:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.493573:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.493576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.493807:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.493809:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.493810:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.493810:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.493812:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.493814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.493948:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.493950:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.493951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.493953:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.493955:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.493956:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.493957:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.493958:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.493959:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.493960:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.493963:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.493964:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.493967:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.493968:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.493968:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.493970:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.493970:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.493971:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.493972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.493976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.493984:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.493987:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.493987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.493988:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.493990:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.493993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.494364:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.494366:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.494367:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.494369:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.494371:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.494373:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.494374:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.494375:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.494376:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.494377:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.494380:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.494382:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.494384:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.494385:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.494386:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.494388:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.494388:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.494389:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.494389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.494394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.494403:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.494405:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.494406:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.494408:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.494409:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.494410:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.494902:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.494905:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.494906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.494908:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.494911:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.494913:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.494914:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.494916:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.494917:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.494918:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.494921:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.494923:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.494925:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.494925:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.494926:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.494927:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.494928:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.494929:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.494930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.494941:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.494943:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.494943:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.494944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.494945:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.494947:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.494949:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.494952:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.494954:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.494955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.494956:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.494957:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.494958:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.494960:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.494961:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.494962:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.494965:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.494967:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.494969:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.494970:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.494971:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.494973:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.494973:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.494974:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.494975:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:12.0:1619677472.495960:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677472.495964:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677472.495968:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.111@o2ib7 : GET - for me
00000800:00000200:13.0:1619677472.495969:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677472.495972:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.111@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677472.495974:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.111@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677472.495976:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677472.495977:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677472.495980:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677472.495986:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677472.495989:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.495995:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.495996:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677472.495997:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.495999:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677472.496000:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.496011:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.496014:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.496015:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.496016:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: REPLY: OK
00000800:00000200:13.0:1619677472.496018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.496019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677472.496149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.496156:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.496159:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.496160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.496162:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.496165:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.496166:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.496167:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.496169:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.496170:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.496171:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.496175:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.496177:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.496181:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.496182:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.496183:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.496185:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.496185:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.496186:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.496187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.497016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.497024:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.497027:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.497028:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.497029:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.497031:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.497032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.497039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.497047:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.497049:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.497050:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.497051:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.497052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.497053:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.497696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.497703:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.497706:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.497706:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.497708:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.497712:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.497713:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.497714:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.497716:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.497717:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.497718:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.497721:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.497723:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.497726:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.497727:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.497728:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.497729:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.497730:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.497731:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677472.497732:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677472.497733:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677472.497738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.497743:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677472.497746:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677472.497746:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.497747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677472.498109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.498115:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.498118:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.498118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.498120:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.498123:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.498124:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.498125:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.498127:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.498128:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.498129:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.498133:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.498134:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.498137:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.498138:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.498138:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.498140:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.498140:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.498141:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.498141:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677472.498157:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.498160:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.498163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.498163:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.498165:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.498166:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.498168:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.498169:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.498170:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.498171:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.498175:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.498176:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.498179:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.498180:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.498180:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.498182:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.498183:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.498184:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.498185:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.498185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.498196:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.498198:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.498199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.498199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:17.0:1619677472.498667:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.498670:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.498670:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.498672:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.498674:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.498676:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.498677:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.498679:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.498680:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.498681:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.498685:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.498687:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.498689:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.498690:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.498690:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.498692:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.498693:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.498694:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.498695:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.498696:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.498699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.498708:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.498709:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677472.498710:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.498710:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.498711:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.498712:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.498713:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.498714:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.498715:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.498717:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.498828:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.498831:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.498832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.498832:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.498833:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.498836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.498901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.498909:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.498912:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.498912:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.498913:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.498915:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.498918:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.499033:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.499036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.499036:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677472.499038:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.499040:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.499042:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677472.499043:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.499044:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.499045:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.499046:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677472.499049:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.499051:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.499053:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.499054:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.499054:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.499056:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.499056:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.499058:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677472.499059:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677472.499059:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.499070:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677472.499072:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:8.2:1619677472.499072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.499075:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.499076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677472.499076:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.499078:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.499078:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.499079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.499080:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.499081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.499094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.499101:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.499104:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.499105:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.499105:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.499105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.499106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.499187:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.499190:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.499191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.499193:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.499195:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.499196:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.499198:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.499199:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.499200:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.499201:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.499204:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.499206:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.499208:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.499208:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.499209:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.499210:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.499210:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.499212:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.499213:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.499213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.499224:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.499225:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.499225:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.499227:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.499228:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677472.499228:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.499229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.499229:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.499230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.499230:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.500103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.500113:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.500116:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.500117:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.500119:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.500120:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.500121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.500122:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.500123:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.500123:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (23)++
00000800:00000200:21.0:1619677472.500124:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.500124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (24)--
00000800:00000200:21.0:1619677472.500125:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.500126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.500494:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.500496:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.500497:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.500499:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.500501:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.500502:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.500504:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.500505:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.500506:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.500507:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.500509:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.500511:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.500513:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.500514:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.500515:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.500516:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.500517:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.500517:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.500518:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.500522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.500531:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.500534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.500534:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.500535:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.500537:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.500540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.500660:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.500662:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.500665:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.500665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.500668:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.500669:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.500671:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.500672:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.500673:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.500674:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.500677:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.500679:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.500683:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.500683:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.500684:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.500686:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.500686:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.500687:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.500688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677472.500961:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.500963:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.500965:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.500966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.500968:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.500969:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.500971:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.500972:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.500973:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.500974:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.500977:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.500979:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.500981:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.500982:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.500983:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.500984:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.500985:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.500986:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.500987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.500997:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.500999:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.501000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.501001:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.501003:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.501005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.501019:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.501022:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.501022:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.501023:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.501024:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.501024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.501339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677472.501385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:13.0:1619677472.501388:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.501391:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.501392:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.501393:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.501395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.501396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677472.501407:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677472.501409:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677472.501410:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.501411:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677472.501411:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.501412:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.2:1619677472.502143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.502151:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.502154:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.502155:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.502156:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.502157:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.502160:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677472.503212:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.503215:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.503215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.503218:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.503219:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.503221:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.503222:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.503224:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.503225:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.503226:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.503229:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.503231:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.503233:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.503234:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.503235:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.503236:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.503237:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.503238:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.503238:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.503243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.503252:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.503254:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.503255:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.503257:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.503258:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.503259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.503855:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.503858:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.503861:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.503861:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.503864:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.503865:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.503867:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.503868:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.503869:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.503870:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.503874:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.503875:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.503879:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.503879:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.503880:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.503886:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.503887:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.503888:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.503888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.503895:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.503897:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.503899:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.503899:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.503902:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.503904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.503923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.503931:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.503934:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.503935:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.503937:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.503937:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.503940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.505033:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.505035:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.505036:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.505038:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.505041:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.505042:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.505044:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.505045:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.505046:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.505047:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.505050:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.505051:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.505054:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.505055:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.505055:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.505057:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.505057:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.505058:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.505059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.505063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.505071:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.505074:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.505075:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.505075:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.505078:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.505081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.505519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.505525:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.505528:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.505529:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.505532:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.505535:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.505536:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.505537:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.505538:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.505540:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.505541:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.505544:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.505545:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.505547:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.505548:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.505548:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.505550:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.505551:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.505552:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.505554:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.505554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.505559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.505566:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.505567:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.505567:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677472.505568:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.505569:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677472.505570:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.505570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.505571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.505571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.505572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.505588:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.505591:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.505592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.505592:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.505592:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.505593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677472.505955:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.505958:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.505958:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.505960:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.505962:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.505963:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677472.506161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.506167:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.506170:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.506170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.506172:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.506175:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.506176:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.506177:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.506179:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.506180:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.506181:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.506185:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.506186:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.506189:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.506190:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.506191:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.506193:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.506194:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.506195:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.506195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.0:1619677472.506583:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.506586:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.506588:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.506588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.506590:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.506592:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.506593:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.506595:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.506596:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.506597:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.506600:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.506602:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.506604:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.506605:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.506606:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.506608:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.506608:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.506609:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.506610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.506632:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.506635:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.506636:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.506636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.506638:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.506641:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.506759:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.506762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.506762:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.506764:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.506767:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.506768:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.506769:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.506770:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.506771:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.506772:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.506775:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.506776:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.506778:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.506779:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.506780:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.506781:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.506781:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.506782:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.506782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.506803:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.506805:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.506805:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.506807:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.506807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.506808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.506847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.506856:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.506859:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.506859:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.506860:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.506862:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.506865:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.507037:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.507040:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.507042:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.507043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.507045:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.507046:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.507047:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.507049:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.507050:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.507051:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.507055:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.507057:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.507059:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.507060:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.507061:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.507062:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.507063:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.507064:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.507066:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.507066:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677472.507074:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.507078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.507078:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677472.507079:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:8.0:1619677472.507079:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.507081:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:8.0:1619677472.507081:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.507082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.507083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.507084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.507096:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.507099:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.507100:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.507100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.507101:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.507101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.508719:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.508721:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.508724:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.508724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.508726:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.508728:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.508729:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.508731:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.508732:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.508733:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.508736:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.508737:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.508740:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.508741:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.508742:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.508743:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.508744:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.508745:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.508745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677472.509007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.509016:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.509019:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.509020:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.509022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.509022:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.509025:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.509481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.509489:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.509492:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.509493:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.509494:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.509495:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.509498:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.509502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.509510:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.509512:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.509514:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.509514:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.509515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.509516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.509804:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.509807:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.509809:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.509810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.509812:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.509813:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.509815:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.509816:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.509817:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.509818:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.509821:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.509823:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.509825:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.509825:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.509826:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.509828:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.509829:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.509829:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.509830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.510255:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.510258:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.510259:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.510260:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.510262:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.510264:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.510265:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.510266:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.510267:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.510268:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.510271:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.510272:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.510275:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.510276:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.510276:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.510278:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.510279:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.510280:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.510281:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.510282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.510284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.510293:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.510295:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.510298:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:8.0:1619677472.510299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.510302:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.510302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677472.510302:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.510303:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.510304:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.510307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.510311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.510313:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.510315:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.510316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.510316:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.510317:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.510317:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.510493:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.510496:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.510496:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.510498:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.510500:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.510502:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.510503:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.510504:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.510505:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.510506:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.510509:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.510511:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.510514:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.510515:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.510516:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.510518:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.510518:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.510519:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.510519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.510524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.510532:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.510535:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.510536:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.510537:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.510539:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.510540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.510814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.510821:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.510823:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.510825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.510827:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.510829:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.510831:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.510832:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.510834:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.510836:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.510837:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.510841:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.510843:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.510847:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.510848:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.510848:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.510850:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.510851:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.510852:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.510853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.511303:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.511306:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.511307:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.511309:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.511312:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.511314:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.511315:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.511316:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.511317:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.511318:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.511321:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.511323:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.511325:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.511325:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.511326:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.511327:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.511327:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.511328:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.511329:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.511339:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.511341:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.511342:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.511343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.511344:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.511344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.512069:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.512071:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677472.512072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677472.512073:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677472.512075:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.512076:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677472.512077:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.512078:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.512079:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.512080:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677472.512082:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677472.512084:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.512086:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.512087:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677472.512087:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677472.512089:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.512089:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.512090:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677472.512092:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677472.512092:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:20.2:1619677472.512096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.512104:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677472.512105:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:18.0:1619677472.512106:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:24.0:1619677472.512107:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677472.512107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:2.0:1619677472.512107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677472.512109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:18.0:1619677472.512110:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.512113:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.512114:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.512115:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.512117:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.512118:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.512119:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.512122:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.512124:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.512125:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.512126:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.512127:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.512128:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.512129:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.512130:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.512131:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.512142:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.512144:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.512145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.512145:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.512147:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.512148:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.512217:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.512220:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.512221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.512223:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.512225:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.512227:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.512228:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.512229:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.512230:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.512231:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.512234:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.512236:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.512239:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.512239:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.512240:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.512242:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.512242:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.512243:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.512243:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.512248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.512256:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.512259:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.512260:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.512260:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.512262:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.512265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.512816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.512824:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.512827:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.512828:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.512828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.512830:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.512833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.513584:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.513586:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.513587:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.513588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.513589:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.513592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.513796:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.513798:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.513801:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.513802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.513803:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.513804:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.513805:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.513807:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.513808:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.513809:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.513812:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.513814:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.513816:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.513817:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.513817:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.513819:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.513819:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.513820:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.513821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.515573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.515583:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.515588:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.515590:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.515593:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.515597:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.515599:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:8.2:1619677472.515600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:18.0:1619677472.515601:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.515603:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.515605:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.515606:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.515612:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:18.0:1619677472.515612:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.515614:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.515615:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.515615:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:21.0:1619677472.515617:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.515617:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:18.0:1619677472.515619:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.515620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.515620:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.515621:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.515623:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.515624:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.515626:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.515627:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.515628:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.515629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.515635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677472.515640:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.515643:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.515643:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.515643:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:20.0:1619677472.515645:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.515646:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.515646:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.515647:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677472.515648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:2.0:1619677472.515649:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677472.515919:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:1.0:1619677472.515922:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.14.8@o2ib3
00000800:00000200:17.0:1619677472.515923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000400:00000200:1.0:1619677472.515925:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.8@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.515928:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.515929:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:1.0:1619677472.515931:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.515932:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.515933:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.515934:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:1.0:1619677472.515938:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.515939:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:1.0:1619677472.515942:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.515943:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:1.0:1619677472.515944:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:1.0:1619677472.515945:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:1.0:1619677472.515946:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:1.0:1619677472.515947:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (22)--
00000800:00000200:1.0:1619677472.515947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)--
00000800:00000200:24.2:1619677472.515951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:24.2:1619677472.515975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:8.0:1619677472.516200:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.516203:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.516204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.516206:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.516209:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.516210:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.516211:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.516213:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.516214:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.516215:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.516218:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.516219:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.516222:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.516223:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.516224:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.516226:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.516226:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.516227:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.516227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.516242:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.516246:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.516248:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.516248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.516252:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.516256:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.516481:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:17.0:1619677472.516484:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.8@o2ib3
00000800:00000200:1.0:1619677472.516485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000400:00000200:17.0:1619677472.516487:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.8@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.516491:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.516493:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677472.516494:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)++
00000400:00000200:17.0:1619677472.516495:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677472.516496:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.8@o2ib3
00000400:00000200:17.0:1619677472.516497:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.516498:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.516499:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677472.516502:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.8@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.516503:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.516504:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.516505:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677472.516505:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677472.516507:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.516508:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.516508:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.516509:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:17.0:1619677472.516512:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.516514:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.516514:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677472.516514:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.516515:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677472.516516:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.516517:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (23)++
00000800:00000200:17.0:1619677472.516518:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (24)--
00000800:00000200:17.0:1619677472.516519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (23)--
00000800:00000200:1.0:1619677472.516519:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.516520:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)++
00000800:00000200:1.0:1619677472.516520:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677472.516520:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677472.516521:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.8@o2ib3
00000400:00000200:17.0:1619677472.516523:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.8@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.516523:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677472.516523:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (23)++
00000800:00000200:1.0:1619677472.516524:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (24)--
00000800:00000200:1.0:1619677472.516524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (23)--
00000400:00000200:17.0:1619677472.516525:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.516525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000400:00000200:17.0:1619677472.516526:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677472.516528:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.516529:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.516530:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.516530:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677472.516533:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.516535:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677472.516537:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.516538:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:17.0:1619677472.516538:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (23)++
00000800:00000200:17.0:1619677472.516540:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (24)--
00000800:00000200:17.0:1619677472.516540:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:17.0:1619677472.516542:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc3a00] (22)++
00000800:00000200:17.0:1619677472.516543:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (23)--
00000800:00000200:12.0:1619677472.516543:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (23)++
00000800:00000200:17.0:1619677472.516544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000800:00000200:12.0:1619677472.516546:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (24)--
00000400:00000200:12.0:1619677472.516547:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.516548:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.516552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677472.516553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677472.516554:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)++
00000800:00000200:1.0:1619677472.516557:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc3a00] (23)--
00000800:00000200:17.0:1619677472.516558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)--
00000800:00000200:1.0:1619677472.516558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000800:00000200:12.0:1619677472.516561:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.516564:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.516565:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.516566:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.516569:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.516570:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:1.0:1619677472.516691:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:1.0:1619677472.516693:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.8@o2ib3
00000800:00000200:17.0:1619677472.516694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000400:00000200:1.0:1619677472.516696:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.8@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.516699:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.516700:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.516701:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.516703:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.516704:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.516705:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.516707:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.516709:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.516711:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.516712:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.516712:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.516714:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.516714:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:1.0:1619677472.516715:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (22)--
00000800:00000200:1.0:1619677472.516715:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)--
00000800:00000200:8.2:1619677472.516721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.516729:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.516732:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.516732:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.516734:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.516735:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.516736:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.516796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:2.0:1619677472.516845:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.516848:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.516849:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.516849:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.516851:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.516855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677472.517089:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:17.0:1619677472.517092:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.8@o2ib3
00000800:00000200:1.0:1619677472.517093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000400:00000200:17.0:1619677472.517095:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.8@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.517097:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.517098:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.517100:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.517101:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.517102:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.517103:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.517106:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.517107:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.517110:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.517110:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.517111:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.517113:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.517114:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:17.0:1619677472.517115:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (22)--
00000800:00000200:17.0:1619677472.517115:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)--
00000800:00000200:1.0:1619677472.517212:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:1.0:1619677472.517215:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.8@o2ib3
00000800:00000200:17.0:1619677472.517216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000400:00000200:1.0:1619677472.517217:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.8@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.517220:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.517221:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677472.517222:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.517223:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.517224:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.517225:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677472.517228:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.517230:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677472.517232:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.517233:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677472.517233:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677472.517235:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677472.517236:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:1.0:1619677472.517237:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (23)--
00000800:00000200:1.0:1619677472.517238:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000800:00000200:1.0:1619677472.517239:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:1.0:1619677472.517241:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.8@o2ib3
00000800:00000200:17.0:1619677472.517242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000400:00000200:1.0:1619677472.517243:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.8@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.517245:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.517246:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677472.517247:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.517248:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.517249:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.517250:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677472.517253:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.517254:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677472.517256:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.517257:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677472.517257:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677472.517259:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677472.517259:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:1.0:1619677472.517260:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc3a00] (22)++
00000800:00000200:1.0:1619677472.517262:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (24)--
00000800:00000200:1.0:1619677472.517263:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (23)--
00000800:00000200:1.0:1619677472.517264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)++
00000800:00000200:8.2:1619677472.517265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677472.517266:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (23)++
00000800:00000200:17.0:1619677472.517267:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc3a00] (24)--
00000800:00000200:1.0:1619677472.517267:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.8@o2ib3
00000800:00000200:17.0:1619677472.517268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (23)--
00000800:00000200:17.0:1619677472.517269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000800:00000200:20.0:1619677472.517272:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:1.0:1619677472.517272:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.8@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.517274:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.517275:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.517275:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.517277:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:21.0:1619677472.517278:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.517278:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:1.0:1619677472.517279:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.517281:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.517282:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.517284:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.517286:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.517287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000400:00000200:1.0:1619677472.517289:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.517291:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.517299:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677472.517300:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)++
00000800:00000200:1.0:1619677472.517300:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.517301:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.8@o2ib3
00000800:00000200:1.0:1619677472.517301:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677472.517302:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677472.517302:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.517303:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.517304:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.517304:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677472.517304:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.517305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:17.0:1619677472.517305:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.8@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.517305:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (23)++
00000800:00000200:1.0:1619677472.517306:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (24)--
00000800:00000200:1.0:1619677472.517306:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (23)--
00000800:00000200:1.0:1619677472.517307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000400:00000200:17.0:1619677472.517308:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.517310:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677472.517311:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.517313:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.517314:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.517315:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677472.517318:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.517320:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.517321:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.517323:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:17.0:1619677472.517323:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.517323:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:18.0:1619677472.517324:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.517324:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.517325:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.517326:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.517326:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677472.517326:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:17.0:1619677472.517327:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (22)--
00000800:00000200:17.0:1619677472.517328:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)--
00000800:00000200:18.0:1619677472.517329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677472.517354:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:1.0:1619677472.517356:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.8@o2ib3
00000800:00000200:17.0:1619677472.517357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (22)--
00000400:00000200:1.0:1619677472.517360:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.8@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.517363:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.517364:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677472.517366:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.517367:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.517369:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.517370:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677472.517375:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.517378:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.517380:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.517380:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.517381:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.517383:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.517383:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (21)++
00000800:00000200:1.0:1619677472.517384:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc3a00] (22)--
00000800:00000200:1.0:1619677472.517385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc3a00] (21)--
00000800:00000200:8.2:1619677472.517389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.517397:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.517399:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677472.517399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:17.0:1619677472.517400:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.517401:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.517402:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.517402:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.517403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.517403:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.517404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:2.0:1619677472.517405:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.517411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.517413:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.517415:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.517416:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.517416:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.517417:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.517418:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.517663:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.517666:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.517667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.517669:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.517672:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.517674:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.517675:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.517676:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.517677:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.517678:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.517682:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.517683:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.517685:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.517685:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.517686:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.517687:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.517687:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.517688:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.517688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:21.0:1619677472.517705:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.517708:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:8.0:1619677472.517708:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:21.0:1619677472.517710:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677472.517710:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.517710:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.517711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677472.517712:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677472.517712:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.517713:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:21.0:1619677472.517714:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.517715:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.517715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:21.0:1619677472.517717:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.517718:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.517719:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.517722:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.517724:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.517727:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.517728:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.517728:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.517730:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.517730:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.517731:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.517732:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.517753:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.517755:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.517756:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.517756:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.517758:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.517761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.517884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.517892:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.517895:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.517895:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.517897:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.517898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.517900:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.518181:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.518184:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.518185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.518187:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.518189:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.518191:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.518192:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.518193:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.518194:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.518195:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.518198:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.518199:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.518201:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.518202:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.518202:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.518204:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.518204:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.518206:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.518207:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.518207:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.518218:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.518220:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.518220:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.518222:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.518222:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.518223:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.518224:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.518224:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.518225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.518225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.518237:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.518239:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.518240:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.518241:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.518241:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.518242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.518886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.518894:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.518897:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.518898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.518898:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677472.518901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:1.0:1619677472.518901:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.518904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.518906:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.518908:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.518909:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.518910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.518910:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.518910:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.519835:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.519837:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.519839:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.519840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.519842:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.519844:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.519845:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.519846:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.519847:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.519848:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.519852:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.519853:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.519856:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.519857:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.519857:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.519859:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.519860:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.519861:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.519862:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.519863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677472.519872:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.519872:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.519874:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:24.0:1619677472.519875:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677472.519875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:24.0:1619677472.519876:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.519876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.519876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.519878:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.519881:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.520149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.520157:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.520160:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.520161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.520163:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.520165:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.520166:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.520168:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.520169:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.520170:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.520171:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.520174:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.520176:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.520178:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.520179:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.520179:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.520181:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.520182:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.520183:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.520183:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.520194:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.520196:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.520197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.520197:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.520199:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.520200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.520704:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.520706:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.520708:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.520709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.520711:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.520712:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.520714:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.520715:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.520716:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.520717:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.520721:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.520723:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.520725:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.520726:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.520727:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.520728:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.520729:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.520730:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.520731:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677472.521214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.521217:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.521218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.521219:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.521221:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.521223:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677472.521224:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.521225:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.521226:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.521227:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677472.521230:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.521232:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.521234:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.521234:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.521235:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.521237:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.521237:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.521239:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677472.521240:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677472.521240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677472.521244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.521252:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.521253:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677472.521253:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.521254:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:1.0:1619677472.521254:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.521255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.521256:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.521256:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:1.0:1619677472.521257:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.521259:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.521343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.521351:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.521354:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.521355:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.521355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.521357:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.521357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.521523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.521531:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.521534:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.521535:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.521537:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.521537:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.521540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.521543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.521550:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.521553:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.521554:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.521554:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.521556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.521556:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.521967:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.521969:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.521971:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.521972:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.521973:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.521975:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.521976:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.521978:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.521979:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.521980:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.521983:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.521984:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.521986:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.521987:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.521988:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.521989:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.521990:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.521990:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.521991:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.522380:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.522382:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.522383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.522385:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.522387:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.522389:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.522390:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.522392:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.522393:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.522394:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.522398:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.522399:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.522402:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.522403:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.522404:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.522405:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.522406:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.522407:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.522407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.522422:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.522425:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.522425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.522426:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.522428:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.522431:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:12.0:1619677472.523153:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.523157:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.523161:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.523163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.523164:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.523166:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677472.523168:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.523171:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.523172:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.523174:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677472.523179:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.523182:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677472.523185:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.523187:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677472.523188:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677472.523190:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677472.523192:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.523193:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.523194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677472.523204:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.523206:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.523208:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.523208:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.523210:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.523214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.523408:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.523411:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.523412:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.523414:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.523418:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.523419:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.523420:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.523422:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.523423:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.523423:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.523427:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.523429:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.523431:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.523431:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.523432:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.523434:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.523434:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.523435:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.523436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.523446:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.523448:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.523448:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.523449:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.523450:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.523451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.523521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.523528:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.523531:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.523531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.523533:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.523535:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.523537:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.523538:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.523540:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.523541:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.523542:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.523545:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.523547:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.523550:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.523551:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.523552:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.523554:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.523554:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.523555:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.523556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.524478:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.524480:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.524482:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.524482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.524484:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.524486:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.524919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.524927:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.524930:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.524931:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.524931:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.524934:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.524938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.525534:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.525537:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.525538:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.525538:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.525540:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.525542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.525553:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.525556:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.525557:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.525557:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.525558:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.525558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.526707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.526715:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.526718:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.526719:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.526720:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.526721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.526724:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.526973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.526981:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.526984:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.526985:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.526986:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.526987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.526989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.527520:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.527523:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.527523:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.527524:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.527526:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.527528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.528131:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.528134:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.528135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.528136:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.528138:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.528139:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.528140:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.528141:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.528142:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.528143:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.528145:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.528147:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.528149:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.528149:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.528150:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.528151:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.528152:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.528152:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.528152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.528163:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.528164:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.528165:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.528166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.528167:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.528167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.528305:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.528307:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.528308:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.528309:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.528309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.528312:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.528324:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.528326:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.528327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.528327:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.528328:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.528329:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.528383:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.528385:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.528388:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.528388:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.528390:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.528391:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.528393:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.528394:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.528395:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.528396:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.528399:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.528401:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.528403:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.528404:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.528405:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.528406:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.528407:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.528408:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.528408:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.528608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.528617:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.528622:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.528624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.528626:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.528631:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.528634:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.528636:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.528638:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.528640:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.528641:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.528646:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.528649:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.528652:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.528654:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.528655:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.528657:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.528658:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.528659:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.528660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.528663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.528671:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.528674:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.528674:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.528676:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.528677:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.528678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.529028:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.529031:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.529032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.529035:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.529036:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.529037:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.529039:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96513317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.529040:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.529041:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.529042:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.529045:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.529046:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.529049:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.529050:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.529051:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (23)++
00000800:00000200:8.0:1619677472.529052:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)++
00000800:00000200:24.0:1619677472.529053:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677472.529053:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.529053:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (25)--
00000800:00000200:24.0:1619677472.529054:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.529054:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.529056:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677472.529056:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.529057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.529058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677472.529058:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000800:00000200:8.0:1619677472.529058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)--
00000400:00000200:24.0:1619677472.529059:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.529059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.529060:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.529062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.529070:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:20.2:1619677472.529071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.0:1619677472.529072:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.529073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.529076:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.529078:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.529079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677472.529079:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677472.529079:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.529080:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.529081:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.529082:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677472.529083:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.529085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.529088:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.529091:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.529093:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.529095:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.529098:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.529099:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.529101:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.529106:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.529108:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.529111:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.529113:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.529114:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.529116:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.529117:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.529118:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.529119:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.529131:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.529134:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.529135:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.529136:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.529138:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.529141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.529822:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.529825:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.529826:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.529828:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.529830:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.529831:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677472.529832:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.529834:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.529835:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.529836:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.529839:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.529841:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.529843:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.529844:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.529845:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.529846:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.529847:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.529847:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.529848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677472.530661:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.530664:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677472.530666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677472.530667:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.530669:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.530670:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677472.530672:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.530673:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.530674:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.530675:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677472.530678:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.530680:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677472.530683:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.530683:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677472.530684:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677472.530686:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677472.530686:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.530687:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.530688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.0:1619677472.531033:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.531035:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.531036:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.531036:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.531038:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.531040:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.531094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.531103:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.531105:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.531106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.531107:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.531109:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.531112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.531923:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.531925:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.531928:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.531928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.531931:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.531932:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677472.531934:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.531935:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.531936:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.531937:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677472.531940:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.531942:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.531945:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.531945:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677472.531946:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.531948:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.531948:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.531950:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.531951:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.531952:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.531961:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.531963:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.531963:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.531964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.531964:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.531967:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.531968:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.531969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.531970:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.531973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.531982:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.531984:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.531985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.531986:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.531987:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.531987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.532662:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.532665:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.532666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.532667:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.532670:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.532671:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.532672:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96513306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.532674:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.532675:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.532676:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.532678:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.532679:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.532680:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677472.532680:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.532682:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.532682:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.532684:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.532684:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.532685:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (23)++
00000800:00000200:8.0:1619677472.532685:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.532686:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.532686:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.532687:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.532688:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.532688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.532698:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.532700:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.532700:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.532701:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.532702:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.532703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.532770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.532778:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.532780:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.532781:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.532783:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.532784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.532786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.533404:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.533408:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.533409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.533411:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.533413:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.533414:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.533415:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.533417:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.533418:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.533419:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.533422:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.533424:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.533426:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.533426:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.533427:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.533428:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.533429:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.533430:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.533430:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.533441:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.533442:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.533443:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.533444:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.533445:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.533446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.533460:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.533462:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.533463:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.533463:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.533463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.533464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.533605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.533612:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.533615:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.533615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.533617:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.533620:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.533621:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.533623:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.533624:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.533625:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.533626:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.533629:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.533631:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.533635:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.533635:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.533636:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.533638:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.533638:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.533639:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.533640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.0:1619677472.534059:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.534061:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.534063:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.534064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.534065:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.534067:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.534068:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.534069:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.534070:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.534071:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.534075:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.534076:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.534079:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.534079:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.534080:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.534082:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.534083:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.534084:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.534084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:17.0:1619677472.534215:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677472.534218:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:1.0:1619677472.534218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:17.0:1619677472.534220:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.534222:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.534224:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677472.534225:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.534227:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.534228:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.534229:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677472.534232:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.534234:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677472.534237:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.534238:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677472.534239:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677472.534240:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677472.534241:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677472.534241:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:17.0:1619677472.534242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:20.0:1619677472.534253:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.534256:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.534256:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.534258:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.534259:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.534261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677472.534522:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.534524:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.534526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.534526:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.534528:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.534531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.534699:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.534701:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.534702:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.534702:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.534704:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.534706:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.535036:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.535039:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.535040:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.535041:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.535043:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.535044:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.535045:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.535047:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.535047:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.535048:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.535052:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.535053:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.535056:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.535057:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.535057:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.535059:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.535059:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.535060:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.535060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.535073:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.535075:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.535076:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.535077:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.535079:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.535082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.535484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.535492:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.535495:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.535496:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.535497:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.535498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.535500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.535502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.535510:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.535512:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.535513:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.535514:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.535515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.535516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.535750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.535756:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.535759:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.535760:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.535761:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.535763:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.535765:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.535766:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.535767:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.535768:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.535769:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.535772:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.535774:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.535775:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.535776:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.535777:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.535778:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.535779:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.535780:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.535781:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.535782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.535787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.535793:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.535794:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.535794:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677472.535795:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.535795:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.535795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.535797:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.535797:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.535798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.535798:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677472.535870:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.535873:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.535874:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.535876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.535877:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.535878:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.535880:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.535881:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.535882:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.535883:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.535886:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.535888:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.535890:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.535891:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.535891:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.535894:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.535894:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.535895:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.535896:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.535897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.535908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.535910:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.535911:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.535912:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.536300:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.536303:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.536304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.536304:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.536306:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.536309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.536600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.536609:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.536611:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.536612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.536613:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.536615:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.536618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.537076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.537084:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.537087:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.537088:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.537088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.537090:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.537093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.537118:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.537120:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.537123:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.537123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.537125:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.537127:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.537128:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.537129:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.537130:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.537131:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.537135:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.537136:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.537139:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.537140:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.537140:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.537142:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.537143:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.537144:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.537145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677472.537911:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.537913:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.537914:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.537915:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.537916:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.537919:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.538034:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677472.538036:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:24.0:1619677472.538037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.538037:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:1.0:1619677472.538039:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.538040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:8.0:1619677472.538040:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.538042:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.538043:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.538044:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.538045:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.538046:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.538046:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677472.538048:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.538048:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.538050:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.538050:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.538052:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.538053:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677472.538053:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677472.538058:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677472.538060:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.538061:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677472.538063:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.538064:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.538065:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:8.0:1619677472.538066:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.538066:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.0:1619677472.538067:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.538067:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.538068:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677472.538068:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:8.0:1619677472.538069:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.538069:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.538069:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677472.538070:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.538071:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.538072:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.538073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.538075:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.538082:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.538085:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.538085:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.538086:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.538086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.538087:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.538088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677472.538088:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.538089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677472.538092:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.538097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.538105:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.538107:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.538108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.538108:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.538109:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.538110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.538374:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.538376:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.538377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.538377:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.538379:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.538382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.538648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.538650:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.538653:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.538654:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.538655:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.538657:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.538658:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.538659:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.538660:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.538661:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.538664:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.538666:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.538668:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.538669:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.538670:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.538672:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.538672:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.538673:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.538674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.538684:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.538686:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.538687:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.538687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.538689:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.538692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.538702:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.538705:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.538705:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.538706:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.538707:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.538707:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.538875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.538888:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.538890:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.538891:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.538893:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.538894:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.538896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:13.0:1619677472.538932:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.538934:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.538935:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.538937:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.538939:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.538940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677472.539007:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.539010:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.539011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.539013:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.539016:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.539017:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.539019:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.539020:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.539021:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.539022:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.539025:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.539026:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.539028:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.539029:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.539029:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.539031:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.539031:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.539032:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.539033:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.539043:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.539045:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.539045:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.539046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.539047:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.539049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.539954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.539960:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.539963:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.539964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.539966:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.539968:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.539970:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.539971:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.539973:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.539974:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.539975:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.539978:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.539980:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677472.539981:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.539983:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:2.0:1619677472.539983:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677472.539984:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.539984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677472.539984:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.539985:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:24.0:1619677472.539986:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.539986:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.539987:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677472.539988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.539988:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.539989:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.539990:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.539995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.540001:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.540003:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.540003:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.540004:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.540103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.540106:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.540108:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.540110:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.540112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.540113:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.540114:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.540116:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.540117:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.540118:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.540119:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.540121:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.540123:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.540125:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.540126:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.540127:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.540128:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.540129:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.540130:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.540131:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.540135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.540143:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.540145:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.540145:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.540147:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.540148:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.540149:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.540155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.540163:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.540166:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.540167:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.540167:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.540168:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.540169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.540518:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.540521:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.540523:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.540524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.540525:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.540526:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.540528:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.540529:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.540530:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.540531:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.540534:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.540535:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.540538:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.540539:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.540539:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.540541:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.540542:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.540542:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.540543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.540555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.540557:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.540558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.540559:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.540561:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.540563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.541466:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.541469:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.541470:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.541471:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.541472:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.541474:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.541475:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.541476:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.541477:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.541478:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.541481:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.541482:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.541485:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.541486:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.541487:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.541488:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.541489:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.541489:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.541490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.541504:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.541507:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.541508:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.541508:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.541510:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.541513:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.541522:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.541525:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.541526:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.541526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.541527:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.541528:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677472.541943:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.541946:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.541947:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.541949:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.541952:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.541955:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677472.541959:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.541960:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.541961:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677472.541962:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.541964:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677472.542006:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.542008:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.542009:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.542009:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.542012:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.542014:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.542030:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.542032:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.542033:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.542033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.542034:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.542034:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.542338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.542340:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.542343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.542343:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.542345:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.542347:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.542348:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.542349:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.542350:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.542351:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.542354:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.542356:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.542359:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.542359:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.542360:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.542362:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.542363:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.542363:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.542364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.542375:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.542378:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.542378:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.542379:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.542381:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.542383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.542557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.542564:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.542567:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.542569:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.542571:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.542574:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.542575:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.542577:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.542578:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.542579:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.542581:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.542585:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.542587:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.542589:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.542590:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.542591:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.542593:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.542593:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.542594:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.542595:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.542606:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.542608:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.542609:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.542610:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.542611:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.542613:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.542866:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.542869:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.542870:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.542871:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.542874:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.542876:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.542877:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.542878:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.542879:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.542880:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.542884:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.542886:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.542892:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.542893:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.542894:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.542895:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.542896:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.542897:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.542898:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.542902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.542909:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677472.542911:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677472.542911:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.542912:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.542913:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677472.542913:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:8.0:1619677472.542914:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.542914:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.542915:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.542916:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:17.0:1619677472.542917:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.542920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.543237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.543245:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.543247:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.543248:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.543250:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.543250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.543253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.543314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677472.543317:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.543320:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677472.543322:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.543323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677472.543324:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.543326:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.543327:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.543329:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.543330:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.543331:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.543334:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.543336:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.543338:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.543339:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.543340:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.543341:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.543342:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.543342:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677472.543343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:20.2:1619677472.543912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.543917:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.543920:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.543922:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.543922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.543924:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.543925:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.543927:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.543928:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.543929:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.543930:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.543933:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.543935:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.543936:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.543937:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.543938:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.543939:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.543940:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.543941:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.543942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.543946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.543954:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.543956:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.543957:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.543959:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.543960:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.543961:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.544047:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.544050:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.544051:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.544051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.544053:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.544055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.544413:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.544416:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.544417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.544419:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.544421:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.544423:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.544424:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.544425:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.544426:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.544427:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.544430:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.544432:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.544433:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.544434:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.544435:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.544436:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.544436:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.544438:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.544439:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.2:1619677472.544441:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677472.544445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.544446:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.544448:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.544448:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.544449:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:24.0:1619677472.544450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.544450:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.544451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.544452:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:21.0:1619677472.544452:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:24.0:1619677472.544453:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:21.0:1619677472.544453:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.544454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.544454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:21.0:1619677472.544455:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.544455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.544457:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.544804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.544811:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.544814:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.544815:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.544817:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.544817:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.544820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.544878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.544881:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.544883:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.544883:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.544886:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.544887:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.544889:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.544890:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.544891:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.544892:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.544895:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.544897:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.544900:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.544900:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.544901:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.544903:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.544903:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.544904:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.544905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677472.545756:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.545758:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.545759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.545759:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.545761:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.545763:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.545773:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.545776:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.545776:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.545777:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.545777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.545778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.546753:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.546755:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.546757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.546758:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.546759:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.546760:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.546761:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.546763:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.546763:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.546764:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.546767:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.546769:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.546772:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.546772:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.546773:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.546775:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.546775:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.546776:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.546776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.546781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.546792:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.546795:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.546797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.546797:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.546800:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.546803:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:1.0:1619677472.546804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:13.0:1619677472.546806:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.546807:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.546809:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.546811:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.546813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677472.546831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.546840:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.546844:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.546846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.546849:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.546852:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.546854:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.546856:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.546859:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.546860:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.546862:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.546868:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.546870:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.546874:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.546875:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.546876:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.546878:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.546879:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.546881:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.546881:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.546884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.546892:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.546894:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.546895:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.546897:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.546898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.546900:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.546956:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.546959:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.546960:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.546960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.546962:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.546964:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677472.547475:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.547478:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.547478:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.547480:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.547482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.547483:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677472.547495:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677472.547497:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677472.547498:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.547498:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677472.547499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.547500:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:18.0:1619677472.547656:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.547659:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.547660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.547663:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.547666:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.547667:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677472.547669:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.547670:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.547671:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.547673:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677472.547677:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.547679:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.547682:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.547683:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.547684:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677472.547686:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677472.547687:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.547688:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.547688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:8.0:1619677472.548004:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.548007:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.548008:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.548010:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.548012:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.548013:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.548014:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.548015:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.548016:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.548017:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.548020:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.548022:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.548024:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.548025:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.548026:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.548027:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.548028:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.548028:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.548029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.548033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.548041:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.548044:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.548045:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.548045:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.548047:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.548050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.548160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.548168:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.548170:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.548172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.548172:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.548174:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.548177:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677472.548406:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.548409:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677472.548411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677472.548412:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.548415:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.548417:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677472.548419:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.548421:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.548422:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.548423:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677472.548427:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.548429:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.548432:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.548433:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.548433:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677472.548435:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.548436:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.548437:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677472.548438:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677472.548439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:13.0:1619677472.548447:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.548449:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677472.548450:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000400:00000200:13.0:1619677472.548450:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.548452:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000400:00000200:13.0:1619677472.548452:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.548453:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:13.0:1619677472.548454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677472.548454:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:12.0:1619677472.548456:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677472.549219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.549228:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.549231:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.549232:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.549232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.549234:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.549237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.549240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.549241:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.549244:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.549244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.549245:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.549246:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.549246:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.549646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:24.0:1619677472.549776:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.549778:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.549780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.549781:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.549784:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.549785:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.549786:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.549787:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.549788:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.549789:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.549792:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.549793:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.549795:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.549796:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.549796:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.549798:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.549798:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.549799:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.549799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.549810:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.549812:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.549812:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.549813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.549814:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.549815:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.549973:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.549975:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.549976:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.549977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.549978:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.549980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.550473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.550485:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.550487:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.550488:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.550490:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.550491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.550493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.550494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.550502:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.550505:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.550506:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.550506:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.550507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.550508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.551205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.551208:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.551210:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.551212:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.551213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.551215:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.551216:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.551217:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.551218:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.551220:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.551220:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.551224:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.551226:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.551228:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.551229:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.551230:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.551231:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.551232:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.551233:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.551233:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.551237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.551244:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.551247:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.551247:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.551249:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.551250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.551251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.551329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.551336:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.551339:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.551340:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.551342:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.551345:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.551347:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.551348:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.551349:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.551350:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.551351:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.551355:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.551357:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.551360:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.551361:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.551362:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.551363:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.551364:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.551365:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.551365:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.551536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677472.551538:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.551541:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677472.551543:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.551545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677472.551545:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.551547:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677472.551548:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.551549:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.551550:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.551551:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677472.551554:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.551556:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.551558:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.551559:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677472.551560:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.551561:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.551562:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.551563:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677472.551564:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677472.551565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677472.551569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.551573:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.551574:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677472.551574:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.551574:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.551576:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677472.551576:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677472.551576:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.551578:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.551579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.551580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:21.0:1619677472.551598:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.551600:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.551601:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.551602:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.551602:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.551603:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.551727:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.551729:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.551730:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.551730:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.551732:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.551734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.551745:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.551747:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.551748:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.551748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.551749:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.551750:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.551837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.551842:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.551845:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.551846:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.551848:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.551850:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.551852:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.551853:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.551854:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.551855:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.551856:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.551860:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.551861:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.551863:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.551864:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.551864:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.551866:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.551867:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.551868:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.551868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.551879:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.551881:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.551882:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.551882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.551883:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.551885:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677472.553131:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.553134:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.553135:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.553137:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.553140:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.553141:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.553143:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.553144:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.553145:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.553146:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.553149:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.553151:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.553153:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.553154:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.553155:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.553156:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.553157:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.553158:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.553158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.553163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.553172:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.553174:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.553175:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.553177:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.553177:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.553179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:13.0:1619677472.553620:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.553622:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.553623:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.553624:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.553626:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.553627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677472.553634:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.553637:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.553638:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.553639:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.553642:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.553644:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.553645:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.553646:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.553647:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.553649:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.553652:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.553654:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.553660:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.553660:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.553661:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.553663:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.553664:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.553665:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.553667:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.553667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677472.553678:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.553681:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677472.553681:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677472.553682:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:18.0:1619677472.553682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.553682:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.553683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.553684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.553684:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:2.0:1619677472.553684:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.553685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.553685:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.553686:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.553688:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.553688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.553691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.554324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.554327:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.554330:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.554332:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.554333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.554334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.554335:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.554337:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.554338:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.554339:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.554340:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.554343:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.554345:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.554347:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.554348:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.554348:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.554350:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.554350:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.554351:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.554353:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.554353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.554356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.554359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.554363:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.554364:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.554364:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.554365:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677472.554365:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.554366:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.554366:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.554368:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.554369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.554370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677472.554392:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.554394:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.554397:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.554397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.554399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.554401:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.554402:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.554403:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.554404:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.554405:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.554408:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.554410:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.554413:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.554414:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.554414:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.554416:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.554417:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.554418:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.554418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677472.554535:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8200] (21)++
00000800:00000200:20.0:1619677472.554537:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.65@o2ib3
00000800:00000200:21.0:1619677472.554540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8200] (22)--
00000400:00000200:20.0:1619677472.554540:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.65@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.554542:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.554543:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677472.554545:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.554547:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.554548:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.554549:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677472.554552:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.554554:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.554556:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.554556:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677472.554557:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.554559:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.554559:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8200] (21)++
00000800:00000200:20.0:1619677472.554560:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8200] (22)--
00000800:00000200:20.0:1619677472.554561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8200] (21)--
00000800:00000200:21.0:1619677472.554572:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.554574:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.554574:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.554576:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.554577:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.554579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.554589:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.554592:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.554593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.554594:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.554597:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.554598:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.554599:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.554601:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.554602:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.554603:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.554605:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.554607:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.554608:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.554609:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.554610:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.554611:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.554611:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.554612:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.554612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.554622:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.554624:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.554624:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.554625:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.554626:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.554627:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.555146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677472.555149:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.555152:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.555152:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.555153:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:18.0:1619677472.555153:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.555155:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.555156:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.555157:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.555159:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.555159:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.555159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.555162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.555167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:2.0:1619677472.555171:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.555174:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.555175:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677472.555175:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.555176:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.555177:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677472.555177:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.555177:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.555178:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.555179:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.555180:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.555181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.555282:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.555284:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.555285:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.555285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.555287:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.555289:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.556502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.556510:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.556513:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.556514:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.556514:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.556516:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.556519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.556869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.556879:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.556883:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.556885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.556888:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.556892:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.556895:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.556897:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.556899:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.556901:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.556902:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.556907:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.556910:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.556914:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.556915:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.556916:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.556919:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.556920:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.556921:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.556922:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.557187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.557190:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.557191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.557191:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.557192:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.557195:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.557212:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.557215:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.557216:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.557216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.557217:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.557217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.557932:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.557936:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.557936:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.557938:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.557940:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.557942:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.557943:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.557946:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.557947:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.557947:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.557950:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.557952:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.557955:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.557955:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.557956:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.557958:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.557958:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.557959:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.557960:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.557961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.557964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.557971:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.557973:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.557974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677472.557975:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.557975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.557977:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.557978:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.557980:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.557980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.557982:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.558548:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.558551:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.558552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.558552:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.558554:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.558557:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.558769:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.558772:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.558772:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.558773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.558774:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.558776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.559446:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.559449:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.559450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.559451:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.559453:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.559455:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.559456:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.559458:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.559459:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.559460:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.559463:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.559465:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.559466:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.559467:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.559468:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.559469:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.559470:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.559470:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.559471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.559482:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.559484:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.559484:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.559485:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.559486:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.559487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.560212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.560220:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.560223:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.560224:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.560225:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.560227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.560228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.560249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.560255:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.560258:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.560259:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.560261:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.560264:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.560265:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.560266:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.560268:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.560269:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.560270:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.560273:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.560275:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.560277:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.560277:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.560278:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.560280:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.560281:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.560281:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.560282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.560293:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.560297:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.560301:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.560301:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.560303:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.560304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.560366:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.560369:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.560370:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.560371:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.560373:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.560374:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.560376:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.560377:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.560378:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.560379:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.560382:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.560383:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.560385:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.560386:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.560386:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.560387:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.560388:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.560388:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.560389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.560399:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.560401:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.560402:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.560403:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.560403:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.560404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.560714:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.560716:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.560717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.560717:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.560719:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.560721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.560735:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.560737:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.560738:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.560738:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.560739:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.560739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.561342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.561351:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.561354:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.561355:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.561355:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.561357:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.561360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.562532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.562541:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.562544:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.562545:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.562545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.562547:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.562550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.562611:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.562613:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.562614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.562614:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.562616:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.562618:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.562729:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.562731:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677472.562732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677472.562734:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.562736:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.562738:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677472.562739:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.562741:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.562742:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.562743:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677472.562746:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.562747:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677472.562751:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.562752:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677472.562753:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677472.562755:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677472.562755:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.562755:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.562756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677472.562768:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.562771:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.562772:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.562774:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.562776:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.562777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677472.562963:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.562966:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.562968:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.562970:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.562971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.562972:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677472.562973:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.562975:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.562976:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.562977:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677472.562980:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.562982:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677472.562985:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.562986:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:13.0:1619677472.562986:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:13.0:1619677472.562988:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677472.562989:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.562990:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.562990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677472.562995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.563003:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.563006:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.563006:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.563007:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.563009:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.563012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677472.563457:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.563460:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.563460:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.563462:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.563465:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.563466:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.563468:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.563469:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.563470:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.563471:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.563474:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.563476:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.563479:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.563480:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.563481:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.563482:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.563483:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.563484:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.563484:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.563489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.563499:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.563501:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.563502:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.563503:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.563506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.563506:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677472.564486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677472.564492:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677472.564495:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677472.564496:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677472.564498:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677472.564500:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.564502:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677472.564504:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.564505:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.564506:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.564507:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677472.564510:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677472.564512:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677472.564515:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.564516:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677472.564517:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677472.564518:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677472.564519:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:21.0:1619677472.564520:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677472.564520:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677472.564520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:21.0:1619677472.564523:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.564525:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.564526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.564527:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.564533:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.564534:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.564536:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.564537:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.564538:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.564542:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.564545:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.564548:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.564549:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.564549:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.564551:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.564552:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.564553:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.564554:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.564555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.564564:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.564566:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.564567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.564568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677472.564578:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.564580:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.564581:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.564581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.564583:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.564585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.564588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.564594:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.564597:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.564598:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.564599:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.564602:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.564604:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.564605:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.564606:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.564607:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.564608:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.564611:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.564613:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.564616:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.564617:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.564618:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.564620:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.564620:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.564622:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.564623:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.564623:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.564629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.564635:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.564637:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.564638:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.564638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.564869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.564872:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.564873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.564875:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.564876:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.564878:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.564879:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.564880:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.564881:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.564882:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.564885:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.564887:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.564889:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.564889:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.564890:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.564891:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.564892:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.564893:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.564894:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.564894:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.564905:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.564906:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.564906:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.564908:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.564909:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.564910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.564910:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.564910:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.564911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.564911:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.565578:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.565581:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.565582:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.565583:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.565584:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.565585:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.565586:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.565587:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.565588:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.565589:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.565591:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.565592:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.565593:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.565594:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.565594:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.565595:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.565596:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.565596:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.565597:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.565607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.565609:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.565610:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.565611:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.565611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.565612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.565625:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.565627:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.565628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.565628:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.565628:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.565629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.566179:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.566181:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.566182:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.566183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.566184:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.566186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677472.566363:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.566364:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.566367:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.566369:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.566372:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.566372:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.566373:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.566375:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.566375:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.566376:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.566376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677472.566594:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:20.0:1619677472.566630:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.566632:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.566633:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.566635:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.566636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.566638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.566892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.566900:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.566902:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.566903:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.566905:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.566905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.566908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.567328:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.567331:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.567332:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.567333:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.567335:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.567336:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.567338:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.567339:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.567340:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.567341:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.567344:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.567346:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.567349:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.567349:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.567350:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.567352:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.567352:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.567353:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.567353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.567368:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.567371:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.567372:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.567372:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.567374:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.567378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.567502:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.567504:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677472.567507:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.567510:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.567511:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.567513:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.567514:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.567515:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.567516:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.567519:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.567520:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.567522:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.567523:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.567523:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.567525:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.567525:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.567526:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.567527:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.567537:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.567539:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.567540:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.567541:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.567542:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.567545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.567559:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.567562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.567562:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677472.567563:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.567564:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.567565:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.567882:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.567884:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.567886:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.567888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.567889:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.567890:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.567891:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.567893:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.567893:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.567894:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.567897:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.567899:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.567901:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.567902:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.567902:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.567904:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.567905:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.567906:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.567907:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.567908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.567918:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.567920:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.567921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.567922:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.568122:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.568124:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.568125:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.568126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.568127:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.568129:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.568263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677472.568266:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.568269:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.568270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.568274:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.568276:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.568278:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.568279:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.568281:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.568284:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.568285:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.568289:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.568291:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.568293:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.568294:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677472.568299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:8.0:1619677472.568301:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677472.568301:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677472.568303:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677472.568303:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.568303:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:24.0:1619677472.568304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677472.568304:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000400:00000200:8.0:1619677472.568304:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.568304:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.568305:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:8.0:1619677472.568306:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.568307:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.568309:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677472.568309:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:21.0:1619677472.568312:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.568312:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:8.0:1619677472.568312:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:21.0:1619677472.568314:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677472.568314:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:26.0:1619677472.568315:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:21.0:1619677472.568315:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.568315:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.568316:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.568317:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677472.568318:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:21.0:1619677472.568318:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.568318:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.568319:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677472.568322:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:26.0:1619677472.568323:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.568323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:21.0:1619677472.568324:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677472.568325:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.568326:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:26.0:1619677472.568327:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.568328:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96513173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.568329:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677472.568330:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677472.568330:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:8.2:1619677472.568330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:26.0:1619677472.568331:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677472.568331:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:26.0:1619677472.568332:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677472.568333:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677472.568334:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.568334:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000400:00000200:26.0:1619677472.568335:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.568335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:21.0:1619677472.568336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677472.568337:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.0:1619677472.568338:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.568338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.568339:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.568340:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.568340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677472.568341:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.568341:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.568342:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.568344:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.568344:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.568345:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.568345:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:12.0:1619677472.568347:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.568350:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.568351:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.568353:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.568355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.568356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:26.0:1619677472.568455:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677472.568481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.568488:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.568491:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.568492:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.568493:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.568494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.568496:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.568501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.568508:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.568511:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.568512:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.568513:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.568513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.568514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.568564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.568572:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.568573:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.568574:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.568576:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.568577:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.568579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677472.568623:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677472.568626:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:26.0:1619677472.568628:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.568632:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.568633:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677472.568635:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.568636:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.568637:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.568638:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677472.568641:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.568642:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677472.568645:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.568646:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677472.568647:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677472.568648:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677472.568649:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677472.568650:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:26.0:1619677472.568651:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:26.0:1619677472.568652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:8.2:1619677472.568654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.568663:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.568665:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.568666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.568667:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.568669:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.568671:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.568678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.568685:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.568688:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.568689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.568689:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.568690:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.568691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.569090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.569097:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.569100:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.569101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.569102:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.569106:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.569107:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.569108:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.569109:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.569110:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.569111:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.569115:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.569117:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.569120:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.569121:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.569122:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.569123:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.569124:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.569125:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.569125:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.569503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.569509:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.569512:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.569512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.569514:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.569517:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.569519:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.569520:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.569521:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.569522:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.569523:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.569525:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.569527:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.569528:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.569529:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.569529:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.569531:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.569531:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.569533:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.569534:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.569534:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.569540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.569546:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.569548:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.569548:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.569548:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.569549:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.569549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.569550:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.569550:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.569551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.569552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.569741:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.569744:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.569745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.569746:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.569749:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.569751:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.569752:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96513166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.569754:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.569755:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.569756:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.569758:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:24.0:1619677472.569759:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.569760:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.569761:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.569761:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.569762:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.569763:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.569764:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.569764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.569765:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (23)++
00000800:00000200:8.0:1619677472.569765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.569766:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.569767:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.569768:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.569768:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.569778:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.569780:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.569780:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.569781:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.569782:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.569783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.569795:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.569797:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.569798:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.569799:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.569799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.569799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.569845:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.569847:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.569848:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.569850:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.569851:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.569852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.570014:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.570016:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.570018:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.570019:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.570021:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.570023:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.570024:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.570025:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.570026:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.570027:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.570031:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.570033:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.570036:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.570037:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.570038:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.570040:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.570040:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.570041:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.570042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677472.570538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.570540:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.570541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.570543:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.570545:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.570547:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.570548:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.570549:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.570550:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.570551:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.570555:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.570556:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.570560:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.570560:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.570561:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.570563:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.570563:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.570564:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.570565:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.570566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.570569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.570577:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.570579:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.570580:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677472.570581:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677472.570581:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.570581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.570582:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.570583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677472.570583:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.570586:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.570589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.570597:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.570599:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.570600:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.570600:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.570601:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.570602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.570771:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.570773:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.570774:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.570776:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.570777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.570778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.570790:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.570792:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.570793:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.570794:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.570794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.570795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.571671:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.571673:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.571674:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.571676:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.571677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.571678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.571764:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.571766:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.571767:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.571768:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.571769:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.571772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.571941:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.571943:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.571946:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.571946:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.571948:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.571950:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.571951:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.571952:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.571953:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.571954:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.571958:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.571959:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.571961:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.571962:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.571963:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.571964:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.571965:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.571966:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.571966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:18.0:1619677472.571972:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.571975:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.571976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.571978:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.571980:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.571982:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677472.571983:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.571985:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.571986:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.571987:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677472.571991:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.571993:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.571996:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.571997:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.571998:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677472.572000:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677472.572000:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.572001:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.572002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:26.0:1619677472.572013:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.572016:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.572017:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.572019:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.572022:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677472.572322:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677472.572356:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.572358:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.572359:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.572360:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677472.572360:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.572361:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:24.0:1619677472.572361:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.572362:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.572363:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.572363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.572610:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.0:1619677472.572736:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.572738:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.572739:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.572741:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.572742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.572743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.573066:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.573069:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677472.573072:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.573075:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.573076:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.573077:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.573079:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.573079:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.573081:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.573084:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.573085:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.573087:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.573088:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.573089:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.573090:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.573091:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.573092:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.573092:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.573103:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.573105:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.573106:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.573107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.573108:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.573110:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.573175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.573177:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.573178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.573179:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.573182:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.573184:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.573185:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.573186:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.573187:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.573188:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.573191:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.573192:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.573195:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.573196:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.573196:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.573198:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.573199:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.573199:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.573200:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.573212:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.573215:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.573216:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.573216:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.573218:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.573221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.573236:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.573239:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.573240:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.573240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.573241:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.573242:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.573276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.573284:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.573285:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677472.573286:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.573288:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.573288:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.573288:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677472.573290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.573290:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:17.0:1619677472.573291:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.573299:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:17.0:1619677472.573300:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.573301:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.573303:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.573304:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.573305:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.573306:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.573309:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.573311:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.573314:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.573314:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.573315:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.573317:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.573317:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.573318:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.573319:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.573323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.573332:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.573334:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.573334:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.573336:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.573338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.573339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.573612:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.573614:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.573615:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.573617:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.573618:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.573619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.573955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.573960:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.573963:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.573965:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.573965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.573968:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.573969:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.573970:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.573972:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.573973:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.573974:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.573977:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.573979:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.573981:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.573982:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.573982:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.573984:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.573984:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.573985:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.573986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.573990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.573997:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.574000:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.574000:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.574002:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.574003:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.574004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.574009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.574017:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.574019:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.574020:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.574020:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.574021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.574022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.574270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.574277:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.574279:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.574281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.574282:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.574285:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.574287:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.574288:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.574289:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.574290:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.574291:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.574300:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.574301:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.574305:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.574305:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.574307:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.574309:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.574310:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.574310:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.574311:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677472.574327:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.574342:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.574344:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.574345:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.574346:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.574347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.0:1619677472.574469:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.574471:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.574472:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.574474:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.574475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.574476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.574494:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.574497:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.574498:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.574498:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.574499:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.574500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.574608:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.574611:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.574612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.574614:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.574617:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.574619:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.574620:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.574621:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.574622:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.574623:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.574626:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.574628:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.574631:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.574632:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.574632:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.574634:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.574634:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.574635:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.574635:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.574640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.574649:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.574651:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.574652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.574653:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.574655:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.574658:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.574763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.574774:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.574777:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.574778:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.574780:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.574780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.574783:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.575353:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.575356:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.575357:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.575357:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.575359:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.575362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.575522:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.575524:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.575525:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.575526:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.575527:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.575529:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.575583:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.575585:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.575587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.575588:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.575592:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.575593:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.575594:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.575596:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.575596:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.575598:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.575600:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.575602:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.575605:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.575606:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.575607:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.575608:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.575609:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.575609:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.575610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.575614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.575623:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.575625:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.575626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.575627:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.575629:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.575631:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.575726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677472.575732:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.575734:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677472.575736:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.575737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677472.575739:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.575741:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677472.575742:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.575743:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.575744:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.575745:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677472.575749:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.575750:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.575753:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.575753:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.575754:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.575756:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.575757:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.575757:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677472.575758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.575762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.575770:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.575772:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.575773:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.575774:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.575775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.575776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.575958:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.575961:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.575962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.575962:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.575964:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.575966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.576167:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.576169:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677472.576171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677472.576172:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.576174:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.576176:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.576177:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.576178:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.576179:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.576180:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.576183:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.576185:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.576187:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.576188:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.576189:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.576190:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.576191:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.576191:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.576192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:2.0:1619677472.576204:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.576207:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.576208:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.576208:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.576211:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.576213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.576230:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677472.576232:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677472.576234:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677472.576235:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.576238:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.576239:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677472.576240:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.576241:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.576242:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.576243:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677472.576245:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.576247:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677472.576248:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.576249:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677472.576249:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.576250:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.576251:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677472.576251:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677472.576251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:24.0:1619677472.576262:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.576263:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.576264:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.576265:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.576266:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.576267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.576286:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.576288:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.576289:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.576289:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.576290:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677472.576290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.576395:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.576397:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.576398:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.576400:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.576401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.576402:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.576411:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.576413:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.576414:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.576414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.576417:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.576419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.576477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.576484:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.576487:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.576487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.576490:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.576492:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.576493:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.576495:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.576496:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.576497:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.576498:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.576502:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.576503:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.576506:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.576507:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.576508:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.576509:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.576510:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.576511:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.576511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677472.576514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:2.0:1619677472.576523:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.576525:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.576526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.576526:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.576529:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.576530:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677472.576549:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.576552:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.576552:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.576554:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.576556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.576557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677472.576860:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.576863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.576863:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677472.576866:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.576868:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.576869:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677472.576871:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.576872:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.576873:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.576874:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677472.576876:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.576878:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:1.0:1619677472.576878:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.576880:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.0:1619677472.576881:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.576881:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677472.576882:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677472.576883:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.576883:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:20.0:1619677472.576884:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:1.0:1619677472.576884:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677472.576885:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:20.0:1619677472.576886:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677472.576886:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:20.0:1619677472.576887:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677472.576887:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.576887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:20.0:1619677472.576888:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.576889:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.576892:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.576894:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.576897:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.576898:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.576898:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677472.576899:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:20.0:1619677472.576900:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.576901:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:18.0:1619677472.576901:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:20.0:1619677472.576902:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.576902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:18.0:1619677472.576902:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.576902:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.576904:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.576907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.576912:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.576916:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.576916:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.576917:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.576919:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.576922:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.576932:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.576934:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.576935:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.576935:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.576936:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.576937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.577258:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.577260:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.577261:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.577262:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.577263:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.577265:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.577422:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.577425:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677472.577428:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.577430:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.577432:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.577433:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.577435:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.577435:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.577437:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.577440:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.577442:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.577445:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.577445:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.577446:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.577448:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.577448:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.577449:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.577449:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677472.577649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.577681:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.577683:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.577684:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.577685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.577686:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.577688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.577719:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.577721:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:26.0:1619677472.577723:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.577726:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.577727:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677472.577728:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.577730:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.577731:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.577731:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677472.577734:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.577735:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.577737:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.577737:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677472.577738:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.577739:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.577739:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.577741:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.577742:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.577742:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.577757:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.577758:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.577759:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.577760:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.0:1619677472.578322:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.578325:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.578326:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.578328:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677472.578328:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.578331:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.578338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677472.578340:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.578345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.578347:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.578347:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.578348:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.578349:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677472.578349:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.578352:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.578353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.578356:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.578358:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.578360:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677472.578361:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.578363:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.578363:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.578364:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677472.578367:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.578369:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677472.578371:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677472.578372:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.578373:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:17.0:1619677472.578373:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677472.578373:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:24.0:1619677472.578374:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.578374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.578376:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.578376:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677472.578376:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.578377:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.578377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677472.578379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.578388:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.578390:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.578392:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.578394:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.578397:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677472.578584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.578586:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.578589:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.578591:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.578592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.578594:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.578595:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.578596:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.578598:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.578599:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.578600:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.578604:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.578605:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.578607:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.578608:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.578608:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.578610:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.578611:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.578611:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.578612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.578616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.578624:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.578626:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.578627:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.578629:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.578629:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.578631:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.578767:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.578770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.578770:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.578771:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.578772:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.578775:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.579174:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.579176:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.579177:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.579179:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.579180:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.579181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677472.579291:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.579294:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.579299:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.579302:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.579305:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.579307:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.579308:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.579309:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.579310:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.579311:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.579315:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.579316:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.579319:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.579320:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.579320:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.579322:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.579322:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.579324:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.579325:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.579325:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.579328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.579337:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.579339:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677472.579340:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.579340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.579340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.579342:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.579343:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.579344:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.579346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.579347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.579408:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.579412:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.579413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.579414:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.579416:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.579418:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.579419:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.579421:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.579422:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.579423:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.579426:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.579428:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.579431:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.579432:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.579432:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.579434:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.579434:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.579436:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.579437:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.579438:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.579440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.579449:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677472.579450:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677472.579451:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.579452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.579452:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677472.579452:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.579453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677472.579454:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.579455:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677472.579457:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.579963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.579971:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.579973:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.579974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.579975:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.579977:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.579979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.579982:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.579983:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.579986:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.579987:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.579987:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.579987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.579988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.580020:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.580023:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.580023:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.580025:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.580026:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.580027:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.580415:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.580418:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.580419:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.580419:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.580421:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.580423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.580446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677472.580451:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.580454:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.580456:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.580457:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.580459:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.580460:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677472.580462:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.580463:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.580464:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.580465:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677472.580468:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.580470:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677472.580473:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.580474:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677472.580475:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677472.580476:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677472.580477:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.580478:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677472.580479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:26.0:1619677472.580493:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:8.0:1619677472.580674:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.580676:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.580677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.580679:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.580681:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.580683:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.580684:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.580685:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.580686:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.580687:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.580690:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.580692:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.580695:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.580696:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.580696:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.580698:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.580699:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.580699:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.580700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.580712:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.580715:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.580716:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.580717:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.580718:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.580722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.580732:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.580734:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.580735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.580736:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.580737:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.580737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.581057:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.581059:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.581060:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.581062:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.581063:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.581064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:12.0:1619677472.581064:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.581068:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.581072:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.581074:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.581076:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.581078:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677472.581080:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.581083:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.581084:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.581086:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677472.581091:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.581095:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677472.581098:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.581100:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677472.581101:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677472.581103:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677472.581104:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.581106:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677472.581107:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677472.581108:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.581123:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677472.581126:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677472.581127:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.581129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:26.0:1619677472.581137:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.581140:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.581141:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.581143:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.581145:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677472.581161:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.2:1619677472.581621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.581629:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.581632:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.581632:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.581634:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.581635:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.581638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.581640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.581648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.581650:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.581651:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.581652:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.581652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.581653:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.581747:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.581749:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.581751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.581752:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.581756:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.581757:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.581758:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677472.581758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:24.0:1619677472.581760:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.581761:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.581761:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677472.581762:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.581764:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:24.0:1619677472.581766:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.581766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.581767:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.581769:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.581770:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677472.581770:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.581771:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.581772:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677472.581772:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:24.0:1619677472.581773:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.581773:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677472.581773:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.581774:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677472.581774:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677472.581775:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677472.581775:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.581776:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.581780:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.581781:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.581784:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.581784:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.581785:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677472.581785:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.581787:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.581787:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677472.581787:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677472.581788:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677472.581788:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.581789:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.581789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677472.581790:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.581792:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677472.581797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.581800:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.581803:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.581803:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.581805:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.581806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.581807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.581922:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.581924:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.581925:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.581926:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.581927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.581929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.581939:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.581941:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.581942:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.581943:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.581944:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.581944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.582403:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.582406:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677472.582409:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.582411:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.582413:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.582414:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.582415:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.582417:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.582417:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.582421:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.582423:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.582425:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.582426:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.582427:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.582428:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.582429:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.582430:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.582431:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.582431:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.2:1619677472.582436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677472.582441:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:2.0:1619677472.582443:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677472.582444:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.582445:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.582445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677472.582447:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.582447:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:24.0:1619677472.582448:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.582450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:2.0:1619677472.582450:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.582453:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.582455:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.582456:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.582458:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.582459:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.582460:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.582463:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.582465:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.582468:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.582469:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.582470:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.582471:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.582472:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.582473:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.582473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.582474:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.582477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.2:1619677472.582477:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677472.582482:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.582483:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.582485:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.582486:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677472.582487:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.582489:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.582489:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.582491:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.582492:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.582494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.582495:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.582497:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.582498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.582499:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.582500:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.582500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.582799:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.582801:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.582802:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.582804:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.582805:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.582806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.583155:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.583158:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677472.583160:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.583163:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.583164:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677472.583166:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.583167:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.583168:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.583169:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.583173:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.583174:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677472.583177:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.583178:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.583179:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.583181:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.583181:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.583182:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.583182:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677472.583195:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677472.583196:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677472.583198:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.583198:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.583199:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.583202:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.583210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.583211:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.583212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.583213:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.583214:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.583215:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.583216:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677472.583217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.583223:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.583226:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.583227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.583229:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.583232:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.583233:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.583234:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.583236:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.583237:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.583239:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.583242:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.583244:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.583246:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.583247:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.583247:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.583249:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.583249:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.583250:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.583251:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.583262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.583264:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.583265:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.583265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.583267:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.583268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677472.583545:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.583549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.583549:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677472.583552:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.583554:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.583556:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677472.583557:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.583558:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.583560:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.583561:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677472.583564:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.583566:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677472.583569:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.583570:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:1.0:1619677472.583571:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677472.583573:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677472.583573:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.583574:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677472.583575:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677472.583576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:13.0:1619677472.583586:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:17.0:1619677472.583587:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:13.0:1619677472.583588:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:17.0:1619677472.583589:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:13.0:1619677472.583589:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.583590:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.583591:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:13.0:1619677472.583591:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.583593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.583594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677472.583604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:12.0:1619677472.583607:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677472.583610:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.583611:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.583612:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677472.583612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.583613:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677472.583651:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.583654:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.583655:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.583655:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.583657:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.583661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.583845:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.583848:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.583848:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.583850:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.583851:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.583853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677472.584633:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.584637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.584637:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.584639:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.584642:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.584644:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.584647:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.584649:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.584650:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.584651:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.584654:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.584657:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.584659:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.584660:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.584661:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.584662:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.584663:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.584663:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.584664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.584669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.584678:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.584680:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.584681:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.584683:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.584685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.584686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.584687:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.584689:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.584690:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.584691:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.584691:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.584694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.584798:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.584800:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.584801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.584801:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.584803:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.584805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.585151:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.585154:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677472.585157:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.585159:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.585161:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.585162:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.585163:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.585164:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.585165:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.585168:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.585170:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.585173:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.585174:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.585174:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.585176:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.585176:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.585177:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.585179:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.585179:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.585195:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677472.585415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.585418:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.585420:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.585422:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.585423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.585424:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.585426:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.585427:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.585428:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.585429:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.585431:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.585434:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.585436:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.585438:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.585439:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.585439:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.585441:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.585442:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.585443:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.585444:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.585445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.585447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.585450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.585453:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.585455:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.585455:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.585455:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.585456:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.585457:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.585457:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.585458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.585459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.585460:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677472.585571:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.585574:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.585574:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.585576:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.585577:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.585578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.585590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.585592:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.585593:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.585594:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.585595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.585595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.2:1619677472.585905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.585913:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.585916:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.585917:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.585917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.585919:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.585922:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.585983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.585986:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.585987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.585989:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.585991:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.585992:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.585994:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.585995:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.585996:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.585997:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.586000:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.586002:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.586003:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.586004:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.586005:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.586006:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.586006:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.586007:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.586007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.586018:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.586020:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.586021:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.586021:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.586022:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.586023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.586557:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.586559:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.586560:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.586562:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.586563:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.586564:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.586582:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.586585:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.586586:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.586586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.586588:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.586590:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.586922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.586929:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.586932:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.586933:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.586935:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.586937:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.586939:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.586940:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.586942:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.586943:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.586944:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.586947:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.586950:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.586952:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.586953:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.586954:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.586955:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.586956:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.586957:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.586958:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.586959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.586961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.586964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677472.586970:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677472.586970:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:21.0:1619677472.586972:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:2.0:1619677472.586972:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:21.0:1619677472.586973:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.586973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.586974:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:21.0:1619677472.586975:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.586975:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677472.586976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677472.586977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677472.586985:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.586988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.586988:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.586989:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.586991:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.586994:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.587006:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.587009:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.587009:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.587011:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.587013:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.587014:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.587015:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.587017:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.587018:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.587019:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.587022:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.587023:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.587025:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.587025:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.587026:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.587027:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.587028:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.587029:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.587031:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.587031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.587041:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.587043:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.587043:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.587045:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.587045:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.587046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.587046:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.587047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.587047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.587048:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.587062:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.587064:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.587065:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.587065:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.587065:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.587066:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.587390:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.587392:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.587393:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.587395:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.587396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.587397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.587535:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.587580:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.587582:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.587583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.587583:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.587585:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.587587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.587602:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.587605:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.587605:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.587606:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.587606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.587607:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.588230:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.2:1619677472.588231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:21.0:1619677472.588232:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.588232:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.588234:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.588236:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.588237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677472.588239:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.588242:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.588243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.588245:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.588247:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.588249:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677472.588251:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.588252:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.588253:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.588254:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677472.588258:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.588260:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677472.588263:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.588264:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677472.588265:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677472.588267:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677472.588267:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.588268:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.588269:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:13.0:1619677472.588280:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.588283:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.588284:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.588286:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.588287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.588289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677472.588491:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.588493:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.588494:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.588494:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.588496:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.588498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.588585:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.588588:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677472.588590:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.588593:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.588594:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.588595:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.588596:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.588597:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.588598:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.588601:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.588603:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.588605:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.588605:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.588606:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.588608:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.588608:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.588609:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.588609:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.588629:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.588631:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.588632:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.588633:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.588634:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.588636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.588751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.588754:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.588756:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.588758:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.588759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.588760:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.588761:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.588762:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.588763:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.588765:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.588765:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.588769:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.588770:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.588773:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.588773:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.588774:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.588775:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.588776:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.588777:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.588777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.588781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.588789:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.588791:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.588792:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.588794:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.588794:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.588796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.588800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.588807:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.588810:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.588811:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.588811:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.588812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.588813:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.589218:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.589220:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.589221:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.589223:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.589223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.589224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.589236:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.589238:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.589239:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.589240:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.589240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.589241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.589939:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.0:1619677472.590079:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.590081:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.590082:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.590084:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.590084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.590086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.590170:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.590172:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.590173:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.590174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.590175:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.590177:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.590296:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.590299:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.590301:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.590301:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.590303:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.590304:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.590306:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.590307:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.590308:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.590309:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.590312:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.590314:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.590316:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.590317:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.590318:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.590319:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.590320:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.590321:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.590321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.590902:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.590904:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.590905:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.590905:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.590907:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.590910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.590959:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.590961:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.590962:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.590964:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.590964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.590966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677472.591088:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.591091:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.591091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.591093:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.591096:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.591098:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.591099:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.591100:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.591101:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.591102:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.591106:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.591107:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.591110:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.591111:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.591111:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.591113:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.591113:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.591114:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.591115:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.591119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.591127:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.591129:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.591131:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.591132:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.591133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.591135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.591552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.591558:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.591561:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.591562:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.591564:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.591567:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.591569:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.591570:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.591571:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.591572:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.591573:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.591576:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.591578:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.591580:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.591581:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.591581:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.591583:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.591584:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.591585:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.591585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.591598:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.591600:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.591600:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.591601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.591602:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.591603:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.591665:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.591668:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.591670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.591671:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.591674:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.591675:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.591676:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.591678:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.591679:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.591680:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.591682:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.591684:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.591686:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.591687:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.591688:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.591689:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.591690:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.591690:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.591691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.591700:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.591702:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.591703:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.591704:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.591705:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.591706:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.591712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.591717:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.591720:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.591721:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.591723:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.591725:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.591726:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.591727:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.591729:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.591730:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.591731:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.591734:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.591736:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.591738:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.591739:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.591740:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.591741:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.591742:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.591743:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.591743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.591748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.591756:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.591758:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.591759:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.591761:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.591761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.591763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.591768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.591776:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.591778:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.591779:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.591780:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.591781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.591782:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677472.591796:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.591798:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.591800:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.591801:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.591802:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.591804:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.591805:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.591806:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.591807:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.591808:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.591811:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.591813:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.591816:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.591817:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.591817:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.591819:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.591820:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.591820:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.591821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:21.0:1619677472.591972:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.591974:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.591975:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.591977:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.591978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.591979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.592107:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.592109:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.592110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.592110:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.592113:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.592115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.592126:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677472.592128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.592129:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.592129:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.592131:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.592131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.592132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.592137:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.592140:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.592141:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.592141:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.592143:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.592146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.592281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.592284:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.592286:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.592289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.592289:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.592291:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.592292:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.592299:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.592300:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.592301:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.592302:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.592305:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.592307:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.592310:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.592310:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.592311:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.592312:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.592313:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.592314:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.592314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.592319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.592327:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.592330:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.592330:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.592332:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.592333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.592334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.592554:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.592557:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.592559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.592560:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.592562:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.592563:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.592565:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.592566:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.592567:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.592568:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.592571:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.592573:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.592574:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.592575:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.592576:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.592577:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.592578:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.592578:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.592579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.592589:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.592591:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.592592:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.592593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.592593:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.592595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.592655:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.592657:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:26.0:1619677472.592660:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.592663:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.592664:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677472.592665:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.592667:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.592668:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.592668:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677472.592671:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.592673:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.592674:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.592675:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677472.592675:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.592676:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.592677:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.592678:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.592678:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677472.592692:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.592693:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.592693:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.592695:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.592696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.592697:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.592698:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677472.592722:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.592724:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.592725:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.592726:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.592727:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677472.592727:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.592729:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677472.592729:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.592730:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.592731:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677472.592830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.592833:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.592833:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.592835:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.592837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.592837:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.592852:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.592855:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.592856:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.592856:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.592857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.592858:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.593006:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.593009:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.593010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.593010:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.593012:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.593014:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.593029:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.593045:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.593048:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.593049:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.593050:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.593051:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677472.593096:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.593099:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.593100:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.593100:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.593100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.593101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.593220:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.593223:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.593225:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.593225:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.593227:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.593228:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.593230:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.593231:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.593232:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.593233:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.593237:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.593238:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.593241:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.593241:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.593242:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.593243:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.593244:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.593245:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.593246:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.593247:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.593256:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.593258:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.593259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.593259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.0:1619677472.593676:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.593678:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.593679:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.593681:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.593682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.593683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.593760:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.593763:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.593764:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.593764:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.593766:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.593768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.594191:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:21.0:1619677472.594193:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.16@o2ib3
00000400:00000200:21.0:1619677472.594196:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.16@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.594196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000400:00000200:21.0:1619677472.594198:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.594200:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.594201:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.594203:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.594204:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.594205:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.594208:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.594210:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 344 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.594213:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.594214:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.594214:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.594216:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.594217:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:21.0:1619677472.594218:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e84c1e00] (22)++
00000800:00000200:21.0:1619677472.594220:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1e00] (23)--
00000800:00000200:21.0:1619677472.594221:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000800:00000200:24.0:1619677472.594228:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.594229:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)++
00000800:00000200:24.0:1619677472.594230:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677472.594231:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e84c1e00] (23)--
00000800:00000200:8.0:1619677472.594231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.594232:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.594232:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000800:00000200:20.0:1619677472.594233:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (21)--
00000400:00000200:24.0:1619677472.594234:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.594236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.594360:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:20.0:1619677472.594363:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.16@o2ib3
00000400:00000200:20.0:1619677472.594365:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.16@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.594366:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000400:00000200:20.0:1619677472.594367:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.594369:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677472.594370:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.594372:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.594373:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.594374:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.594377:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.594379:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.594381:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.594382:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.594382:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.594384:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.594385:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:20.0:1619677472.594386:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1e00] (22)--
00000800:00000200:20.0:1619677472.594386:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (21)--
00000800:00000200:21.0:1619677472.594400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:18.0:1619677472.594400:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677472.594403:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.16@o2ib3
00000800:00000200:18.0:1619677472.594403:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.594404:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.594404:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:21.0:1619677472.594405:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.16@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.594405:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000400:00000200:18.0:1619677472.594406:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.594407:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.594409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:21.0:1619677472.594410:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:21.0:1619677472.594411:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.594412:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.594413:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.594414:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677472.594418:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.594418:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.594419:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677472.594421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.594421:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.594422:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677472.594423:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.594423:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.594423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.594424:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677472.594424:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677472.594426:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677472.594427:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:21.0:1619677472.594428:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1e00] (22)--
00000800:00000200:21.0:1619677472.594428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (21)--
00000800:00000200:20.0:1619677472.594434:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:20.0:1619677472.594436:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.16@o2ib3
00000400:00000200:20.0:1619677472.594438:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.16@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.594439:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)++
00000800:00000200:12.0:1619677472.594439:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:21.0:1619677472.594440:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.16@o2ib3
00000400:00000200:20.0:1619677472.594441:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.594442:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:21.0:1619677472.594443:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.16@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.594443:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:12.0:1619677472.594443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.594444:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.594445:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:20.0:1619677472.594446:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.594447:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677472.594447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:20.0:1619677472.594448:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677472.594452:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.594452:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.594453:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677472.594453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:21.0:1619677472.594454:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.594456:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.594457:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.594458:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.594459:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.594462:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.594464:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.594466:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.594467:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.594467:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.594467:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677472.594468:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.594468:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677472.594470:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.594470:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677472.594471:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1e00] (23)++
00000800:00000200:20.0:1619677472.594471:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1e00] (23)++
00000800:00000200:20.0:1619677472.594472:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1e00] (25)--
00000800:00000200:21.0:1619677472.594473:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e84c1e00] (24)++
00000800:00000200:20.0:1619677472.594473:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (24)--
00000800:00000200:20.0:1619677472.594474:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1e00] (24)++
00000800:00000200:21.0:1619677472.594475:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1e00] (24)--
00000800:00000200:21.0:1619677472.594475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (24)--
00000800:00000200:20.0:1619677472.594475:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.16@o2ib3
00000800:00000200:21.0:1619677472.594477:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (23)--
00000800:00000200:8.2:1619677472.594477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:20.0:1619677472.594478:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.16@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.594481:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.594482:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:20.0:1619677472.594483:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:13.0:1619677472.594484:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:20.0:1619677472.594485:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96513047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.594486:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:20.0:1619677472.594486:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.594486:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.594487:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.594487:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.594487:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677472.594488:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:21.0:1619677472.594489:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:20.0:1619677472.594489:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677472.594489:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.594492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.594493:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1e00] (23)++
00000400:00000200:20.0:1619677472.594493:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.594494:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e84c1e00] (24)--
00000800:00000200:21.0:1619677472.594494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (23)--
00000800:00000200:13.0:1619677472.594494:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.0:1619677472.594495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.594495:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 344 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:21.0:1619677472.594496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000800:00000200:20.0:1619677472.594498:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.594498:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:20.0:1619677472.594499:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.0:1619677472.594502:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:20.0:1619677472.594503:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:20.0:1619677472.594504:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1e00] (23)--
00000800:00000200:20.0:1619677472.594504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000800:00000200:20.0:1619677472.594505:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:12.0:1619677472.594505:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677472.594507:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.16@o2ib3
00000800:00000200:12.0:1619677472.594508:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:21.0:1619677472.594509:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000800:00000200:12.0:1619677472.594509:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:20.0:1619677472.594510:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.16@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.594510:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677472.594511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:20.0:1619677472.594512:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.594512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:20.0:1619677472.594514:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.594515:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.594516:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.594517:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677472.594518:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:20.0:1619677472.594518:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.594522:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.594525:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.594527:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.594527:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.594528:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.594530:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.594531:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1e00] (22)++
00000800:00000200:20.0:1619677472.594532:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1e00] (23)--
00000800:00000200:20.0:1619677472.594533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000800:00000200:20.0:1619677472.594534:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:21.0:1619677472.594535:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000800:00000200:20.0:1619677472.594535:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.16@o2ib3
00000400:00000200:20.0:1619677472.594537:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.16@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.594540:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.594541:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677472.594543:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.594544:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.594545:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.594546:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677472.594550:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.594552:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677472.594555:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.594556:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:20.0:1619677472.594556:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677472.594558:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677472.594559:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:20.0:1619677472.594560:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1e00] (22)--
00000800:00000200:20.0:1619677472.594560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (21)--
00000800:00000200:21.0:1619677472.594570:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:21.0:1619677472.594573:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.16@o2ib3
00000400:00000200:21.0:1619677472.594575:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.16@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.594575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000400:00000200:21.0:1619677472.594577:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.594578:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677472.594579:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.594581:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.594582:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.594583:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677472.594586:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.594588:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.594590:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.594591:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.594591:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.594593:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.594593:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1e00] (21)++
00000800:00000200:21.0:1619677472.594595:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e84c1e00] (22)++
00000800:00000200:21.0:1619677472.594596:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1e00] (23)--
00000800:00000200:21.0:1619677472.594597:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000800:00000200:8.2:1619677472.594599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.594608:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.594609:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)++
00000800:00000200:20.0:1619677472.594609:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.594610:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e84c1e00] (23)--
00000400:00000200:20.0:1619677472.594610:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.594611:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (22)--
00000800:00000200:21.0:1619677472.594612:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.594612:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.594613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1e00] (21)--
00000800:00000200:20.0:1619677472.594615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677472.594693:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.594695:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.594696:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.594698:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.594699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.594700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.594741:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.594744:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677472.594746:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.594748:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.594750:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.594751:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.594752:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.594753:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.594754:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.594758:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.594759:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.594763:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.594763:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.594764:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.594766:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.594766:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.594767:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.594767:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677472.595209:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677472.595236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.595245:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677472.595247:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677472.595248:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677472.595249:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:20.0:1619677472.595249:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.595250:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677472.595250:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.595251:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677472.595252:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.595253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677472.595552:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.595554:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.595555:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.595557:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.595558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.595559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677472.596052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.596055:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.596056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.596058:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.596060:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.596061:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.596063:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.596064:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.596065:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.596066:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.596069:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.596071:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.596074:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.596075:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.596075:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.596077:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.596077:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.596078:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.596079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.596083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.596091:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.596094:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.596094:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.596096:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.596097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.596100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.596101:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.2:1619677472.596103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.596104:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.596106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.596106:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.596107:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:20.0:1619677472.596108:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.596109:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.596110:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.596110:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.596110:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:21.0:1619677472.596111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.596111:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.596112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:20.0:1619677472.596112:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.596113:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.596114:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.596118:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.596119:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.596122:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.596123:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.596124:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.596125:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.596126:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.596127:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.596127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.596138:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.596141:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.596142:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.596142:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.596145:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.596147:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.596155:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.596158:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.596159:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.596159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.596160:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.596160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.596241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.596246:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.596249:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.596251:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.596251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.596253:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.596255:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.596256:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.596258:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.596259:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.596260:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.596263:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.596265:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.596267:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.596268:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.596269:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.596270:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.596271:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.596272:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.596272:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.596276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.596284:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.596286:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.596287:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.596289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.596289:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.596291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.596369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677472.596375:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.596377:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677472.596379:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.596380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677472.596382:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.596387:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.596388:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96513035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.596388:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:21.0:1619677472.596389:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.596390:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.596390:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.596390:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.596391:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.596392:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.596394:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.596396:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.596397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.596398:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.596399:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.596399:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.596400:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.596402:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.596402:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.596403:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677472.596404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:21.0:1619677472.596405:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.596406:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.596408:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.596408:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.596408:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.2:1619677472.596408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.596409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.596410:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.596412:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.596413:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.596414:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.596414:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.596416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.596582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677472.596587:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.596590:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.596591:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.596592:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.596594:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.596595:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.596596:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.596598:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.596599:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.596600:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.596602:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.596604:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.596606:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.596606:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.596607:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.596608:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.596609:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.596609:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677472.596610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:26.0:1619677472.596671:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.596927:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.596929:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.596931:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.596932:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.596934:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.596935:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.596936:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.596938:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.596939:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.596939:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.596943:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.596944:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.596947:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.596948:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.596948:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.596950:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.596951:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.596951:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.596952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.596965:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.596968:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.596969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.596970:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.596971:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.596975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.597068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.597076:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.597079:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.597080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.597080:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.597083:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.597086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.597087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.597095:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.597097:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.597098:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.597098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.597099:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.597100:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.597661:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.597663:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.597664:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.597665:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677472.597666:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.597667:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677472.597668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677472.597668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677472.597669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:24.0:1619677472.597670:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.597674:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.597675:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.597677:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.597678:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.597679:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.597680:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.597683:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.597684:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.597686:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.597687:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.597688:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.597689:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.597689:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.597691:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.597692:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.597693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.597703:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.597704:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.597705:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.597706:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.597707:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.597708:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.597708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.597708:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.597709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.597709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.598072:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.598075:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.598075:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.598076:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.598077:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.598079:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.598304:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.598306:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.598309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.598309:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.598311:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.598312:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.598314:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.598315:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.598316:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.598317:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.598320:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.598322:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.598325:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.598326:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.598326:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.598328:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.598329:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.598330:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.598330:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.598341:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.598344:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.598345:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.598345:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.598347:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.598350:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677472.598360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677472.598367:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677472.598371:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677472.598371:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000400:00000200:1.0:1619677472.598373:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.598376:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.598378:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677472.598379:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.598380:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.598381:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.598382:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677472.598386:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.598387:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677472.598390:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.598391:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677472.598391:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677472.598393:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677472.598394:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.598394:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677472.598395:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.0:1619677472.598406:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.2:1619677472.598408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.0:1619677472.598408:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.598409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.598409:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.598411:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.598414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.598415:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.598417:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.598419:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.598420:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.598423:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.598424:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:2.0:1619677472.598424:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.598425:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.598427:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677472.598428:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.598428:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:2.0:1619677472.598428:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677472.598429:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.598430:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:2.0:1619677472.598430:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.598431:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:2.0:1619677472.598434:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.598436:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.598438:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.598439:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.598440:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.598442:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.598442:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.598443:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.598444:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.598445:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.598450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.598457:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.598458:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.598458:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677472.598459:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.598459:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.598460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.598461:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.598461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.598462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.598463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.598474:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.598477:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.598478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.598478:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.598480:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.598482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.598591:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.598594:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.598594:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.598596:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.598599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.598600:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.598603:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.598606:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.598607:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.598609:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.598610:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.598612:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.598613:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.598614:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.598615:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.598616:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.598620:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.598622:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.598625:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.598626:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.598626:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.598628:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.598629:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.598629:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.598630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.598634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.598643:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.598646:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.598646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.598647:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.598649:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.598652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.598760:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.598763:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677472.598766:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.598769:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.598770:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.598772:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.598773:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.598774:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.598775:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.598778:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.598779:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.598781:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.598782:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.598782:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.598784:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.598784:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.598785:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.598785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.598806:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.598808:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.598809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.598809:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.598811:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.598814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.599142:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.599145:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.599147:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.599148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.599150:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.599151:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.599152:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.599154:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.599155:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.599156:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.599160:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.599161:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.599164:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.599165:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.599165:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.599167:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.599168:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.599169:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.599169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677472.599374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.599380:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.599383:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.599384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.599385:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.599387:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.599389:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677472.599390:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.599391:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.599392:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.599393:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677472.599396:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.599398:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.599400:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.599401:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677472.599402:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677472.599404:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677472.599404:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.599405:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.599406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.599492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.599501:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.599504:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.599505:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.599505:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.599507:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.599510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.599622:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.2:1619677472.599711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.599718:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.599721:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.599721:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.599723:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.599726:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.599728:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.599729:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.599731:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.599732:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.599733:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.599736:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.599738:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.599741:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.599742:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.599742:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.599744:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.599745:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.599745:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.599746:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677472.599757:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.599759:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.599761:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.599763:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677472.599766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677472.599794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.599800:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.599802:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.599804:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.599805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.599807:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.599809:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.599810:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.599811:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.599813:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.599814:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.599817:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.599819:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.599820:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.599821:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.599822:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.599823:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.599824:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.599825:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.599826:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.599827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.599829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.599833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.599840:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.599841:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.599842:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677472.599842:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.599843:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.599844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.599844:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.599845:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.599846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.599847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677472.599882:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.599885:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.599885:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.599887:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.599888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.599889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.600557:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.600560:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.600562:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.600563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.600565:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.600566:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.600568:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.600569:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.600570:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.600571:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.600574:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.600575:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.600578:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.600579:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.600580:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.600581:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.600582:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.600583:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.600584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.600595:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.600598:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.600599:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.600599:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.600601:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.600604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.600714:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.600716:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.600717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.600717:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.600719:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.600721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.601066:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.601069:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.601070:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.601071:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.601073:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.601075:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677472.601076:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.601078:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.601079:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.601079:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.601083:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.601085:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677472.601088:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.601088:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677472.601089:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677472.601091:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677472.601091:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.601092:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.601092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:17.0:1619677472.601136:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.601139:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.601139:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.601142:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.601144:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.601146:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.601147:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.601148:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.601149:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.601150:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.601154:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.601155:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.601158:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.601159:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.601159:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.601161:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.601161:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.601163:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.601164:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.601165:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.601168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.601175:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677472.601177:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.601177:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.601177:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677472.601179:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677472.601179:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677472.601180:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.601181:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.601182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.601184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.601189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.601197:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.601200:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.601201:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.601202:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.601202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.601204:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.601285:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.601287:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.601289:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.601290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.601292:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.601298:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.601299:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.601300:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.601301:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.601302:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.601306:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.601308:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.601311:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.601311:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.601312:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.601314:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.601315:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.601316:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.601317:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.601318:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.601327:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677472.601327:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.601329:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.601330:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677472.601330:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.601331:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.601331:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677472.601331:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.601334:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.601337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.601564:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.601566:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.601567:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.601569:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.601570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.601571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.601584:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.601587:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.601588:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.601588:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.601589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.601590:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.601701:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677472.602197:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.602200:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.602201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.602202:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.602204:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.602206:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.602207:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.602209:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.602210:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.602210:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.602214:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.602215:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.602218:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.602219:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.602219:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.602221:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.602221:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.602222:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.602222:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.602227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.602235:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.602238:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.602239:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.602239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.602241:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.602244:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677472.602608:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.602611:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677472.602612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677472.602614:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.602617:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.602619:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677472.602620:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96513005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.602622:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.602623:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.602624:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677472.602627:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.602629:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.602632:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.602633:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.602634:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677472.602636:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.602636:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.602637:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.602638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:26.0:1619677472.602650:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.602652:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.602654:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.602656:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.602659:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.0:1619677472.602799:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.602801:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.602802:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.602802:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.602804:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.602807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.602821:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.602823:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.602824:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.602824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.602825:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.602826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677472.602894:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.602897:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.602899:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677472.602901:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.602902:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.602903:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677472.602904:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96513003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.602906:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.602907:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.602908:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677472.602911:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.602912:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677472.602915:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.602916:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:12.0:1619677472.602917:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:12.0:1619677472.602919:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:12.0:1619677472.602919:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.602920:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.602921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677472.602925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.602934:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.602936:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.602937:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.602937:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.602939:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.602943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.602944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.602951:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.602954:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.602955:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.602955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.602956:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.602957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.603017:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.603019:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.603020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.603022:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.603025:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.603026:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.603028:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96513002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.603029:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.603030:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.603031:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.603034:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.603036:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.603039:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.603040:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.603040:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.603042:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.603042:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.603043:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.603044:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.603056:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.603059:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.603059:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.603060:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.603062:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.603065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.603104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.603113:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.603115:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.603116:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.603118:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.603118:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.603121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.603421:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.603423:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.603424:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.603426:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.603427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.603428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.603518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.603521:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.603524:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.603526:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.603527:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.603528:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.603530:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.603531:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.603533:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.603534:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.603535:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.603538:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.603540:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.603542:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.603543:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.603544:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.603545:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.603546:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.603547:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.603547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.603552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.603560:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.603562:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.603562:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.603564:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:21.0:1619677472.603564:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.603565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.603567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.603598:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.603600:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.603601:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.603602:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677472.603602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.603603:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.603604:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.603605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.603606:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.603606:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.604359:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.604362:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.604363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.604366:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:8.0:1619677472.604370:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.604373:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.604374:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:8.0:1619677472.604374:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.604375:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96512995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.604377:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:8.0:1619677472.604377:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.604378:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.604378:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.604379:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.604379:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.604382:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.604383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.604384:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.604385:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.604388:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.604389:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.604390:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.604390:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.604391:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.604392:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.604392:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.604392:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677472.604394:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.604394:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.604396:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.604396:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.604397:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.604397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.604406:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.604409:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677472.604409:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.604410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.604410:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.604411:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.604412:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677472.604412:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.604413:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.604416:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.604426:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.604429:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.604430:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.604430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.604431:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.604432:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.604486:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.604522:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.604525:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.604525:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677472.604795:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.604797:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.604798:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.604800:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.604801:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.604802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.604810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.604816:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.604819:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.604820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.604822:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.604824:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.604826:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.604827:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.604828:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.604829:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.604830:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.604834:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.604836:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.604840:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.604841:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.604841:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.604843:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.604844:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.604845:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.604845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.0:1619677472.605331:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.605333:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.605336:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.605336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.605338:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.605340:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.605341:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.605343:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.605344:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.605345:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.605347:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.605349:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.605352:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.605353:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.605353:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.605355:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.605356:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.605356:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.605357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.605369:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.605372:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.605373:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.605373:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.605375:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.605378:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.605528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677472.605533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677472.605533:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.605535:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.605538:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:1.0:1619677472.605538:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.605540:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677472.605540:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.605541:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.605541:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.605543:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:17.0:1619677472.605543:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.605543:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:1.0:1619677472.605543:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677472.605545:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.605546:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.605547:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.605548:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.605549:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.605549:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.605550:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.605552:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.605553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:18.0:1619677472.605554:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677472.605554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677472.605554:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677472.605555:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677472.605555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:18.0:1619677472.605557:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677472.605557:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677472.605560:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677472.605561:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.605561:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677472.605562:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677472.605563:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677472.605563:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677472.605564:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:18.0:1619677472.605565:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.605565:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.605565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677472.605566:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.605566:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.605568:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.605569:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.605569:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.605570:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.605581:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:26.0:1619677472.605583:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:2.0:1619677472.605583:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.605584:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.605585:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677472.605586:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:2.0:1619677472.605587:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:26.0:1619677472.605589:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.605589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:26.0:1619677472.605592:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.605594:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677472.605595:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.605596:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.605597:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.605599:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677472.605602:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.605604:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677472.605607:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.605608:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677472.605608:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677472.605610:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677472.605610:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.605611:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.605612:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677472.605754:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.605757:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.605757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.605760:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.605762:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.605764:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.605765:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.605766:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.605767:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.605768:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.605772:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.605774:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.605776:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.605777:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.605778:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.605780:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.605780:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.605781:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.605782:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.605786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.605794:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.605797:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.605798:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.605799:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.605800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.605802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.605803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.605812:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.605814:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.605815:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.605815:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.605817:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.605817:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.606432:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.606434:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.606435:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.606437:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.606438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.606439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.606759:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677472.606781:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.606783:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677472.606786:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.606786:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.606788:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.606790:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677472.606791:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.606792:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.606793:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.606794:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.606797:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.606799:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.606801:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.606802:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.606802:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.606804:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.606805:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.606806:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.606807:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.606808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.606819:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:18.0:1619677472.606819:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677472.606821:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.606821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.606822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.606822:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.606823:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.606823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.606826:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.606828:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.606991:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.606994:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.606994:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.606996:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.606999:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.607000:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.607002:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.607003:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.607004:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.607005:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.607008:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.607010:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.607012:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.607013:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.607013:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.607015:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.607015:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.607016:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.607016:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677472.607026:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.607028:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.607029:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.607030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.607031:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.607032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.607615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.607621:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.607624:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.607626:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.607626:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.607628:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.607630:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.607631:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.607633:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.607634:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.607635:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.607638:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.607639:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.607641:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.607642:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.607642:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.607644:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.607645:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.607646:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.607646:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.607651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.607659:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.607661:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.607661:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.607663:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.607664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.607665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.608056:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.608058:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.608059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.608061:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.608063:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.608064:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.608066:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.608067:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.608068:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.608069:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.608072:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.608074:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.608077:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.608078:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.608078:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.608080:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.608080:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.608081:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.608082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.608086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.608095:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.608098:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.608099:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.608099:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.608101:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.608104:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.608249:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.608251:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.608252:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.608253:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.608254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.608255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.608271:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.608273:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.608274:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.608275:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.608276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.608276:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.608503:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.608506:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.608508:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.608509:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.608510:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.608512:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.608514:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.608515:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.608516:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.608517:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.608520:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.608522:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.608524:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.608525:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.608526:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.608528:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.608528:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.608529:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.608529:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.608541:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.608544:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.608545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.608545:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.608548:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.608550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.608559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.608561:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.608562:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.608562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.608563:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.608564:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.608672:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.608674:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.608675:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.608677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677472.608680:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.608683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677472.608719:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.608722:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.608723:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.608724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.608725:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.608728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677472.608774:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.608777:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677472.608780:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.608784:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.608785:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.608787:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.608788:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.608789:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.608790:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.608793:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.608795:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.608798:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.608799:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.608800:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.608801:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.608802:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.608802:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.608803:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.608813:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.608816:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.608817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.608817:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.608818:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.608821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.608830:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.608832:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.608833:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.608834:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677472.608834:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.608835:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.608959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.608967:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.608970:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.608971:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.608972:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.608973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.608976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.609307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.609314:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.609317:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.609318:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677472.609319:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677472.609321:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.609323:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.609325:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.609326:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.609328:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.609330:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.609331:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.609336:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.609338:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.609341:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.609342:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.609343:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.609344:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.609345:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.609347:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.609348:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.609349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.609351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.609354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677472.609359:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677472.609359:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:21.0:1619677472.609361:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:2.0:1619677472.609361:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:21.0:1619677472.609362:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.609362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:21.0:1619677472.609363:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.609363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677472.609365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.609366:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.609497:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.609499:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.609501:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.609502:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.609506:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.609510:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.609512:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.609513:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.609514:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.609515:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.609518:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.609519:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.609522:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.609523:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.609524:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.609525:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.609526:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.609527:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.609527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.609540:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.609542:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:18.0:1619677472.609542:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.609543:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.609544:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:18.0:1619677472.609544:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.609545:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.609545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:18.0:1619677472.609546:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.609548:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.609550:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.609552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.609663:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.609666:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.609666:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.609668:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.609669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.609670:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.609932:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.609935:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677472.609936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677472.609937:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677472.609939:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.609941:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.609942:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.609943:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.609944:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.609946:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.609948:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677472.609950:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.609952:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.609953:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.609953:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.609954:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.609955:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.609956:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677472.609957:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677472.609958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.609966:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.609967:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.609968:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.609970:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.609971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.609972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.609972:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677472.609973:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677472.609974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.609975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.2:1619677472.610279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.610287:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.610289:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.610290:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.610292:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.610292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.610299:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.610478:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.610480:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.610482:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.610483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.610485:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.610486:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.610488:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.610489:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.610490:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.610491:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.610495:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.610497:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.610499:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.610499:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.610500:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.610502:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.610502:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.610503:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.610503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.611066:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.611069:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.611070:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.611071:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.611074:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.611075:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.611076:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.611077:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.611078:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.611079:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.611082:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.611084:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.611087:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.611087:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.611088:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.611090:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.611090:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.611091:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.611091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.611105:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.611108:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.611109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.611109:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.611111:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.611115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.611117:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.611119:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.611119:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.611120:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.611121:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.611121:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.611319:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.611321:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.611322:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.611324:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.611325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.611326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.611468:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.611471:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677472.611474:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.611477:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.611478:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677472.611480:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.611481:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.611482:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.611483:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.611486:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.611488:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677472.611490:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.611491:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.611492:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.611494:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.611494:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.611495:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.611496:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677472.611509:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.611512:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.611513:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.611513:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.611515:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677472.611519:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677472.611582:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.611589:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.611591:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.611592:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.611592:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.611595:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.611597:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.611600:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.611600:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.611601:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.611602:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.611603:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.611603:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.611604:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677472.611625:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.611628:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.611630:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.611631:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.611632:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.611633:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.611635:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.611636:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.611637:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.611638:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.611642:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.611643:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.611646:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.611647:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.611647:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.611649:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.611649:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.611650:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.611651:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677472.611723:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.611725:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.611726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.611726:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.611728:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.611731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.611732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.611737:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.611740:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.611742:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.611742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.611745:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.611746:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.611747:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.611749:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.611750:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.611751:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.611754:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.611756:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.611758:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.611758:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.611759:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.611761:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.611761:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.611762:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.611762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.611767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.611774:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.611776:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.611777:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.611779:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.611780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.611781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.611786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.611794:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.611797:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.611798:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.611798:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.611799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.611800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.613136:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.613139:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.613140:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.613140:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.613142:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.613142:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.613143:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.613144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.613157:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.613159:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.613160:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.613161:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.613163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.613164:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.613174:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.613177:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.613178:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.613178:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.613179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.613180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.613544:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.613547:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.613548:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.613550:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.613553:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.613554:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.613555:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.613556:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.613557:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.613558:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.613562:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.613563:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.613565:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.613566:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.613567:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.613568:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.613569:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.613570:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.613571:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.613572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677472.613582:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.613585:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.613586:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.613586:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677472.613586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.613587:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:24.0:1619677472.613588:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.613588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.613589:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.613590:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.613599:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.613602:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.613603:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.613604:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.613607:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.613608:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.613609:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.613610:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.613611:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.613612:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.613615:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.613616:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.613618:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.613619:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.613620:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.613621:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.613622:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.613622:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.613623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.613639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:8.0:1619677472.613643:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.613645:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.613646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.613648:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.613650:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.613652:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.613653:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.613654:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.613655:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.613656:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.613659:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.613661:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.613663:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.613664:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.613665:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.613666:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.613667:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.613668:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.613669:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.613669:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.613672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.613680:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.613682:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.613683:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677472.613684:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.613684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677472.613684:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.613684:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.613686:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677472.613686:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.613689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.613721:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.613723:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.613724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.613724:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.613726:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.613729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.613741:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.613743:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.613744:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.613744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.613745:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.613746:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.614477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.614483:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.614486:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.614487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.614489:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.614492:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.614493:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.614495:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.614496:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.614497:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.614498:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.614501:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.614503:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.614505:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.614506:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.614506:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.614508:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.614508:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.614509:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.614510:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.614522:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.614524:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.614525:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.614526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.614528:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.614529:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.614533:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.614536:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.614539:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.614541:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.614542:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.614544:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.614545:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677472.614545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:22.0:1619677472.614546:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.614547:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.614550:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.614552:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.614555:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.614555:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.614556:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.614557:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:22.0:1619677472.614558:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.614558:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.614559:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677472.614559:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:22.0:1619677472.614560:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:21.0:1619677472.614560:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.614561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677472.614562:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.614563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.614568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.614570:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.614572:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.614572:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.614572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677472.614574:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.614576:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.614720:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.614723:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.614725:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.614726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.614727:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.614729:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.614731:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.614732:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.614733:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.614735:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.614738:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.614740:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.614742:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.614743:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.614744:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.614746:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.614746:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.614748:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.614749:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.614750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.614761:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.614764:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.614764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.614765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.614771:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.614774:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.614775:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.614777:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.614780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.615223:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.615226:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.615227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.615229:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.615232:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.615233:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.615235:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.615236:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.615237:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.615238:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.615241:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.615243:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.615245:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.615245:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.615246:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.615247:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.615248:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.615249:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.615249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.615259:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.615260:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.615261:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.615262:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.615263:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.615264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.615291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.615297:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.615299:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.615301:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.615302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.615304:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.615305:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.615306:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.615307:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.615309:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.615310:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.615313:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.615315:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.615317:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.615317:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.615318:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.615320:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.615320:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.615321:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.615322:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.615323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.615326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.615330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.615336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.615337:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.615337:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.615338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677472.615339:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.615339:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.615339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.615340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.615341:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.615342:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677472.615570:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.615572:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.615573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.615573:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.615576:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.615577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.615781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.615789:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.615792:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.615793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.615794:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.615795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.615797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.616122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.616156:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.616158:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.616159:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.616161:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.616162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.616163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677472.616496:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.616499:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.616500:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.616503:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.616506:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.616507:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.616508:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.616510:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.616511:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.616512:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.616515:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.616516:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.616519:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.616520:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.616521:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.616523:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.616523:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.616524:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.616524:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.616537:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.616540:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.616541:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.616543:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.616547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:13.0:1619677472.616877:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.616880:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.616883:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.616886:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.616886:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.616887:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677472.616889:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.616890:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.616891:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.616892:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677472.616895:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.616897:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677472.616900:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.616901:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677472.616901:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677472.616903:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677472.616903:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.616904:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.616905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677472.616920:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.616924:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.616926:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.616927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.616929:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.616934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.617163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.617171:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.617175:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.617176:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.617180:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.617182:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.617185:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.617187:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.617189:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.617191:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.617193:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.617198:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.617201:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.617205:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.617206:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.617207:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.617210:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.617211:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.617212:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.617213:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677472.617222:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.617225:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.617226:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.617226:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.617229:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677472.617231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677472.617273:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.617275:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.617277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.617278:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.617280:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.617281:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.617282:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.617284:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.617285:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.617286:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.617289:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.617290:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.617294:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.617295:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.617300:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.617302:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.617302:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.617303:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.617304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.617308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.617318:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.617322:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.617324:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:1.0:1619677472.617324:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.617327:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.617328:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)++
00000400:00000200:1.0:1619677472.617328:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.617330:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (24)--
00000800:00000200:17.0:1619677472.617331:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.617332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.617333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.617528:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.617531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.617531:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.617532:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.617534:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.617536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.617806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.617813:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.617817:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.617818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.617820:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.617823:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.617825:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677472.617826:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.617829:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.617830:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.617831:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677472.617835:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.617837:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.617840:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.617841:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677472.617842:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677472.617844:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677472.617845:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.617846:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.617846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:22.0:1619677472.617862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.617876:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.617879:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.617880:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.617881:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.617882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.617936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.617971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.617973:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.617974:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.617976:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.617977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.617978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.618158:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.618160:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.618163:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.618163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.618164:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.618166:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.618167:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.618169:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.618170:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.618171:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.618174:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.618176:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.618179:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.618180:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.618181:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.618183:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.618183:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.618184:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.618185:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.618195:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.618198:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.618199:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.618199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.618201:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.618203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.618209:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.618211:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.618212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.618212:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:10.0:1619677472.618214:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.618215:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.618216:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.618216:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000400:00000200:10.0:1619677472.618217:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.618217:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.618217:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:10.0:1619677472.618218:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.618219:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.618220:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.618221:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.618222:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.618225:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.618226:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.618229:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.618230:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.618230:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.618232:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.618232:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.618233:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.618233:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.618243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.618245:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.618246:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.618247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.618248:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.618250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.618351:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.618354:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.618355:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.618356:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.618359:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.618360:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677472.618361:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.618363:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.618363:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.618364:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677472.618367:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.618368:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677472.618371:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.618371:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677472.618372:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677472.618373:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677472.618374:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.618375:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.618375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677472.618387:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.618390:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.618390:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.618392:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.618394:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.618395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677472.618508:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.618510:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.618511:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.618512:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.618515:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.618517:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.618518:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.618519:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.618520:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.618521:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.618524:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.618525:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.618528:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.618529:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.618529:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.618531:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.618531:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.618532:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.618532:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.618545:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.618547:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.618549:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.618551:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.618554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.618567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:20.2:1619677472.618968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.618971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.618973:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.618976:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.618976:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.618978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.618980:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.618981:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.618982:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.618983:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.618984:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.618987:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.618988:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.618990:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.618991:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.618992:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.618993:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.618994:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.618995:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.618996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.619000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.619008:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.619010:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.619011:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.619013:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.619014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.619015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.619019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.619027:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.619030:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.619031:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.619031:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.619032:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.619033:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.619224:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.619226:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.619227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.619227:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.619229:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.619231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.619283:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.619285:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.619288:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.619290:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.619292:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.619298:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.619299:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.619300:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.619301:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.619304:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.619306:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.619308:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.619309:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.619309:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.619311:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.619312:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.619313:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.619313:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.619326:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.619328:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.619329:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.619330:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.619332:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.619333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.619351:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.619354:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.619355:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.619356:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.619356:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.619357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.619835:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.619838:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.619838:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.619841:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.619843:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.619844:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677472.619846:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.619847:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.619848:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.619849:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.619852:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.619854:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677472.619857:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.619858:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.619859:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.619860:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.619861:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.619861:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.619862:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677472.619875:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.619878:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.619878:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.619879:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.619881:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677472.619884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.619887:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.619890:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.619891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.619892:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.619895:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.619896:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.619898:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.619899:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.619900:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.619901:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.619903:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.619905:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.619906:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.619907:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.619907:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.619909:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.619910:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.619910:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.619911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.619921:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.619923:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.619923:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.619924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.619925:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.619926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.620456:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.620458:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.620459:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.620461:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.620464:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.620466:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677472.620467:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.620468:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.620469:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.620470:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677472.620473:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.620474:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677472.620477:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.620478:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677472.620478:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677472.620480:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677472.620480:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.620481:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677472.620483:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677472.620483:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677472.620487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.620496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677472.620498:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:1.0:1619677472.620499:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:26.0:1619677472.620500:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:17.0:1619677472.620500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.620500:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.620501:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.620502:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:1.0:1619677472.620502:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.620505:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.620838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.620847:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.620849:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.620850:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.620850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.620853:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.620855:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677472.620884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.620918:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677472.620919:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677472.620920:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.620921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.620922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.620922:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.620922:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.620924:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.620925:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.620926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.621104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.621111:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.621113:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.621114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.621117:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.621119:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.621121:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.621122:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.621124:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.621125:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.621126:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.621129:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.621130:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.621133:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.621133:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.621134:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.621136:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:2.0:1619677472.621136:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.621136:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.621137:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677472.621138:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:2.0:1619677472.621138:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:24.0:1619677472.621139:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.621140:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.621142:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.621144:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677472.621149:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.621151:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.621152:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.621153:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.621154:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.621156:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.621377:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.621380:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.621381:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.621383:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.621386:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.621387:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.621389:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.621391:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.621392:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.621392:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.621396:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.621398:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.621401:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.621402:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.621402:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.621404:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.621404:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.621405:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.621406:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.621421:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.2:1619677472.622343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.622352:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.622354:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.622355:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.622356:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.622358:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.622360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.622530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.622536:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.622539:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.622540:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.622541:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.622544:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.622545:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.622546:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.622548:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.622549:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.622550:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.622553:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.622554:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.622558:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.622558:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.622559:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.622561:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.622562:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.622562:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.622563:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677472.622633:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.622635:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.622638:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.622641:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.622642:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.622644:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.622645:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.622646:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.622648:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.622652:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.622655:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.622657:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.622658:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.622659:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.622661:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.622661:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.622662:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.622663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677472.622670:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.622673:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.622674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.622674:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.622675:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677472.622676:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.622678:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.622678:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.622678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:21.0:1619677472.622680:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.622681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.622682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.622688:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.622691:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.622692:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.622692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.622694:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.622694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.622792:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.622794:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.622795:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.622797:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.622800:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.622801:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.622803:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.622804:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.622805:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.622806:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.622810:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.622811:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.622814:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.622815:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.622816:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.622817:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.622818:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.622819:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.622820:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.622821:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.622833:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677472.622835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.622835:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.622836:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.622837:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677472.623026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.623029:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.623031:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.623033:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.623034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.623035:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.623037:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.623038:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.623039:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.623040:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.623041:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.623045:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.623047:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.623049:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.623050:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.623051:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.623052:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.623053:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.623053:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.623054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.623058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.623066:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.623069:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.623069:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.623071:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.623072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.623073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.623241:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.623244:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.623246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.623246:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.623248:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.623250:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.623251:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.623253:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.623253:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.623254:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.623258:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.623259:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.623262:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.623263:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.623264:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.623265:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.623266:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.623267:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.623269:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.623269:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.623279:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.623281:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.623282:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677472.623283:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677472.623283:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.623283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.623284:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.623285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.623285:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.623289:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.623300:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.623303:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.623304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.623304:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.623305:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.623305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.623448:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.623451:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.623451:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.623454:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.623456:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.623457:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.623459:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.623460:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.623461:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.623462:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.623464:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.623466:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.623468:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.623469:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.623469:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.623471:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.623471:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.623472:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.623474:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.623474:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.623484:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.623486:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:10.0:1619677472.623487:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677472.623487:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.623488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.623489:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677472.623489:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.623490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.623491:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.623492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.623684:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.623687:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.623687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.623690:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.623692:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.623694:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.623695:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.623696:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.623697:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.623699:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.623701:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.623703:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.623706:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.623707:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.623707:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.623709:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.623710:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.623711:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.623711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.623716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.623724:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.623726:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.623727:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.623729:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.623730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.623731:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.623824:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.623827:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.623829:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.623829:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.623831:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.623833:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.623834:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.623836:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.623837:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.623838:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.623842:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.623843:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.623846:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.623847:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.623848:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.623850:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.623850:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.623851:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.623852:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:22.0:1619677472.623975:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.623978:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.623981:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.623983:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.623984:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.623986:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.623987:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.623988:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.623990:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.623993:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.623995:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.623997:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.623998:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.623998:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.624000:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.624000:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.624001:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.624001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.624013:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.624015:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.624016:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.624018:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.624019:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.624020:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.624436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.624444:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.624447:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.624448:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.624449:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.624451:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.624454:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.624589:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.624591:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.624592:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.624592:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.624594:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.624596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.624874:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.624877:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.624879:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.624879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.624881:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.624883:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.624884:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.624886:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.624886:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.624888:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.624891:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.624893:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.624895:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.624896:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.624896:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.624898:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.624898:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.624899:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.624900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.624915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.625162:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.625165:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.625166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.625167:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.625170:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.625171:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.625173:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.625174:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.625175:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.625176:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.625179:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.625181:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.625183:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.625184:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.625184:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.625186:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.625186:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.625187:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.625187:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.625197:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.625199:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.625200:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.625201:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.625201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.625203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.625582:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.625584:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.625587:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.625589:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.625590:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.625592:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.625593:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.625594:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.625595:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.625598:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.625600:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.625602:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.625602:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.625603:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.625605:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.625605:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.625606:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.625607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.625620:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.625622:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.625622:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.625624:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.625625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.625626:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.625635:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.625638:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.625639:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.625639:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.625640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.625641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.626183:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.626185:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.626186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.626189:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.626191:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.626193:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.626194:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.626195:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.626196:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.626197:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.626200:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.626202:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.626205:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.626205:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.626206:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.626208:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.626208:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.626209:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.626210:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.626211:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.626214:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.626215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.626215:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.626217:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.626219:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.626225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.2:1619677472.626315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.626326:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.626328:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.626330:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.626332:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.626332:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.626335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.626586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.626590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.626592:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.626594:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.626595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.626596:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.626598:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.626599:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.626600:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.626601:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.626602:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.626605:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.626607:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.626609:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.626610:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.626611:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.626612:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.626613:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.626613:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.626614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.626618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.626625:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.626628:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.626628:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.626630:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.626631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.626632:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.626638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.626647:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.626649:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.626650:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.626651:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.626651:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.626653:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.627013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.627020:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.627023:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.627023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.627026:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.627028:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.627030:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.627032:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.627033:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.627034:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.627035:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.627039:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.627040:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.627044:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.627044:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.627045:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.627047:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.627047:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.627048:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.627049:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677472.627351:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.627354:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.627357:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.627359:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.627361:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.627362:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.627364:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.627365:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.627366:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.627369:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.627371:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.627373:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.627374:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.627375:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.627376:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.627377:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.627378:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.627379:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.627379:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.627392:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.627395:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.627395:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.627397:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.627398:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.627399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.627908:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.627910:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.627912:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.627913:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.627914:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.627916:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.627917:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.627919:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.627920:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.627921:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.627924:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.627926:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.627929:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.627930:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.627930:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.627932:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.627933:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.627934:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.627934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.627945:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.627948:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.627949:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.627949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.627951:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.627954:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.627968:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.627970:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.627971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.627971:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.627972:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.627973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.628015:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:26.0:1619677472.628018:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:10.0:1619677472.628018:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.12.5@o2ib3
00000400:00000200:10.0:1619677472.628021:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.628025:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.628026:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:10.0:1619677472.628027:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.628028:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.628029:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.628030:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:10.0:1619677472.628033:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.628034:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:10.0:1619677472.628036:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.628037:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:10.0:1619677472.628038:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:10.0:1619677472.628039:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:10.0:1619677472.628039:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:10.0:1619677472.628040:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:10.0:1619677472.628041:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)--
00000800:00000200:24.2:1619677472.628045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:26.0:1619677472.628052:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)++
00000800:00000200:26.0:1619677472.628054:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2fa00] (23)--
00000400:00000200:26.0:1619677472.628055:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.628056:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.51@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.628057:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (21)--
00000800:00000200:10.0:1619677472.628057:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:10.0:1619677472.628058:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:26.0:1619677472.628060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:10.0:1619677472.628060:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000400:00000200:10.0:1619677472.628062:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.628063:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.628065:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677472.628066:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.628067:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.628068:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.628068:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677472.628071:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.628072:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677472.628075:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.628075:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677472.628076:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:10.0:1619677472.628077:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677472.628078:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:10.0:1619677472.628078:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:10.0:1619677472.628078:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)--
00000800:00000200:12.0:1619677472.628090:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.628092:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.628093:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.628095:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.628097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.628098:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:26.0:1619677472.628107:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:26.0:1619677472.628111:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000800:00000200:10.0:1619677472.628111:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000400:00000200:26.0:1619677472.628114:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.628116:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.628118:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677472.628119:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.628121:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.628122:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.628122:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677472.628125:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.628126:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:26.0:1619677472.628127:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.2:1619677472.628127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677472.628128:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.628128:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677472.628129:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677472.628129:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.628130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:26.0:1619677472.628131:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.628131:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:26.0:1619677472.628132:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (22)--
00000400:00000200:17.0:1619677472.628132:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677472.628133:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)--
00000400:00000200:17.0:1619677472.628134:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.628135:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:17.0:1619677472.628136:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.628137:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96512891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.628138:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:17.0:1619677472.628138:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.628139:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.628139:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.628141:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.628141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:17.0:1619677472.628141:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.628144:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.628144:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:26.0:1619677472.628145:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.628146:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:17.0:1619677472.628146:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677472.628146:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000400:00000200:26.0:1619677472.628147:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.628148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:26.0:1619677472.628149:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.628149:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677472.628149:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.628150:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:26.0:1619677472.628151:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677472.628151:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:10.0:1619677472.628151:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.628152:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:26.0:1619677472.628153:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:17.0:1619677472.628153:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.628153:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:10.0:1619677472.628153:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677472.628154:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.628154:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:10.0:1619677472.628155:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.628156:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.628157:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.628158:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.2:1619677472.628158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:10.0:1619677472.628162:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.628163:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677472.628166:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.628167:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:10.0:1619677472.628167:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677472.628168:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677472.628169:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.628170:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.628170:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677472.628170:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:10.0:1619677472.628171:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f3a7a00] (22)++
00000400:00000200:20.0:1619677472.628172:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.628173:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:10.0:1619677472.628173:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (23)--
00000800:00000200:10.0:1619677472.628173:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:20.0:1619677472.628174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.628175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.628184:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677472.628186:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)++
00000800:00000200:1.0:1619677472.628186:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.628187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.628188:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.628189:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f3a7a00] (23)--
00000800:00000200:26.0:1619677472.628189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:10.0:1619677472.628190:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)--
00000400:00000200:1.0:1619677472.628190:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.628193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.628195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.628202:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.628205:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.628205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.628206:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.628207:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.628207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.628212:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.628214:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.628215:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.628217:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.628219:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.628220:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.628222:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.628223:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.628224:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.628224:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.628227:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.628229:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.628232:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.628232:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.628233:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.628234:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.628235:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.628236:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.628236:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.628246:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.628248:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.628249:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.628249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.628251:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.628253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.628269:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.628272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.628272:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677472.628272:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.628273:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.628274:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.628288:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:10.0:1619677472.628291:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000800:00000200:26.0:1619677472.628292:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000400:00000200:10.0:1619677472.628300:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.628302:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.628304:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:10.0:1619677472.628305:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.628306:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.628307:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.628308:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:10.0:1619677472.628310:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.628312:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677472.628314:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.628314:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:10.0:1619677472.628315:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677472.628316:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677472.628317:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:10.0:1619677472.628318:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:10.0:1619677472.628318:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)--
00000800:00000200:24.0:1619677472.628329:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:26.0:1619677472.628330:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:26.0:1619677472.628332:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000800:00000200:24.0:1619677472.628332:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.628332:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.628333:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)++
00000800:00000200:8.0:1619677472.628333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.628334:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677472.628335:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.628335:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000800:00000200:24.0:1619677472.628337:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:10.0:1619677472.628337:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.628338:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.628339:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.628340:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.628342:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.628342:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.628343:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.628344:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.628346:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.628347:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677472.628347:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677472.628349:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.628349:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.628350:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.628350:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.628353:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677472.628353:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.628354:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.628355:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.628355:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.628357:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.628357:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.628358:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (23)++
00000800:00000200:10.0:1619677472.628358:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.628358:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.628359:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (24)--
00000800:00000200:26.0:1619677472.628359:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (23)--
00000800:00000200:10.0:1619677472.628360:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677472.628360:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (22)++
00000800:00000200:26.0:1619677472.628361:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:10.0:1619677472.628361:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (23)--
00000800:00000200:10.0:1619677472.628362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)--
00000800:00000200:22.0:1619677472.628371:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677472.628374:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:22.0:1619677472.628375:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.628377:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.628381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.628382:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.628384:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.628385:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.628386:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.628386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.628387:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:26.0:1619677472.628390:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000800:00000200:10.0:1619677472.628390:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)++
00000800:00000200:10.0:1619677472.628391:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000400:00000200:26.0:1619677472.628392:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.628394:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.628396:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.628396:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.628397:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677472.628398:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.628400:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.628401:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.628402:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677472.628403:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677472.628404:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.628406:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677472.628406:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677472.628407:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677472.628407:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.628407:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677472.628410:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677472.628410:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.628411:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677472.628411:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677472.628412:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677472.628413:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677472.628414:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (23)++
00000800:00000200:10.0:1619677472.628414:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.628415:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677472.628415:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (22)++
00000800:00000200:26.0:1619677472.628416:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f3a7a00] (24)++
00000800:00000200:26.0:1619677472.628417:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (25)--
00000800:00000200:10.0:1619677472.628417:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (23)--
00000800:00000200:10.0:1619677472.628417:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (25)++
00000800:00000200:26.0:1619677472.628418:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (24)--
00000800:00000200:10.0:1619677472.628418:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (25)--
00000800:00000200:10.0:1619677472.628418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (23)--
00000800:00000200:26.0:1619677472.628419:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)++
00000800:00000200:26.0:1619677472.628421:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000800:00000200:10.0:1619677472.628422:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (23)++
00000800:00000200:10.0:1619677472.628422:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f3a7a00] (24)--
00000400:00000200:26.0:1619677472.628423:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.628423:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (23)--
00000800:00000200:10.0:1619677472.628424:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000400:00000200:26.0:1619677472.628426:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.628427:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677472.628428:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96512883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.628429:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.628430:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.628431:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677472.628433:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.628435:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677472.628436:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.628437:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677472.628437:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677472.628438:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.628439:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (22)++
00000800:00000200:26.0:1619677472.628441:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (23)--
00000800:00000200:26.0:1619677472.628441:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:26.0:1619677472.628443:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:26.0:1619677472.628445:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000800:00000200:10.0:1619677472.628446:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000400:00000200:26.0:1619677472.628447:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.628447:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.628448:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.628449:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.628449:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.628450:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.628451:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:26.0:1619677472.628452:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.628453:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.628454:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.628455:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.628457:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677472.628458:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.628458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677472.628460:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.628462:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.628463:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.628463:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.628465:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.628466:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:26.0:1619677472.628466:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:26.0:1619677472.628467:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)--
00000800:00000200:26.0:1619677472.628468:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677472.628469:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677472.628469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.628470:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677472.628471:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.628472:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.0:1619677472.628476:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.628478:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.628479:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.628480:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.628481:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.628482:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:8.0:1619677472.628483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.628484:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000800:00000200:26.0:1619677472.628486:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)++
00000800:00000200:26.0:1619677472.628487:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000400:00000200:10.0:1619677472.628487:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.628489:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.628490:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.628490:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.628491:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.628493:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.628494:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.628494:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.628496:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.628498:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677472.628498:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677472.628499:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.628500:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.628501:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677472.628501:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.628502:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677472.628505:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.628505:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.628506:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.628507:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677472.628507:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.628509:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.628510:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.628510:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (23)++
00000800:00000200:26.0:1619677472.628511:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677472.628511:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677472.628511:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f3a7a00] (24)++
00000800:00000200:10.0:1619677472.628512:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (25)--
00000800:00000200:26.0:1619677472.628513:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677472.628513:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (24)--
00000800:00000200:26.0:1619677472.628514:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (24)++
00000800:00000200:26.0:1619677472.628514:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (24)--
00000800:00000200:10.0:1619677472.628514:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (24)++
00000800:00000200:26.0:1619677472.628515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (24)--
00000800:00000200:26.0:1619677472.628516:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (23)++
00000800:00000200:10.0:1619677472.628516:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000800:00000200:26.0:1619677472.628517:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f3a7a00] (24)--
00000800:00000200:26.0:1619677472.628518:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (23)--
00000400:00000200:10.0:1619677472.628518:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677472.628520:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)++
00000800:00000200:26.0:1619677472.628520:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (23)--
00000400:00000200:10.0:1619677472.628520:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.628521:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677472.628522:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000400:00000200:10.0:1619677472.628522:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96512879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.628524:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.628525:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.628525:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677472.628526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:12.0:1619677472.628530:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:10.0:1619677472.628530:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.628532:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677472.628532:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:12.0:1619677472.628533:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.628533:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.628534:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677472.628534:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:12.0:1619677472.628535:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.628536:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:13.0:1619677472.628537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.628538:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677472.628538:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (22)++
00000800:00000200:10.0:1619677472.628539:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (23)--
00000800:00000200:26.0:1619677472.628540:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:10.0:1619677472.628540:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:26.0:1619677472.628543:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000400:00000200:26.0:1619677472.628545:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.628545:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000400:00000200:26.0:1619677472.628547:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.628547:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.628547:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677472.628548:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.628549:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677472.628550:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.628550:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:26.0:1619677472.628552:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.628553:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.628554:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677472.628555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.628556:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:26.0:1619677472.628557:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.628559:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677472.628561:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.628562:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677472.628563:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677472.628565:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677472.628566:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:26.0:1619677472.628566:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:10.0:1619677472.628566:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.628567:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)--
00000800:00000200:10.0:1619677472.628567:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.628568:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.628568:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.628569:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677472.628569:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677472.628719:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.628722:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.628724:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.628727:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.628728:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.628730:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.628731:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.628732:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.628733:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.628736:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.628737:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.628740:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.628740:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.628741:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.628743:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.628743:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.628744:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.628744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.628756:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.628759:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.628759:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.628761:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.628762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.628763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.628868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.628874:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.628877:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.628878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.628880:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.628884:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.628885:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.628886:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.628888:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.628889:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.628890:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.628893:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.628895:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.628897:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.628898:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.628898:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.628900:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.628901:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.628902:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.628903:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.628903:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.628909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.628915:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.628916:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.628916:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677472.628917:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.628917:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.628917:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.628918:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.628919:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.628919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.628920:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.629240:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.629243:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.629244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.629246:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.629248:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.629249:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.629251:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.629252:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.629253:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.629254:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.629257:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.629258:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.629260:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.629261:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.629261:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.629263:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.629263:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.629264:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.629264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.629274:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.629276:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.629277:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.629278:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.629279:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.629280:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.629417:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:26.0:1619677472.629420:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.5@o2ib3
00000800:00000200:10.0:1619677472.629421:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000400:00000200:26.0:1619677472.629422:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.5@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.629424:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.629426:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677472.629427:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.629428:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.629429:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.629430:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677472.629433:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.629434:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677472.629437:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.629438:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677472.629438:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677472.629440:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677472.629441:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (21)++
00000800:00000200:26.0:1619677472.629441:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f3a7a00] (22)++
00000800:00000200:26.0:1619677472.629442:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f3a7a00] (23)--
00000800:00000200:26.0:1619677472.629443:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000800:00000200:8.2:1619677472.629447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.629455:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:10.0:1619677472.629456:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)++
00000800:00000200:10.0:1619677472.629458:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f3a7a00] (23)--
00000800:00000200:1.0:1619677472.629458:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.629459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:10.0:1619677472.629459:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (22)--
00000400:00000200:1.0:1619677472.629459:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.629461:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f3a7a00] (21)--
00000400:00000200:1.0:1619677472.629461:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.629464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677472.630315:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.630351:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.630353:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.630354:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.630355:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.630357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.630358:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.630429:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.630432:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.630434:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.630434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.630436:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.630437:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.630439:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.630440:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.630441:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.630442:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.630445:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.630447:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.630449:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.630449:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.630450:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.630451:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.630452:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.630453:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.630454:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.630455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.630464:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677472.630465:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677472.630466:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.630466:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677472.630467:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677472.630467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:22.0:1619677472.630468:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.630470:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.630473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.630572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.630575:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.630578:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.630580:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.630581:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.630582:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.630583:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.630584:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.630586:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.630587:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.630588:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.630591:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.630593:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.630594:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.630596:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.630596:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.630598:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.630598:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.630599:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.630600:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.630601:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.630604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.630607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.630610:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.630611:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.630612:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677472.630612:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.630613:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677472.630613:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.630614:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.630615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.630616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.630617:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677472.631551:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.631554:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.631555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.631557:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.631559:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.631561:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.631562:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.631563:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.631564:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.631565:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.631568:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.631570:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.631573:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.631574:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.631575:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.631576:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.631577:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.631577:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.631578:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.631595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677472.631611:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.631614:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.631615:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.631615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.631617:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.631620:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.631868:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.631871:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.631872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.631872:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.631873:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.631875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.631883:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.631886:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.631886:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:24.0:1619677472.631889:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:17.0:1619677472.631889:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.631891:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677472.631891:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.631892:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.631892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.631893:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:17.0:1619677472.631893:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:24.0:1619677472.631894:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:17.0:1619677472.631894:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.631895:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.631896:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.631897:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.631901:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.631902:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.631905:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.631905:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.631906:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.631908:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.631908:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.631910:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.631911:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.631911:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.631914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.631922:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677472.631924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.631924:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.631925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677472.631926:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677472.631926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:21.0:1619677472.631927:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.631929:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.631929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.631932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.631932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.631940:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.631943:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.631944:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.631944:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.631945:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.631946:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.632014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.632021:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.632024:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.632025:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.632026:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.632027:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.632029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.632070:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.632073:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.632075:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.632078:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.632079:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.632080:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.632082:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.632083:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.632084:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.632087:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.632089:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.632091:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.632092:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.632093:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.632094:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.632095:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.632095:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.632096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.632108:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.632110:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.632111:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.632112:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.632113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.632115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.632125:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.632127:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.632128:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.632129:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.632129:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.632131:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677472.632383:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.632386:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.632386:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677472.632389:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.632391:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.632393:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.632394:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.632395:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.632396:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.632397:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.632400:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.632402:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.632405:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.632405:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.632406:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.632408:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.632408:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.632409:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.632409:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.632426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677472.632492:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.632494:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.632497:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.632497:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.632498:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.632500:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.632501:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.632502:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.632503:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.632505:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.632508:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.632509:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.632512:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.632513:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.632513:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.632515:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.632516:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.632517:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.632517:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.632529:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.632531:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.632532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.632533:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.632535:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.632538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.632688:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.632691:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.632693:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.632694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.632695:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.632697:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.632698:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.632700:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.632701:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.632702:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.632706:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.632708:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.632710:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.632711:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.632712:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.632714:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.632714:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.632715:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.632716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677472.632857:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.632860:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.632861:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.632863:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.632865:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.632867:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677472.632868:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.632869:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.632870:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.632871:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.632874:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.632876:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677472.632879:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.632880:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.632881:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.632882:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.632883:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.632883:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.632884:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677472.632897:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.632900:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.632901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.632901:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.632903:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.632906:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.632910:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.632913:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.632914:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.632915:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000400:00000200:24.0:1619677472.632916:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.632918:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.632918:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677472.632919:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:18.0:1619677472.632919:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.632919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.632920:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000400:00000200:24.0:1619677472.632921:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.632921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:24.0:1619677472.632922:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.632923:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.632924:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.632928:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.632929:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.632932:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.632933:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.632933:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.632935:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.632936:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.632937:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.632937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.632949:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.632951:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.632953:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.632953:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.632954:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.632957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.633523:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.633525:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.633528:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.633530:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.633530:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677472.633532:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677472.633533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677472.633533:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677472.633533:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:22.0:1619677472.633534:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.633535:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.633536:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677472.633537:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.633540:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677472.633540:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.633542:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:10.0:1619677472.633542:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.633543:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.633544:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677472.633545:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.633545:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:10.0:1619677472.633545:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677472.633546:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:10.0:1619677472.633546:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677472.633548:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.633548:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.633549:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:10.0:1619677472.633550:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.633551:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.633551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677472.633552:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.633554:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.633555:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.633556:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.633557:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.633557:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.633558:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.633559:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.633560:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677472.633562:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.633565:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.633566:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.633568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677472.633569:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.633571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.633573:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.633576:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.633577:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.633578:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.633580:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.633582:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.633583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.633583:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.633585:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.633589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.633591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.633591:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.633591:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.633592:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.633594:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.633596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.634146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.634155:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.634157:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.634158:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.634160:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.634160:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.634163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.634268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.634271:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.634273:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.634275:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.634276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.634278:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.634279:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.634281:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.634282:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.634283:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.634284:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.634287:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.634288:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.634291:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.634291:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.634292:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.634298:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.634299:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.634300:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.634300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.634303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677472.634304:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.634305:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.634306:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.634307:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.634308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.634309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.634456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.634463:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.634466:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.634467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.634469:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.634471:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.634473:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.634475:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.634476:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.634478:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.634479:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.634482:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.634483:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.634486:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.634487:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.634488:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.634490:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.634490:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.634491:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.634492:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.634493:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.634498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677472.634503:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:10.0:1619677472.634503:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:2.0:1619677472.634504:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:20.0:1619677472.634505:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:10.0:1619677472.634506:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:2.0:1619677472.634506:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:26.0:1619677472.634507:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:20.0:1619677472.634507:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677472.634507:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677472.634508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:10.0:1619677472.634508:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.634509:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.634509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:20.0:1619677472.634510:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.634510:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000400:00000200:20.0:1619677472.634511:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.634513:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677472.634513:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:20.0:1619677472.634514:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.634515:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.634519:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.634520:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.634522:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.634523:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.634524:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.634526:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.634526:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.634527:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.634527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.634539:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.634542:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.634543:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.634546:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.634548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.634561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.634796:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.634799:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.634799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.634802:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.634804:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.634806:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677472.634807:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.634809:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677472.634810:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677472.634810:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.634811:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.634813:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.634813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677472.634815:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.634817:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.634818:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.634820:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677472.634821:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.634821:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677472.634822:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:24.0:1619677472.634823:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.634824:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.634824:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677472.634824:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:24.0:1619677472.634825:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677472.634825:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.634825:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:24.0:1619677472.634826:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.634827:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.634831:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.634833:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.634837:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.634837:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.634838:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.634839:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.634839:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.634840:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.634841:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.634842:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:8.0:1619677472.634842:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.634844:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.634845:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.634845:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.634846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.634847:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.634850:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.634854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.634857:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677472.634858:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.634858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.634859:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.634860:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.634861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677472.634861:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.634862:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677472.634864:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677472.635121:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.635123:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.635126:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.635129:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.635130:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.635131:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.635133:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.635134:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.635135:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.635137:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.635139:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.635141:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.635142:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.635143:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.635144:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.635145:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.635145:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.635146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.635158:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.635160:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.635161:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.635163:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.635163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.635165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.635738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.635745:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.635748:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.635749:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.635751:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.635754:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.635756:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.635757:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.635759:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.635760:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.635761:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.635764:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.635766:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.635768:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.635769:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.635769:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.635771:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.635772:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.635773:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.635774:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.635785:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.635788:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.635788:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.635789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.635790:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.635791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.636372:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.636374:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.636375:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.636377:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.636380:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.636381:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.636382:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.636384:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.636385:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.636386:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.636389:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.636390:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.636393:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.636394:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.636395:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.636397:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.636397:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.636398:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.636398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.636415:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.636681:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.636683:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.636684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.636684:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.636686:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.636688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.636700:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.636702:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.636703:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.636703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.636704:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.636705:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.636903:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.636906:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.636908:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.636911:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.636912:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.636914:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.636915:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.636916:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.636917:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.636922:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.636924:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.636926:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.636927:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.636928:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.636929:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.636930:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.636931:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.636931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.636943:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.636946:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.636946:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.636948:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.636949:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.636950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677472.637019:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.637021:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.637022:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.637024:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.637027:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.637029:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.637030:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.637031:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.637032:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.637033:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.637036:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.637038:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.637041:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.637041:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.637042:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.637044:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.637044:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.637045:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.637045:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677472.637060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.637129:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.637131:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.637132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.637134:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.637136:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.637137:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.637139:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.637140:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.637141:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.637142:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.637144:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.637146:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.637147:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.637148:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.637149:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.637150:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.637150:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.637151:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.637152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.637161:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.637163:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.637164:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.637165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.637166:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.637167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.637171:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.637174:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.637175:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.637177:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.637179:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.637180:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.637181:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.637183:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.637184:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.637184:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.637188:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.637189:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.637192:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.637193:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.637193:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.637195:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.637195:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.637196:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.637196:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.637208:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.637211:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.637212:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.637214:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.637217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.637231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.637269:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.637272:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.637273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677472.637274:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:17.0:1619677472.637275:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.637276:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:17.0:1619677472.637277:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.637278:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.637279:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:17.0:1619677472.637279:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.637280:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.637281:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.637282:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677472.637282:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.637283:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.637284:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.637285:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.637285:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.637286:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.637287:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.637297:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.637300:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:17.0:1619677472.637300:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.637303:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.637304:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.637304:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.637305:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677472.637305:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.637306:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.637307:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677472.637307:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.637308:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:17.0:1619677472.637308:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.637309:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.637309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:17.0:1619677472.637309:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.637310:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.637311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.637315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.637323:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677472.637323:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.637326:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:8.0:1619677472.637326:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.637327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:20.0:1619677472.637327:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.637327:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.637330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.637330:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.637331:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.637333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.637334:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.637337:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.637345:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.637348:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.637349:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.637349:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.637350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.637351:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.637442:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.637445:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677472.637446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677472.637448:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.637450:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.637452:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677472.637453:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.637454:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.637455:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.637456:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677472.637460:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.637461:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.637463:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.637463:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.637464:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.637465:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.637466:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.637467:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.637467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.637472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.637479:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.637481:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.637482:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.637483:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.637484:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.637485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.638060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.638066:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.638069:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677472.638071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.638072:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.638074:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.638076:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.638078:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.638079:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.638081:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.638082:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.638086:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.638088:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.638094:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.638095:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.638095:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.638097:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.638098:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.638099:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.638100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.638103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.638112:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:20.0:1619677472.638114:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:20.0:1619677472.638115:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.638116:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.638117:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:21.0:1619677472.638118:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:21.0:1619677472.638118:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.638119:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.638120:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.638120:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.638122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.638269:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.638272:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.638275:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.638277:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.638279:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.638280:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.638281:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.638282:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.638283:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.638287:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.638289:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.638291:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.638292:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.638292:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.638299:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.638299:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.638300:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.638301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.638313:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.638315:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.638316:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.638318:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.638319:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.638320:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.638320:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.638321:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.638321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.638322:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.638322:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.638720:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.638723:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.638724:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.638724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.638726:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.638728:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.638808:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.638810:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.638811:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.638813:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.638816:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.638817:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.638818:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.638819:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.638820:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.638821:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.638823:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.638825:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.638827:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.638828:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.638828:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.638830:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.638830:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.638831:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.638831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.638841:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.638844:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.638845:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.638845:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.638847:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.638849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:13.0:1619677472.639020:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.639023:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.639025:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.639028:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.639028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.639029:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677472.639031:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.639032:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.639033:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.639034:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677472.639037:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.639038:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677472.639041:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.639042:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:13.0:1619677472.639042:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:13.0:1619677472.639044:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677472.639045:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.639045:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.639046:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677472.639059:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.639062:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.639063:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.639063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.639065:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.639068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.639160:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.639163:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.639164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.639166:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.639167:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.639169:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.639170:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.639171:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.639172:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.639173:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.639176:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.639178:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.639181:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.639181:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.639182:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.639183:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.639184:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.639185:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.639185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.639189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.639198:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.639200:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.639201:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.639202:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.639204:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.639206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:23.0:1619677472.639306:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:22.0:1619677472.639886:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.639922:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677472.639923:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677472.639924:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677472.639925:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.639925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677472.639925:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.639927:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.639928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.639930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.639955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.639964:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.639967:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.639968:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.639968:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.639971:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.639973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.640187:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.640190:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.640190:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.640192:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.640195:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.640196:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.640197:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.640199:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.640200:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.640201:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.640203:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.640205:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.640207:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.640208:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.640208:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.640210:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.640210:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.640212:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677472.640213:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677472.640213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.0:1619677472.640222:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.640224:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.640225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.640225:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.640226:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:8.0:1619677472.640227:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.640228:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677472.640229:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.0:1619677472.640229:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.640230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:24.0:1619677472.640244:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.640246:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.640247:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.640247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.640248:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.640249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.640423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.640430:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.640433:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.640434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677472.640435:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677472.640435:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.640437:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:2.0:1619677472.640437:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.640438:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677472.640439:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:8.0:1619677472.640441:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.640441:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.640442:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.640443:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.640444:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.640444:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677472.640445:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.640448:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.640449:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.640449:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677472.640450:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.640451:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.640451:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.640454:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.640455:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:8.0:1619677472.640456:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.640456:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677472.640458:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.640458:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.640458:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.640459:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.640460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.640461:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.640462:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.2:1619677472.640464:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677472.640467:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.640469:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.640470:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.640471:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.640471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677472.640472:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.640474:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.640475:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677472.640475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:21.0:1619677472.640476:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.640477:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.640479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677472.640483:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.2:1619677472.640484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677472.640486:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.640487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.640487:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.640490:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.640491:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677472.640492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677472.640494:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.640495:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:1.0:1619677472.640495:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.0:1619677472.640496:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.640496:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677472.640496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677472.640498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677472.640499:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.640500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.640500:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.640501:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.640501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.640712:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.640714:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.640715:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.640715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.640717:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.640719:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.640731:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.640733:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.640734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.640734:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.640735:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.640736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.641171:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.641173:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.641174:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.641176:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.641178:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.641179:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.641180:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.641181:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.641182:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.641183:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.641186:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.641188:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.641191:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.641191:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.641192:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.641194:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.641194:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.641195:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.641195:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.641208:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.641211:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.641212:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.641214:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.641217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.641272:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.641274:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.641275:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.641277:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.641280:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.641281:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677472.641283:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.641284:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.641285:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.641286:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677472.641290:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.641291:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677472.641298:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677472.641300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677472.641300:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.641300:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677472.641302:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.641302:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.641303:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.641303:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677472.641306:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.641308:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.641311:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.641311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.641314:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.641315:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.641317:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96512822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.641317:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:21.0:1619677472.641318:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.641319:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.641319:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:21.0:1619677472.641320:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677472.641320:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.641320:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.641323:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.641324:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.641327:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677472.641327:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677472.641328:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.641329:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.641330:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.641331:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.641332:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.641333:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:24.0:1619677472.641334:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677472.641334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677472.641337:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.641338:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.641338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.641340:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.641342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.641348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.641356:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.641358:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.641359:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.641361:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.641362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.641363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677472.641542:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.641545:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.641546:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.641548:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.641551:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.641552:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.641554:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.641555:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.641556:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.641557:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.641560:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.641562:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.641565:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.641566:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.641567:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.641568:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.641569:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.641569:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.641570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.641586:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.641745:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.641748:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.641751:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.641753:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.641754:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.641755:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.641756:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.641758:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.641758:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.641761:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.641763:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.641765:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.641766:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.641767:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.641768:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.641769:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.641769:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.641770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.641782:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.641784:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.641785:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.641787:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.641787:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.641789:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.642008:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.642011:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.642012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.642013:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.642016:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.642017:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.642019:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.642020:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.642021:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.642022:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.642025:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.642027:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.642029:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.642030:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.642030:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.642032:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.642032:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.642034:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.642035:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.642035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.642046:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.642047:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.642048:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.642050:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.642050:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.642051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.642051:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.642051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.642052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.642052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677472.642084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.642089:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.642092:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.642093:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.642095:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.642097:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.642099:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.642100:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.642101:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.642102:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.642103:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.642106:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.642108:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.642109:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.642110:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.642111:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.642112:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.642113:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.642114:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.642114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677472.642120:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.642122:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677472.642125:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.642125:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.642126:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:20.0:1619677472.642127:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.642128:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:18.0:1619677472.642128:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.642129:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.642129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:20.0:1619677472.642130:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.642131:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.642131:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:20.0:1619677472.642133:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.642133:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:20.0:1619677472.642134:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.642137:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.642139:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.642142:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.642142:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.642143:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.642144:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.642145:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.642146:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677472.642146:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.642147:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:18.0:1619677472.642147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.642148:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.642148:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.642149:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.642149:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677472.642149:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.642160:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.642162:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.642163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.642164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677472.642331:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.642334:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.642334:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.642337:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.642338:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.642340:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.642341:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.642343:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.642344:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.642346:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.642350:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.642353:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.642355:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.642356:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.642356:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.642358:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.642359:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.642359:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.642360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:24.0:1619677472.642369:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.642371:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.642372:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.642372:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.642374:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.642375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677472.642377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.642384:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.642386:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.642387:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.642388:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.642390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.642391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.642402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.642410:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.642412:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.642413:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.642414:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.642415:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.642415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:22.0:1619677472.643380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.643417:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.643419:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.643420:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.643422:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.643423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.643424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.643516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.643522:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.643524:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.643526:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.643527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.643529:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.643531:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.643532:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.643533:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.643535:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.643535:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.643539:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.643541:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.643543:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.643544:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.643545:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.643546:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.643547:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.643547:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.643549:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.643549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.643553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.643556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677472.643560:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677472.643562:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:10.0:1619677472.643563:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.643564:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677472.643564:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.643564:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.643564:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.643565:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677472.643566:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.643566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.643567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:10.0:1619677472.643567:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.643568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.643569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677472.643570:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.643571:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.643572:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.643573:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.643574:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.643575:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.643579:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.643580:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.643583:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.643583:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.643584:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.643585:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.643586:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.643587:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.643588:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.643589:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.643597:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.643600:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.643601:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.643601:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.643602:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677472.643603:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.643604:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.643605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.643605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.643606:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677472.643855:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.643857:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.643858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.643860:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.643862:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.643863:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677472.643865:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.643866:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.643867:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.643867:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677472.643870:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.643872:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677472.643875:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.643875:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677472.643876:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677472.643877:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677472.643878:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.643878:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.643879:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:18.0:1619677472.643893:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.643895:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.643896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.643897:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.643899:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.643902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.644243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.644246:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.644247:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.644247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.644249:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.644250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.644590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.644599:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.644601:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.644602:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.644604:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.644604:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.644607:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.644849:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.644852:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.644853:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.644854:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.644856:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.644857:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.644859:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.644860:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.644861:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.644862:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.644865:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.644867:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.644869:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.644870:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.644871:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.644872:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.644873:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.644873:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.644874:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.644878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.644887:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.644889:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.644890:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.644891:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.644893:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.644895:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.645024:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.645027:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.645027:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.645029:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.645032:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.645033:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677472.645034:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.645036:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.645037:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.645038:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677472.645041:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.645043:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677472.645046:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.645046:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:17.0:1619677472.645047:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677472.645049:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677472.645049:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.645051:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677472.645052:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677472.645052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.645063:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677472.645065:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677472.645065:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.645066:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:22.0:1619677472.645333:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.645336:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.645339:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.645342:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.645343:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.645344:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.645346:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.645347:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.645348:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.645351:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.645353:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.645356:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.645356:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.645357:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.645359:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.645359:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.645360:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.645360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.645373:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.645375:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.645376:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.645377:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.645379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.645380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.645394:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.645397:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.645398:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.645398:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.645399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.645400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.645785:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.645787:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.645788:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.645788:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.645790:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.645792:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.645805:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.645808:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.645808:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.645809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.645809:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.645810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.645927:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:17.0:1619677472.645930:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.45@o2ib3
00000800:00000200:1.0:1619677472.645930:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000400:00000200:17.0:1619677472.645932:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.45@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.645935:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.645937:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:17.0:1619677472.645938:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.645939:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.645940:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.645941:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:17.0:1619677472.645944:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.645946:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:17.0:1619677472.645949:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.645950:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:17.0:1619677472.645950:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677472.645952:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:17.0:1619677472.645953:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:17.0:1619677472.645953:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:17.0:1619677472.645954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)--
00000800:00000200:24.2:1619677472.645958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.645961:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.645964:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677472.645965:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.645965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.645967:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.645970:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677472.645981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677472.645984:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.645987:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677472.645988:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677472.645988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677472.645989:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677472.645989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677472.646003:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.646005:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.646006:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.646008:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.646011:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.646012:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.646014:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.646015:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.646016:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.646017:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.646020:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.646022:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.646025:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.646026:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.646027:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.646029:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.646029:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.646030:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.646031:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.646031:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677472.646031:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.646032:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.646033:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677472.646035:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.646036:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677472.646038:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.646039:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.646039:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.646040:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677472.646043:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677472.646044:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677472.646046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.646047:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.646048:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.646048:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677472.646050:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.646050:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.646051:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.646052:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:22.0:1619677472.646062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.646065:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.646065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:22.0:1619677472.646066:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.646067:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.646067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677472.646067:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.646069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.646069:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.646071:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.646074:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.646194:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.646197:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.646198:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.646199:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.646202:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.646204:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.646205:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.646206:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.646207:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.646208:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.646211:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.646212:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.646215:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.646216:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.646216:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.646218:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.646218:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.646219:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.646220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.646232:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.646234:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.646235:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.646236:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.646238:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.646241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.646398:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.646401:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.646402:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.646403:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.646406:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.646407:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.646409:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.646410:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.646411:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.646412:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.646415:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.646416:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.646419:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.646420:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.646420:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.646422:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.646422:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.646423:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.646424:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.646425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.646439:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677472.646440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.646441:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.646442:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.646443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677472.646493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.646499:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.646502:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.646503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.646504:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.646506:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.646508:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677472.646509:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.646511:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.646512:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.646513:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.646517:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.646520:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677472.646523:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.646524:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677472.646524:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.646526:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.646527:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.646528:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.646528:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.646533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677472.646537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677472.646546:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:1.0:1619677472.646546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:21.0:1619677472.646549:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:1.0:1619677472.646549:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:21.0:1619677472.646550:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.646550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.646551:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.646552:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.646552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:1.0:1619677472.646553:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.646555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677472.646556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.647171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.647174:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.647176:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.647178:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.647179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.647181:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.647183:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.647184:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.647185:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.647186:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.647188:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.647191:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.647193:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.647195:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.647196:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.647197:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.647199:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.647199:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.647200:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.647201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677472.647205:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:8.2:1619677472.647205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.647208:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.647209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.647211:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.647213:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:17.0:1619677472.647213:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.647215:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.647215:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.647215:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.647216:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.647217:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.647218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:17.0:1619677472.647218:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677472.647219:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:17.0:1619677472.647219:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.647220:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.647223:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.647225:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.647228:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.647228:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.647229:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.647231:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.647231:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.647232:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.647233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.647236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.647245:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.647247:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.647248:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.647249:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.647251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.647252:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.647253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.647260:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.647263:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.647264:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.647264:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.647265:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.647266:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.647314:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.647317:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.647319:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.647322:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.647323:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.647325:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.647326:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.647327:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.647328:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.647331:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.647333:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.647336:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.647337:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.647337:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.647339:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.647339:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.647340:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.647342:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.647342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.647355:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.647357:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.647357:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.647359:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.647360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.647362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.647439:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.647442:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.647443:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.647445:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.647448:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.647449:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.647450:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.647452:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.647453:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.647453:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.647456:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.647458:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.647460:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.647460:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.647461:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.647462:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.647463:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.647464:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.647464:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.647474:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.647477:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.647477:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.647478:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.647479:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.647480:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.647869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.647872:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.647873:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.647873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.647874:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.647876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.648274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.648282:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.648285:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.648286:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.648287:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.648288:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.648290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677472.648478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.648481:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.648483:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.648484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.648485:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.648487:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.648488:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.648489:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.648490:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.648491:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.648495:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.648497:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.648500:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.648501:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.648502:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.648503:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.648504:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.648505:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.648505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677472.648784:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.648787:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.648788:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.648790:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.648792:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.648794:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.648795:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.648796:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.648797:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.648798:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.648801:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.648802:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.648804:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.648805:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.648806:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.648807:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.648808:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.648808:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.648809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.648820:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.648822:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.648823:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.648823:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.648825:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.648827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.648908:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.648911:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.648913:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.648916:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.648917:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.648918:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.648920:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.648920:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.648921:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.648925:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.648926:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.648928:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.648929:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.648930:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.648931:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.648932:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.648933:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.648933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.648945:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.648948:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.648949:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.648950:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.648951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.648953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677472.649490:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.649492:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.649493:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.649495:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.649498:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.649499:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677472.649500:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.649502:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.649503:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.649504:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677472.649507:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.649509:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677472.649512:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.649512:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677472.649513:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677472.649515:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677472.649515:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.649516:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.649517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677472.649522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.649531:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.649533:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.649534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.649535:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.649537:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.649540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.649541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.649549:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.649551:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.649552:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.649552:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.649553:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.649554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.650043:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.650045:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.650046:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.650047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.650049:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.650051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.650061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.650069:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.650072:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.650073:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.650075:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.650075:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.650078:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.650202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.650208:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.650211:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.650212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.650214:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.650216:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.650218:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.650219:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.650221:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.650222:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.650223:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.650226:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.650228:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.650230:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.650231:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.650231:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.650233:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.650234:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.650234:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.650235:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.650246:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.650248:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.650249:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.650250:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.650251:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.650252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.650645:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.650648:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.650649:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.650650:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.650652:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.650654:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.650655:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.650656:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.650657:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.650658:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.650662:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.650663:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.650666:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.650667:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.650668:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.650669:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.650670:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.650671:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.650672:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.650673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.650684:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.650686:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.650687:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677472.650688:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677472.650688:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.650688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.650689:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.650690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.650690:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.650694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.650703:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.650705:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.650707:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.650707:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.650708:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.650708:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677472.650924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677472.650930:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.650933:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677472.650934:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677472.650936:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.650939:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.650941:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677472.650942:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.650944:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.650945:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.650946:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677472.650948:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.650950:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677472.650952:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.650953:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:1.0:1619677472.650954:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677472.650955:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677472.650956:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.650958:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677472.650959:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677472.650959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:5.2:1619677472.650966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:18.0:1619677472.650969:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.650972:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:17.0:1619677472.650972:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000400:00000200:18.0:1619677472.650973:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.650973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677472.650974:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:22.0:1619677472.650975:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:18.0:1619677472.650975:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.650975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677472.650976:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:18.0:1619677472.650977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.651011:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.651013:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.651014:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.651015:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.651016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.651017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.651149:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.651152:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.651152:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.651155:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.651158:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.651159:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.651161:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.651162:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.651163:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.651164:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.651167:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.651169:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.651171:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.651172:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.651173:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.651174:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.651175:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.651176:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.651176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.651188:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.651191:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.651192:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.651194:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.651197:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.651340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.651345:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.651348:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.651350:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.651350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.651353:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.651354:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.651356:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.651357:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.651358:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.651359:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.651362:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.651364:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.651367:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.651367:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.651368:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.651370:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.651370:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.651371:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.651372:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.651376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.651384:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.651386:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.651387:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.651389:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.651390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.651391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.651674:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.651677:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.651678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.651678:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.651680:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.651682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.651695:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.651698:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.651698:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.651699:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.651699:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.651700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.651800:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.651803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.651803:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.651805:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.651808:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.651809:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.651810:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.651812:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.651813:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.651813:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.651816:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.651818:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.651821:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.651822:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.651822:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.651824:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.651824:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.651825:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.651826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.651838:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.651841:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.651842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.651842:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.651844:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.651847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.652551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677472.652560:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677472.652560:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.652563:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677472.652563:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677472.652563:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:26.0:1619677472.652564:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677472.652564:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.652566:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677472.652567:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.652570:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.652570:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:10.0:1619677472.652572:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.652573:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.652574:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.652575:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.652576:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.652580:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.652581:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.652583:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.652584:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.652585:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.652586:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.652586:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.652587:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.652588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.652598:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.652601:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.652602:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.652602:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.652603:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.652605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.652618:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.652620:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.652621:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.652621:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.652622:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.652623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.652670:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.652672:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.652673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.652675:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.652677:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.652679:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.652680:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.652681:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.652682:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.652682:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.652685:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.652687:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.652688:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.652689:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.652690:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.652691:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.652691:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.652692:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.652692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.652703:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.652704:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.652705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.652705:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.652706:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.652708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.652883:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.652886:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.652887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.652889:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.652891:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.652894:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.652895:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.652897:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.652898:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.652899:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.652902:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.652904:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.652906:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.652907:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.652908:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.652909:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.652910:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.652911:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.652912:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.652912:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.652916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.652923:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677472.652925:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.652925:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.652925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.652926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.652927:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.652928:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.652930:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.652931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.652933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.652970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.653005:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.653007:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.653008:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.653010:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.653011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.653012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.653023:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.653026:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.653027:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.653027:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.653028:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.653029:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.653643:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.653646:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.653647:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.653648:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.653649:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.653651:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.653679:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.653682:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.653683:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.653684:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.653687:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.653688:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.653689:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.653690:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.653691:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.653691:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.653694:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.653696:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.653697:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.653698:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.653699:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.653700:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.653701:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.653702:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.653703:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.653704:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.653712:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.653714:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.653715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.653715:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.653716:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677472.653717:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.653718:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.653719:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.653719:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.653720:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677472.653968:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.653971:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677472.653971:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677472.653973:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.653975:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.653976:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677472.653977:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.653979:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.653979:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.653980:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677472.653983:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.653985:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677472.653987:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.653988:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677472.653989:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677472.653991:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677472.653991:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.653992:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677472.653992:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:21.0:1619677472.654004:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.654007:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.654007:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.654009:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.654010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.654012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.654026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.654033:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.654037:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.654037:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.654039:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.654041:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.654043:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.654044:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.654046:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.654047:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.654048:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.654052:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.654053:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.654056:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.654057:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.654057:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.654059:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.654060:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.654061:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.654061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677472.654110:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.654113:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.654114:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.654114:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.654116:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.654120:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.654536:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.654539:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.654541:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.654544:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.654545:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.654547:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.654548:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.654549:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.654550:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.654553:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.654555:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.654557:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.654558:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.654558:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.654560:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.654560:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.654561:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.654562:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.654563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.654574:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.654577:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.654577:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.654579:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.654580:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.654581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.654924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.654929:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.654931:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.654934:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.654934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.654936:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.654937:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.654939:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.654940:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.654941:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.654942:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.654946:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.654947:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.654950:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.654950:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.654951:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.654953:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.654953:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.654954:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.654954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677472.654956:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.654959:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:8.2:1619677472.654959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677472.654960:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.654962:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677472.654965:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.654966:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:10.0:1619677472.654966:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677472.654967:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.654968:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:10.0:1619677472.654968:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.654969:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.654969:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.654971:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:10.0:1619677472.654971:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.0:1619677472.654972:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.654974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:10.0:1619677472.654975:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677472.654977:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.2:1619677472.654978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:10.0:1619677472.654979:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.654980:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.654981:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677472.654982:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.654983:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.654983:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.654984:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677472.654986:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.654988:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.654989:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.654989:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.654990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.654991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.654997:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.655000:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.655001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.655002:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.655004:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677472.655007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677472.655016:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.655019:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.655019:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677472.655020:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.655021:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.655021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.655275:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.655277:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.655278:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.655279:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.655280:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.655282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.655416:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.655418:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.655419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.655421:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.655424:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.655425:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.655426:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.655427:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.655428:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.655429:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.655432:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.655434:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.655436:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.655437:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.655438:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.655439:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.655440:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.655441:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.655441:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.655454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.655457:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.655458:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.655460:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.655463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.655475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.655476:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.655479:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.655480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.655482:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.655485:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.655487:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677472.655488:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.655489:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.655490:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.655491:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677472.655494:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.655496:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677472.655499:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.655500:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677472.655500:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677472.655502:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677472.655503:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.655504:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.655504:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:10.0:1619677472.655514:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.655517:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677472.655518:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677472.655519:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.655521:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.655524:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677472.655877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:8.2:1619677472.655881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.655889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.655891:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.655892:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.655895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.655895:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.655901:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.655920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:12.0:1619677472.655923:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.655925:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.655926:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.655928:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.655931:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.655932:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:26.0:1619677472.656005:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.656008:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.656009:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.656011:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.656013:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.656015:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677472.656016:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.656017:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.656018:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.656019:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677472.656022:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.656024:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677472.656026:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.656027:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677472.656027:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677472.656029:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677472.656029:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.656031:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.656032:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.656032:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:12.0:1619677472.656041:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.656043:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.656044:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.656045:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:12.0:1619677472.656046:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.656047:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:13.0:1619677472.656048:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677472.656048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:12.0:1619677472.656049:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677472.656049:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:13.0:1619677472.656064:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677472.656067:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677472.656068:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.656069:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677472.656069:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.656070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677472.656553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.656555:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.656558:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.656561:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.656562:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.656563:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.656565:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.656566:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.656567:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.656571:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.656572:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.656575:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.656575:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.656576:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.656578:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.656578:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.656579:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.656579:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.656593:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.656595:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.656596:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.656597:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.656598:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.656599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.656838:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677472.656841:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677472.656842:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677472.656844:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.656846:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.656848:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677472.656849:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.656851:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.656852:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.656853:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677472.656856:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.656858:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677472.656861:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.656862:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677472.656862:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677472.656864:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677472.656865:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677472.656865:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677472.656866:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:24.0:1619677472.657213:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.657216:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.657217:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.657217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.657218:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.657220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.657235:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.657237:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.657238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.657238:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.657239:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.657240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.657348:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.657351:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.657352:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.657354:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.657356:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.657357:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.657358:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.657358:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677472.657359:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.657360:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677472.657360:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677472.657361:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.657362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.657363:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.657365:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.657365:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.657366:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.657367:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.657368:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677472.657369:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677472.657369:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677472.657370:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:24.0:1619677472.657370:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677472.657371:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:24.0:1619677472.657371:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677472.657372:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.657373:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.657374:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.657374:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677472.657374:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.657376:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.657379:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.657380:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.657380:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.657382:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.657382:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.657383:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.657383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677472.657391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677472.657396:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.657399:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.657400:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.657400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.657402:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.657405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.657416:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.657419:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.657420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.657420:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.657421:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.657421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.657486:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.657489:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.657491:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.657492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.657493:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.657495:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.657496:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.657498:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.657499:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.657500:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.657503:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.657505:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.657508:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.657509:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.657509:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.657511:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.657512:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.657513:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.657513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677472.657880:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.657883:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.657884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.657886:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.657889:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.657891:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.657892:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.657893:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.657895:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.657896:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.657899:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.657901:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.657904:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.657905:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.657905:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.657907:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.657907:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.657908:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.657908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.657913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.657921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.657923:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.657924:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.657926:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.657929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.657930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:13.0:1619677472.657934:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.657937:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.657939:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.657941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.657942:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.657943:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677472.657945:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.657946:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.657947:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.657948:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677472.657951:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.657953:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677472.657956:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.657956:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677472.657957:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677472.657959:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677472.657959:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.657961:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677472.657962:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677472.657963:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.657976:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677472.657978:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677472.657979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.657980:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677472.658468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.658474:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.658476:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.658478:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.658479:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.658481:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.658482:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.658484:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.658485:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.658486:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.658487:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.658490:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.658492:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.658493:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.658494:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.658495:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.658496:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.658497:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.658498:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.658499:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.658500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.658503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.658506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.658513:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.658514:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.658514:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.658514:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677472.658516:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.658516:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.658516:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.658517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.658518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.658519:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.658563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.658571:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.658574:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.658575:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.658575:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677472.658575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.658577:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.658578:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.658579:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677472.658581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.658582:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000400:00000200:8.0:1619677472.658586:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.658589:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.658590:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677472.658590:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000400:00000200:8.0:1619677472.658592:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677472.658592:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.658593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:8.0:1619677472.658593:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677472.658593:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000400:00000200:8.0:1619677472.658594:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.658595:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677472.658595:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.658595:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:8.0:1619677472.658598:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.658599:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.658602:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.658603:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.658603:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.658605:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.658605:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.658606:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677472.658608:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.0:1619677472.658608:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.658608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677472.658611:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.658614:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.658616:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.658618:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.658619:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.658620:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677472.658621:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:22.0:1619677472.658621:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.658622:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677472.658623:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.658623:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.658624:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.658625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677472.658625:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.658627:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677472.658627:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.658628:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.658628:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677472.658630:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.658631:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:18.0:1619677472.658631:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.658632:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.658633:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.658634:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.658634:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677472.658634:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.658635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.658647:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.658649:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.658650:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.658652:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.658653:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.658654:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.658671:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.658674:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.658675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:21.0:1619677472.658675:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.658676:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.658676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.658678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.658703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.658710:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.658713:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.658714:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.658714:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.658716:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.658718:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.658777:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.658779:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.658780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.658780:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.658782:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.658784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.658891:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.658894:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.658894:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677472.658896:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.658899:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.658900:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.658901:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.658902:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.658903:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.658904:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.658907:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.658909:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.658911:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.658911:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.658912:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.658913:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.658914:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.658914:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.658915:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.658925:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.658927:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.658928:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.658928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.658929:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.658932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.659197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.659202:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.659205:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.659206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.659209:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.659211:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.659212:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.659214:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.659215:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.659216:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.659217:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.659220:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.659222:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.659224:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.659225:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.659225:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.659227:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.659228:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.659229:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.659230:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.659231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.659236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.659242:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.659243:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.659243:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.659243:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.659244:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.659244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.659245:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.659245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.659246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.659247:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:12.0:1619677472.659764:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.659767:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.659769:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.659772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.659772:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.659773:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677472.659775:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.659776:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.659777:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.659778:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677472.659781:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.659783:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677472.659786:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.659786:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677472.659787:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677472.659789:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677472.659789:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.659790:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.659791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:26.0:1619677472.659802:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.659805:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.659806:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.659806:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677472.659808:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.659810:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677472.660196:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.660198:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.660199:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.660201:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.660203:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.660204:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.660205:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.660206:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.660207:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.660208:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.660211:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.660213:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.660216:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.660216:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.660217:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.660219:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.660219:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.660219:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.660220:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.660232:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.660235:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.660236:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.660238:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.660241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.660382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.660389:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.660392:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.660393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.660394:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.660396:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.660397:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.660399:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.660400:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.660401:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.660402:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.660405:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.660407:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.660410:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.660411:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.660412:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.660413:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.660414:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.660415:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.660416:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.660416:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.660421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677472.660427:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:18.0:1619677472.660427:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.660429:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:26.0:1619677472.660430:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:18.0:1619677472.660430:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677472.660431:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:2.0:1619677472.660431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:26.0:1619677472.660432:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.660434:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677472.660437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:8.0:1619677472.660714:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.660716:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.660717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.660717:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.660719:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.660721:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.661008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.661042:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.661045:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.661045:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.661047:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.661048:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.661049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.661149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.661157:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.661159:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.661160:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.661162:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.661162:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.661165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.661209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.661217:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.661220:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.661220:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.661221:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.661222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.661223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677472.661549:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.661552:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.661553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.661554:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.661557:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.661558:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677472.661559:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.661561:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.661562:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.661563:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677472.661566:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.661568:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677472.661570:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.661571:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677472.661571:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677472.661573:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677472.661573:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.661574:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.661574:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:13.0:1619677472.661587:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.661589:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.661590:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.661592:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.661594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677472.661594:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677472.662137:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.662140:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.662141:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.662142:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.662145:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.662146:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.662147:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.662148:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.662149:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.662150:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.662153:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.662154:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.662157:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.662157:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.662158:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.662159:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.662160:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.662161:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.662161:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.662173:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.662176:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.662177:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.662179:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.662182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.662221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.662224:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.662227:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.662229:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.662231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.662231:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.662233:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.662234:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.662236:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.662237:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.662239:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.662243:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.662246:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.662249:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.662250:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.662252:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.662253:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.662254:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.662255:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.662256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.662260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677472.662267:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:21.0:1619677472.662268:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.662270:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:18.0:1619677472.662270:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:21.0:1619677472.662271:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.662271:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.662271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:20.0:1619677472.662273:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677472.662274:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.662274:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.662276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.662278:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.662357:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.662360:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.662361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.662364:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.662367:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.662368:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.662369:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.662370:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.662371:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.662372:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.662375:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.662377:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.662380:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.662381:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.662381:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.662382:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (24)++
00000800:00000200:24.0:1619677472.662383:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (25)--
00000800:00000200:24.0:1619677472.662384:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.662384:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (25)--
00000800:00000200:24.0:1619677472.662385:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.662385:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.662386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:8.0:1619677472.662386:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.662387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.662389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.662396:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.662397:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.662398:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.662399:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.662399:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.662401:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.662414:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.662416:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.662417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.662417:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.662418:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.662418:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.662720:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.662722:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.662723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.662725:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.662727:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.662729:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.662730:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.662731:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.662732:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.662733:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.662736:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.662737:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.662740:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.662741:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.662742:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.662743:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.662744:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.662744:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.662745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677472.662750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.662757:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.662759:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.662760:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.662762:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.662763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.662766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677472.663221:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.663225:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.663225:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.663228:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.663230:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.663231:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.663233:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.663234:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.663235:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.663236:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.663239:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.663241:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.663243:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.663244:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.663245:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.663247:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.663247:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.663248:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.663248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.663253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.663261:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.663263:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.663264:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.663266:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.663266:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.663268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677472.663278:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.663281:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.663282:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.663284:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.663287:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.663288:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677472.663290:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.663291:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.663292:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.663299:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.663302:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.663304:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677472.663307:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.663308:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.663308:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.663310:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.663310:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.663312:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677472.663313:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677472.663314:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677472.663325:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677472.663328:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677472.663328:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.663329:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.663330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.663330:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677472.663330:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.663331:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677472.663332:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.663335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677472.663612:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677472.663623:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.663626:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.663627:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.663630:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.663632:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.663634:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.663635:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.663636:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.663638:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.663638:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.663642:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.663643:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.663646:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.663647:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.663647:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677472.663647:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677472.663648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677472.663649:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.663649:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677472.663649:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:26.0:1619677472.663650:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677472.663650:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677472.663651:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677472.663651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.663651:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.663651:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.663652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677472.663653:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.663654:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.663656:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.663662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.663664:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.663665:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.663665:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677472.663666:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.663667:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:24.0:1619677472.663668:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.663668:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.663669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.663670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.664127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.664135:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.664138:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.664138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.664139:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.664141:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.664145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.664287:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.664289:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.664290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.664290:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.664292:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.664300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.664624:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.664627:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.664628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.664629:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.664630:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.664632:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.664633:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.664634:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.664635:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.664636:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.664640:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.664642:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.664645:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.664645:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.664646:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.664648:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.664648:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.664649:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.664649:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.664663:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.664666:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.664667:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.664667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.664669:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.664673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.665020:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.665023:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.665024:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.665025:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.665027:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.665029:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.665030:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.665031:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.665032:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.665033:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.665036:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.665038:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.665040:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.665041:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.665042:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.665043:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.665044:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.665045:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.665046:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.665046:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.665059:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677472.665061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.665061:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.665062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.665063:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.2:1619677472.665473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.665479:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.665482:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.665483:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.665485:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.665488:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.665489:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.665491:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.665492:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.665493:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.665494:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.665497:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.665498:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.665500:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.665501:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.665502:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.665503:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.665504:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.665505:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.665505:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.665518:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.665520:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.665521:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.665521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.665522:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.665523:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.665534:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.665536:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.665537:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.665538:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.665538:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.665538:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.665726:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.665728:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.665731:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.665733:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.665735:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.665736:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.665737:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.665738:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.665739:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.665742:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.665744:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.665746:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.665747:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.665748:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.665749:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.665750:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.665751:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.665751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.665763:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.665765:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.665766:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.665768:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.665769:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.665770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.665880:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.665883:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.665884:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.665884:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.665886:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.665889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.665901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.665903:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.665906:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.665908:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.665909:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.665910:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.665912:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.665913:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.665914:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.665915:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.665916:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.665919:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.665921:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.665923:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.665924:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.665925:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.665927:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.665928:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.665928:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.665929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.665934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.665942:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.665945:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.665945:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.665947:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.665948:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.665949:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.666596:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.666599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.666599:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677472.666602:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.666604:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.666606:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677472.666607:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.666609:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.666610:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.666611:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677472.666614:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.666616:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677472.666619:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.666620:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677472.666621:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677472.666622:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677472.666623:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.666624:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677472.666625:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677472.666626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:26.0:1619677472.666636:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:17.0:1619677472.666638:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:26.0:1619677472.666639:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:17.0:1619677472.666640:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:10.0:1619677472.666640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.666641:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.666641:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.666642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:26.0:1619677472.666643:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.666647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677472.666943:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.666946:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.666946:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677472.666948:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.666950:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.666952:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.666953:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.666954:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.666955:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.666956:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.666960:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.666962:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.666965:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.666966:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.666967:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.666968:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.666969:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.666970:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.666970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.666986:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.667296:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.667298:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.667301:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.667303:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.667304:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.667305:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.667307:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.667308:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.667309:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.667312:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.667314:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.667316:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.667317:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.667317:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.667319:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.667320:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.667320:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.667321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.667334:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.667336:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.667337:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.667338:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.667340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.667341:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.667351:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.667354:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.667355:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.667356:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.667356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.667357:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.667622:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.667625:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.667626:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.667628:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.667631:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.667632:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.667633:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.667634:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.667635:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.667636:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.667639:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.667641:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.667642:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.667643:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.667644:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.667645:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.667646:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.667646:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.667647:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.667657:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.667659:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.667660:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.667660:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.667661:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.667663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.667777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.667785:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.667787:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.667788:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.667790:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.667790:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677472.667794:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.667797:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677472.667797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:24.0:1619677472.667798:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677472.667798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:24.0:1619677472.667800:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.667804:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.667805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.667806:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.667809:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.667810:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.667811:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.667811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.667812:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.667816:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.667819:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.667820:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.667820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.667821:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.667821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.667874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.667881:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.667883:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.667884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.667886:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.667889:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.667890:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.667892:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.667893:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.667894:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.667895:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.667899:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.667900:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.667903:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.667904:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.667904:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.667906:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.667907:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.667908:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.667908:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.667912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.667921:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.667923:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.667924:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.667925:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.667927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.667929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.667937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.667945:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.667948:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.667949:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.667950:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.667950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.667951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.668060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.668065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.668068:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.668069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.668072:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.668075:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.668077:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677472.668078:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.668079:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.668081:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.668082:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677472.668085:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.668086:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677472.668090:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.668090:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677472.668091:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677472.668093:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677472.668094:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.668095:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677472.668096:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677472.668096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677472.668101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:8.0:1619677472.668105:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677472.668107:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:8.0:1619677472.668108:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.668109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677472.668109:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:8.0:1619677472.668109:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.668110:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.668111:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:8.0:1619677472.668111:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.668114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.668127:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.668129:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.668130:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.668130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.668131:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.668132:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.668331:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.668334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.668334:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.668337:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.668340:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.668342:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.668343:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.668344:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.668345:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.668347:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.668350:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.668351:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.668354:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.668354:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.668355:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.668357:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.668357:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.668358:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.668358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.668364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.668372:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.668375:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.668375:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.668377:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.668378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.668379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677472.668830:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.668833:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.668833:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.668835:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.668838:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.668840:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.668841:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.668842:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.668843:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.668844:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.668848:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.668849:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.668852:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.668852:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.668853:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.668855:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.668855:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.668856:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.668856:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.668867:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.668869:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.668870:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.668870:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.668872:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.668875:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.669011:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.669014:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.669015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.669016:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.669019:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.669020:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.669021:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.669023:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.669023:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.669024:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.669027:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.669028:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.669031:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.669032:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.669033:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.669035:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.669035:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.669036:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.669036:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677472.669041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.669048:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.669051:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.669052:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.669053:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.669055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.669056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.669324:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:26.0:1619677472.669326:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677472.669327:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.669328:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.669328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677472.669329:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:24.0:1619677472.669330:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.669330:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.669332:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.669332:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677472.669335:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.669336:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.669337:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.669338:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.669339:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.669340:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.669343:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.669345:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.669348:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.669348:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.669349:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.669351:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.669351:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.669352:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.669352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.669368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.669397:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.669399:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.669402:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.669404:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.669405:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.669406:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.669408:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.669409:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677472.669409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677472.669410:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.669413:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.669415:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.669415:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.669417:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677472.669418:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.669419:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.669419:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677472.669420:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.669420:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677472.669421:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.669421:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677472.669421:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.669422:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.669422:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677472.669423:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.669424:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.669425:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.669426:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.669427:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.669430:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.669432:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.669434:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.669434:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.669435:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.669435:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.669436:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:20.0:1619677472.669436:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.669437:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.669438:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677472.669438:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.669439:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.669439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677472.669441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.669442:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.669444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.669452:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.669453:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.669454:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.669456:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.669457:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.669458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677472.670234:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.670237:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.670237:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677472.670240:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.670243:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.670244:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677472.670245:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.670247:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.670248:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.670249:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677472.670252:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.670253:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677472.670256:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.670256:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677472.670257:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677472.670258:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677472.670259:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.670260:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677472.670261:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677472.670262:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.670275:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.670276:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.670277:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.670278:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677472.670332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.670341:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.670343:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.670344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.670345:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.670347:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.670350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.670908:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.670911:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.670912:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.670914:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.670916:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.670917:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.670918:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.670920:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.670921:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.670922:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.670925:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.670926:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.670929:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.670930:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.670930:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.670932:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.670933:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.670934:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.670935:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.670936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.670938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.670946:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.670948:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.670949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.670949:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677472.670950:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677472.670950:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.670951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.670952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677472.670952:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.670955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677472.671128:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.671131:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.671132:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.671134:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.671136:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.671137:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677472.671138:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.671140:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.671141:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.671142:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677472.671144:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.671146:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677472.671148:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.671149:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677472.671149:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677472.671151:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677472.671151:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.671152:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.671152:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677472.671165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.671168:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.671168:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.671170:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.671172:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.671174:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677472.671228:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.671230:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.671231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.671231:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.671233:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.671235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.671654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.671690:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.671691:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.671692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.671692:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677472.671693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.671694:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.671695:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.671696:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.671698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.671699:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.671772:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.671775:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.671776:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.671778:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.671780:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.671782:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.671783:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.671784:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.671785:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.671787:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.671790:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.671792:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.671795:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.671795:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.671796:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.671798:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.671798:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.671799:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.671799:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.671815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.671830:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.671832:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.671833:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.671835:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.671835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.672058:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.672061:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.672061:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677472.672063:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.672066:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.672067:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677472.672069:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.672070:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.672071:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.672071:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677472.672074:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.672076:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677472.672078:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.672079:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.672080:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677472.672081:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.672082:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.672083:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.672083:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677472.672096:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.672098:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.672100:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.672100:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.672102:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.672105:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677472.672203:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.672206:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.672207:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.672209:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.672211:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.672212:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677472.672214:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.672215:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.672216:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.672217:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677472.672219:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.672220:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.672222:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.672222:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677472.672223:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.672224:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.672224:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.672225:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.672226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677472.672236:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.672238:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677472.672239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677472.672239:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.672240:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.672242:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677472.672257:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.672260:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.672260:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.672260:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.672262:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.672262:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677472.672360:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.672362:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.672363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.672365:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.672367:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.672368:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.672369:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.672370:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.672371:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.672372:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.672375:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.672376:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.672378:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.672379:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.672379:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.672381:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.672381:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.672382:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.672382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.672393:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.672395:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.672395:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.672396:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.672397:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.672398:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.672875:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.672878:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.672879:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.672879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.672880:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.672882:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.672892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.672895:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.672897:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.672899:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.672900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.672901:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.672903:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:8.0:1619677472.672903:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677472.672904:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.672906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677472.672906:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677472.672906:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.672907:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677472.672907:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677472.672908:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.0:1619677472.672908:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.672909:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:20.0:1619677472.672911:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.672913:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.672915:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.672916:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.672916:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.672918:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.672919:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.672920:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.672921:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.672922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.672924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.672927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.672930:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.672932:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.672932:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.672932:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.672933:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.672934:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.672934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.672935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.672936:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.672937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677472.673218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.673252:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.673254:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.673255:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.673257:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.673258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.673259:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677472.673535:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.673538:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.673539:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.673541:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.673543:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.673544:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.673545:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.673546:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.673547:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.673549:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.673552:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.673553:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.673556:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.673556:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.673557:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.673558:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.673559:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.673560:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.673561:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.673561:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677472.673572:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.673574:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677472.673575:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677472.673575:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.673576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.673577:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.673577:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677472.673577:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.673578:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.673580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.673599:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.673602:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.673603:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.673604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:17.0:1619677472.673605:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.673607:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.673609:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.673611:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.673612:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.673613:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:17.0:1619677472.673613:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.673614:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.673616:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.673616:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.673617:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677472.673618:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.673619:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.673620:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.673621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677472.673622:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677472.673622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.673623:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.673624:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.673625:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677472.673626:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.673628:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677472.673629:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.673629:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.673629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677472.673632:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.673635:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:21.0:1619677472.673637:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (24)++
00000800:00000200:21.0:1619677472.673637:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (25)--
00000800:00000200:20.0:1619677472.673637:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000800:00000200:21.0:1619677472.673638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.673638:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.673639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.673639:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000400:00000200:20.0:1619677472.673640:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.673642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.673642:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.673642:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.673643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677472.673777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.673784:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.673786:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.673787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.673789:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.673792:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.673793:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.673795:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.673796:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.673797:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.673798:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.673801:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.673803:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.673805:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.673806:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.673806:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.673808:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.673809:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.673810:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.673810:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.673822:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.673824:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.673825:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.673826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.673828:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.673829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.674209:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.674212:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.674213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.674215:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.674217:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.674219:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.674220:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.674222:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.674222:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.674224:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.674227:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.674229:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.674232:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.674233:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.674233:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.674235:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.674236:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.674236:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.674237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.674249:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.674252:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.674253:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.674255:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.674258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.674271:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:8.0:1619677472.674813:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.674816:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.674817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.674817:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.674819:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.674821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.674933:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.674936:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677472.674937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677472.674938:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.674941:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.674942:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677472.674943:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.674944:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.674946:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.674946:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677472.674949:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.674951:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677472.674953:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.674954:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677472.674954:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677472.674956:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677472.674957:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.674957:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.674958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:20.2:1619677472.675241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.675246:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.675249:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.675251:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.675251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.675253:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.675254:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.675256:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.675257:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.675262:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.675263:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677472.675265:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677472.675266:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.675268:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.0:1619677472.675268:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:22.0:1619677472.675270:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.675271:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.675272:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.675272:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:22.0:1619677472.675273:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.675274:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.0:1619677472.675274:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.675274:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.675275:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677472.675276:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.675276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677472.675277:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.675278:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.675279:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.2:1619677472.675280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:22.0:1619677472.675282:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.675284:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.675287:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.675287:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.675288:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.675288:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:22.0:1619677472.675290:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.675290:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677472.675290:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:22.0:1619677472.675291:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.675291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677472.675291:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.675292:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.675293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.675300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.675301:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.675303:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.675304:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677472.675304:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.675305:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.675307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.675332:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.675334:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.675335:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.675336:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.675336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.675337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.675624:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.675627:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.675627:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.675630:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.675632:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.675634:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.675635:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.675637:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.675638:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.675639:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.675642:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.675644:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.675647:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.675648:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.675648:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.675650:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.675651:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.675651:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.675652:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.675665:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.675668:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.675669:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.675669:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.675671:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.675675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.675894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.675903:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.675905:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.675906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.675907:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.675909:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.675911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.675914:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.675916:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.675918:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.675919:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.675919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.675920:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.675920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.676264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.676270:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.676273:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.676274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.676275:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.676278:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.676279:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.676280:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.676281:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.676282:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.676283:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.676287:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.676288:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.676291:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.676292:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.676292:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.676302:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.676303:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.676304:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.676304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.676376:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.676379:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.676380:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.676380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.676381:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.676384:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.676513:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:1.0:1619677472.676516:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.45@o2ib3
00000800:00000200:17.0:1619677472.676517:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000400:00000200:1.0:1619677472.676519:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.45@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.676521:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.676522:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:1.0:1619677472.676524:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.676525:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.676526:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.676527:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:1.0:1619677472.676530:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.676532:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:1.0:1619677472.676536:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.676536:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:1.0:1619677472.676537:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677472.676539:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:1.0:1619677472.676539:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:1.0:1619677472.676540:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:1.0:1619677472.676540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)--
00000800:00000200:24.2:1619677472.676545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.676548:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.676551:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677472.676552:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.676552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.676554:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.676557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677472.676765:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.676767:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.676769:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.676769:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677472.676771:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.676771:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.676772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677472.676774:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.676774:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.676775:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.676776:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.676776:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.676778:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.676778:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677472.676779:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.676780:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677472.676780:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.676781:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.676782:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.676783:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677472.676784:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.676786:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677472.676787:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.676789:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.676789:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677472.676790:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.676790:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.676792:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.676792:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.676793:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677472.676793:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.676793:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677472.676794:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.676794:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.676795:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.676796:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.676796:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.676797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.676802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.676809:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:22.0:1619677472.676811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.676812:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.676813:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.676814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.676816:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.676819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677472.676881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907b38d400] (20)++
00000800:00000200:18.0:1619677472.676887:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b38d400] (21)++
00000800:00000200:18.0:1619677472.676891:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.11@o2ib3
00000800:00000200:2.0:1619677472.676891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38d400] (22)--
00000400:00000200:18.0:1619677472.676894:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.11@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.676896:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.676898:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677472.676899:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.676901:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.676902:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.676903:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677472.676906:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.676908:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677472.676911:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.676912:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677472.676913:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677472.676914:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677472.676915:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907b38d400] (21)++
00000800:00000200:18.0:1619677472.676916:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907b38d400] (22)--
00000800:00000200:18.0:1619677472.676917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38d400] (21)--
00000800:00000200:21.0:1619677472.676932:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:1.0:1619677472.676933:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:21.0:1619677472.676934:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:1.0:1619677472.676935:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.45@o2ib3
00000400:00000200:21.0:1619677472.676936:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.676936:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000400:00000200:21.0:1619677472.676938:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.676938:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:1.0:1619677472.676939:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.45@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.676941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677472.676942:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.676943:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:1.0:1619677472.676945:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.676946:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.676947:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.676948:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:1.0:1619677472.676952:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.676953:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:1.0:1619677472.676956:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.676957:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:1.0:1619677472.676958:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677472.676960:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:1.0:1619677472.676960:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:1.0:1619677472.676961:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce0bf7800] (22)++
00000800:00000200:1.0:1619677472.676963:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf7800] (23)--
00000800:00000200:1.0:1619677472.676963:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:24.2:1619677472.676965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.676968:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.676970:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:8.0:1619677472.676971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.676972:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.676973:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)++
00000400:00000200:24.0:1619677472.676974:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.676976:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce0bf7800] (23)--
00000800:00000200:24.0:1619677472.676977:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:17.0:1619677472.676977:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:1.0:1619677472.676978:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)--
00000800:00000200:22.0:1619677472.677129:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.677132:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.677135:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.677137:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.677138:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.677140:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.677141:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.677142:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.677143:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.677146:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.677148:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.677150:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.677151:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.677151:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.677153:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.677154:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.677154:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.677155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.677167:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.677169:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.677170:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.677172:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.677173:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.677174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677472.677321:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:17.0:1619677472.677324:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.45@o2ib3
00000800:00000200:1.0:1619677472.677325:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000400:00000200:17.0:1619677472.677327:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.45@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.677330:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.677331:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:17.0:1619677472.677333:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.677334:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.677335:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.677336:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:17.0:1619677472.677340:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.677342:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:17.0:1619677472.677345:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.677345:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:17.0:1619677472.677346:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677472.677348:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:17.0:1619677472.677349:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:17.0:1619677472.677349:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:17.0:1619677472.677350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)--
00000800:00000200:24.2:1619677472.677354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.677357:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.677360:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677472.677361:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.677361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.677363:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.677366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:21.0:1619677472.677369:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.677371:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.677374:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.677374:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.677376:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.677378:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.677379:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.677380:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.677381:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.677382:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.677386:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.677387:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.677390:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.677390:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.677391:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.677393:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.677393:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.677394:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.677395:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.677396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.677405:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.677407:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.677408:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.677409:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.677412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677472.677412:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.677415:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.677416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677472.677418:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.677421:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.677422:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677472.677424:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.677425:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.677426:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.677427:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677472.677431:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.677432:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677472.677435:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.677435:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677472.677436:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677472.677437:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677472.677438:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.677439:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.677439:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677472.677451:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.677454:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.677455:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.677457:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.677460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677472.677515:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:17.0:1619677472.677518:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.45@o2ib3
00000800:00000200:1.0:1619677472.677518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000400:00000200:17.0:1619677472.677520:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.45@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.677523:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.677524:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:17.0:1619677472.677525:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.677527:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.677528:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.677528:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:17.0:1619677472.677532:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.677533:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:17.0:1619677472.677536:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.677537:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:17.0:1619677472.677537:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677472.677539:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:17.0:1619677472.677539:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:17.0:1619677472.677540:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:17.0:1619677472.677540:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)--
00000800:00000200:24.2:1619677472.677544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.677547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.677550:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:8.0:1619677472.677550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.677551:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.677553:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.677555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677472.677564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677472.677567:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.677570:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677472.677571:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677472.677571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677472.677572:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677472.677572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:1.0:1619677472.677641:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:17.0:1619677472.677644:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:1.0:1619677472.677644:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.45@o2ib3
00000400:00000200:1.0:1619677472.677646:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.45@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.677649:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.677650:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:1.0:1619677472.677651:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.677653:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.677654:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.677654:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:1.0:1619677472.677658:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.677659:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:1.0:1619677472.677662:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.677662:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:1.0:1619677472.677663:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677472.677665:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:1.0:1619677472.677665:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:1.0:1619677472.677666:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:1.0:1619677472.677667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)--
00000800:00000200:24.2:1619677472.677670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.677673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.677676:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677472.677677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.677677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677472.677679:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.677681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.0:1619677472.677744:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.677747:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.677747:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.677749:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.677752:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.677753:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.677754:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.677756:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.677757:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.677757:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.677760:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.677762:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.677763:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.677764:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.677765:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.677766:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.677766:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.677768:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.677769:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.677769:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.677780:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.677781:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.677782:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.677783:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.677784:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.677785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.677785:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.677785:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.677786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.677786:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677472.677918:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.677921:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.677922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.677924:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.677927:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.677928:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677472.677929:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.677930:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.677931:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.677932:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677472.677935:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.677936:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677472.677937:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.677938:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677472.677939:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677472.677940:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677472.677940:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.677941:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.677941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677472.677952:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.677953:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.677954:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.677955:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.677955:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.677957:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:17.0:1619677472.678209:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.678212:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.678212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.678215:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.678217:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.678219:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.678220:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.678222:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.678223:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.678224:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.678227:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.678229:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.678231:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.678232:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.678233:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.678234:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.678235:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.678235:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.678236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.678241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.678249:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.678251:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.678252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.678254:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.678255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.678256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.678364:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.678366:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.678367:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.678368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.678369:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.678371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.678382:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.678384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.678384:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.678385:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.678386:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.678387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.678533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.678567:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.678569:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677472.678569:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677472.678570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.678571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.678571:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.678572:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.678574:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.678575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.678576:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677472.678762:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.678765:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.678766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.678768:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.678770:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.678771:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.678772:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.678774:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.678775:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.678776:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.678779:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.678781:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.678783:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.678784:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.678785:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.678787:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.678787:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.678788:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.678788:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.678798:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.678800:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.678801:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.678801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.678803:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.678805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.679051:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.679053:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.679054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.679056:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.679058:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.679059:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.679061:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.679062:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.679063:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.679063:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.679066:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.679068:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.679071:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.679072:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.679072:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.679074:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.679074:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.679075:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.679075:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.679091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:8.2:1619677472.679200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.679208:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.679211:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.679212:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.679213:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.679215:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.679216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.679222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.679229:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.679232:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.679233:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.679233:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.679234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.679235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.679485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.679491:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.679493:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.679495:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.679496:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.679497:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.679499:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.679500:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.679501:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.679502:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.679503:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.679506:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.679508:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.679510:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.679511:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.679512:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.679513:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.679514:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.679515:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.679515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.679519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.679527:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.679529:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.679530:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.679531:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.679533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.679533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.679972:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.679975:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.679976:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.679976:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.679978:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.679980:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.680185:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.680188:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.680190:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.680193:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.680194:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.680195:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.680197:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.680198:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.680199:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.680202:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.680204:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.680206:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.680207:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.680207:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.680209:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.680209:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.680210:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.680211:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.680223:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.680225:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.680226:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.680228:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.680228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.680230:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.680239:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.680242:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.680243:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.680244:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.680245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.680246:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677472.680467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677472.680501:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:10.0:1619677472.680501:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:22.0:1619677472.680503:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677472.680504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677472.680505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:10.0:1619677472.680505:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677472.680506:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677472.680506:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.680508:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.680511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677472.680822:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.680825:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.680826:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.680827:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.680830:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.680831:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677472.680832:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.680834:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.680835:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.680836:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.680839:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.680840:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677472.680843:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.680844:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.680845:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.680847:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.680847:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.680848:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.680848:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677472.680862:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.680865:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.680866:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.680866:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.680868:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.680873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.680875:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.680877:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.680878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.680878:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.680879:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.680880:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.681022:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.681025:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.681026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.681027:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.681029:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.681030:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.681031:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.681034:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.681034:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.681035:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.681038:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.681040:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.681043:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.681043:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.681044:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.681046:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.681046:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.681047:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.681047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.681060:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.681063:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.681063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.681064:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.681066:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.681069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.681228:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.681231:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.681233:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.681234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.681235:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.681236:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.681238:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.681239:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.681240:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.681241:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.681245:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.681246:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.681249:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.681250:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.681250:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.681252:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.681253:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.681253:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.681254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.681269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.681283:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.681287:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.681288:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.681289:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.681289:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.681717:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.681720:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.681720:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677472.681723:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.681725:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.681726:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.681728:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.681729:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.681730:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.681731:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.681734:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.681735:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.681738:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.681739:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.681740:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.681742:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.681742:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.681743:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.681743:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.681760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.2:1619677472.681822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.681831:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.681834:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.681835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.681835:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.681837:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.681840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.681941:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.681944:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.681945:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.681948:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.681950:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.681952:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.681953:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.681955:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.681956:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.681956:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.681961:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677472.681962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.0:1619677472.681962:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.681965:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.681966:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.681966:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677472.681967:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.681967:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000400:00000200:24.0:1619677472.681968:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.681968:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.681968:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677472.681970:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.681971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.681971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.681971:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677472.681971:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.681972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677472.681972:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.681973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677472.681974:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677472.681976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:2.0:1619677472.681976:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.681978:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.681979:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.681980:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.681981:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.681982:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677472.681983:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:2.0:1619677472.681985:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.681986:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.681987:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.681987:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677472.681987:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.681989:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677472.681990:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.681990:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.681990:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.681992:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677472.681993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677472.681993:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.681994:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.681995:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.682006:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.682008:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.682009:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.682009:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.682011:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.682013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.682022:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.682025:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.682026:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.682026:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.682026:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.682027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.682060:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.682063:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.682066:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.682069:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.682070:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.682071:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.682073:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.682074:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.682075:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.682078:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.682080:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.682082:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.682083:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.682084:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.682086:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.682086:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.682087:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.682087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.682099:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.682101:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.682102:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.682104:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.682105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.682106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.682549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.682556:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.682558:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.682560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.682561:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.682564:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.682565:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.682566:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.682568:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.682569:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.682570:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.682574:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.682576:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.682579:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.682579:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.682580:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.682582:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.682582:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.682583:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.682584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677472.682595:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.682598:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.682599:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.682599:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677472.682601:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677472.682604:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677472.682805:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.682808:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.682810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.682811:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.682815:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.682816:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.682818:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.682819:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.682820:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.682821:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.682824:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.682826:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.682829:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.682830:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.682830:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.682832:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.682832:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.682833:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.682834:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.682846:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.682848:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.682850:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.682850:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.682852:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.682855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.683044:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.683047:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.683049:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.683050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.683051:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.683053:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.683054:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.683055:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.683056:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.683057:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.683061:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.683063:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.683065:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.683066:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.683066:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.683068:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.683068:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.683069:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.683070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.683087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.2:1619677472.683175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.683181:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.683183:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.683185:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.683186:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.683187:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.683189:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.683190:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.683192:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.683193:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.683194:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.683197:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.683199:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.683201:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.683202:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.683202:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.683204:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.683204:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.683205:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.683206:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.683210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.683218:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.683220:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.683221:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.683222:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.683223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.683224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.683459:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.683462:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.683464:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.683467:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.683468:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.683469:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.683471:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.683472:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.683473:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.683476:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.683478:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.683480:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.683481:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.683481:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.683483:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.683483:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.683484:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.683484:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:10.0:1619677472.683486:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.683489:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.683490:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.683492:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.683495:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.683496:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.683497:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.683498:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:10.0:1619677472.683499:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.683499:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677472.683500:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:10.0:1619677472.683500:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:21.0:1619677472.683501:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.683503:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.683503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.683505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677472.683505:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.683507:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.683509:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.683510:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.683511:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.683512:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.683512:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.683514:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.683515:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.683516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.683524:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.683526:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.683527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.683528:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677472.683528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.683529:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.683530:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.683531:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.683532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.683532:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.683541:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.683543:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.683544:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.683544:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.683545:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.683545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.683726:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.683729:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677472.683731:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.683731:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.683733:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.683735:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677472.683736:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.683737:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.683738:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.683739:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677472.683742:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.683744:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677472.683747:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.683747:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.683748:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.683750:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.683750:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.683751:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.683752:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.683765:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.683768:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.683768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.683769:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.683771:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.683773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.683850:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.683852:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.683853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.683855:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.683858:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.683859:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.683860:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.683861:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.683862:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.683863:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.683866:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.683868:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.683871:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.683871:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.683872:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.683874:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.683874:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.683875:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677472.683876:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.683877:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.683890:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677472.683891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.683892:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.683892:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.683893:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677472.684273:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.684275:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.684276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.684276:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.684278:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.684280:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.684364:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.684367:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.684368:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.684368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.684370:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.684372:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.684508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.684514:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.684517:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.684518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.684520:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.684523:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.684525:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677472.684526:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.684527:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.684529:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.684530:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677472.684533:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.684535:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.684539:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.684539:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.684540:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677472.684542:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.684542:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.684543:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.684544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:10.0:1619677472.684555:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.684557:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677472.684558:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677472.684559:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.684561:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.684564:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677472.684576:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677472.684579:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677472.684579:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.684580:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.684581:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.684581:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677472.684912:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.684914:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.684916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.684917:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.684919:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.684920:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677472.684921:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.684922:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.684923:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.684924:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677472.684927:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.684928:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677472.684930:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.684931:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677472.684931:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677472.684932:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.684933:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.684933:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677472.684935:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677472.684935:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.684946:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.684947:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.684948:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.684949:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.684949:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.684950:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.684950:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.684951:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677472.684951:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.684952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677472.685102:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.685104:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.685107:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.685110:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.685111:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.685112:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.685113:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.685114:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.685115:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.685119:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.685120:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.685122:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.685123:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.685124:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.685126:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.685126:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.685127:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.685128:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.685128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.685140:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.685143:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.685143:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.685145:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.685146:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.685147:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.685423:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.685425:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.685426:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.685426:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.685428:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.685430:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.685985:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.685987:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.685991:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.685993:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.685994:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.685996:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.685997:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.685998:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.685999:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.686001:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.686003:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.686005:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.686006:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.686007:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.686009:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.686009:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.686010:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.686010:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.686022:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.686025:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.686025:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.686027:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.686028:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.686029:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.686041:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.686044:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.686044:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.686045:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.686046:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.686047:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.686243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.686246:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.686247:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.686247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.686249:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.686251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.686268:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.686270:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.686271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.686271:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.686272:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.686272:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.686345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.686351:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.686353:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.686355:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.686356:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.686358:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.686360:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.686361:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.686362:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.686363:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.686365:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.686368:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.686370:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.686372:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.686373:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.686374:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.686375:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.686376:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.686377:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.686378:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.686379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.686382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.686385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.686391:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.686393:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.686393:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.686393:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.686394:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677472.686395:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.686395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.686396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.686397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.686398:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677472.686553:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.686556:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.686557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.686559:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.686562:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.686564:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677472.686565:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.686567:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.686567:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677472.686569:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677472.686569:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.686572:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.686573:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.686573:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677472.686574:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.686576:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.686577:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677472.686578:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.686578:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677472.686579:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677472.686580:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677472.686580:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677472.686581:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.686581:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000400:00000200:26.0:1619677472.686582:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677472.686582:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.686582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:26.0:1619677472.686583:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.686584:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.686587:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.686589:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.686592:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.686593:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.686593:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:26.0:1619677472.686595:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:26.0:1619677472.686595:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.686596:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.686597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.686599:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:22.0:1619677472.686600:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:22.0:1619677472.686601:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.686603:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.686607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.686626:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.686629:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.686630:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.686631:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.686632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.686761:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.686764:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.686765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.686767:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.686769:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.686771:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.686772:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.686773:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.686774:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.686775:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.686778:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.686780:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.686783:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.686784:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.686784:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.686786:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.686787:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.686788:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.686790:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.686790:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.686793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.686801:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677472.686803:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677472.686804:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.686805:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.686805:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677472.686805:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.686806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.686806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677472.686808:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.686810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.686813:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.686815:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.686817:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.686818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.686818:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.686819:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.686819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677472.686848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.686881:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.686883:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.686884:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.686885:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.686887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.686888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.686959:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.686961:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.686964:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.686964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.686965:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.686967:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.686968:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.686970:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.686970:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.686972:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.686975:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.686977:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.686979:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.686980:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.686981:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.686983:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.686983:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.686984:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.686985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.686995:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.686998:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.686998:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.686999:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.687001:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.687003:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.687520:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.687523:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.687524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.687525:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.687527:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.687529:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.687530:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.687532:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.687532:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.687534:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.687537:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.687539:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.687541:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.687542:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.687543:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.687545:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.687545:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.687546:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.687546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.687560:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.687563:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.687564:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.687565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.687566:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.687570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.687860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:18.0:1619677472.687866:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677472.687869:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:2.0:1619677472.687870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:18.0:1619677472.687872:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.687875:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.687876:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677472.687878:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.687879:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.687880:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.687881:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.2:1619677472.687881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:18.0:1619677472.687884:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.687886:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677472.687889:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.687889:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677472.687890:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677472.687890:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677472.687892:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677472.687892:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:17.0:1619677472.687893:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677472.687894:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000400:00000200:17.0:1619677472.687894:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.687894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.687895:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000400:00000200:17.0:1619677472.687896:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.687899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.687906:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.687908:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.687909:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.687911:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.687913:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.687914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677472.687916:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.687919:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.687921:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.687924:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.687925:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.687926:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.687928:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.687929:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.687930:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.687933:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.687935:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.687937:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.687938:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.687938:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.687940:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.687941:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.687941:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.687942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.687953:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.687955:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.687956:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.687957:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.687959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.687960:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.688137:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.688140:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.688142:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.688143:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.688144:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.688146:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.688147:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.688149:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.688150:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.688151:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.688154:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.688156:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.688159:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.688160:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.688160:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.688162:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.688163:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.688164:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.688165:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.688166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.688176:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.688178:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.688178:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.688179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.688182:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.688185:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.688186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.688187:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.688188:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.688191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.688338:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.688340:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.688341:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.688341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.688343:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.688345:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.688616:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.688619:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.688620:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.688621:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.688624:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.688625:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.688626:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.688628:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.688628:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.688629:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.688632:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.688633:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.688635:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.688635:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.688636:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.688637:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.688638:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.688638:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.688639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.688648:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.688650:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.688650:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.688652:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.688652:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.688653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.688662:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.688665:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.688666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.688668:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.688668:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:10.0:1619677472.688670:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.688671:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.688672:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:10.0:1619677472.688672:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677472.688672:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.688672:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000400:00000200:10.0:1619677472.688673:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.688673:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:10.0:1619677472.688674:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.688675:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.688675:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.688678:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.688680:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.688683:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.688683:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.688684:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.688686:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.688686:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.688687:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.688688:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.688703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.688717:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.688720:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.688721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.688723:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.688725:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.688726:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.688728:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.688729:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.688730:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.688731:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.688733:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.688735:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.688736:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.688737:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.688738:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.688739:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.688740:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.688740:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.688741:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.688751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.688754:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.688754:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.688755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.688756:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.688759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.688837:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.688839:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.688842:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.688845:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.688846:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.688847:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.688849:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.688850:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.688851:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.688855:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.688857:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.688860:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.688861:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.688861:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.688863:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.688864:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.688865:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.688866:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.688866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.688879:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.688881:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.688882:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.688884:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.688885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.688886:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.688897:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.688900:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.688901:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.688901:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.688902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.688903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.689529:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.689532:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.689533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.689533:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.689535:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.689537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.689547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.689549:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.689550:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.689551:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677472.689551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.689552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.689602:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.689604:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.689605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.689607:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.689609:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.689610:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.689611:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.689613:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.689613:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.689614:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.689617:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.689619:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.689622:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.689622:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.689623:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.689625:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.689625:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.689626:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.689626:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677472.689632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.689640:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.689642:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.689643:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.689645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.689645:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.689648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677472.689688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.689691:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.689694:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.689696:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.689698:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.689699:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.689700:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.689701:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.689702:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.689705:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.689707:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.689709:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.689710:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.689711:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.689712:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.689713:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.689713:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.689714:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.689726:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.689728:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.689729:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.689731:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.689732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.689733:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.689783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677472.689787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.689792:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.689794:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.689795:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.689796:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677472.689797:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.689799:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.689800:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.689801:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.689803:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.689804:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.689805:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.689807:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.689808:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.689811:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677472.689813:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:2.0:1619677472.689813:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.689814:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.689815:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.689816:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.689816:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.689819:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677472.689820:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.689820:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.689821:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.689823:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.689823:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.689824:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.689825:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.689826:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.689826:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (23)++
00000800:00000200:18.0:1619677472.689827:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.689827:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (24)--
00000800:00000200:2.0:1619677472.689828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.689829:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.689829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.689830:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.689831:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (23)++
00000800:00000200:2.0:1619677472.689831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.689832:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.2:1619677472.689833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.689833:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.689833:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.689834:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.689834:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.689835:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.689836:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677472.689836:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.689837:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.689837:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.689838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.689838:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677472.689839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.689845:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.689848:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.689849:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.689854:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677472.689856:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.689858:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.689859:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.689859:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.689859:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.689860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.689916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.689919:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.689922:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.689924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.689924:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.689926:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.689928:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.689929:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.689930:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.689932:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.689933:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.689936:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.689937:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.689940:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.689940:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.689941:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.689942:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.689943:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.689944:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.689944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.689948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.689956:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:26.0:1619677472.689957:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.0:1619677472.689958:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.689959:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.689960:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.689960:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:21.0:1619677472.689961:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.689961:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:26.0:1619677472.689963:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.689963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:26.0:1619677472.689966:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.689968:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677472.689969:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.689970:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.689971:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677472.689971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:26.0:1619677472.689972:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677472.689976:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.689977:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.689978:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677472.689979:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.689980:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677472.689981:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.689981:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677472.689982:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.689982:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:26.0:1619677472.689983:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677472.689983:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.689983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:26.0:1619677472.689984:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.689984:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.0:1619677472.689984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.690277:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.690279:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.690280:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.690280:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.690282:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.690284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.690690:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.690725:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.690728:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.690728:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.690730:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.690731:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.690732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.691168:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.691170:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.691172:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.691173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.691175:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.691176:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.691177:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.691179:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.691180:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677472.691180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:21.0:1619677472.691181:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.691184:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.691186:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.691188:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.691189:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.691189:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.691191:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.691192:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.691192:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.691193:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.691204:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.691207:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.691208:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.691210:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.691213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677472.691228:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.691231:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.691232:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.691233:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.691235:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.691238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:10.0:1619677472.691427:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.691430:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.691431:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.691433:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.691436:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.691437:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.691439:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.691440:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.691441:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.691442:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.691445:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.691447:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.691450:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.691450:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.691451:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.691453:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.691453:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.691454:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.691454:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.691472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.691532:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.691535:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.691537:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.691540:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.691541:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.691542:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.691543:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.691545:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.691546:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.691549:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.691550:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.691553:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.691553:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.691554:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.691556:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.691556:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.691557:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.691557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.691569:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.691571:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.691572:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.691574:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.691575:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.691576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.691653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.691661:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.691663:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.691664:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.691666:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.691666:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.691669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.691873:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.691876:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.691877:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.691878:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.691879:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.691881:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.692167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.692170:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.692173:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.692174:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.692175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.692177:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.692178:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.692180:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.692181:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.692182:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.692183:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.692186:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.692187:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.692190:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.692191:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.692192:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.692193:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.692194:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.692194:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.692195:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.692200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.692210:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.692213:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.692215:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.692216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.692218:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.692222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677472.692406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677472.692429:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.692432:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.692432:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.692434:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.692437:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.692439:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677472.692440:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.692441:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677472.692441:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:26.0:1619677472.692442:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677472.692442:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677472.692442:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:26.0:1619677472.692443:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677472.692443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.692444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.692444:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.692445:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.692447:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:26.0:1619677472.692448:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.692448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.692450:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.692451:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677472.692454:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.692455:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677472.692455:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677472.692457:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677472.692457:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.692458:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.692459:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.0:1619677472.692464:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.2:1619677472.692464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.692466:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.692467:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.692468:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.692469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.692470:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677472.692474:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.692478:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.692480:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.692480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.692483:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.692487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.692845:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.692848:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.692849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.692851:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.692853:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.692854:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.692855:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.692857:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.692858:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.692859:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.692862:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.692864:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.692867:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.692868:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.692868:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.692870:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.692871:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.692872:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.692872:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.692885:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.692888:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.692889:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.692889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.692892:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.692896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.693140:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.693143:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.693143:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677472.693146:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.693148:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.693149:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677472.693151:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.693152:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.693153:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.693154:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677472.693156:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.693158:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677472.693161:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.693161:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677472.693162:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677472.693164:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677472.693164:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.693165:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.693165:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:13.0:1619677472.693178:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.693180:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.693181:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.693182:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.693185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.693186:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677472.693426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677472.693448:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.693450:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.693451:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.693453:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.693456:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.693457:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.693458:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.693459:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.693460:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.693461:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677472.693461:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.693464:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.693464:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.693465:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.693467:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677472.693467:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.693468:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677472.693470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.693471:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.693471:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.693472:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.693474:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.693474:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.693475:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.693476:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.693487:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.693490:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.693491:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.693491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.693493:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.693497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.693505:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677472.693508:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.693509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.693509:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.693510:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.693511:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677472.693548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.693556:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.693559:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.693560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.693560:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.693562:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.693565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677472.693924:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.693927:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.693928:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.693930:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.693932:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.693933:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.693935:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.693936:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.693937:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.693938:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.693941:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.693942:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.693944:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.693945:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.693946:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.693947:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.693947:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.693949:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.693950:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.693950:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677472.693959:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.693961:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.693962:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.693963:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677472.693963:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.693964:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.693965:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.693965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.693967:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.693967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.694196:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.694198:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.694199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.694199:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.694201:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.694202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.694231:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.694234:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.694234:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677472.694236:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.694238:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.694239:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677472.694240:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.694242:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.694243:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.694244:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677472.694246:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.694248:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677472.694250:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.694251:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677472.694251:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677472.694253:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677472.694253:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.694254:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677472.694256:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677472.694256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:12.0:1619677472.694266:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677472.694269:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:12.0:1619677472.694269:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.694269:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.694271:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.694271:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:12.0:1619677472.694271:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.694272:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677472.694273:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.694274:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677472.694283:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677472.694284:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:12.0:1619677472.694285:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.694286:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:22.0:1619677472.694287:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:12.0:1619677472.694287:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677472.694287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677472.694288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:12.0:1619677472.694289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:22.0:1619677472.694290:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.694292:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.694299:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.694300:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.694301:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.694303:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.694303:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.694307:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.694309:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.694312:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.694313:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.694313:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.694315:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.694315:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.694316:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.694317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.694330:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.694332:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.694333:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.694334:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.694335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.694337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.694462:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.694465:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.694466:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.694468:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.694470:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.694472:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.694473:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.694475:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.694476:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.694477:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.694480:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.694482:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.694485:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.694486:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.694486:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.694488:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.694489:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.694490:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.694491:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.694492:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677472.694503:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.694504:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677472.694505:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677472.694506:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:2.0:1619677472.694506:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.694507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.694507:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.694509:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677472.694509:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.694512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.694818:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.694821:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.694822:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.694824:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.694827:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.694828:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677472.694830:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.694831:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.694832:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.694833:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.694836:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.694838:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677472.694841:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.694841:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.694842:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.694844:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.694844:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.694845:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677472.694847:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677472.694847:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677472.694857:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.694860:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:10.0:1619677472.694860:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:18.0:1619677472.694861:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.694861:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.694862:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677472.694862:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:18.0:1619677472.694863:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:10.0:1619677472.694864:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677472.694866:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.694901:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.694903:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.694904:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.694904:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.694906:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.694909:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677472.695133:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.695136:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.695137:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.695139:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.695141:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.695142:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.695144:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.695145:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.695146:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.695147:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.695151:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.695152:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.695154:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.695155:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.695156:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.695158:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.695158:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.695159:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.695159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.695171:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.695173:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.695174:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.695176:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.695177:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.695178:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.695349:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.695352:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677472.695353:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677472.695355:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677472.695357:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.695358:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677472.695360:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.695361:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.695362:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.695363:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677472.695366:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677472.695368:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.695371:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.695372:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677472.695372:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677472.695374:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.695375:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.695375:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.695376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677472.695388:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.695391:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.695392:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.695392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.695395:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.695398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677472.695737:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:24.0:1619677472.695738:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.695740:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:17.0:1619677472.695740:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.695740:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:24.0:1619677472.695741:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.695741:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.695743:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:17.0:1619677472.695743:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.695745:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.695746:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:17.0:1619677472.695747:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677472.695748:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.695749:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.695751:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.695751:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677472.695755:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.695756:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.695757:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677472.695758:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.695759:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.695760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677472.695760:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.695760:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677472.695761:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.695761:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677472.695762:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:8.0:1619677472.695762:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.695762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.695763:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.695764:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.695768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.695776:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.695778:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.695779:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.695780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.695781:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.695782:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677472.696154:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.696157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.696157:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677472.696159:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.696161:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.696163:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.696164:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.696165:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.696167:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.696168:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.696171:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.696173:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.696175:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.696176:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.696176:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.696178:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.696178:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.696180:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677472.696181:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677472.696181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.696191:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.696193:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.696194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.696194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.696194:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.696196:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.696197:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.696198:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.696200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.696200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.696211:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.696213:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.696214:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.696215:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.696216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.696217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.696236:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.696238:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.696239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.696241:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.696244:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.696246:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.696247:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.696248:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.696249:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.696250:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.696253:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.696255:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.696258:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.696259:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.696259:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.696261:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.696262:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.696262:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.696263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.696275:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.696277:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.696278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.696279:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.696281:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.696284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.696299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.696302:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.696302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.696303:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.696304:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.696304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.696731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.696737:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.696740:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.696741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.696743:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.696745:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.696747:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.696748:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.696749:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.696750:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.696751:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.696755:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.696756:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.696759:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.696759:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.696760:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.696762:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.696762:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.696763:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.696764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.696776:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.696778:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.696779:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.696779:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.696781:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.696782:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677472.697015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.697017:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.697018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.697020:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.697022:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.697024:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.697025:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.697026:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.697027:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.697028:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.697031:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.697033:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.697035:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.697036:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.697037:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.697039:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.697039:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.697040:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.697040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.697053:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.697055:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.697056:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.697058:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.697059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.697060:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.697454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.697468:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.697471:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.697472:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.697473:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.697474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.697477:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.697848:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.697851:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.697852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.697854:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.697857:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.697858:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.697859:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.697861:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.697862:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.697863:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.697867:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.697869:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.697871:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.697872:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.697873:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.697874:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.697875:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.697875:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.697876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.697888:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.697890:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.697891:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.697893:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.697894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.697896:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677472.698072:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.698076:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.698076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.698079:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.698080:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.698082:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.698083:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.698084:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.698085:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.698086:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.698089:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.698091:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.698093:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.698094:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.698095:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.698097:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.698097:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.698098:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.698098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.698103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.698110:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.698113:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.698113:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.698115:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.698116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.698118:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677472.698219:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.698222:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.698223:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.698225:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.698227:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.698229:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.698230:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.698231:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.698233:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.698234:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.698237:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.698239:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.698242:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.698243:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.698243:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.698245:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.698245:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.698246:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.698246:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.698258:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.698261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.698261:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.698262:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.698264:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.698267:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.698628:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.698631:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.698632:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.698633:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.698636:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.698637:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.698638:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.698639:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.698640:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.698641:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.698644:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.698645:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.698648:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.698648:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.698649:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.698650:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.698651:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.698652:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.698652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.698663:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.698666:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.698667:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.698667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.698669:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.698671:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.698881:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.698884:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.698885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.698886:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.698889:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.698890:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.698891:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.698893:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.698894:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.698895:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.698898:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.698900:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.698902:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.698903:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.698904:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.698905:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.698905:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.698906:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.698907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.698919:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.698921:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.698922:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.698923:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.698925:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.698926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.699024:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.699027:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.699029:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.699030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677472.699030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677472.699031:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.699033:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:21.0:1619677472.699033:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:8.0:1619677472.699033:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:21.0:1619677472.699035:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.699037:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.699037:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.699038:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.699039:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.699040:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.699043:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.699045:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.699046:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.699047:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.699048:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.699049:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:8.0:1619677472.699049:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677472.699050:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:8.0:1619677472.699050:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.699051:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.699052:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.699052:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.699053:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.699054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677472.699054:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.699056:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.699059:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.699059:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.699060:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.699062:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.699062:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.699063:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.699063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677472.699073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677472.699074:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677472.699076:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:24.0:1619677472.699076:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.699077:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000400:00000200:24.0:1619677472.699078:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.699078:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:8.0:1619677472.699078:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.699080:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.699080:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677472.699082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.699082:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.699083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.699083:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.699084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.699086:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.699088:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.699090:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.699091:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.699092:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.699093:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.699095:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.699098:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.699100:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.699103:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.699104:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.699105:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.699106:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.699107:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.699108:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.699108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677472.699119:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.699122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677472.699122:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677472.699123:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.699125:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677472.699127:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677472.699128:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.699130:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.699131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.699133:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.699135:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.699136:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.699138:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.699139:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.699140:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.699141:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677472.699144:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:24.0:1619677472.699144:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.699146:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677472.699146:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677472.699147:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:24.0:1619677472.699147:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.699147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.699148:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.699148:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677472.699148:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.699149:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.699150:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.699150:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.699150:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.699151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677472.699172:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.699174:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.699174:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.699175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.699176:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.699178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.699203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.699211:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.699214:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.699215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.699215:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.699217:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.699220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.699222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.699229:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.699232:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.699232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.699233:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.699234:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.699234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677472.699752:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.699755:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.699756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.699758:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.699760:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.699761:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.699763:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.699764:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.699765:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.699766:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.699769:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.699771:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.699774:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.699775:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.699775:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.699777:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.699777:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.699779:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677472.699780:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677472.699780:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.699791:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.699792:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677472.699792:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677472.699793:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.699794:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677472.699794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677472.699795:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.699796:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.699797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.699799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.699809:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.699812:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.699813:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.699813:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.699814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.699815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.700258:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.700260:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.700261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.700264:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.700269:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.700270:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.700272:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.700273:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.700274:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.700275:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.700279:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.700282:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.700287:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.700287:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.700288:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.700289:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (24)++
00000800:00000200:8.0:1619677472.700289:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.700289:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.700291:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (24)--
00000400:00000200:8.0:1619677472.700291:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.700292:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.700299:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677472.700300:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.700301:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677472.700302:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.700302:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.700303:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:8.0:1619677472.700303:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.700304:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.700305:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.700305:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:24.0:1619677472.700306:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.700306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.700307:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.700308:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.700309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.700309:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.700310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.700310:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.700310:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.700311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.700325:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.700329:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677472.700329:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.700330:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677472.700331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.700331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.700524:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.700527:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677472.700529:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.700529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.700530:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.700532:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.700533:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.700534:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.700535:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.700536:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.700539:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.700541:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.700543:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.700544:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.700545:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.700547:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.700547:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.700548:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.700548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.700562:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.700565:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.700566:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.700567:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.700569:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.700572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.700589:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.700591:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.700592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.700592:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.700593:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.700594:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.700649:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.700652:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.700653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.700655:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.700658:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.700659:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.700660:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.700661:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.700662:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.700663:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.700666:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.700668:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.700670:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.700671:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.700671:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.700673:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.700673:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.700674:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.700674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.700687:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.700689:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.700690:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.700692:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.700693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.700694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677472.701038:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.701041:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.701043:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.701044:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.701046:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.701048:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.701049:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.701051:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.701051:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.701053:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.701056:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.701058:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.701061:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.701061:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.701062:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.701064:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.701064:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.701065:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.701065:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.701077:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.701081:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.701082:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.701082:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.701084:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.701087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677472.701219:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.701222:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.701224:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.701225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.701226:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.701228:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.701229:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.701230:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.701231:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.701233:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.701236:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.701238:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.701240:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.701241:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.701241:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.701243:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.701244:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.701244:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.701245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.701257:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.701259:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.701260:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.701260:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.701262:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.701265:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.701674:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.701677:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.701678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.701679:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.701681:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.701683:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.701684:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.701685:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.701686:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.701687:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.701690:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.701692:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.701694:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.701695:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.701695:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.701697:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.701697:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.701698:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.701698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.701712:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.701714:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.701715:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.701716:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.701718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.701719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.701792:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.701795:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.701796:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.701796:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.701798:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.701800:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.701911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677472.701914:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.701916:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677472.701918:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.701919:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677472.701921:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.701922:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.701924:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.701925:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.701926:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.701927:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.701931:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.701933:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.701935:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.701936:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.701936:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.701938:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.701939:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.701940:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677472.701941:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677472.701942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677472.701947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:22.0:1619677472.701951:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677472.701952:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:22.0:1619677472.701954:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677472.701954:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000400:00000200:22.0:1619677472.701955:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.701955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:6.0:1619677472.701955:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.701957:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.701957:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:22.0:1619677472.701959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.702496:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.702499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.702499:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677472.702501:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.702503:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.702505:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.702506:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.702507:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.702508:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.702509:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.702513:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.702514:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.702517:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.702518:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.702518:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.702520:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.702520:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.702521:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.702521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.702535:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.702537:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.702538:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.702539:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.702540:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.702542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.702823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.702830:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.702832:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.702833:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.702835:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.702838:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.702839:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.702841:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.702842:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.702843:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.702844:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.702847:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.702849:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.702850:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.702851:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.702852:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.702853:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.702854:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.702855:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.702856:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.702867:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.702869:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.702870:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.702871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.702872:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.702873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.703040:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677472.703040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677472.703043:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.703044:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.703046:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.703048:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.703048:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:26.0:1619677472.703050:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.703051:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.703051:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:26.0:1619677472.703052:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.703052:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.703053:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677472.703054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.703054:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677472.703055:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677472.703057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:26.0:1619677472.703059:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677472.703059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:26.0:1619677472.703060:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.703063:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.703064:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.703065:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.703066:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677472.703066:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677472.703067:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.703068:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.703069:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:21.0:1619677472.703069:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677472.703070:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.0:1619677472.703070:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.703071:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.703072:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.703072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.703080:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.703083:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677472.703083:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677472.703084:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.703084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677472.703085:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.703085:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677472.703086:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.703087:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.703089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677472.703141:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.703145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.703145:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.703147:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.703150:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.703151:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.703153:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.703154:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.703155:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.703156:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.703159:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.703161:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.703164:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.703165:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.703165:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.703167:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.703168:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.703168:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.703169:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.703173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.703180:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.703182:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.703183:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.703185:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.703185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.703188:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677472.703290:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.703293:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.703294:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.703301:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.703304:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.703305:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677472.703306:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.703307:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.703309:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.703310:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677472.703313:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.703315:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677472.703318:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.703319:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677472.703320:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677472.703321:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677472.703322:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.703323:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.703323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677472.703328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.703336:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.703338:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.703339:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.703341:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.703341:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.703344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677472.703357:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.703360:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.703361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.703363:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.703365:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.703366:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.703368:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.703369:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.703370:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.703371:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.703374:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.703376:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.703379:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.703379:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.703380:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.703382:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.703382:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.703383:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677472.703384:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677472.703385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.703395:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.703397:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.703397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.703397:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677472.703398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.703399:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.703400:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.703401:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.703402:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.703404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.703415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.703418:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.703419:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.703419:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.703420:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.703421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677472.703881:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.703884:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.703885:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.703887:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.703890:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.703891:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.703893:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.703894:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.703895:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.703896:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.703899:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.703901:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.703903:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.703904:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.703905:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.703906:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.703906:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.703908:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.703909:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.703910:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.703923:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.703924:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.703925:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.703926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.703929:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.703931:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.703932:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.703932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.703934:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.703936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.703961:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.703963:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.703964:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.703965:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.703968:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.703969:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.703970:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.703971:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.703972:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.703973:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.703975:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.703977:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.703978:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.703979:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.703979:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.703981:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.703981:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.703982:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.703982:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.703992:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.703994:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.703995:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.703996:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.703996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.703997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.704064:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.704066:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.704067:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.704067:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.704069:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.704070:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.704081:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.704083:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.704084:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.704084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.704085:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.704085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677472.704360:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.704363:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.704363:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677472.704366:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.704368:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.704369:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.704371:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.704372:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.704373:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.704374:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.704377:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.704379:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.704382:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.704383:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.704383:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.704385:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.704385:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.704386:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.704386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.704400:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.704402:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.704403:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.704405:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.704406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.704407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.704650:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.704653:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.704654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.704654:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.704656:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.704658:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.705222:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.705225:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.705226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.705228:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.705230:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.705232:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.705233:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.705234:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.705236:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.705237:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.705240:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.705241:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.705244:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.705245:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.705245:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.705247:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.705247:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.705248:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.705249:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.705261:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.705263:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.705264:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.705265:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.705266:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.705268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.705353:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.705356:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.705357:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.705359:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.705361:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.705362:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.705363:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.705365:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.705366:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.705367:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.705370:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.705372:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.705375:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.705375:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.705376:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.705378:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.705378:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.705379:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.705381:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.705381:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.705393:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.705395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.705396:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.705397:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.705397:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.705397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677472.705397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.705398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.705399:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.705403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.705816:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.705819:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.705820:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.705822:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.705824:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.705826:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.705827:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.705828:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.705829:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.705830:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.705833:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.705835:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.705837:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.705838:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.705839:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.705841:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.705841:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.705842:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.705842:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.705855:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.705858:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.705859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.705859:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.705861:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.705864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.705875:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.705878:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.705878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.705879:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.705880:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.705880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.706084:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.706087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.706087:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677472.706089:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.706091:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.706093:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.706093:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.706095:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.706096:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.706097:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.706101:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.706102:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.706105:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.706105:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.706106:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.706108:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.706108:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.706109:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.706109:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.706121:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.706124:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.706124:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.706126:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.706127:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.706128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.706185:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.706188:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.706190:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.706191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.706193:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.706195:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.706196:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.706197:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.706198:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.706199:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.706203:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.706204:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.706208:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.706208:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.706209:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.706211:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.706212:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.706213:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.706214:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.706215:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.706223:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677472.706224:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.706225:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.706226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677472.706227:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677472.706227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:24.0:1619677472.706228:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.706228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.706230:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.706233:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677472.706471:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.706474:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.706476:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.706478:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.706479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.706480:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677472.706481:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.706482:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.706483:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.706484:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677472.706487:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.706489:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677472.706492:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.706493:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:13.0:1619677472.706493:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:13.0:1619677472.706495:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677472.706496:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.706497:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.706497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677472.706502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.706511:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.706514:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.706515:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.706515:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.706517:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.706520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.706526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.706533:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.706535:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.706536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.706537:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.706538:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.706539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.706580:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.706582:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.706583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.706585:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.706587:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.706588:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.706590:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.706591:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.706592:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.706593:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.706596:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.706597:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.706600:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.706601:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.706601:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.706603:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.706603:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.706604:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.706605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.706618:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.706621:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.706622:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.706623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.706625:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.706628:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.707132:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.707135:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.707136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.707138:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.707140:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.707141:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.707143:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.707144:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.707146:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.707147:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.707150:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.707151:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.707154:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.707155:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.707155:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.707157:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.707157:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.707159:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.707160:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.707160:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.707170:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677472.707170:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677472.707172:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677472.707172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.707173:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677472.707173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677472.707174:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.707175:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.707176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.707178:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.707192:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.707194:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.707195:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.707196:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.707196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.707198:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677472.707418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677472.707424:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677472.707428:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677472.707428:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677472.707430:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677472.707433:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.707434:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.707436:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.707437:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.707438:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.707439:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.707442:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677472.707444:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.707446:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.707447:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.707448:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.707449:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.707450:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677472.707450:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677472.707451:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:22.0:1619677472.707464:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.707466:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.707467:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.707467:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.707469:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.707472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.707888:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.707891:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.707892:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.707894:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.707897:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.707898:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.707900:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.707901:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.707902:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.707903:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.707907:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.707908:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.707912:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.707912:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.707913:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.707915:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.707915:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.707916:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.707916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.707928:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.707930:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.707931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.707932:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.707934:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.707937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.707945:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.707948:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.707949:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.707949:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.707950:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.707951:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.707966:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.707968:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.707969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.707971:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.707973:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.707974:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.707975:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.707976:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.707978:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.707979:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.707982:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.707984:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.707986:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.707987:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.707988:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.707990:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.707990:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.707991:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.707991:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.708003:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.708005:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.708006:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.708008:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.708009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.708010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.708582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.708589:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.708592:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.708593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.708595:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.708597:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.708599:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.708600:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.708602:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.708603:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.708604:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.708607:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.708609:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.708612:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.708613:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.708614:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.708616:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.708616:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.708618:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.708618:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677472.708630:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.708633:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677472.708633:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677472.708634:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.708636:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677472.708639:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677472.708782:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.708785:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.708786:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.708788:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.708790:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.708791:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.708792:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.708794:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.708795:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.708796:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.708799:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.708800:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.708803:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.708803:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.708804:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.708806:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.708806:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.708807:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.708807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.708820:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.708822:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677472.708822:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.708823:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.708824:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.708824:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.708825:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.708827:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.708829:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677472.708830:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.708831:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.708832:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.708833:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.708834:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.708835:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677472.708836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:24.0:1619677472.708837:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.708838:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677472.708841:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.708842:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677472.708843:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677472.708844:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677472.708845:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.708846:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677472.708846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:22.0:1619677472.708860:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.708861:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.708862:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.708863:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.708863:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.708865:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677472.708865:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677472.708869:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.708869:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.708870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:6.0:1619677472.708870:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677472.708871:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.708871:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.708873:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.708874:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.708875:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.708878:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.708879:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.708882:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.708882:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.708883:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.708885:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.708885:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.708886:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.708886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.708899:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.708900:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.708901:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.708901:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.708902:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.708903:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.708905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677472.708905:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.708906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.708907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.708907:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.708909:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.708910:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.708911:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.708912:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.708913:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.708917:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.708919:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.708921:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.708922:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.708923:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.708925:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.708925:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.708926:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.708927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.708937:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.708940:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.708941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.708941:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.708943:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.708946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.709344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.709353:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.709356:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.709357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.709357:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.709359:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.709362:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.709381:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.709384:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.709385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.709387:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.709389:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.709390:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677472.709392:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.709393:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.709394:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.709395:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677472.709399:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.709401:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677472.709403:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.709404:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677472.709405:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677472.709406:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677472.709407:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.709408:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.709408:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677472.709421:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.709424:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.709425:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.709425:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.709427:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.709430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.709434:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.709436:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.709437:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.709439:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.709442:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.709443:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.709445:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.709446:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.709447:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.709448:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.709451:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.709453:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.709455:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.709456:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.709456:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.709458:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.709458:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.709459:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.709459:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.709481:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.709483:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.709484:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.709485:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.709485:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.709487:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.709733:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.709736:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.709737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.709739:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.709741:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.709743:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.709744:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.709745:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.709746:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.709747:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.709751:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.709752:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.709755:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.709755:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.709756:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.709758:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.709758:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.709760:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.709761:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.709761:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.709764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.709772:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.709774:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677472.709775:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.709775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.709776:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677472.709777:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.709778:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.709780:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.709781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.709782:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.709846:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.709848:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.709849:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.709849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.709851:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.709853:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.709870:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.709872:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.709873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.709873:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.709874:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.709875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.709913:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.709915:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.709916:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.709918:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.709921:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.709922:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.709923:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.709924:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.709925:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.709927:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.709930:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.709931:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.709934:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.709935:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.709935:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.709937:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.709937:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.709938:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.709938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.709950:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.709952:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.709953:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.709954:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.709956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.709956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.710366:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.710368:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.710371:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.710371:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.710373:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.710375:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677472.710376:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.710378:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.710379:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.710380:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677472.710383:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.710385:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.710388:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.710388:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677472.710389:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.710391:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.710391:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.710393:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.710394:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.710395:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.710403:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.710406:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.710406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677472.710406:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677472.710407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677472.710409:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.710410:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.710410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.710412:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.710415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.710424:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.710427:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.710428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.710428:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.710429:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.710430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.710599:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.710601:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.710602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.710604:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.710607:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.710608:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.710610:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.710611:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.710612:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.710613:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.710616:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.710618:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.710621:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.710622:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.710622:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.710624:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.710624:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.710625:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.710626:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.710630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.710638:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.710641:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.710642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.710642:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.710644:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.710647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677472.710773:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.710776:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.710777:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.710779:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.710781:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.710783:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.710784:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.710785:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.710786:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.710787:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.710790:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.710791:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.2:1619677472.710793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677472.710794:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.710795:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.710796:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.710798:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.710798:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.710799:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.710799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677472.710801:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.710804:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.710804:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.710806:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.710807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.710809:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677472.710812:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.710814:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.710816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.710816:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.710817:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.710818:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677472.710818:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677472.710822:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677472.710822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.710823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.710825:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.710827:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.710828:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.710829:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.710830:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.710831:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.710835:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.710836:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.710839:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.710840:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.710840:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.710842:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.710842:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.710843:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.710844:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.710845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.710855:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677472.710857:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677472.710857:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677472.710858:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.710859:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677472.710859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677472.710860:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.710862:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.710863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.710864:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.710880:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.710882:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.710883:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.710884:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.710885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.710885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.711694:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.711697:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.711698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.711700:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.711702:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.711704:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.711705:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.711706:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.711707:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.711708:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.711711:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.711713:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.711715:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.711716:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.711716:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.711718:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.711719:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.711719:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.711720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677472.711734:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.711735:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.0:1619677472.711736:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.711737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.711737:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.711739:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:8.0:1619677472.711739:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677472.711740:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.711742:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.711742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.711743:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677472.711743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.711744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.711746:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.711746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.711749:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.711751:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.711753:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677472.711754:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.711755:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.711756:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.711757:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677472.711761:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.711763:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677472.711766:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.711767:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677472.711768:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677472.711769:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677472.711770:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.711770:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.711771:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677472.711775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.711783:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.711785:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.711786:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.711788:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.711788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.711791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.711793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.711801:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.711804:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.711805:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.711805:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.711806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.711807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.712009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.712016:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.712019:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.712020:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.712022:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.712024:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.712026:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.712027:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.712028:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.712029:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.712030:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.712034:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.712035:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.712037:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.712038:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.712039:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.712040:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.712041:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.712042:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.712042:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.712053:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.712056:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.712056:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.712057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.712058:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.712060:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.712125:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.712127:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.712128:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.712130:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.712133:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.712134:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.712136:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.712137:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.712138:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.712139:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.712142:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.712144:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.712147:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.712148:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.712148:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.712150:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.712150:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.712152:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.712153:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.712153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.712163:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.712166:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677472.712166:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:18.0:1619677472.712167:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.712167:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.712168:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677472.712169:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.712169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.712170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.712173:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.712308:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.712311:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.712312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.712314:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.712317:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.712318:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677472.712319:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.712320:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.712321:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.712322:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.712326:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.712327:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.712330:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.712331:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.712331:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.712333:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.712333:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.712334:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.712335:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677472.712347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.712350:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.712351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.712351:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.712354:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.712356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677472.712358:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.712361:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.712363:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.712364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.712365:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.712367:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.712368:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.712369:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.712370:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.712371:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.712374:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.712376:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.712378:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.712379:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.712380:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.712382:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.712382:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.712383:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.712384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.712405:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.712408:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.712409:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.712409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.712412:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.712414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.712779:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.712782:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.712783:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.712785:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.712788:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.712789:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.712790:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.712791:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677472.712792:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677472.712792:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.712793:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.712796:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:10.0:1619677472.712797:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.712797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677472.712799:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:22.0:1619677472.712800:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.712802:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677472.712803:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.712803:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.712803:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677472.712804:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.712805:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677472.712805:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.712805:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677472.712806:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677472.712806:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.712807:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677472.712808:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.712808:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.712812:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.712814:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.712817:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.712818:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.712819:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.712820:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.712821:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.712821:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.712822:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.712822:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.712822:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.712823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677472.712824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.712824:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.712827:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677472.712835:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.712837:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.712838:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.712839:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.712840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.712842:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677472.713679:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.713682:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.713682:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.713685:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.713687:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.713689:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:1.0:1619677472.713689:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.713690:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.713691:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677472.713692:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.713692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:1.0:1619677472.713692:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.713694:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:22.0:1619677472.713696:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.713698:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677472.713699:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.713700:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:1.0:1619677472.713700:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:22.0:1619677472.713701:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.713703:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677472.713703:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677472.713704:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677472.713704:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:22.0:1619677472.713705:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677472.713705:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.713706:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.713707:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:22.0:1619677472.713708:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.713708:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.713708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:22.0:1619677472.713710:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.713713:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677472.713713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677472.713714:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.713715:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.713717:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.713717:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.713717:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.713718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.713721:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.713724:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.713725:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.713727:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.713728:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.713729:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.713730:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.713732:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.713733:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677472.713733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:21.0:1619677472.713735:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.713735:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.713737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.713737:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.713738:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.713739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.713739:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.713740:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.713741:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.713875:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.713877:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.713878:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.713878:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.713880:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.713882:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.714141:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.714143:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.714144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.714146:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.714149:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.714150:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.714151:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.714152:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.714153:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.714154:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.714157:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.714159:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.714161:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.714161:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.714162:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.714164:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.714164:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.714165:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.714167:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.714167:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677472.714175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.714177:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.714178:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.714179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.714180:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.714180:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677472.714182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.714182:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.714182:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.714184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.714203:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.714206:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.714207:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.714207:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.714208:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.714208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677472.714556:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.714559:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.714560:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.714561:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.714564:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.714565:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.714567:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.714568:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.714569:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.714570:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.714573:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.714575:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.714577:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.714578:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.714579:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.714580:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.714581:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.714582:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677472.714583:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:24.0:1619677472.714584:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677472.714584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677472.714586:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.714587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.714589:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.714592:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.714594:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.714595:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677472.714595:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:24.0:1619677472.714596:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677472.714596:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677472.714596:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:24.0:1619677472.714597:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677472.714597:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677472.714598:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677472.714598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.714598:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.714599:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.714601:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.714602:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.714602:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.714603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.714605:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.714607:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.714607:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.714608:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.714610:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.714610:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.714611:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.714611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677472.714614:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.714616:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.714617:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.714618:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.714619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.714620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.714621:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.714623:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.714624:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.714625:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.714626:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.714628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.714640:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.714643:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.714644:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.714644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.714645:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.714645:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677472.714723:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677472.714726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677472.714726:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677472.714729:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.714731:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.714733:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677472.714734:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.714735:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.714736:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.714737:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677472.714741:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.714743:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677472.714746:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.714747:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677472.714747:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677472.714749:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677472.714750:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.714750:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677472.714751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:17.0:1619677472.714785:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.714788:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.714789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.714791:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.714794:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.714795:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677472.714797:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.714798:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.714799:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.714800:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677472.714804:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.714806:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677472.714809:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.714810:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:17.0:1619677472.714810:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677472.714812:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677472.714813:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.714814:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.714814:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.715018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.715026:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.715029:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.715029:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.715030:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.715032:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.715035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.715532:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.715534:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.715535:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.715536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.715537:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.715539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.715578:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.715580:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.715581:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.715583:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.715585:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.715587:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.715588:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.715589:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.715590:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.715591:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.715593:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.715595:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.715598:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.715598:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.715599:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.715601:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.715601:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.715601:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.715602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.715616:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.715616:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.715617:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.715618:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.715618:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.715621:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677472.715621:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.715624:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.715625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.715626:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677472.715626:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677472.715627:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.715628:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:6.0:1619677472.715628:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677472.715629:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.715629:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677472.715630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.715630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.715630:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.715631:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.715632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:6.0:1619677472.715635:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.715637:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.715640:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.715642:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.715643:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.715645:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.715645:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.715646:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.715646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.715658:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.715660:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.715661:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.715663:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.715663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.715664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.715934:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.715936:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.715937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.715939:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.715941:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.715942:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.715944:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.715945:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.715946:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.715947:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.715950:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.715952:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.715954:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.715955:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.715956:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.715958:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.715958:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.715959:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.715959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.715964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.715972:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.715975:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.715975:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.715976:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.715978:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.715981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.715982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.715990:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.715992:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.715993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.715993:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.715994:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.715995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.716236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.716244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.716246:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.716247:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.716249:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.716249:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.716252:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.716516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.716518:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.716519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.716521:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.716524:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.716525:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.716526:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.716528:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.716529:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.716530:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.716533:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.716535:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.716538:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.716539:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.716539:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.716541:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.716542:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.716542:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.716542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.716556:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.716558:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.716559:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.716560:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.716562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.716562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.716724:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.716726:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677472.716727:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677472.716729:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.716731:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.716733:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677472.716734:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677472.716736:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:6.0:1619677472.716736:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.716737:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.716737:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677472.716739:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.716740:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:6.0:1619677472.716741:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677472.716742:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677472.716743:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677472.716745:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.716746:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677472.716747:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677472.716747:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:6.0:1619677472.716747:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:10.0:1619677472.716748:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.716749:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677472.716749:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677472.716749:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000400:00000200:10.0:1619677472.716750:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677472.716750:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000400:00000200:10.0:1619677472.716751:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677472.716751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:10.0:1619677472.716755:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677472.716757:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677472.716760:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.716760:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.716761:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677472.716763:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.716763:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677472.716764:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677472.716764:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.716765:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677472.716767:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.716769:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.716771:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.716772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.716775:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677472.716778:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.716781:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.716782:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.716782:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.716784:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.716790:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.716791:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.716793:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677472.716794:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.716795:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.716795:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.716796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677472.717439:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.717442:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.717443:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.717445:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.717448:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.717450:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.717451:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.717452:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.717454:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.717455:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.717458:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.717460:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.717462:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.717463:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.717464:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.717465:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.717466:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.717466:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.717467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.717480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.717482:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.717483:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.717485:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.717486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.717487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.717523:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.717526:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.717527:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.717529:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.717532:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.717533:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.717535:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.717536:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.717537:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.717538:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.717541:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.717543:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.717546:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.717547:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.717547:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.717549:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.717550:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.717550:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.717551:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.717563:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.717565:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.717566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.717567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.717569:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.717572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677472.717854:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.717857:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.717858:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.717858:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.717860:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.717862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.717886:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.717889:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.717891:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.717891:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.717893:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.717895:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.717896:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.717898:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.717899:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.717900:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.717903:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.717905:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.717908:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.717909:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.717910:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.717911:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.717912:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.717913:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.717914:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.717925:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.717927:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.717928:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.717929:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.717931:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.717933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.717963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.717970:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.717973:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.717974:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.717976:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.717978:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.717980:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.717981:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.717982:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.717983:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.717984:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.717987:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.717989:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.717992:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.717993:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.717993:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.717995:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.717996:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.717997:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.717997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677472.718008:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.718011:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677472.718011:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677472.718012:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.718014:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677472.718016:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677472.718048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.718053:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.718056:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.718057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.718059:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.718062:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.718063:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.718065:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.718066:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.718067:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.718068:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.718070:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.718072:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.718073:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.718074:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.718074:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.718076:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.718076:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.718078:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.718079:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.718079:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.718085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.718091:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.718092:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.718092:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.718092:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677472.718093:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.718093:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.718094:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.718094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.718095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.718096:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.718511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.718513:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.718514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.718516:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.718518:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.718520:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.718521:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.718522:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.718523:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.718524:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.718527:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.718529:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.718531:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.718532:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.718533:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.718535:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.718535:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.718536:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.718537:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.718537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.718548:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677472.718549:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677472.718550:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677472.718551:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677472.718551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677472.718552:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.718552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677472.718554:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.718555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.718556:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.718566:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.718569:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.718570:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.718570:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.718571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.718572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677472.718588:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:17.0:1619677472.718591:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.45@o2ib3
00000800:00000200:1.0:1619677472.718591:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000400:00000200:17.0:1619677472.718594:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.45@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.718596:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.718598:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:17.0:1619677472.718599:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.718600:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.718601:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.718602:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:17.0:1619677472.718606:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.718607:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:17.0:1619677472.718610:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.718612:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:17.0:1619677472.718612:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677472.718614:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:17.0:1619677472.718615:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf7800] (21)++
00000800:00000200:17.0:1619677472.718616:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce0bf7800] (22)++
00000800:00000200:17.0:1619677472.718617:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf7800] (23)--
00000800:00000200:17.0:1619677472.718618:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:24.2:1619677472.718621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677472.718624:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677472.718627:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:8.0:1619677472.718628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:1.0:1619677472.718628:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)++
00000400:00000200:24.0:1619677472.718629:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.718630:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce0bf7800] (23)--
00000400:00000200:24.0:1619677472.718631:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.718631:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (22)--
00000800:00000200:1.0:1619677472.718632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf7800] (21)--
00000800:00000200:24.0:1619677472.718634:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:17.0:1619677472.718779:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.718782:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.718783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.718784:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.718787:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.718788:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677472.718789:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.718791:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.718792:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.718792:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677472.718795:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.718796:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677472.718799:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.718799:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677472.718800:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677472.718801:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677472.718802:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.718803:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677472.718804:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677472.718804:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.718815:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:2.0:1619677472.718817:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677472.718817:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677472.718817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.718818:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677472.718819:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.718820:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.718821:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.718822:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.718825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677472.718936:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.718939:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.718940:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.718941:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.718944:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.718945:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.718946:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.718948:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.718949:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.718950:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.718953:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.718954:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.718957:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.718958:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.718958:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.718960:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.718960:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.718961:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.718962:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.718972:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.718974:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.718975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.718975:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.718977:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.718979:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.719387:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.719389:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.719390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.719392:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.719394:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.719396:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.719397:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.719398:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.719399:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.719400:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.719404:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.719405:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.719408:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.719408:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.719409:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.719411:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.719411:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.719412:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.719412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.719426:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.719428:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.719429:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.719430:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.719432:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.719432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677472.719633:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.719636:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.719637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.719638:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.719641:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.719643:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.719644:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.719645:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.719646:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.719647:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.719651:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.719652:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.719654:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.719654:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.719655:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.719656:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.719657:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.719657:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.719658:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.719668:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.719670:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.719670:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.719671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.719672:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.719673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.719689:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.719691:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.719692:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.719692:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.719692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.719693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.720153:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.720155:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.720157:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.720158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.720160:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.720162:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.720163:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.720165:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.720165:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.720166:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.720169:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.720171:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.720173:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.720174:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.720175:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.720176:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.720177:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.720178:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.720179:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.720179:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.720190:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677472.720190:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677472.720192:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.720193:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677472.720193:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.720194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.720194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677472.720194:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.720196:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.720198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.720303:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.720305:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:8.0:1619677472.720305:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:6.0:1619677472.720306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677472.720307:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.720308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:22.0:1619677472.720308:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.720308:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.720310:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.720311:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:22.0:1619677472.720312:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.720313:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.720313:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:22.0:1619677472.720314:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.720316:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.720316:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.720320:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.720322:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.720324:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.720325:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.720326:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.720327:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.720328:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.720329:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.720329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.720341:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.720343:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.720344:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.720346:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.720347:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.720348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.720360:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677472.720361:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677472.720363:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677472.720364:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.720365:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677472.720365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.720366:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.720368:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.720368:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.720370:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677472.720370:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.720372:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.720372:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.720374:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.720374:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.720375:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.720376:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677472.720376:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.720377:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.720378:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.720381:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.720383:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677472.720383:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.720385:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.720386:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.720386:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.720387:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677472.720388:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.720389:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677472.720389:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.720389:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.720390:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.720391:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.720391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677472.720391:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.720392:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.720393:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.720393:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677472.720404:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:6.0:1619677472.720405:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677472.720406:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.720407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.720408:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.720408:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.720409:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.720409:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.720410:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000400:00000200:6.0:1619677472.720412:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.720413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677472.720415:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677472.720422:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.720425:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.720426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.720426:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.720427:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.720427:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.720617:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.720620:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.720621:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.720623:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.720624:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.720626:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.720627:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.720629:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.720630:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.720631:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.720634:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.720636:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.720639:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.720639:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.720640:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.720641:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.720642:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.720643:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.720644:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.720645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.720648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.720656:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677472.720658:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.720659:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.720660:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.720660:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.720660:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677472.720660:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.720661:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677472.720662:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.720665:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.720919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.720928:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.720930:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.720931:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.720931:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.720933:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.720936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677472.721403:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.721406:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.721406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.721408:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.721410:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.721411:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.721413:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.721414:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.721415:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.721416:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.721419:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.721421:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.721423:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.721424:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.721425:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.721426:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.721427:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.721428:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.721428:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.721441:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.721443:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.721444:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.721446:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.721447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.721448:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677472.722316:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.722319:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.722320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.722321:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677472.722322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.722324:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:6.0:1619677472.722324:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.722326:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:6.0:1619677472.722326:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677472.722327:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.722327:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96512462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.722328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:6.0:1619677472.722329:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.722331:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.722331:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:6.0:1619677472.722331:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.722332:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677472.722333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.722334:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.722335:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.722335:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.722335:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.722337:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.722337:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.722340:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.722341:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677472.722341:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.722342:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.722343:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.722344:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.722345:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677472.722346:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677472.722347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.722357:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677472.722357:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677472.722359:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677472.722359:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677472.722360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677472.722360:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.722361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677472.722362:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.722363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.722364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.722383:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.722386:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.722386:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.722387:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.722388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.722389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.722391:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.722394:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.722395:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.722398:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.722400:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.722401:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.722403:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.722404:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.722405:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.722406:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.722410:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.722411:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.722414:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.722415:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.722415:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.722417:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.722418:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.722419:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.722420:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.722421:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677472.722431:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677472.722433:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677472.722434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677472.722434:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677472.722434:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.722435:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.722436:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:10.0:1619677472.722436:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.722436:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677472.722437:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677472.722438:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.722438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.0:1619677472.722439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.722440:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.722440:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:20.0:1619677472.722442:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.722443:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.722444:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.722445:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.722446:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.722449:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.722450:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.722453:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.722453:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.722454:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.722456:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.722456:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.722457:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.722457:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.722469:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.722472:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.722473:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.722473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.722475:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.722477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.722929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.722936:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.722939:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.722940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.722942:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.722945:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.722946:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.722948:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.722949:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.722950:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.722951:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.722954:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.722956:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.722959:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.722959:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.722960:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.722962:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.722963:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.722963:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.722964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.722968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.722977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.722979:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.722980:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.722982:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.722982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.722984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677472.723197:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.723199:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.723200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.723202:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.723204:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.723205:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.723207:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.723208:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.723209:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.723210:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.723213:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.723215:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.2:1619677472.723216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:6.0:1619677472.723217:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.723218:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677472.723219:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:6.0:1619677472.723219:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.723220:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.723221:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.723221:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.723222:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:6.0:1619677472.723222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677472.723224:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.723225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677472.723226:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.723228:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677472.723229:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.723230:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.723231:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.723232:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.723233:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.723234:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.723235:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.723236:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677472.723237:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.723238:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.723239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.723240:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.723240:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.723241:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677472.723242:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677472.723243:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.723244:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.723245:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677472.723245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:2.0:1619677472.723257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.723259:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.723260:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.723261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.723263:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.723265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.723305:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.723308:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.723309:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.723309:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.723311:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.723315:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.723735:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.723738:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.723738:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000400:00000200:2.0:1619677472.723741:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.723742:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.723743:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677472.723745:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.723746:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.723747:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.723748:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677472.723751:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.723753:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.723757:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.723757:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.723758:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677472.723760:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.723761:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.723762:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677472.723763:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677472.723764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:26.0:1619677472.723773:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:18.0:1619677472.723775:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:26.0:1619677472.723776:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:18.0:1619677472.723776:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:10.0:1619677472.723777:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.723778:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.723778:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.723778:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:26.0:1619677472.723780:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.723783:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.0:1619677472.723904:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.723907:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.723908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.723908:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.723910:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.723912:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.723926:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.723928:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.723931:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.723931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.723933:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.723934:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.723936:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.723937:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.723938:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.723939:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.723942:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.723944:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.723946:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.723947:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.723948:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.723949:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.723950:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.723951:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.723951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.723963:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.723966:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.723967:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.723967:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.723969:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.723971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.723980:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.723982:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.723983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.723983:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.723984:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.723985:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.724080:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.724083:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.724083:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677472.724086:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.724088:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.724089:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.724091:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.724092:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.724093:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.724094:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.724097:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.724098:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.724101:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.724102:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.724103:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.724105:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.724105:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.724106:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.724107:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.724108:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677472.724120:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.724121:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677472.724122:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677472.724123:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677472.724123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.724124:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677472.724124:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.724125:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677472.724126:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.724128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677472.724157:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.724160:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.724161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677472.724163:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.724165:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.724167:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677472.724168:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.724169:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.724170:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.724171:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677472.724175:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.724176:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.724178:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.724179:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.724180:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.724181:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.724182:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.724183:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.724183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677472.724198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.724205:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.724207:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.724208:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.724209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.724210:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.724211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.724222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.724229:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.724231:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.724232:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.724232:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.724233:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.724233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677472.724293:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.724299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.724301:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677472.724304:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.724306:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.724307:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.724309:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.724310:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.724311:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.724312:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.724315:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.724317:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.724320:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.724321:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.724321:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.724323:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.724323:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.724324:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.724325:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.724338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.724340:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.724341:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.724343:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.724344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.724345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.724698:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.724701:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.724702:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.724704:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.724706:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.724707:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.724709:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.724710:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.724711:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.724712:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.724715:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.724717:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.724719:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.724720:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.724720:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.724722:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.724722:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.724723:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.724724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.724734:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.724736:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.724737:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.724737:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.724738:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.724740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.724754:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.724756:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.724756:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.724757:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.724757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.724758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.725222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677472.725222:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.725224:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.725225:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.725227:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.725229:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677472.725230:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.725231:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677472.725232:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.725232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677472.725233:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.725234:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.725235:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.725235:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.725236:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.725238:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.725239:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677472.725240:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.725241:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677472.725241:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.725243:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.725244:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677472.725244:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677472.725245:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677472.725245:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.725246:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.725247:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:18.0:1619677472.725248:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.725248:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.725249:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.725249:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677472.725250:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.725252:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.725253:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.725254:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.725255:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.725256:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.725257:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.725257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677472.725263:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.725263:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.725264:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.725265:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.725265:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677472.725268:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677472.725268:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.725268:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677472.725271:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.725271:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.725271:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.725271:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.725272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:6.0:1619677472.725272:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677472.725274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:6.0:1619677472.725274:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.725274:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.725275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.725276:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677472.725276:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:6.0:1619677472.725278:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.725279:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.725282:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.725284:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.725286:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.725287:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.725288:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.725290:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.725290:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.725291:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.725291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.725305:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.725307:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.725308:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.725309:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.725310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.725312:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.725524:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.725527:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.725528:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.725530:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.725533:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.725534:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677472.725535:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.725537:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.725538:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.725539:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677472.725542:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.725543:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677472.725547:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.725548:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677472.725548:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677472.725550:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677472.725550:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.725551:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677472.725553:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677472.725553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677472.725557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.725564:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677472.725566:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677472.725566:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.725568:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677472.725568:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:10.0:1619677472.725568:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.725569:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.725570:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677472.725570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.725570:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.725571:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.0:1619677472.725571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.725571:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677472.725572:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677472.725573:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:10.0:1619677472.725574:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.725575:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.725576:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.725577:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677472.725579:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.725581:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677472.725582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677472.725582:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.725583:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677472.725583:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677472.725585:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.725585:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.725586:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.725586:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677472.725597:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.725598:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.725599:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.725600:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677472.725601:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.725603:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:17.0:1619677472.725727:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.725729:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.725730:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.725732:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.725734:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.725735:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677472.725736:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.725738:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.725739:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.725740:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677472.725743:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.725745:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.725746:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.725747:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.725748:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.725749:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.725750:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.725750:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.725751:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.725755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.725763:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.725764:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.725765:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.725766:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.725767:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.725768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.725835:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.725837:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.725838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.725838:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.725840:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.725843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.726184:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.726187:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.726188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.726190:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.726192:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.726194:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.726195:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.726197:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.726198:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.726199:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.726202:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.726204:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.726206:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.726207:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.726208:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.726210:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.726210:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.726211:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.726212:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.726212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.726223:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677472.726225:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677472.726225:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677472.726225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.726226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.726228:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.726228:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.726230:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.726231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.726232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.726242:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.726245:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.726246:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.726247:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.726247:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.726248:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.727027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.727036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.727038:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.727039:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.727040:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.727042:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.727045:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677472.727308:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.727310:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.727311:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.727313:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.727316:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.727317:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.727318:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.727320:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.727321:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.727322:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.727325:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.727327:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.727330:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.727330:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.727331:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.727333:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.727333:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.727334:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.727334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.727347:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.727350:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.727350:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.727352:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.727354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.727354:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.727399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.727406:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.727409:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.727409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.727411:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.727414:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.727415:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.727417:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.727418:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.727419:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.727420:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.727423:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.727425:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.727428:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.727428:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.727429:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.727431:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.727431:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.727433:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.727434:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.727434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.727440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677472.727444:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:2.0:1619677472.727445:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:26.0:1619677472.727447:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677472.727447:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:2.0:1619677472.727447:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:10.0:1619677472.727448:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.727448:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.727449:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:10.0:1619677472.727450:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677472.727453:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677472.727461:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.727463:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.727464:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.727464:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.727465:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.727466:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677472.727551:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.727553:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.727554:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.727555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.727556:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.727558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.728142:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.728144:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.728146:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.728147:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.728148:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.728150:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.728151:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.728153:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.728154:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.728155:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.728158:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.728160:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.728162:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.728163:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.728163:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.728165:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.728166:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.728167:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.728167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677472.728178:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.728181:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.728182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.728182:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.728184:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677472.728186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.728320:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.728322:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.728323:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.728325:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.728327:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.728328:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.728329:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.728331:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.728332:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.728333:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.728336:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.728337:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.728340:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.728341:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.728342:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.728343:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.728344:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.728344:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.728345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.728358:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:1.0:1619677472.728358:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677472.728360:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.728361:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.728361:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.728362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677472.728363:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.728364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.728365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677472.728365:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.728366:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677472.728368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:1.0:1619677472.728368:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.728369:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.728371:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.728372:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.728374:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.728377:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:1.0:1619677472.728377:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.728378:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.728379:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.728379:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.728381:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.728382:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.728383:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.728384:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.728384:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.728385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.728386:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.728387:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.728388:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.728388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.728392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.728400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.728402:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.728403:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.728405:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.728406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.728407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.728848:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.728850:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.728853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.728853:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.728855:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.728856:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.728857:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.728859:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.728860:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.728861:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.728863:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.728865:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.728868:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.728869:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.728869:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.728871:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.728872:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.728873:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.728874:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.728875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677472.728884:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677472.728886:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677472.728887:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.728887:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677472.728888:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:8.0:1619677472.728888:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.728889:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.728890:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677472.728890:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.728893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.728981:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.728983:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.728984:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.728986:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.728988:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.728989:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.728990:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.728992:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.728992:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.728994:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.728996:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.728998:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.729001:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.729001:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.729002:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.729004:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.729004:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.729005:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.729005:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.729015:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.729018:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.729019:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.729019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.729020:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.729022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677472.729268:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.729271:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.729272:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.729274:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.729276:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.729278:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.729279:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.729280:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.729281:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.729282:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.729286:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677472.729288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:6.0:1619677472.729288:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.729290:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.729291:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.729291:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.729293:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:6.0:1619677472.729298:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.729299:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.729299:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.729300:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:6.0:1619677472.729300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.729301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677472.729302:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.729303:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.729304:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.729305:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.729305:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.729307:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677472.729307:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.729308:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.729310:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.729310:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.729312:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677472.729315:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.729317:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.729317:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.729319:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.729319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.729320:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.729321:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.729322:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.729323:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.729324:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:21.0:1619677472.729325:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677472.729325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677472.729329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.729332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.729339:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.729340:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.729340:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:21.0:1619677472.729341:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.729341:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000400:00000200:21.0:1619677472.729342:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.729342:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677472.729344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.729345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677472.729346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.0:1619677472.729538:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.729541:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.729541:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.729543:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.729545:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.729546:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.729548:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.729549:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.729550:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.729551:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.729554:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.729556:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.729558:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.729559:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.729559:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.729561:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.729562:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.729562:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.729563:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.729573:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.729575:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.729575:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.729576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.729577:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.729578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.729593:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.729596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.729596:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.729597:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.729597:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.729598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.729762:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.729764:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.729765:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.729766:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.729767:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.729769:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.730369:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.730371:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.730372:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.730374:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.730375:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.730377:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.730378:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.730379:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.730380:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.730381:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.730384:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.730385:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.730388:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.730389:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.730389:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.730391:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.730392:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.730392:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.730392:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.730407:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677472.730409:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:18.0:1619677472.730409:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.730410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.730411:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.730412:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:18.0:1619677472.730413:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.730413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.730415:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.730417:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.730419:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:18.0:1619677472.730419:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:22.0:1619677472.730420:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.730421:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:22.0:1619677472.730422:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.730423:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.730423:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677472.730424:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677472.730424:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.730424:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.730426:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.730426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:22.0:1619677472.730427:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.730429:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.730431:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.730432:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.730432:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.730434:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.730434:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.730435:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.730436:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.730437:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.730447:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677472.730449:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677472.730449:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677472.730450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.730451:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677472.730451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677472.730452:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.730454:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.730455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.730456:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.730470:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.730472:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.730473:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.730474:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.730474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.730476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.730804:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.730807:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.730808:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.730810:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.730813:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.730814:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.730815:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.730816:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.730818:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.730818:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.730822:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.730824:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.730827:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.730828:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.730829:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.730830:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.730831:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.730831:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.730832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.730844:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.730847:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.730848:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.730848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.730850:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.730853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.730864:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.730867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.730867:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.730868:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.730869:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.730870:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677472.731025:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.731028:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.731029:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.731030:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.731032:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.731034:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.731300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.731303:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.731303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.731305:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.731308:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.731309:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.731310:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.731312:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.731313:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.731314:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.731317:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.731318:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.731321:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.731322:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.731323:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.731325:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.731325:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.731326:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.731326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.731340:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.731342:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.731343:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.731345:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.731346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.731347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.731410:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.731412:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.731413:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.731414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.731415:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.731417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.732012:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.732015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.732015:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677472.732017:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.732020:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.732021:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677472.732022:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.732024:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.732024:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.732025:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677472.732028:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.732030:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677472.732032:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.732033:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677472.732033:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677472.732035:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677472.732035:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.732036:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.732036:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:13.0:1619677472.732048:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.732051:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.732051:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.732053:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.732055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.732056:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677472.732172:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.732175:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.732176:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.732177:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.732180:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.732181:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.732182:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.732184:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.732185:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.732185:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.732189:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.732191:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.732193:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.732194:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.732194:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.732196:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.732196:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.732197:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.732198:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.732210:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.732213:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.732214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.732214:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.732216:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.732219:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.732227:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.732230:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.732230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.732231:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.732232:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.732232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677472.732601:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.732604:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.732605:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.732607:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.732610:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.732611:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.732612:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.732614:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.732615:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.732616:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.732619:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.732621:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.732623:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.732624:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.732625:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.732626:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.732627:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.732628:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.732628:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.732634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.732642:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.732644:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.732645:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.732647:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.732649:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.732650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.732686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.732694:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.732696:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.732697:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.732699:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.732699:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.732702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.732706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.732714:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.732716:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.732717:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.732718:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.732719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.732719:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677472.732764:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.732766:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677472.732767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677472.732769:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.732771:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.732773:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677472.732774:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.732776:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.732777:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.732778:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677472.732781:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.732783:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677472.732786:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.732787:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677472.732788:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677472.732790:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677472.732791:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.732792:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677472.732792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.0:1619677472.732814:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.732817:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.732818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.732818:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.732820:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.732823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.733347:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677472.733350:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677472.733350:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.733350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677472.733352:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:22.0:1619677472.733352:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.733353:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.733354:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.733355:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:22.0:1619677472.733356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.733357:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:22.0:1619677472.733357:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.733358:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.733360:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.733361:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.733362:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.733365:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.733367:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.733369:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.733370:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.733371:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.733372:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.733373:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.733374:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.733374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.733386:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.733388:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.733389:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.733391:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.733392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.733393:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.733584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.733591:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.733594:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.733594:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.733596:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.733599:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.733600:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.733601:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.733603:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.733604:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.733605:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.733608:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.733610:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.733612:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.733613:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.733613:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.733615:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.733616:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.733617:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.733617:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.733628:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.733630:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.733631:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.733632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.733633:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.733634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.733767:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.733770:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.733771:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.733773:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.733776:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.733777:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.733778:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.733780:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.733781:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.733782:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.733786:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.733788:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.733792:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.733793:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.733793:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.733795:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.733795:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.733797:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.733798:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.733798:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.733807:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.733809:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.733810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.733810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.733812:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677472.733812:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.733813:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.733814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.733814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.733815:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.733828:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.733830:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.733831:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.733831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.733832:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.733833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.733986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.733995:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.733998:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.733999:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.733999:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.734002:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.734005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677472.734204:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.734206:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.734207:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.734209:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677472.734211:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.734213:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677472.734214:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.734215:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.734216:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.734217:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677472.734220:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677472.734222:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677472.734224:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.734225:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.734226:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677472.734227:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.734228:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.734229:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.734229:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677472.734241:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.734244:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.734245:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.734245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.734248:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677472.734250:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.734410:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.734413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.734413:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.734414:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.734415:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.734417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.734429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.734432:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.734433:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.734433:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.734433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.734434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.734601:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.734603:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.734604:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.734606:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.734609:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.734610:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.734611:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.734612:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.734613:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.734614:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.734616:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.734618:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.734619:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.734620:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.734621:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.734622:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.734623:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.734624:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.734625:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.734626:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.734636:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.734637:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.734638:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.734640:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.734640:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.734641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.734641:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.734641:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.734642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.734642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677472.735205:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.735207:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.735208:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.735210:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.735212:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.735214:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.735215:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.735216:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.735217:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.735218:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.735218:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677472.735221:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.735221:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.735222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677472.735224:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:6.0:1619677472.735225:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677472.735226:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.735227:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.735227:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677472.735228:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.735229:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.735229:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677472.735230:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.735230:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677472.735230:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677472.735231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677472.735231:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.735232:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.735234:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.735237:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.735239:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.735243:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.735244:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.735244:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.735245:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.735245:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:22.0:1619677472.735246:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.735247:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.735248:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.735249:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.735250:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.735250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677472.735251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.735253:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677472.735261:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.735263:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.735264:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.735265:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.735267:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:20.0:1619677472.735268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.735269:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677472.735270:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677472.735271:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677472.735273:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.735275:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.735277:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677472.735278:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.735279:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.735280:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.735282:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.735282:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.735284:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:6.0:1619677472.735285:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.735286:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677472.735286:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.735286:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.735288:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.735289:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.735290:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677472.735290:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.735291:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677472.735292:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677472.735293:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677472.735299:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677472.735300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:18.0:1619677472.735308:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.735310:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.735311:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.735312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.735313:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.735315:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.735453:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.735456:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.735457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.735459:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.735461:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.735462:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.735464:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.735465:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.735466:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.735467:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.735471:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.735472:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.735475:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.735476:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.735477:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.735478:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.735479:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.735480:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.735480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.735492:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.735495:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.735496:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.735496:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.735498:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.735501:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.735511:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.735513:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.735514:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.735514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.735515:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.735516:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.736283:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.736285:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.736286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.736286:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.736288:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.736290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.736715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.736721:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.736724:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.736725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.736727:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.736729:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.736730:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.736732:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.736734:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.736735:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.736736:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.736739:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.736741:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.736744:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.736745:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.736745:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.736747:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.736748:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.736749:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.736749:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:24.0:1619677472.736822:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.736824:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.736825:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.736825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.736827:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.736829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.736918:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.736921:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.736922:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.736924:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.736927:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.736928:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.736930:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.736931:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.736932:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.736933:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.736936:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.736938:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.736941:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.736942:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.736942:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.736944:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.736944:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.736946:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.736947:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.736947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677472.736947:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.736949:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.736950:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.736952:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.736956:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.736957:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677472.736957:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677472.736958:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:10.0:1619677472.736958:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.736959:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677472.736959:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677472.736960:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677472.736960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677472.736961:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.736961:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.736961:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.736961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677472.736963:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.736964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677472.736966:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.736968:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.736971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677472.736972:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.0:1619677472.736972:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677472.736972:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.736973:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677472.736973:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.736973:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.736975:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.736976:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.736976:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677472.736976:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.736976:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677472.736977:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.106@o2ib7 : GET - for me
00000800:00000200:20.0:1619677472.736977:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677472.736977:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677472.736978:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677472.736978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:26.0:1619677472.736980:0:19941:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.106@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:10.0:1619677472.736980:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677472.736982:0:19941:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.106@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:26.0:1619677472.736984:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677472.736985:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677472.736986:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677472.736989:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : REPLY try# 0
00000800:00000200:22.0:1619677472.736989:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.736991:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677472.736992:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.736992:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.736992:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.736993:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677472.736993:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:22.0:1619677472.736993:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.736994:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677472.736995:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:22.0:1619677472.736995:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.736998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.737006:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.737008:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677472.737009:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.737010:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677472.737011:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: REPLY: OK
00000800:00000200:10.0:1619677472.737013:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.0:1619677472.737294:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.737299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.737302:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.737304:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.737307:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.737308:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.737310:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.737311:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.737312:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.737313:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.737316:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.737317:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.737320:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.737321:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.737321:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.737323:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.737324:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.737325:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.737325:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.737329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.737338:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.737341:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.737342:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.737342:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.737344:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.737347:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.737350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.737357:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.737360:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.737361:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.737361:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.737362:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.737362:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.737684:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.737687:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.737688:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.737688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.737690:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.737692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.737952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.737959:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.737963:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.737964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.737966:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.737969:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.737970:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677472.737971:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.737973:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.737974:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.737975:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.737978:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.737980:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.737982:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.737983:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677472.737984:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677472.737986:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.737986:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.737987:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.737988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677472.737999:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.738003:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.738004:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.738004:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677472.738006:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677472.738009:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677472.738339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.738342:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.738343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.738344:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.738347:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.738348:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.738349:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.738351:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.738352:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.738352:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.738355:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.738357:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.738359:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.738360:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.738361:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.738362:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.738363:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.738364:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.738364:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.738377:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.738379:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.738380:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.738382:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.738383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.738384:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.738403:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.738406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.738406:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.738407:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.738409:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.738411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.738549:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.738551:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.738552:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.738554:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.738556:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.738557:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.738559:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.738560:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.738561:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.738562:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.738565:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.738566:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.738569:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.738569:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.738570:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.738571:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.738572:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.738572:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.738573:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.738582:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.738585:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.738586:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.738586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.738587:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.738589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.739088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.739097:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.739100:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.739100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.739101:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.739103:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.739106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.739287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.739290:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.739298:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677472.739299:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.739300:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.739302:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.739304:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.739305:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.739306:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.739307:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.739308:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.739312:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.739313:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.739316:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.739316:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.739317:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.739319:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.739320:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.739321:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.739321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.739325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.739333:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.739335:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.739336:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.739337:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.739338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.739340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.739799:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.739802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.739802:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.739804:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.739807:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.739808:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.739809:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.739811:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.739812:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.739813:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.739817:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.739819:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.739821:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.739822:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.739823:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.739825:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.739825:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.739826:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.739826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.739839:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.739841:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.739841:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.739843:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.739843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.739845:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.739858:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.739860:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.739861:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.739861:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.739861:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.739862:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.740018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.740024:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.740027:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.740028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.740030:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.740033:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.740034:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.740035:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.740037:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.740038:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.740039:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.740042:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.740044:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.740046:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.740046:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.740047:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.740049:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.740049:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.740050:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.740051:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.740062:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.740064:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.740065:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.740065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.740067:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.740068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.740091:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.740093:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.740094:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.740096:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.740098:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.740099:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.740101:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.740102:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.740103:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.740104:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.740108:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.740109:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.740112:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.740113:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.740113:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.740115:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.740115:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.740116:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.740116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.740129:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.740132:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.740133:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.740133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.740136:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.740138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.740193:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.740195:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.740196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.740198:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.740200:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.740201:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.740202:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.740203:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.740204:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.740205:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.740208:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.740209:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.740212:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.740212:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.740213:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.740215:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.740215:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.740216:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.740216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.740228:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.740230:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.740231:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.740233:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.740234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.740235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.740314:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.740316:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.740317:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.740317:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.740319:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.740321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.740354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677472.740359:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.740362:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.740364:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.740365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.740366:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.740368:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677472.740369:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.740370:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.740371:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.740372:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677472.740376:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.740378:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.740380:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.740381:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677472.740382:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.740383:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.740384:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.740385:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677472.740385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:2.0:1619677472.740397:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.740400:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.740401:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.740401:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.740403:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.740406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677472.740456:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.740459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.740459:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677472.740462:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.740464:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.740466:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677472.740467:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.740468:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.740470:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.740471:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677472.740474:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.740475:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677472.740479:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.740480:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677472.740481:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677472.740483:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677472.740483:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.740484:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.740485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677472.740497:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.740499:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.740500:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.740502:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.740505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.740505:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.2:1619677472.740506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.740514:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.740517:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.740518:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.740518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.740520:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.740523:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.740869:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.740872:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.740872:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.740875:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.740876:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.740878:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.740879:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.740880:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.740881:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.740882:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.740886:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.740888:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.740891:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.740892:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.740892:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.740894:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.740894:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.740896:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.740897:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.740897:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.740908:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.740910:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.740911:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677472.740912:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677472.740912:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.740912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.740913:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.740914:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.740914:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.740918:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.741872:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.741875:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.741876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.741879:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.741881:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.741883:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.741884:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.741887:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.741888:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.741890:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.741894:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.741897:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.741900:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.741901:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.741902:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.741903:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.741904:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.741905:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.741905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.741919:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.741922:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.741922:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.741924:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.741925:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.741926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.741928:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.741931:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.741932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.741932:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.741934:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.741936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.742159:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.742161:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.742162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.742164:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.742166:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.742168:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.742169:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.742171:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.742172:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.742173:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.742176:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.742178:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.742181:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.742182:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.742182:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.742184:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.742185:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.742185:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.742186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.742191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.742199:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.742202:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.742203:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.742203:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.742206:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.742209:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.742849:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.742852:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.742853:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.742853:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.742855:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.742857:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.742870:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.742872:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.742873:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.742873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.742874:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.742875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.742974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.742980:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.742983:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.742985:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.742986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.742988:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.742989:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.742990:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.742992:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.742993:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.742994:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.742997:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.742999:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.743001:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.743002:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.743002:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.743004:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.743004:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.743005:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.743007:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.743007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.743010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.743013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.743020:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.743021:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.743022:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.743022:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677472.743023:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.743023:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.743024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677472.743024:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677472.743025:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.743026:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.743027:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677472.743027:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.743028:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.743030:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677472.743033:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.743034:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677472.743035:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.743037:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.743038:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.743039:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677472.743041:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677472.743043:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677472.743046:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.743046:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.743047:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677472.743049:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.743049:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.743050:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.743050:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677472.743063:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.743066:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.743067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.743067:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.743069:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.743071:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.743083:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.743086:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.743087:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.743087:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.743088:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.743089:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677472.743254:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.743256:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.743257:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.743259:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.743261:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.743263:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.743264:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.743265:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.743266:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.743267:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.743271:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.743273:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.743275:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.743276:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.743277:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.743278:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.743279:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.743280:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677472.743281:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677472.743281:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.743291:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677472.743291:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677472.743296:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.743298:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.743298:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.743298:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677472.743299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677472.743299:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.743300:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.743303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.743319:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.743321:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.743322:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.743323:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.743323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.743325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.743388:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.743391:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.743392:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.743394:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.743397:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.743398:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.743400:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.743401:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.743403:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.743404:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.743404:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.743406:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677472.743408:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.743409:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677472.743410:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.743411:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.743411:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.743412:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.743413:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.743414:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.743415:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.743417:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677472.743417:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.743418:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.743419:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.743419:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.743421:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.743422:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.743422:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.743422:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.743423:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.743423:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.743424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677472.743424:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677472.743425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.743425:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.743426:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.743426:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.743435:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677472.743438:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:6.0:1619677472.743438:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677472.743439:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677472.743439:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.743440:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.743441:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677472.743441:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.743445:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.743445:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.743446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.743973:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.743975:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.743976:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.743977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.743978:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.743980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.744138:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.744141:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.744142:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.744144:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.744147:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.744148:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.744149:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.744150:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.744151:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.744152:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.744155:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.744157:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.744160:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.744160:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.744161:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.744162:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.744163:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.744164:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.744165:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.744166:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677472.744176:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677472.744178:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677472.744179:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.744179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.744180:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.744180:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.744181:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.744182:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677472.744182:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.744184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.744197:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.744200:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.744201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.744201:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.744202:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.744202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.744272:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:22.0:1619677472.744275:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:6.0:1619677472.744276:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:22.0:1619677472.744278:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.744280:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.744283:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:22.0:1619677472.744284:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.744285:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.744286:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.744287:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:22.0:1619677472.744291:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.744292:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:22.0:1619677472.744300:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.744301:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:22.0:1619677472.744302:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:6.0:1619677472.744303:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)++
00000800:00000200:22.0:1619677472.744304:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:22.0:1619677472.744305:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (23)++
00000800:00000200:6.0:1619677472.744305:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:22.0:1619677472.744306:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (24)--
00000800:00000200:22.0:1619677472.744306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (23)--
00000800:00000200:22.0:1619677472.744307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:6.0:1619677472.744307:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.744309:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.2:1619677472.744311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000400:00000200:6.0:1619677472.744311:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:6.0:1619677472.744312:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.744314:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.744314:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.744315:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677472.744318:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.744319:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)++
00000800:00000200:6.0:1619677472.744319:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677472.744322:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (23)--
00000800:00000200:10.0:1619677472.744322:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2fa00] (23)--
00000800:00000200:6.0:1619677472.744322:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677472.744323:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.744323:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677472.744323:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:10.0:1619677472.744325:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.51@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.744326:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677472.744326:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:6.0:1619677472.744327:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:10.0:1619677472.744328:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (21)--
00000800:00000200:6.0:1619677472.744328:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000800:00000200:24.0:1619677472.744340:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.744342:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.744343:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.744343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.744346:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.744348:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.744360:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.744363:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.744364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.744364:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.744365:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.744366:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.744370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.744379:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.744382:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.744382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.744383:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.744385:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000400:00000200:17.0:1619677472.744385:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.744388:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:17.0:1619677472.744388:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.744389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:6.0:1619677472.744389:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:22.0:1619677472.744391:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.744394:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.744395:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677472.744397:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677472.744397:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000400:00000200:22.0:1619677472.744398:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.744399:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.744400:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677472.744400:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.744401:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.744402:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.744403:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.744403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:22.0:1619677472.744404:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.744406:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.744409:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.744409:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677472.744410:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.744412:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.744412:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:22.0:1619677472.744414:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d6a00] (22)++
00000800:00000200:22.0:1619677472.744415:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (23)--
00000800:00000200:22.0:1619677472.744416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:6.0:1619677472.744425:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.744427:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)++
00000800:00000200:6.0:1619677472.744427:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.744427:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.744428:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d6a00] (23)--
00000800:00000200:22.0:1619677472.744428:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:22.0:1619677472.744429:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.744429:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.744430:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:22.0:1619677472.744431:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:6.0:1619677472.744431:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.744432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:22.0:1619677472.744433:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.744435:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.744436:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677472.744437:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.744438:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.744439:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.744440:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677472.744443:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.744444:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677472.744447:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.744447:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677472.744448:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677472.744450:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677472.744450:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:22.0:1619677472.744451:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:22.0:1619677472.744451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000800:00000200:6.0:1619677472.744527:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:6.0:1619677472.744529:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:22.0:1619677472.744530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:6.0:1619677472.744532:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.744535:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.744537:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677472.744538:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.744539:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.744540:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.744541:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677472.744544:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.744545:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677472.744548:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.744549:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:6.0:1619677472.744549:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:6.0:1619677472.744551:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677472.744551:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:6.0:1619677472.744552:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:6.0:1619677472.744553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000800:00000200:13.0:1619677472.744575:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.744578:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.744579:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.744580:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.744583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.744584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677472.744801:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:22.0:1619677472.744804:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:6.0:1619677472.744805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:22.0:1619677472.744807:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.744810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.744811:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.744812:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.744814:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.744815:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.744815:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.744818:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.744820:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.744822:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.744822:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.744823:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.744825:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.744825:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:22.0:1619677472.744826:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:22.0:1619677472.744826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000800:00000200:20.0:1619677472.744838:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.744841:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.744841:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.744843:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.744843:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:21.0:1619677472.744844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.744845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677472.744845:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:22.0:1619677472.744846:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:6.0:1619677472.744847:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.744851:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.744852:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677472.744853:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.744854:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.744855:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.744856:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677472.744858:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.744859:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.744861:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.744861:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677472.744861:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.744863:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.744863:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:6.0:1619677472.744864:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d6a00] (22)++
00000800:00000200:6.0:1619677472.744865:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (23)--
00000800:00000200:6.0:1619677472.744866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:22.0:1619677472.744877:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.744878:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.744878:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.744878:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)++
00000800:00000200:6.0:1619677472.744879:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d6a00] (23)--
00000800:00000200:6.0:1619677472.744879:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:22.0:1619677472.744880:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.744880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.744881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000800:00000200:22.0:1619677472.744882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.744901:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.744904:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.744905:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.744907:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.744909:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.744910:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677472.744910:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000400:00000200:26.0:1619677472.744911:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.744913:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677472.744913:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:6.0:1619677472.744913:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000400:00000200:26.0:1619677472.744914:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.744915:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677472.744916:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.744919:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.744920:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:6.0:1619677472.744921:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.744922:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677472.744923:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.744924:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:6.0:1619677472.744924:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677472.744925:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677472.744925:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.744926:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677472.744927:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.744927:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677472.744927:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677472.744928:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.744929:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.744930:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677472.744930:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.744932:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677472.744936:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.744937:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:6.0:1619677472.744938:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:6.0:1619677472.744940:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677472.744941:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)++
00000800:00000200:22.0:1619677472.744942:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677472.744942:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677472.744942:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (23)--
00000800:00000200:6.0:1619677472.744943:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:22.0:1619677472.744944:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.744944:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)++
00000400:00000200:22.0:1619677472.744945:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.744945:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677472.744945:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.62@o2ib3
00000800:00000200:10.0:1619677472.744946:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677472.744947:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.744947:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677472.744948:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.62@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.744949:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.744950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.744951:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:6.0:1619677472.744951:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677472.744952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (22)--
00000400:00000200:6.0:1619677472.744952:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.744954:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.744955:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.744956:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677472.744958:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.744960:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.744961:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.744962:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677472.744962:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:13.0:1619677472.744964:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:6.0:1619677472.744964:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.744964:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d6a00] (21)++
00000800:00000200:6.0:1619677472.744964:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d6a00] (22)--
00000800:00000200:6.0:1619677472.744965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d6a00] (21)--
00000800:00000200:13.0:1619677472.744967:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.744968:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.744969:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.744971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.744972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677472.744976:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.744978:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.744979:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.744979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.744982:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.744984:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:12.0:1619677472.744990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677472.744993:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.744994:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.744995:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677472.744995:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677472.744996:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:12.0:1619677472.744997:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677472.744999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.744999:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.745000:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.745001:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.745002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.745057:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.745059:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.745060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.745062:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.745064:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.745065:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.745066:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.745068:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.745069:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.745069:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.745072:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.745074:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.745076:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.745076:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.745077:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.745079:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.745079:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.745080:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.745080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.745092:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.745095:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.745095:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.745097:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.745098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.745099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677472.745123:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39e000] (21)++
00000800:00000200:2.0:1619677472.745126:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.16@o2ib3
00000800:00000200:18.0:1619677472.745127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39e000] (22)--
00000400:00000200:2.0:1619677472.745129:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.16@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.745131:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.745133:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677472.745134:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.745136:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.745137:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.745138:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677472.745141:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.745143:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.745146:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.745147:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.745147:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677472.745150:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.745150:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39e000] (21)++
00000800:00000200:2.0:1619677472.745151:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39e000] (22)--
00000800:00000200:2.0:1619677472.745152:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39e000] (21)--
00000800:00000200:26.0:1619677472.745162:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.745165:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.745166:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.745166:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.745168:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.745171:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:18.0:1619677472.745194:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39e000] (21)++
00000800:00000200:18.0:1619677472.745197:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.16@o2ib3
00000800:00000200:2.0:1619677472.745197:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39e000] (22)--
00000400:00000200:18.0:1619677472.745199:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.16@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.745202:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.745204:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677472.745205:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.745206:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.745207:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.745208:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677472.745212:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.745213:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.745214:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677472.745216:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.745217:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677472.745217:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677472.745218:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.745218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.745220:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677472.745220:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39e000] (21)++
00000400:00000200:24.0:1619677472.745221:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.745221:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39e000] (22)--
00000800:00000200:18.0:1619677472.745222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39e000] (21)--
00000400:00000200:24.0:1619677472.745223:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.745225:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.745226:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.745228:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.745229:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.745230:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.745234:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.745234:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.745235:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.745236:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.745236:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.745238:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.745239:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.745239:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.745240:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677472.745242:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (24)--
00000800:00000200:8.0:1619677472.745242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677472.745243:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.745243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.745244:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.745244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.745245:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677472.745246:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.745246:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.745247:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.745247:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.745248:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.745249:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.745250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.745251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677472.745251:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:24.0:1619677472.745252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:8.0:1619677472.745252:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.745253:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.745255:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.745256:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677472.745258:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.745260:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677472.745262:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.745263:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677472.745263:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677472.745265:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677472.745265:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.745266:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677472.745268:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677472.745268:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:10.0:1619677472.745276:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.745278:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677472.745279:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677472.745279:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.745280:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000400:00000200:10.0:1619677472.745281:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.745283:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:10.0:1619677472.745283:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:24.0:1619677472.745284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.745285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677472.745501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.745510:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.745513:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.745514:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.745514:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.745516:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.745519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.745607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.745610:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.745610:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.745611:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.745613:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.745615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.746090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.746093:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.746095:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.746098:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.746099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.746100:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.746101:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.746103:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.746104:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.746105:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.746106:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.746109:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.746111:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.746113:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.746114:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.746115:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.746116:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.746117:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.746118:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.746118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.746122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.746130:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.746133:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.746133:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.746135:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.746136:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.746137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.746238:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.746241:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.746242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.746244:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.746246:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.746248:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.746249:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.746250:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.746251:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.746252:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.746256:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.746257:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.746261:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.746261:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.746262:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.746264:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.746264:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.746265:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.746266:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.746270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.746278:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.746281:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.746282:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.746282:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.746284:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.746287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677472.746594:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.746597:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677472.746599:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677472.746600:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.746603:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.746604:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677472.746605:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.746607:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.746608:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.746609:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677472.746612:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.746614:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677472.746617:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.746618:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:6.0:1619677472.746619:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677472.746620:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677472.746621:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.746622:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677472.746623:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677472.746623:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677472.746634:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677472.746635:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677472.746636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677472.746637:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:6.0:1619677472.746731:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.746734:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.746735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.746737:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.746740:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.746742:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.746743:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.746744:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.746745:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.746746:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.746749:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.746751:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.746753:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.746753:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.746754:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.746755:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.746756:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.746757:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.746757:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.746769:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.746772:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.746772:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.746774:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.746775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.746776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.747101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.747108:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.747110:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.747111:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.747113:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.747116:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.747118:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.747119:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.747120:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.747121:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.747122:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.747126:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.747127:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.747130:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.747131:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.747132:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.747134:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.747134:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.747135:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.747136:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.747173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.747179:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.747182:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.747182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.747184:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.747187:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.747189:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.747190:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.747191:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.747192:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.747193:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.747196:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.747197:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.747199:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.747199:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.747200:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.747201:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.747202:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.747203:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.747204:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.747204:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.747210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.747216:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.747217:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.747217:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.747218:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.747218:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.747218:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.747219:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.747220:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677472.747220:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.747221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.747238:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.747240:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.747241:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.747241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.747242:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.747243:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.747516:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.747519:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.747520:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.747522:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.747525:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.747526:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.747527:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.747528:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.747529:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.747530:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.747533:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.747535:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.747538:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.747539:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.747539:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.747541:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.747541:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.747543:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.747544:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.747544:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.747547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677472.747554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:17.0:1619677472.747555:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.747558:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.747558:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.747559:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.747559:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.0:1619677472.747560:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:24.0:1619677472.747561:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677472.747561:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.747562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677472.747562:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:24.0:1619677472.747563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:21.0:1619677472.747564:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.747564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.747565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677472.747566:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.747567:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.747568:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.747570:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.747571:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.747572:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.747574:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.747576:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.747578:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.747579:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.747580:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.747581:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.747582:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.747583:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677472.747584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.747587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.747595:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.747598:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.747598:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.747600:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.747601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.747602:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.747864:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.747867:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.747868:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.747868:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.747870:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.747872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.747883:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.747885:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.747886:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.747886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.747887:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.747888:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.747983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.747991:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.747994:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.747995:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.747995:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.747996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.747997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677472.748016:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.748019:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.748020:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.748022:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.748024:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.748025:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.748027:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.748028:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.748029:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.748030:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.748033:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.748035:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.748038:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.748039:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.748039:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.748041:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.748041:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.748042:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.748042:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.748055:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.748058:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.748058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.748059:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.748061:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.748064:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.748119:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.748122:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.748123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.748125:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.748127:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.748128:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.748129:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.748131:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.748132:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.748133:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.748136:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.748138:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.748140:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.748141:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.748141:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.748143:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.748143:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.748144:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.748144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.748156:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.748158:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.748159:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.748161:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.748162:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.748163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.748179:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.748181:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.748182:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.748183:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.748184:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.748185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677472.748498:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.748501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.748501:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677472.748503:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.748506:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.748508:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677472.748509:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.748510:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.748511:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.748512:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677472.748515:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.748517:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677472.748520:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.748520:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677472.748521:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677472.748523:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677472.748523:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.748525:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677472.748526:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677472.748527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:22.0:1619677472.748538:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.748540:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677472.748540:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:22.0:1619677472.748541:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.748542:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:6.0:1619677472.748542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.748543:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.748543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.748544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677472.748546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.748556:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.748559:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.748560:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.748560:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.748562:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.748562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.748748:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.748750:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.748751:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.748754:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.748757:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.748758:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.748759:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.748760:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.748762:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.748763:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.748766:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.748768:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.748770:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.748771:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.748772:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.748773:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.748774:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.748774:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.748775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.748786:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.748788:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.748789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.748789:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.748791:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.748793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.749373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.749387:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.749389:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.749391:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.749392:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.749393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.749396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.749428:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.749431:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.749432:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.749432:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.749434:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.749436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.749705:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.749707:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.749708:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.749710:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.749712:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.749713:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.749715:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.749716:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.749717:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.749717:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.749720:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.749722:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.749725:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.749726:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.749727:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.749728:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.749729:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.749729:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.749730:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.749742:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.749745:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.749745:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.749746:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.749748:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.749751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.749935:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.749938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.749938:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677472.749940:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.749942:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.749944:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.749945:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.749946:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.749947:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.749948:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.749950:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.749952:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.749954:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.749955:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.749956:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.749958:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.749958:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.749959:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677472.749960:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677472.749960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.749971:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677472.749971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677472.749972:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.749973:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.749974:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.749974:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.749975:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.749977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677472.749977:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.749979:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.750280:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.750283:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.750284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.750286:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.750289:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.750291:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.750292:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.750299:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.750300:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.750301:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.750304:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.750306:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.750309:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.750310:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.750310:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.750312:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.750313:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.750313:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.750314:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.750327:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.750330:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.750331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.750331:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.750333:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.750336:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.751341:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.751344:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.751344:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677472.751347:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.751349:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.751351:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677472.751352:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.751353:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.751354:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.751355:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677472.751358:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.751359:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677472.751362:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.751363:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.751364:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677472.751365:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.751366:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.751367:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.751367:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677472.751381:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.751384:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.751384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.751385:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.751387:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.751390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677472.751598:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.751601:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.751601:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677472.751604:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.751606:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.751607:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.751609:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.751610:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.751611:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.751612:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.751614:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.751616:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.751619:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.751620:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.751621:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.751622:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.751623:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.751623:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.751624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.751636:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.751638:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.751639:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.751641:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.751642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.751643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.751692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.751701:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.751704:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.751704:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.751705:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.751707:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.751710:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.751712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.751719:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.751722:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.751723:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.751723:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.751724:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.751724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.752137:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.752140:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.752141:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.752143:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.752145:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.752147:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.752148:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.752150:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.752151:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.752152:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.752155:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.752157:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.752159:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.752160:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.752161:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.752162:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.752163:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.752164:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.752164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.752168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.752176:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.752179:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.752180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.752180:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.752182:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.752185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.752578:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.752581:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.752583:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.752584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.752585:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.752587:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.752588:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.752590:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.752591:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.752592:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.752596:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.752597:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.752600:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.752600:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.752601:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.752603:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.752604:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.752604:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.752605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.752617:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.752620:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.752621:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.752621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.752623:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.752626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.752791:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.752794:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.752795:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.752797:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.752800:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.752801:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.752802:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.752804:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.752805:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.752806:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.752809:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.752811:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.752813:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.752814:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.752815:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.752816:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.752817:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.752817:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.752818:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.752829:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.752831:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.752832:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.752833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.752834:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.752837:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677472.753021:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.753024:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.753025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.753027:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.753029:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.753030:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.753031:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.753033:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.753034:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.753035:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.753038:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.753040:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.753043:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.753043:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.753044:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.753046:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.753046:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.753047:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.753047:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.753060:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.753062:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.753063:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.753064:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.753065:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.753067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.753345:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.753347:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.753348:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.753350:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.753353:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.753354:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.753355:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.753357:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.753358:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.753359:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.753361:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.753363:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.753365:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.753365:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.753366:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.753367:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.753368:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.753368:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.753369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.753406:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.753408:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.753409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.753409:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.753410:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.753412:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.753492:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.753494:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.753495:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.753496:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.753499:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.753500:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.753501:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.753502:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.753503:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.753504:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.753507:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.753508:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.753510:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.753511:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.753511:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.753513:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.753513:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.753514:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.753516:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.753516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677472.753524:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.753526:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.753527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.753528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.753529:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677472.753529:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.753531:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677472.753531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.753532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.753533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677472.754048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.754054:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.754057:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.754058:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.754060:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.754062:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.754064:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.754066:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.754067:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.754068:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.754069:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.754072:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.754074:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.754077:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.754077:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.754078:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.754080:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.754080:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.754081:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.754081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.754093:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.754096:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.754096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.754097:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.754099:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.754101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677472.754216:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.754219:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.754220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.754220:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.754222:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.754224:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.754465:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.754467:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.754468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.754470:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.754472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.754474:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.754475:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.754476:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.754477:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.754478:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.754481:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.754482:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.754485:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.754486:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.754487:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.754488:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.754489:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.754489:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.754490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.754503:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.754506:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.754507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.754507:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.754509:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.754513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.754615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.754618:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.754621:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.754623:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.754624:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.754625:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.754627:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.754628:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.754629:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.754630:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.754631:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.754635:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.754636:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.754639:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.754639:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.754640:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.754642:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.754642:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.754643:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.754644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.754648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.754657:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.754659:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.754660:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.754661:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.754662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.754663:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.754849:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.754852:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.754853:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.754855:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.754857:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.754858:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.754860:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.754861:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.754862:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.754863:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.754867:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.754869:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.754871:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.754872:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.754872:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.754874:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.754875:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.754875:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.754876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.754887:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.754889:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.754890:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.754892:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.754893:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.754894:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.754906:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.754909:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.754910:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.754911:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.754912:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.754912:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.755328:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.755330:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.755332:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.755333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.755334:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.755336:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.755337:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.755338:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.755339:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.755340:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.755343:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.755345:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.755348:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.755349:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.755350:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.755351:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.755352:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.755353:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.755353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:13.0:1619677472.755698:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.755701:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.755703:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.755705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.755706:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.755708:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677472.755709:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.755711:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.755711:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.755712:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677472.755715:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.755717:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677472.755720:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.755721:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677472.755721:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677472.755723:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677472.755724:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.755724:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.755725:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677472.755730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.755739:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.755741:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.755742:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.755743:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.755745:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.755748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.755779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.755785:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.755788:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.755789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.755791:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.755793:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.755795:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677472.755796:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.755797:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.755798:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.755799:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677472.755802:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.755804:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677472.755807:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.755808:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677472.755808:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677472.755810:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677472.755811:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.755812:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.755812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677472.755823:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.755826:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.755826:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.755828:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.755829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.755832:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.755878:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.755880:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.755881:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.755881:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.755883:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.755885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.755904:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.755906:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.755907:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.755907:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.755908:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.755909:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677472.756437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.756444:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.756446:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.756447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.756449:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.756451:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.756453:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.756454:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.756455:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.756457:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.756458:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.756461:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.756463:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.756465:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.756466:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.756467:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.756468:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.756469:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.756470:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677472.756471:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677472.756471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677472.756477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677472.756483:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677472.756483:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677472.756484:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677472.756485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677472.756486:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677472.756486:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.756487:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.756487:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.756489:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.756492:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.756502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.756504:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.756505:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.756505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.756506:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.756507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.756516:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.756519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.756519:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677472.756521:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.756524:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.756525:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.756526:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.756528:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.756528:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.756530:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.756533:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.756534:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.756537:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.756538:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.756538:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.756540:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.756540:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.756542:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677472.756543:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677472.756543:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.756553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.756555:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.756555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677472.756555:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677472.756556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.756558:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.756558:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.756560:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.756561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.756562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.756679:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.756682:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.756684:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.756684:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.756686:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.756687:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.756688:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.756690:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.756691:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.756692:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.756695:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.756697:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.756699:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.756700:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.756701:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.756702:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.756703:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.756704:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.756704:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677472.756716:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.756718:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.756719:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.756719:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.756721:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677472.756723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.757377:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.757380:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.757381:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.757383:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.757386:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.757387:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677472.757389:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.757390:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.757391:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.757392:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677472.757396:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.757397:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677472.757401:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.757401:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677472.757402:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677472.757404:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677472.757404:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.757405:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677472.757406:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677472.757407:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677472.757410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.757418:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677472.757420:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:21.0:1619677472.757421:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.757422:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.757422:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677472.757423:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677472.757424:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.757424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.757424:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677472.757426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.757429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.757437:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.757439:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.757440:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.757441:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.757442:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677472.757442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677472.757457:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.757459:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.757460:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.757460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.757462:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.757465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.757899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.757907:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.757910:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.757911:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.757911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.757914:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.757916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677472.757968:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.757971:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.757972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.757974:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.757977:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.757978:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.757980:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.757981:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.757982:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.757983:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.757987:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.757988:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.757991:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.757992:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.757992:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.757994:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.757995:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.757995:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.757996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.758007:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.758009:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.758010:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.758012:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.758013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.758014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.758045:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.758048:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.758049:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.758051:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.758053:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.758055:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.758056:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.758057:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.758058:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.758059:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.758062:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.758064:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.758066:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.758067:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.758068:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.758069:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.758070:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.758071:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.758071:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.758081:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.758083:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.758084:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.758085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.758086:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.758088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.758101:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.758104:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.758105:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.758105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.758106:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.758106:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.758119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.758125:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.758128:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.758129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.758131:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.758134:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.758135:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.758136:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.758138:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.758139:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.758140:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.758142:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.758144:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.758146:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.758146:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.758147:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.758149:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.758149:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.758150:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.758151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.758162:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.758164:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.758165:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.758166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.758167:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.758168:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.758213:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.758216:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.758217:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.758218:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.758221:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.758222:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.758224:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.758225:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.758226:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.758227:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.758230:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.758231:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.758234:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.758235:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.758236:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.758237:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.758238:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.758239:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.758240:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.758241:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.758251:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.758253:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.758253:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.758254:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.758254:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.758255:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.758256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677472.758256:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.758257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.758260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677472.758645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.758651:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.758654:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.758655:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.758656:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.758659:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.758660:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.758661:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.758663:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.758664:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.758665:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.758668:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.758670:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.758673:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.758673:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.758674:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.758675:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.758676:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.758677:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.758677:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.758690:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.758693:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.758694:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.758694:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.758696:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.758698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677472.758758:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.758761:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.758762:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.758764:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.758766:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.758768:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.758769:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.758770:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.758771:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.758772:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.758776:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.758778:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.758780:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.758781:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.758782:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.758784:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.758784:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.758785:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.758786:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.758797:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.758800:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.758800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.758801:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.758803:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.758806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677472.758856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907b38d400] (20)++
00000800:00000200:2.0:1619677472.758863:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b38d400] (21)++
00000800:00000200:2.0:1619677472.758866:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.11@o2ib3
00000800:00000200:18.0:1619677472.758867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38d400] (22)--
00000400:00000200:2.0:1619677472.758868:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.2.11@o2ib3 : GET - for me
00000400:00000200:2.0:1619677472.758871:0:19932:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.2.11@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:2.0:1619677472.758873:0:19932:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.2.11@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:2.0:1619677472.758874:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907b38d400] (21)++
00000800:00000200:2.0:1619677472.758876:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907b38d400] (22)++
00000800:00000200:2.0:1619677472.758877:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907b38d400] (23)--
00000400:00000200:2.0:1619677472.758879:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.2.11@o2ib3 local destination
00000800:00000200:5.2:1619677472.758883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907b38d400] (22)++
00000400:00000200:2.0:1619677472.758883:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.2.11@o2ib3(10.51.2.11@o2ib3:10.51.2.11@o2ib3) : REPLY try# 0
00000800:00000200:2.0:1619677472.758886:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.2.11@o2ib3
00000800:00000200:2.0:1619677472.758888:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c610ff80] -> 10.51.2.11@o2ib3 (2) version: 12
00000800:00000200:18.0:1619677472.758889:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b38d400] (23)++
00000800:00000200:2.0:1619677472.758890:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91907b38d400] (23)++
00000800:00000200:2.0:1619677472.758890:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907b38d400] (25)++
00000800:00000200:18.0:1619677472.758892:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907b38d400] (26)--
00000800:00000200:2.0:1619677472.758892:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91907b38d400] (26)--
00000800:00000200:2.0:1619677472.758892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38d400] (24)--
00000800:00000200:18.0:1619677472.758893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38d400] (23)--
00000800:00000200:2.0:1619677472.758893:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38d400] (22)--
00000800:00000200:5.2:1619677472.758897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907b38d400] (21)++
00000800:00000200:18.0:1619677472.758903:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b38d400] (22)++
00000800:00000200:18.0:1619677472.758904:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907b38d400] (23)--
00000400:00000200:18.0:1619677472.758905:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.758906:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38d400] (22)--
00000400:00000200:18.0:1619677472.758907:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.2.11@o2ib3: REPLY: OK
00000800:00000200:18.0:1619677472.758909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b38d400] (21)--
00000800:00000200:26.0:1619677472.759241:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.759244:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.759245:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.759247:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.759250:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.759251:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.759253:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.759254:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.759255:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.759256:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.759259:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.759260:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.759263:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.759264:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.759265:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.759266:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.759267:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.759267:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.759268:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.759280:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.759283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.759283:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.759284:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.759286:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.759288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:23.0:1619677472.759304:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:21.0:1619677472.759508:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.759511:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.759513:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.759513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.759515:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.759517:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.759518:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.759520:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.759521:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.759521:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.759524:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.759526:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.759529:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.759530:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.759530:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.759532:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.759533:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.759534:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.759534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.759546:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.759549:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.759550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.759550:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.759552:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.759555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.759642:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.759644:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.759645:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.759646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.759647:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.759649:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.759832:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.759835:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.759836:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.759838:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.759840:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.759841:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.759843:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.759844:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.759845:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.759846:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.759850:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.759851:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.759854:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.759855:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.759855:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.759857:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.759857:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.759859:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.759859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.759870:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.759872:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.759873:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.759875:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.759876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.759877:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.759888:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.759891:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.759891:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.759892:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.759893:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.759894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.760101:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.760104:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.760105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.760106:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.760109:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.760111:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.760112:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.760113:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.760114:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.760115:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.760119:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.760120:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.760123:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.760124:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.760124:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.760126:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.760126:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.760127:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.760128:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.760140:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.760143:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.760144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.760144:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.760146:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.760149:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.760158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.760161:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.760162:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.760162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.760163:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.760164:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677472.760910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.760916:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.760918:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.760919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.760921:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.760924:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.760925:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.760927:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.760928:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.760929:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.760930:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.760933:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.760935:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.760938:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.760938:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.760939:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.760941:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.760942:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.760942:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.760943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.760956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.760959:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.760960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.760960:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.760962:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.760964:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677472.761054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.761062:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.761065:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.761065:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677472.761066:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.761068:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.761071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677472.761314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677472.761320:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677472.761323:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677472.761324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677472.761325:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677472.761328:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.761329:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677472.761330:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.761331:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.761332:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.761333:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677472.761337:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677472.761338:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677472.761341:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.761342:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677472.761343:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677472.761345:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677472.761345:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677472.761346:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677472.761346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:10.0:1619677472.761370:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.761372:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677472.761373:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677472.761374:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.761376:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:10.0:1619677472.761379:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677472.761512:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.761515:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.761515:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.761518:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.761520:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.761521:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.761523:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.761524:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.761525:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.761526:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.761529:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.761531:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.761533:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.761534:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.761535:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.761536:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.761537:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.761538:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.761538:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.761550:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.761553:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.761553:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.761555:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.761556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.761557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.762249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.762254:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.762257:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.762259:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.762260:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.762261:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:2.0:1619677472.762263:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.762264:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:2.0:1619677472.762264:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677472.762266:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.762267:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.762268:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:2.0:1619677472.762268:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.762269:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.762270:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.762270:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:12.0:1619677472.762272:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677472.762273:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.762274:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677472.762275:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.762277:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677472.762277:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:12.0:1619677472.762278:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.762280:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.762281:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677472.762281:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:12.0:1619677472.762282:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.762283:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:12.0:1619677472.762284:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.762284:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.762285:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.762286:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:12.0:1619677472.762287:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.762287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677472.762288:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677472.762289:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677472.762290:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677472.762291:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:20.2:1619677472.762292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677472.762292:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:21.0:1619677472.762300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:12.0:1619677472.762300:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:18.0:1619677472.762301:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677472.762301:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:21.0:1619677472.762302:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:18.0:1619677472.762303:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:21.0:1619677472.762304:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.762304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:21.0:1619677472.762305:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.762305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677472.762306:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:20.0:1619677472.762306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.762308:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.762309:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.762310:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.762310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677472.762312:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.762314:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677472.762326:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677472.762328:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677472.762329:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677472.762329:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.762330:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677472.762330:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:13.0:1619677472.762350:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677472.762352:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677472.762353:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.762354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:24.0:1619677472.762435:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.762437:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.762438:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.762438:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.762440:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.762443:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.762457:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.762459:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.762460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.762460:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.762461:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.762462:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.762873:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.762875:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.762876:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.762878:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.762880:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.762882:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.762883:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.762884:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.762885:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.762886:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.762889:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.762891:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.762893:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.762894:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.762895:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.762896:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.762897:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.762897:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.762898:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.762910:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.762914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.762914:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.762915:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.762917:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.762920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.762928:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.762931:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.762931:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.762931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.762933:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.762933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.762945:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.762948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.762948:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677472.762950:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.762952:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.762954:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.762955:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.762956:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.762957:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.762958:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.762961:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.762963:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.762966:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.762966:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.762967:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.762969:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.762969:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.762970:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677472.762971:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677472.762972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.762982:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677472.762982:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677472.762983:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.762984:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.762984:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677472.762985:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677472.762985:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.762987:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.762988:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.762989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677472.763292:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677472.763298:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677472.763300:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677472.763302:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.763304:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.763306:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677472.763307:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.763309:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.763310:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.763310:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677472.763314:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.763315:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677472.763318:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.763319:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677472.763319:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677472.763321:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677472.763321:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677472.763322:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677472.763323:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677472.763323:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:8.2:1619677472.763327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677472.763335:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:1.0:1619677472.763336:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677472.763337:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677472.763338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:1.0:1619677472.763338:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:22.0:1619677472.763339:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:1.0:1619677472.763339:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.763340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.763341:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.763344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.763520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.763527:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.763530:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.763531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.763533:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.763535:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.763537:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.763538:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.763540:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.763541:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.763542:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.763546:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.763547:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.763550:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.763551:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.763552:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.763554:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.763555:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.763556:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.763556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677472.763644:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.763647:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.763648:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.763650:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.763653:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.763654:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.763655:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.763656:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.763657:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.763658:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.763661:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.763663:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.763666:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.763667:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.763667:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.763669:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.763669:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.763671:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.763672:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.763672:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677472.763683:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.763685:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677472.763685:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.763686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.763687:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677472.763687:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.763688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.763689:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677472.763689:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.763692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.763850:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.763853:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.763854:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.763855:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.763858:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.763859:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.763860:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.763862:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.763863:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.763864:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.763866:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.763868:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.763869:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.763870:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.763871:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.763872:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.763872:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.763873:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.763873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.763884:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.763886:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.763886:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.763887:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.763888:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.763889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.764038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.764047:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.764050:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.764050:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.764051:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.764053:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.764055:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.764060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.764067:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.764070:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.764071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.764071:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.764072:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.764072:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.764226:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.764228:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.764229:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.764229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.764231:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.764233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.764247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.764253:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.764256:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.764257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.764259:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.764262:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.764263:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.764264:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.764266:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.764267:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.764268:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.764271:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.764273:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.764276:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.764277:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.764277:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.764279:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.764280:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.764281:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.764281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677472.764468:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.764470:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.764471:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.764473:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.764476:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.764477:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677472.764478:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.764480:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.764481:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.764482:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677472.764484:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.764486:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677472.764489:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.764490:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.764490:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677472.764492:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.764492:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.764493:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.764493:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677472.764508:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.764510:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.764511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.764512:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.764514:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.764517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677472.764773:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.764776:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.764777:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.764779:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.764781:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.764782:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.764784:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.764785:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.764786:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.764787:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.764791:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.764792:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.764795:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.764795:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.764796:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.764797:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.764798:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.764799:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.764799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.764812:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.764814:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.764815:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.764816:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.764817:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.764819:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.764832:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.764834:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.764835:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.764836:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.764837:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.764838:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.764909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.764915:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.764918:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.764919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.764920:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.764923:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.764924:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.764926:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.764927:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.764928:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.764929:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.764932:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.764934:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.764936:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.764937:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.764937:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.764939:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.764939:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.764940:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.764941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.764952:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.764954:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.764955:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.764956:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.764957:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.764958:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.765292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.765302:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.765305:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.765305:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.765306:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.765308:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.765311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.765313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.765321:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.765324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.765324:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.765325:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.765325:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.765326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677472.765343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.765348:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.765350:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.765351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.765353:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.765355:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.765357:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.765358:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.765359:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.765360:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.765361:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.765364:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.765366:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.765368:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.765369:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.765370:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.765371:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.765372:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.765372:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.765373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.765386:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.765389:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.765390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.765390:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.765392:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.765395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.765403:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.765405:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.765406:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.765406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.765407:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.765408:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.765462:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.765465:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.765466:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.765468:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.765471:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.765472:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.765474:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.765475:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.765476:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.765477:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.765480:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.765482:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.765485:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.765486:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.765486:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.765488:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.765488:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.765489:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.765491:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.765491:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677472.765501:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.765503:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.765504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.765504:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677472.765505:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.765506:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.765507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677472.765507:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.765509:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.765510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677472.765755:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.765757:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.765758:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.765759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.765760:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.765763:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.765847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.765853:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.765856:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.765857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.765859:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.765861:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.765863:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677472.765864:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.765866:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.765867:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.765868:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677472.765871:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.765873:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677472.765876:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.765877:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677472.765878:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677472.765880:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677472.765880:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.765881:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.765882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677472.765893:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.765896:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.765897:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.765898:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677472.765900:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.765901:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677472.766459:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.766462:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.766463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.766466:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.766469:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.766471:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.766472:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.766473:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.766474:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.766476:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.766479:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.766481:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.766483:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.766484:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677472.766485:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677472.766485:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.766486:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.766487:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.766487:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677472.766488:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:22.0:1619677472.766488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:24.0:1619677472.766489:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.766489:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.766491:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.766493:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.766500:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.766502:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.766503:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.766505:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.766505:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.766506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.766507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.766508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.766508:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.766509:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.766510:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.766511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.766906:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.766908:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.766910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.766911:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.766913:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.766914:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.766915:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.766917:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.766918:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.766919:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.766922:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.766924:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.766926:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.766927:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.766928:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.766930:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.766930:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.766931:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.766931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.766944:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.766947:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.766948:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.766948:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.766950:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.766953:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:23.0:1619677472.767299:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:23.0:1619677472.767301:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:20.2:1619677472.767451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.767456:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.767459:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.767460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.767462:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.767465:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.767466:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677472.767467:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.767469:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.767470:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.767471:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677472.767474:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.767476:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677472.767478:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.767479:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677472.767480:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677472.767482:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677472.767482:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.767484:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.767485:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.767485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.767491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677472.767497:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677472.767497:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.767498:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.767499:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677472.767500:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.767500:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.767500:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:21.0:1619677472.767502:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.767503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.767505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.767697:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.767699:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.767700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.767700:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.767702:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.767704:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.767816:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.767818:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.767819:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.767821:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.767824:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.767825:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.767826:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.767827:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.767828:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.767829:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.767832:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.767834:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.767837:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.767837:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.767838:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.767840:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.767840:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.767840:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.767841:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.767865:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.767866:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.767866:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.767867:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677472.767868:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.767870:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677472.767871:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677472.767873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.767874:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.767874:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.767875:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.767876:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.767877:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.767879:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.767880:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.767880:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.767883:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.767885:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.767887:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.767888:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.767889:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.767890:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.767891:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.767891:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.767892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.767903:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.767906:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.767906:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.767908:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.767909:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.767911:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.768653:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.768656:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.768657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.768658:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.768662:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.768663:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.768664:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.768665:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.768666:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.768667:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.768670:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.768672:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.768673:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.768674:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.768675:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.768676:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.768677:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.768678:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.768679:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.768679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.768690:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.768692:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.768692:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.768694:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.768694:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.768695:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.768695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.768696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.768697:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.768697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677472.769037:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.769039:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.769040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.769042:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.769045:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.769046:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.769047:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.769048:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.769049:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.769050:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.769053:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.769054:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.769057:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.769058:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.769058:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.769060:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.769060:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.769061:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.769061:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.769074:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.769076:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.769077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.769078:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.769080:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.769083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677472.769141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.769149:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.769152:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.769153:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.769155:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.769155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.769158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.769429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.769432:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.769433:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.769433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.769435:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.769437:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.769688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.769691:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.769692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.769694:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.769697:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.769698:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.769699:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.769700:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.769702:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.769702:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.769706:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.769708:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.769710:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.769711:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.769711:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.769713:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.769713:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.769715:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677472.769715:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677472.769716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677472.769726:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677472.769726:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677472.769728:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677472.769729:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677472.769729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677472.769730:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.769730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677472.769731:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.769733:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.769734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.769750:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.769753:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.769753:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.769754:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.769755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.769756:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.770287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:2.0:1619677472.770293:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677472.770298:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:2.0:1619677472.770302:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000400:00000200:2.0:1619677472.770304:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.770306:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.770307:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677472.770308:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.770309:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.770310:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.770312:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677472.770315:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.770316:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 600 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677472.770319:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.770320:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677472.770320:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677472.770322:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677472.770323:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.770324:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:2.0:1619677472.770324:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:20.0:1619677472.770337:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.770339:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.770340:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.770341:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.770343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.770344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677472.770807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.770813:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.770816:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.770817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.770820:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.770822:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.770824:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.770825:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.770826:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.770827:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.770828:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.770831:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.770833:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.770835:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.770836:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.770837:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.770838:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.770839:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.770841:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677472.770842:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677472.770842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677472.770847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677472.770852:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677472.770853:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677472.770855:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677472.770855:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677472.770856:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.770856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.770857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.770857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.770858:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.770861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.770883:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.770886:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:26.0:1619677472.770889:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.770889:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.770890:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677472.770892:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.770892:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677472.770894:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.770895:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677472.770896:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.770897:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.770899:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.770900:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677472.770902:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.770904:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677472.770904:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677472.770906:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677472.770906:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.770907:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.770908:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.770909:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.770912:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677472.770912:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.770913:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677472.770913:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677472.770914:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.770915:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677472.770916:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.770917:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677472.770917:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.770918:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.770918:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.770919:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.770919:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.770920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.770920:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.770921:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.770922:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.770923:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677472.770932:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.770934:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:6.0:1619677472.770934:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677472.770935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.770936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677472.770936:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.770937:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.770938:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.770940:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.770942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.771348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.771351:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677472.771351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677472.771353:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.771355:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.771357:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.771358:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.771359:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.771361:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.771362:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.771364:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.771366:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.771368:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.771369:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.771370:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.771372:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.771372:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677472.771373:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677472.771373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677472.771386:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.771388:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.771389:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.771391:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.771392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.771393:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677472.771907:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.771910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.771910:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.771914:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.771916:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.771918:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.771919:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.771920:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.771922:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.771923:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.771926:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.771928:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.771931:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.771931:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.771932:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.771934:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.771935:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.771936:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.771937:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.771938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.771940:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677472.771944:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.771946:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.771947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.771948:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.771948:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000400:00000200:8.0:1619677472.771948:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.771949:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.771950:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677472.771951:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.771951:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.771951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.771952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677472.771953:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.771953:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.771954:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.771955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.771956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.771957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.771957:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.771958:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.771959:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.771959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.771962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.771969:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.771972:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.771973:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.771973:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.771974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.771975:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.772414:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.772416:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.772417:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.772419:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.772420:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.772422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677472.772544:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.772547:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.772548:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.772549:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.772552:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.772554:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677472.772555:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.772556:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.772557:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.772558:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677472.772562:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.772563:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677472.772565:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.772566:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677472.772566:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677472.772568:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677472.772568:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.772569:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.772569:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677472.772784:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.772787:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677472.772788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677472.772790:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.772793:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.772794:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677472.772795:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.772797:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.772798:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.772799:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677472.772802:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.772803:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677472.772806:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.772806:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677472.772807:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677472.772809:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677472.772809:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677472.772810:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677472.772810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677472.772822:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.772824:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.772825:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.772827:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.772828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.772829:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.772841:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.772844:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.772845:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.772845:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677472.772846:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.772847:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677472.772992:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.772995:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.772995:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.772998:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.773000:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.773001:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.773002:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.773004:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.773005:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.773006:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.773009:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.773011:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.773014:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.773015:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.773015:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.773017:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.773018:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.773018:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.773019:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.773030:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.773033:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.773034:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.773035:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.773037:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.773039:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.773056:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.773059:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.773060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.773060:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.773061:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.773062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.773217:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.773219:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.773221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.773222:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.773224:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.773226:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.773227:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.773228:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.773229:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.773230:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.773234:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.773235:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.773238:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.773239:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.773239:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.773241:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.773241:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.773242:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.773242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.773256:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.773259:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.773260:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.773260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.773262:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.773266:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.773722:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.773725:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.773725:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677472.773728:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.773730:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.773731:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677472.773733:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.773734:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.773735:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.773736:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677472.773739:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.773741:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677472.773744:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.773744:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.773745:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677472.773747:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.773747:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.773748:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.773748:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677472.773762:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.773765:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.773766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.773766:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.773768:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.773771:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677472.773962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.773968:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.773971:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.773971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.773973:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.773976:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.773977:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677472.773979:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.773980:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.773981:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.773982:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677472.773986:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.773987:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677472.773990:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.773991:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677472.773992:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677472.773994:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677472.773994:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.773995:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.773995:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677472.774008:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.774010:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.774011:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.774013:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677472.774016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.774017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677472.774023:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.774026:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.774027:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.774028:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.774031:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.774032:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.774033:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.774035:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.774036:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.774037:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.774040:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.774041:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.774044:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.774045:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.774045:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.774047:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.774047:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.774049:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.774050:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.774050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677472.774062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677472.774063:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677472.774064:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677472.774065:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677472.774065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.774066:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.774066:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.774067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677472.774068:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.774070:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677472.774101:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.774104:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677472.774105:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677472.774107:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.774110:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.774111:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.774112:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.774114:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.774115:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.774116:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.774119:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.774121:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.774123:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.774123:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.774124:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.774126:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.774127:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.774128:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.774128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.2:1619677472.774129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.774137:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.774139:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:18.0:1619677472.774141:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.774142:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:18.0:1619677472.774143:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.774143:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.774143:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.774145:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.774145:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.774146:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.774147:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.774148:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.774149:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.774151:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.774151:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677472.774153:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.774153:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.774154:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677472.774154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677472.774155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.774155:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.774155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.774156:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.774156:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:8.0:1619677472.774157:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.774158:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.774158:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.774159:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.774160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:8.0:1619677472.774160:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.774161:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.774163:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.774164:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.774165:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.774167:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677472.774168:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:8.0:1619677472.774170:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.774171:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.774172:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000800:00000200:18.0:1619677472.774172:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (25)--
00000800:00000200:8.0:1619677472.774172:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677472.774172:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (23)++
00000800:00000200:2.0:1619677472.774172:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (24)--
00000400:00000200:18.0:1619677472.774173:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.774173:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677472.774174:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.774174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.774175:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:18.0:1619677472.774176:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.774176:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.774177:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677472.774178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.774181:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.774182:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.774182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.774189:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677472.774192:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:24.0:1619677472.774193:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.774193:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:8.0:1619677472.774193:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:8.0:1619677472.774194:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.774195:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.774196:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.774197:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.774198:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.774198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.774518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.774526:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.774529:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.774530:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.774530:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.774533:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.774536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:23.0:1619677472.775300:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:23.0:1619677472.775302:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:23.0:1619677472.775304:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:5.2:1619677472.775649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.775654:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.775657:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.775658:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.775660:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.775662:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.775664:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.775665:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.775666:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.775667:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.775668:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.775671:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.775673:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.775675:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.775676:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.775677:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.775678:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.775679:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.775679:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.775680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.775693:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.775696:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.775697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.775697:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.775699:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.775701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677472.775706:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.775709:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.775710:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677472.775710:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677472.775712:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677472.775713:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.775714:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.775714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.775715:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000400:00000200:10.0:1619677472.775715:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.775716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:10.0:1619677472.775716:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.775717:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.775719:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.775720:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.775721:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.775723:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.775725:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.775728:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.775729:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.775729:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.775731:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.775731:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.775732:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.775733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.0:1619677472.775743:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677472.775744:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:21.0:1619677472.775745:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.775747:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677472.775747:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.775748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677472.775748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:6.0:1619677472.775748:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.775750:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.775751:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.775752:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.775753:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677472.775753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:21.0:1619677472.775754:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.775755:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.775756:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.775759:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.775760:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.775763:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.775763:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.775764:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.775766:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.775766:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.775768:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.775769:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.775770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.775778:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677472.775779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.775781:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677472.775781:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:22.0:1619677472.775782:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.775782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677472.775782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.775783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:22.0:1619677472.775784:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.775787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677472.776390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.776398:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.776401:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.776402:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.776403:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.776405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.776407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.776926:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.776929:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.776930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.776932:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.776933:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.776935:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.776936:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.776937:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.776938:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.776939:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.776942:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.776944:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.776947:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.776947:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.776948:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.776950:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.776950:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.776950:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.776951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.776956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.776964:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.776966:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.776967:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.776968:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.776970:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.776973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677472.776981:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.776984:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.776986:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677472.776988:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.776990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.776990:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677472.776991:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.776992:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.776993:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.776994:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677472.776997:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.776999:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677472.777001:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.777002:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677472.777003:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677472.777005:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677472.777006:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.777006:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.777007:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677472.777020:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.777023:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.777025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.777025:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.777027:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.777034:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.777035:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.777037:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.777039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.777039:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.777040:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.777040:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677472.777811:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.777814:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677472.777815:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677472.777817:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.777820:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.777821:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677472.777823:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.777824:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.777825:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.777826:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677472.777829:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.777831:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677472.777833:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.777834:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677472.777835:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677472.777836:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677472.777837:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.777838:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677472.777838:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:8.2:1619677472.777843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.777852:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.777854:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.777855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.777856:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.777858:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.777860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677472.777860:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.777863:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.777864:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.777866:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.777869:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.777870:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.777871:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677472.777871:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:10.0:1619677472.777873:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.777874:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677472.777874:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.777875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:10.0:1619677472.777875:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677472.777878:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.777880:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.777880:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.777882:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:1.0:1619677472.777883:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677472.777884:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677472.777885:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.777885:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.777886:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:1.0:1619677472.777886:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677472.777887:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:1.0:1619677472.777887:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.777887:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677472.777888:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.777888:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.777889:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677472.777891:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.777893:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677472.777896:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.777897:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677472.777897:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677472.777899:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677472.777900:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677472.777900:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.777901:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.0:1619677472.777902:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677472.777902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:8.0:1619677472.777903:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.777904:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.777905:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.777908:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.777911:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.777914:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.777915:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.777915:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677472.777918:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.777920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677472.778937:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.778940:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.778940:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677472.778943:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.778945:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.778947:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.778948:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.778949:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.778950:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.778951:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.778954:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.778956:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.778958:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.778959:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.778960:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677472.778962:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:10.0:1619677472.778963:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.778964:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.778964:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.778969:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677472.778972:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:22.0:1619677472.778972:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677472.778972:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:22.0:1619677472.778973:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.778973:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.778975:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677472.778975:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.778979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.778979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.778980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.778992:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.778994:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.778995:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.778995:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.778996:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.778996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.779147:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.779150:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.779151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.779152:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.779155:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.779156:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.779158:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.779159:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.779160:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.779161:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.779164:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.779166:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.779168:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.779169:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.779170:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.779171:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.779172:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.779173:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.779174:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.779175:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.779187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.779187:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.779189:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677472.779190:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677472.779190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.779191:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.779191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.779192:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.779193:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.779196:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.779302:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.779305:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.779306:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.779308:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.779311:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.779313:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.779314:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.779315:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.779316:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.779317:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.779320:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.779322:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.779324:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.779325:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.779326:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.779327:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.779328:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.779329:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.779329:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.779334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.779342:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.779345:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.779346:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.779347:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.779348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.779350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.779782:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.779785:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.779787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.779787:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.779789:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.779791:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.779792:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.779793:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.779794:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.779795:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.779799:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.779800:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.779803:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.779804:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.779805:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.779806:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.779807:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.779808:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.779808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.779829:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.779832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.779832:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.779833:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.779835:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.779838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.780121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.780127:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.780130:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.780131:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.780133:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.780134:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.780136:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.780137:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.780138:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.780139:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.780141:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.780144:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.780146:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.780148:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.780148:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.780149:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.780151:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.780152:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.780153:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.780154:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.780155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.780160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.780166:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.780167:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.780167:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677472.780168:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.780168:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.780169:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.780170:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.780170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.780171:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.780171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.0:1619677472.780357:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.780359:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.780362:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.780362:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.780365:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.780366:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.780368:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.780369:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.780370:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.780371:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.780374:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.780376:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.780378:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.780379:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.780380:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.780381:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.780382:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.780382:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.780383:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.780397:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.780400:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.780401:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.780401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.780403:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.780405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.780518:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.780521:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.780522:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.780524:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.780527:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.780528:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.780530:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.780531:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.780532:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.780533:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.780537:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.780539:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.780541:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.780542:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.780543:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.780545:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.780545:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.780546:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.780546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.780558:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.780561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.780561:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.780562:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.780564:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.780567:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677472.780654:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677472.780654:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:24.0:1619677472.780656:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.780657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677472.780658:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.780658:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:24.0:1619677472.780661:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.780661:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.780663:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.780664:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.780665:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.780666:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.780667:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.780668:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.780668:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.780669:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.780670:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.780671:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677472.780671:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.780672:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.780676:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.780677:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.780678:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677472.780680:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.780681:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.780682:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.780683:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677472.780683:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.780683:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.780684:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677472.780684:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.780685:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.780686:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.780686:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677472.780686:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677472.780687:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.780688:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.780688:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:2.0:1619677472.780699:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.780702:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.780703:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.780703:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.780705:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.780708:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.780709:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.780712:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.780713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.780713:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.780715:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.780718:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.780718:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677472.780721:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.780722:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.780722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.780723:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.780724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.780733:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.780736:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.780737:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.780737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.780738:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.780739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.780823:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.780826:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.780828:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.780829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.780830:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.780832:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.780833:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.780835:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.780836:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.780837:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.780840:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.780842:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.780845:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.780846:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.780846:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.780848:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.780849:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.780850:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.780850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677472.781021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677472.781027:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:5.2:1619677472.781028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.0:1619677472.781030:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:8.2:1619677472.781031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677472.781032:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.781033:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.781036:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.781036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:21.0:1619677472.781038:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677472.781039:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.781039:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:21.0:1619677472.781041:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677472.781041:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:21.0:1619677472.781042:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.781043:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677472.781043:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.781043:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.781044:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.781045:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.781047:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.781048:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677472.781048:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.781049:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96512219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.781051:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:17.0:1619677472.781051:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.781052:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677472.781053:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.781054:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:17.0:1619677472.781054:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677472.781055:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.781056:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.781057:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:1.0:1619677472.781057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.0:1619677472.781058:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677472.781058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:17.0:1619677472.781058:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.781059:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.781060:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.781060:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.781063:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.781064:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.781065:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.781066:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.781067:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.781067:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.781068:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677472.781070:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.781073:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.781074:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.781074:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.781076:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.781078:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.781080:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.781083:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.781084:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.781084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.781086:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.781088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677472.782051:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.782054:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.782056:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.782056:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.782058:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.782060:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.782061:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.782063:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.782064:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.782064:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.782067:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.782069:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.782071:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.782072:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.782072:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.782074:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.782075:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.782075:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.782076:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677472.782089:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.782091:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.782092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.782092:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.782094:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677472.782096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.782169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.782176:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.782179:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.782179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.782181:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.782183:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.782185:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.782186:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677472.782188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:18.0:1619677472.782188:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.782189:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.782191:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.782194:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:18.0:1619677472.782195:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.782196:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.782197:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.782198:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.782201:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677472.782201:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.782202:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:2.0:1619677472.782202:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677472.782203:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:2.0:1619677472.782204:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.782205:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:2.0:1619677472.782205:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.782206:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.782207:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677472.782208:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.782209:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.782209:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677472.782210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.782210:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.782212:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677472.782216:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.782217:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677472.782217:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677472.782219:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677472.782220:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.782221:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677472.782222:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677472.782222:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677472.782228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:13.0:1619677472.782232:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677472.782234:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:13.0:1619677472.782235:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677472.782236:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:13.0:1619677472.782236:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.782237:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:13.0:1619677472.782237:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.782238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:12.0:1619677472.782239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.782240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677472.782350:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.782353:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.782354:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.782356:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.782358:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.782360:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.782361:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.782362:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.782363:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.782364:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.782368:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.782369:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.782372:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.782373:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.782374:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.782375:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.782376:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.782376:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.782377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.782381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.782390:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.782393:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.782393:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.782394:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.782396:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.782399:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.782402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.782409:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.782412:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.782413:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.782413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.782414:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.782414:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677472.782784:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d0600] (21)++
00000800:00000200:22.0:1619677472.782788:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.28@o2ib3
00000800:00000200:6.0:1619677472.782788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d0600] (22)--
00000400:00000200:22.0:1619677472.782790:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.28@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.782793:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.782794:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677472.782795:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.782797:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.782798:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.782799:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677472.782802:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.28@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.782803:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.782806:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.782807:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677472.782808:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677472.782810:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677472.782810:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d0600] (21)++
00000800:00000200:22.0:1619677472.782811:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d0600] (22)--
00000800:00000200:22.0:1619677472.782811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d0600] (21)--
00000800:00000200:10.0:1619677472.782813:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.782817:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.782818:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.782820:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.782822:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.782824:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:10.0:1619677472.782824:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.782825:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.782826:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677472.782827:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.782827:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.782827:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.782829:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:10.0:1619677472.782829:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677472.782830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.782832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677472.782833:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.782835:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.782837:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.782838:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.782838:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.782840:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.782840:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.782841:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.782842:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.782852:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.782854:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.782855:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.782855:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.782857:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.782859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.783530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:26.0:1619677472.783575:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.783578:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.783579:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.783579:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677472.783580:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.783583:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677472.784329:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.784331:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.784332:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.784333:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.784335:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.784337:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.784338:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.784339:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.784340:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.784341:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.784344:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.784346:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.784349:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.784350:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.784350:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.784352:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.784352:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.784353:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.784355:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.784355:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677472.784366:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.784369:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677472.784369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.784369:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.784370:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.784371:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.784372:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677472.784372:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.784373:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.784375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.784567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677472.784570:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.784573:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677472.784575:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.784576:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677472.784577:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.784579:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.784580:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.784582:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.784583:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.784584:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.784587:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.784589:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.784591:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.784592:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.784592:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.784594:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.784595:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.784595:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677472.784596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.784601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.784609:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.784611:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.784612:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.784614:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.784614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.784616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.784893:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.784896:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.784897:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.784899:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.784900:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.784901:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.784902:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.784904:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.784904:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.784905:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.784908:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.784910:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.784912:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.784912:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.784913:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.784914:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.784915:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.784916:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.784917:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.784918:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677472.784927:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677472.784928:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677472.784930:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.784931:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.784931:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.784931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677472.784932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.784933:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.784933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.784935:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.784945:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.784948:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.784949:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.784949:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.784950:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.784951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.785033:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.785036:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.785036:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.785039:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.785042:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.785043:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.785044:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.785046:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.785047:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.785048:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.785051:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.785053:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.785057:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.785057:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.785058:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.785060:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.785060:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.785061:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.785061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.785074:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.785077:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.785078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.785078:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.785081:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.785084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.785134:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.785136:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.785137:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.785138:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.785139:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.785142:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.785458:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.785460:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.785461:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.785463:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.785466:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.785467:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.785468:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.785470:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.785470:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.785471:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.785474:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.785476:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.785479:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.785480:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.785480:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.785482:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.785482:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.785483:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677472.785485:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.785485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677472.785496:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677472.785499:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677472.785499:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.785500:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.785500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.785501:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.785502:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677472.785502:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.785502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.785505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677472.786082:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.786085:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.786086:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.786088:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.786091:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.786092:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.786094:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.786095:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.786096:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.786097:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.786100:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.786101:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.786104:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.786105:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.786105:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.786107:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.786107:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.786109:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.786110:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.786111:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.786122:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.786123:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.786125:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:18.0:1619677472.786125:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.786126:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.786126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677472.786127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.786127:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.786129:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.786132:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.786827:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.786830:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.786832:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.786832:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.786834:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.786836:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.786837:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.786839:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.786840:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.786841:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.786844:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.786846:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.786849:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.786850:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.786850:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.786852:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.786853:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.786853:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.786854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.786866:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.786868:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.786869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.786870:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.786872:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.786874:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:24.0:1619677472.786875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.786877:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.786878:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.786880:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.786883:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.786884:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677472.786885:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.786887:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.786888:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.786889:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.786892:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.786894:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677472.786896:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.786897:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.786898:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.786900:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.786900:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.786901:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.786901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677472.786914:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.786917:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.786918:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.786918:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.786920:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.786923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677472.787222:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.787226:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.787226:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.787230:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.787232:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.787234:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.787235:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.787236:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.787238:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.787239:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.787242:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677472.787243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.787243:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.787247:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.787247:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.787248:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (23)++
00000800:00000200:1.0:1619677472.787250:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (24)--
00000800:00000200:1.0:1619677472.787251:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677472.787252:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:1.0:1619677472.787252:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.787252:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.0:1619677472.787254:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:20.0:1619677472.787255:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.787257:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.787259:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:20.0:1619677472.787260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (24)--
00000800:00000200:21.0:1619677472.787261:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:21.0:1619677472.787262:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.787262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677472.787264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:21.0:1619677472.787265:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.787267:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.787269:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.787272:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.787272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.787273:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.787274:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.787275:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677472.787576:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.787578:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.787579:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.787581:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.787584:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.787586:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.787587:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.787588:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.787589:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.787590:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.787594:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.787595:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.787598:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.787599:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.787599:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.787601:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.787601:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.787601:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.787602:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.787614:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.787616:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.787617:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.787617:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.787619:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.787621:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.787737:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.787740:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.787741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.787742:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.787745:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.787746:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.787747:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.787748:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.787749:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.787750:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.787754:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.787755:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.787758:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.787759:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.787759:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.787761:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.787761:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.787762:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.787763:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.787767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.787775:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.787778:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.787779:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.787779:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.787781:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.787784:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.787887:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.787890:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677472.787891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677472.787893:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677472.787895:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.787897:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677472.787898:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.787899:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.787900:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.787901:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677472.787904:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677472.787905:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.787908:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.787909:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677472.787909:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677472.787911:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.787911:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677472.787912:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677472.787914:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677472.787914:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:6.0:1619677472.787924:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677472.787927:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:6.0:1619677472.787927:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.787928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.787928:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.787929:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677472.787929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.787930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:6.0:1619677472.787930:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677472.787933:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.788192:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.788195:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.788196:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.788196:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.788197:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.788197:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.788409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.788415:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.788419:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.788419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.788421:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.788424:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.788425:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.788427:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.788428:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.788430:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.788431:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.788434:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.788436:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.788438:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.788439:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.788439:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.788441:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.788442:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.788443:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.788443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.788455:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.788457:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.788458:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.788459:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.788460:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.788461:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.788474:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.788477:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.788477:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.788478:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.788478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.788479:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677472.788550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.788556:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.788559:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.788560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.788562:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.788564:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.788565:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.788566:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.788568:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.788569:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.788570:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.788573:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.788574:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.788578:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.788578:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.788579:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.788580:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.788581:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.788582:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.788582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.788595:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.788598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.788598:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.788599:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.788601:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.788603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.789180:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.789183:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.789183:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.789185:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.789188:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.789189:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.789190:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.789192:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.789193:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.789194:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.789197:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.789199:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.789202:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.789203:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.789203:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.789205:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.789206:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.789206:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.789207:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.789218:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.789221:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.789222:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.789222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.789224:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.789227:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.790039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677472.790042:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.790045:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677472.790047:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.790050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677472.790050:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.790052:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.790053:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.790054:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.790055:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.790056:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.790060:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.790061:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.790064:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.790065:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.790065:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.790067:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.790068:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.790068:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677472.790069:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.790074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.790084:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.790088:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.790089:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.790091:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677472.790092:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.790094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677472.790258:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.790261:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.790261:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677472.790264:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.790267:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.790269:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677472.790270:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.790271:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.790272:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.790273:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677472.790276:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.790278:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677472.790281:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.790282:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677472.790282:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:10.0:1619677472.790284:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677472.790284:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.790285:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.790285:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677472.790300:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.790303:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.790304:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.790307:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.790309:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.790311:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677472.790312:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677472.790312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.790313:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.790314:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.790315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.0:1619677472.790443:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.790446:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.790448:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.790450:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.790450:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.790452:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.790453:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.790455:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.790456:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.790457:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.790461:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.790462:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.790465:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.790466:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.790466:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.790468:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.790468:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.790469:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.790470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.790492:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.790495:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.790496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.790496:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.790498:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.790500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.790653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.790659:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.790662:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.790663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.790665:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.790667:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.790668:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.790670:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.790671:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.790672:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.790673:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.790677:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.790679:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.790682:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.790683:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.790683:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.790685:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.790686:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.790687:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.790687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677472.791226:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.791229:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.791229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.791230:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.791232:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.791236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:23.0:1619677472.791302:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:23.0:1619677472.791304:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:23.0:1619677472.791306:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:23.0:1619677472.791308:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:24.0:1619677472.791313:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677472.791315:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677472.791316:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:21.0:1619677472.791317:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:8.0:1619677472.791317:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.791320:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.791320:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.791321:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677472.791323:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)++
00000400:00000200:24.0:1619677472.791323:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.791324:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.791325:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:26.0:1619677472.791326:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.1.46@o2ib3
00000400:00000200:24.0:1619677472.791326:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.791326:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.791327:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677472.791327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000400:00000200:24.0:1619677472.791329:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.791329:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.791330:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.791331:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.791331:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.791332:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.791332:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677472.791334:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.791336:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677472.791336:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677472.791337:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.791338:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677472.791339:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.791341:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.791342:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677472.791342:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.791344:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.791345:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.791346:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:26.0:1619677472.791347:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.791347:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.791348:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677472.791349:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677472.791349:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.791350:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.791350:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677472.791350:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.791351:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677472.791352:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.791352:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (22)++
00000800:00000200:26.0:1619677472.791353:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677472.791354:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.791354:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677472.791354:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.791355:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:26.0:1619677472.791356:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677472.791356:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (22)++
00000800:00000200:21.0:1619677472.791356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:26.0:1619677472.791358:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (23)--
00000800:00000200:26.0:1619677472.791359:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000800:00000200:26.0:1619677472.791360:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:26.0:1619677472.791361:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.46@o2ib3
00000800:00000200:10.0:1619677472.791361:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000400:00000200:26.0:1619677472.791364:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.791365:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.791367:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:26.0:1619677472.791368:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96512186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.791369:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.791370:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.791370:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:26.0:1619677472.791371:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.791373:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677472.791373:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000400:00000200:20.0:1619677472.791374:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.791375:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677472.791375:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.791376:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)++
00000800:00000200:21.0:1619677472.791376:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677472.791376:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000800:00000200:26.0:1619677472.791377:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.791377:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:8.0:1619677472.791377:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.791378:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.791378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677472.791379:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.791380:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677472.791380:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:26.0:1619677472.791381:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)++
00000800:00000200:8.0:1619677472.791381:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:26.0:1619677472.791382:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (24)++
00000800:00000200:24.0:1619677472.791383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)--
00000800:00000200:8.0:1619677472.791383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:26.0:1619677472.791384:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:26.0:1619677472.791385:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (22)++
00000800:00000200:26.0:1619677472.791386:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (23)--
00000800:00000200:26.0:1619677472.791387:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000800:00000200:26.0:1619677472.791388:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:26.0:1619677472.791390:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.46@o2ib3
00000800:00000200:10.0:1619677472.791391:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000400:00000200:26.0:1619677472.791392:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.791396:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.791397:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677472.791399:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.791400:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677472.791400:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:26.0:1619677472.791401:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.791401:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:24.0:1619677472.791403:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[5] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.791403:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677472.791404:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (24)++
00000800:00000200:24.0:1619677472.791404:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (25)--
00000800:00000200:8.0:1619677472.791404:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:26.0:1619677472.791405:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.791405:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.791405:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.791406:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677472.791406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.791407:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.791408:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.791408:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677472.791409:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677472.791409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.791410:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.791411:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (22)++
00000800:00000200:26.0:1619677472.791412:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (23)--
00000800:00000200:26.0:1619677472.791413:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000800:00000200:26.0:1619677472.791414:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:26.0:1619677472.791416:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.46@o2ib3
00000800:00000200:10.0:1619677472.791416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000400:00000200:26.0:1619677472.791418:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.791418:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.791419:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677472.791419:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.791420:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.791421:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677472.791421:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:26.0:1619677472.791423:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.791424:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.791425:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.791425:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677472.791428:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.791430:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677472.791430:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.791431:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.791431:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677472.791432:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677472.791432:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677472.791434:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.791435:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (22)++
00000800:00000200:26.0:1619677472.791437:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06ffaa800] (23)++
00000800:00000200:26.0:1619677472.791439:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (24)--
00000800:00000200:10.0:1619677472.791439:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (23)++
00000800:00000200:26.0:1619677472.791440:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (24)--
00000800:00000200:26.0:1619677472.791441:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.791441:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.46@o2ib3
00000800:00000200:26.0:1619677472.791442:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.791443:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.791443:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.791444:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.791445:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677472.791445:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.791446:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (23)++
00000800:00000200:26.0:1619677472.791446:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06ffaa800] (24)--
00000800:00000200:26.0:1619677472.791447:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (23)--
00000400:00000200:10.0:1619677472.791447:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677472.791448:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:10.0:1619677472.791448:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677472.791449:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)++
00000800:00000200:26.0:1619677472.791450:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.46@o2ib3
00000400:00000200:10.0:1619677472.791450:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.791451:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.791452:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677472.791454:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.791455:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.791456:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677472.791457:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.791458:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677472.791458:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677472.791459:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.791460:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.791461:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.791464:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.791464:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.791465:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677472.791465:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677472.791466:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.791467:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677472.791468:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.791468:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (23)++
00000800:00000200:10.0:1619677472.791468:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (24)--
00000800:00000200:26.0:1619677472.791469:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.791469:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.791469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (23)--
00000800:00000200:10.0:1619677472.791470:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)++
00000800:00000200:26.0:1619677472.791471:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.791471:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.46@o2ib3
00000800:00000200:26.0:1619677472.791472:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (23)++
00000800:00000200:26.0:1619677472.791472:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (24)--
00000800:00000200:26.0:1619677472.791473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (23)--
00000400:00000200:10.0:1619677472.791473:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677472.791474:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000400:00000200:10.0:1619677472.791475:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.791477:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677472.791478:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96512181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.791479:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.791480:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.791481:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677472.791483:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:10.0:1619677472.791483:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.791485:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:10.0:1619677472.791485:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:24.0:1619677472.791486:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.791486:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.791487:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677472.791487:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.0:1619677472.791487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.791488:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.791488:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.791489:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:10.0:1619677472.791489:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (22)--
00000800:00000200:24.0:1619677472.791490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.791490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)--
00000800:00000200:26.0:1619677472.791500:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.791502:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.791503:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.791505:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.791505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.791506:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:26.0:1619677472.791507:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677472.791508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000800:00000200:10.0:1619677472.791508:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.46@o2ib3
00000800:00000200:8.0:1619677472.791508:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677472.791510:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.791510:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.791511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677472.791511:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.791511:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.791512:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677472.791513:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677472.791513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:10.0:1619677472.791514:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.791515:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.791516:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.791517:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.791521:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:10.0:1619677472.791521:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.791522:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677472.791522:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.791523:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.791525:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677472.791525:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.791526:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.791526:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.791527:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677472.791527:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.791528:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)++
00000800:00000200:26.0:1619677472.791529:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.46@o2ib3
00000800:00000200:10.0:1619677472.791529:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.791530:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (23)++
00000400:00000200:26.0:1619677472.791531:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.791531:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06ffaa800] (24)++
00000800:00000200:10.0:1619677472.791532:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (25)--
00000400:00000200:26.0:1619677472.791533:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.791533:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (24)--
00000400:00000200:26.0:1619677472.791534:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:10.0:1619677472.791534:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (23)--
00000400:00000200:26.0:1619677472.791535:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.791537:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.791538:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.791538:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677472.791541:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.791543:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677472.791543:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677472.791544:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.791545:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677472.791545:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677472.791546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.791546:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (23)++
00000800:00000200:6.0:1619677472.791546:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677472.791547:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.791547:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (24)++
00000800:00000200:10.0:1619677472.791547:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06ffaa800] (24)--
00000400:00000200:6.0:1619677472.791547:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.791548:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (24)--
00000800:00000200:10.0:1619677472.791548:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (24)--
00000800:00000200:26.0:1619677472.791549:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000800:00000200:10.0:1619677472.791549:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)--
00000400:00000200:6.0:1619677472.791549:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.791551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.791558:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.791560:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.791561:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.791562:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677472.791563:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.791565:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677472.791755:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.791757:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.791758:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.791759:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.791762:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.791764:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677472.791765:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.791766:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.791767:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.791768:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.791771:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.791772:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.791772:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:24.0:1619677472.791775:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.791775:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.46@o2ib3
00000800:00000200:26.0:1619677472.791776:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000800:00000200:24.0:1619677472.791776:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677472.791776:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677472.791777:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.791778:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677472.791778:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.791779:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.791779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:10.0:1619677472.791779:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.791780:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.791781:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.791782:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.791783:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.791784:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.791786:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.791788:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.791790:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.791790:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677472.791791:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677472.791791:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.791792:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.791793:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:10.0:1619677472.791793:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (22)--
00000800:00000200:22.0:1619677472.791794:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677472.791794:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)--
00000800:00000200:6.0:1619677472.791794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.791795:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.791797:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.791799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.791804:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.791806:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.791807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.791807:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.791809:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.791812:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.792060:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:26.0:1619677472.792063:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.46@o2ib3
00000800:00000200:10.0:1619677472.792064:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000400:00000200:26.0:1619677472.792065:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.792067:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.792068:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677472.792069:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.792071:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.792071:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.792072:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677472.792075:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.792077:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677472.792079:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.792080:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:26.0:1619677472.792080:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677472.792082:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677472.792082:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:26.0:1619677472.792083:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (22)--
00000800:00000200:26.0:1619677472.792083:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)--
00000800:00000200:12.0:1619677472.792095:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.792098:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.792098:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.792100:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.792102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.792103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:10.0:1619677472.792106:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:10.0:1619677472.792108:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.46@o2ib3
00000800:00000200:26.0:1619677472.792109:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000400:00000200:10.0:1619677472.792111:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.792113:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.792115:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677472.792116:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.792117:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.792118:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.792119:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677472.792122:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.792123:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677472.792125:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.792126:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677472.792126:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677472.792128:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677472.792128:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:10.0:1619677472.792129:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06ffaa800] (22)++
00000800:00000200:10.0:1619677472.792130:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (23)--
00000800:00000200:10.0:1619677472.792131:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000800:00000200:26.0:1619677472.792143:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)++
00000800:00000200:26.0:1619677472.792145:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06ffaa800] (23)--
00000800:00000200:26.0:1619677472.792145:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000800:00000200:26.0:1619677472.792147:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)--
00000800:00000200:10.0:1619677472.792166:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:10.0:1619677472.792168:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.46@o2ib3
00000800:00000200:26.0:1619677472.792169:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (22)--
00000400:00000200:10.0:1619677472.792170:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.46@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.792173:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.792173:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677472.792175:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.792176:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.792177:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.792177:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677472.792180:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.792182:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677472.792183:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.792183:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677472.792184:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677472.792185:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.792185:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaa800] (21)++
00000800:00000200:10.0:1619677472.792186:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaa800] (22)--
00000800:00000200:10.0:1619677472.792187:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaa800] (21)--
00000800:00000200:26.0:1619677472.792197:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.792199:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.792199:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.792200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677472.792201:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.792202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677472.792205:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.792208:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.792209:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.792210:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.792212:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.792213:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.792215:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.792216:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.792217:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.792217:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.792221:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.792222:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.792225:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.792226:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.792226:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.792228:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.792228:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.792229:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.792229:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.792242:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.792245:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.792246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.792246:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.792249:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.792252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.792431:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.792433:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.792434:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.792436:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.792438:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.792439:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.792440:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.792441:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.792442:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.792443:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.792446:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.792447:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.792449:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.792450:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.792450:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.792452:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.792452:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.792453:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.792453:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.792465:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.792467:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.792468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.792468:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.792470:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.792473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677472.792617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.792619:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.792621:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.792624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.792624:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.792625:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:12.0:1619677472.792626:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.792627:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.792628:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.792629:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:12.0:1619677472.792632:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.792634:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677472.792636:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.792637:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:12.0:1619677472.792637:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:12.0:1619677472.792639:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:12.0:1619677472.792639:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.792640:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.792641:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:26.0:1619677472.792651:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.792653:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.792654:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.792654:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677472.792656:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.792659:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677472.792670:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.792672:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.792674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.792675:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.792678:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.792679:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.792680:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.792681:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.792682:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.792683:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.792686:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.792687:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.792689:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.792689:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.792690:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.792691:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.792691:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.792692:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.792693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677472.792702:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.792704:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.792705:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.792706:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.792706:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.792708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.793236:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.793239:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.793240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.793241:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.793243:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.793244:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.793246:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.793247:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.793247:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.793248:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.793252:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.793253:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677472.793253:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677472.793256:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.793256:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.793257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677472.793257:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.793257:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:10.0:1619677472.793258:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.793259:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.793259:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.793260:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:10.0:1619677472.793260:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.793261:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.793262:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677472.793262:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.793263:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.793264:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.793265:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677472.793265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:10.0:1619677472.793266:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.793269:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.793271:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.793274:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:10.0:1619677472.793274:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.793275:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.793275:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677472.793275:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.793276:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:10.0:1619677472.793277:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:8.0:1619677472.793277:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677472.793277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.793278:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.793278:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.793278:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677472.793278:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.793279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677472.793279:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677472.793280:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.793284:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677472.793290:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.793294:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.793298:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.793299:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.793301:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.793304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.793404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677472.793410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.793412:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.793415:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.793415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.793417:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.793419:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677472.793420:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.793422:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.793423:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.793424:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677472.793427:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.793429:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677472.793432:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.793432:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677472.793433:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677472.793435:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677472.793435:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.793437:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677472.793438:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677472.793439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677472.793443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.793446:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:26.0:1619677472.793448:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:20.0:1619677472.793448:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677472.793449:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677472.793450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:26.0:1619677472.793451:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.793452:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.793452:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677472.793454:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.793457:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677472.793466:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.793469:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.793469:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677472.793470:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.793471:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.793471:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677472.794125:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.794128:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.794128:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.794130:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.794133:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.794134:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.794135:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.794136:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.794137:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.794138:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.794141:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.794143:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.794145:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.794146:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.794147:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.794148:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.794149:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.794150:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.794151:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.794151:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677472.794162:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677472.794164:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677472.794165:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.794166:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.794166:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677472.794166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.794167:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677472.794168:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.794168:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.794171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.794182:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.794185:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.794186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.794186:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.794188:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.794189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.794444:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677472.794446:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677472.794447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677472.794449:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.794451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.794453:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677472.794454:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.794456:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.794457:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.794458:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677472.794461:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.794463:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677472.794466:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.794467:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:22.0:1619677472.794467:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677472.794469:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677472.794470:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677472.794470:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677472.794471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:1.0:1619677472.794473:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.794476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.794476:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677472.794479:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.794481:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.794482:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677472.794484:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.794485:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:1.0:1619677472.794485:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.794486:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.794487:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:1.0:1619677472.794487:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677472.794489:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.794489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.794491:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677472.794492:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.794494:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.794495:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677472.794497:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.794498:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677472.794498:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677472.794500:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677472.794501:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.794501:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.794502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:26.0:1619677472.794514:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.794516:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.794518:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.794518:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677472.794520:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.794523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677472.794599:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4fe00] (21)++
00000800:00000200:22.0:1619677472.794602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4fe00] (22)--
00000800:00000200:6.0:1619677472.794602:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.42@o2ib3
00000400:00000200:6.0:1619677472.794605:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.42@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.794607:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.794609:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677472.794610:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.794611:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.794613:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.794614:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677472.794617:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.42@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.794618:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677472.794622:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.794622:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677472.794623:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677472.794625:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677472.794625:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4fe00] (21)++
00000800:00000200:6.0:1619677472.794626:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4fe00] (22)--
00000800:00000200:6.0:1619677472.794626:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4fe00] (21)--
00000800:00000200:5.2:1619677472.795003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.795009:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.795011:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.795012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.795014:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.795017:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.795018:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.795020:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.795021:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.795022:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.795023:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.795027:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.795028:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.795031:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.795032:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.795032:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.795034:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.795034:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.795036:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677472.795038:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677472.795038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677472.795043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677472.795047:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677472.795049:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677472.795050:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.795051:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.795051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.795051:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677472.795052:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677472.795053:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.795053:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.795056:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.795069:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677472.795072:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.795073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.795073:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.795074:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.795075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677472.795078:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.795081:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.795083:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.795084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.795086:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.795088:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.795089:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.795090:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.795091:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.795092:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.795096:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.795097:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.795099:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.795100:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.795101:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.795102:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.795103:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.795104:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.795105:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.795106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677472.795116:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677472.795116:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677472.795118:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.795119:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.795119:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.795119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677472.795120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.795121:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.795121:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.795124:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.795226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.795232:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.795235:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.795236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.795237:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.795240:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.795241:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.795242:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.795244:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.795245:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.795246:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.795248:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.795250:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.795252:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.795253:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.795253:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.795255:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.795256:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.795257:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.795257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.795268:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.795270:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.795271:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.795271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.795273:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.795274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.796248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.796259:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.796262:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.796263:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.796265:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.796265:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.796268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677472.796549:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.796552:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.796552:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677472.796554:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677472.796557:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.796558:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.796559:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.796561:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.796562:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.796563:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.796566:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677472.796568:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.796570:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.796571:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.796572:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.796574:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.796574:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.796575:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677472.796577:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677472.796577:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677472.796580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.796590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677472.796591:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677472.796592:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677472.796593:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:20.0:1619677472.796593:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.796594:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.796594:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677472.796595:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.796596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.796598:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.796793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.796799:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.796802:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.796803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.796804:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.796807:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.796808:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.796810:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.796811:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.796812:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.796813:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.796817:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.796818:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.796821:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.796822:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.796823:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.796825:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.796825:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.796826:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.796827:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.796828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.796833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677472.796838:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:18.0:1619677472.796839:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:26.0:1619677472.796840:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:18.0:1619677472.796841:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:10.0:1619677472.796841:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677472.796842:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.796842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.796843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:26.0:1619677472.796844:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677472.796847:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:8.0:1619677472.796942:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.796945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677472.796945:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:8.0:1619677472.796948:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.796950:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.796952:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.796953:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.796955:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.796955:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.796957:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.796959:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.796961:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.796964:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.796964:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.796965:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.796967:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.796967:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.796968:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677472.796968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:18.0:1619677472.796981:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.796983:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.796985:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.796985:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.796987:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.796990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.797004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.797007:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.797008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.797008:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.797009:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.797010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.797101:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.797104:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.797104:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.797106:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.797109:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.797110:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.797112:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.797113:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.797114:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.797115:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.797118:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.797119:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.797121:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.797121:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.797122:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.797123:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.797124:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.797125:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.797125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.797135:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.797136:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.797137:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.797138:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.797139:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.797140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.797156:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.797159:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.797160:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.797162:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.797164:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.797165:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.797166:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.797168:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.797169:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.797169:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.797172:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.797174:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.797176:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.797177:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.797177:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.797179:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.797179:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.797180:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.797180:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.797190:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.797193:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.797193:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.797194:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.797195:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.797197:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.797402:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.797405:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.797406:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.797407:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.797410:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.797412:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677472.797413:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.797414:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.797415:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.797415:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677472.797419:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.797420:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677472.797423:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.797423:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677472.797424:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677472.797426:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677472.797426:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.797427:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.797427:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677472.797433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.797442:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.797445:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.797445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.797446:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.797448:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.797451:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.797571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677472.797577:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.797580:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677472.797582:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.797582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677472.797585:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.797586:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.797588:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.797589:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.797590:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.797591:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.797594:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.797596:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.797599:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.797599:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.797600:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.797601:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.797602:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.797603:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:21.0:1619677472.797604:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677472.797605:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677472.797607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.797611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.797617:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.797618:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.797618:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.797619:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:21.0:1619677472.797620:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.797620:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677472.797620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677472.797622:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.797623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.797624:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.797626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.797634:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.797637:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.797638:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.797638:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.797639:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.797640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677472.797673:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.797675:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.797676:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.797678:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.797681:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.797683:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.797684:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.797685:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.797686:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.797687:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.797691:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.797692:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.797696:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.797696:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.797697:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.797699:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.797699:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.797700:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.797700:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.797714:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.797716:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.797717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.797718:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.797720:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.797722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677472.797896:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.797899:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.797900:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.797900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.797902:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.797906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.798053:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.798056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.798056:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.798058:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.798060:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.798061:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.798063:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.798064:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.798065:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.798066:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.798068:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.798070:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.798072:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.798072:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.798073:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.798074:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.798074:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.798075:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.798075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.798085:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.798087:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.798087:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.798088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.798089:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.798090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.798112:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.798114:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.798115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.798115:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.798115:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.798116:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.798211:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.798213:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.798214:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.798215:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.798217:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.798221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.798450:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.798453:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677472.798454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677472.798455:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.798457:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.798459:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677472.798460:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.798461:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.798463:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.798463:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677472.798466:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.798468:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677472.798471:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.798471:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677472.798472:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677472.798474:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677472.798474:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.798475:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.798475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677472.798480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.798488:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.798492:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.798493:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.798494:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.798496:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.798499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677472.798736:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.798739:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.798740:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.798742:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.798745:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.798746:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677472.798747:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.798749:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.798750:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.798750:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677472.798753:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.798755:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677472.798758:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.798759:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677472.798759:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677472.798761:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677472.798761:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.798763:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.798764:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.798764:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677472.798767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.798774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.798777:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:10.0:1619677472.798777:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:17.0:1619677472.798778:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.798778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677472.798779:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:17.0:1619677472.798780:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.798780:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.798781:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677472.798783:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.798784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.798791:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.798794:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.798795:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.798795:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.798796:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.798797:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.798910:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.798913:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.798915:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.798917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.798917:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.798919:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.798920:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.798921:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.798922:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.798923:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.798926:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.798928:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.798930:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.798930:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.798931:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.798932:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.798933:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.798934:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.798934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.798947:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.798950:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.798951:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.798951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.798953:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.798955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.798973:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.798976:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.798977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.798979:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.798982:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.798984:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.798985:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.798986:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.798987:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.798988:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.798991:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.798992:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.798995:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.798996:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.798997:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.798998:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.798999:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.799000:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.799000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.799012:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.799014:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.799015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.799016:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.799018:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.799020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:23.0:1619677472.799299:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:23.0:1619677472.799300:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:23.0:1619677472.799301:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:23.0:1619677472.799303:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:18.0:1619677472.799828:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.799830:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.799831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.799832:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.799834:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.799837:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677472.799845:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.799848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.799848:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677472.799849:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.799850:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.799850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677472.800786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.800792:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.800795:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.800796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.800798:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.800800:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.800801:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.800803:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.800804:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.800805:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.800806:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.800810:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.800811:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.800814:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.800815:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.800815:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.800817:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.800818:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.800818:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.800819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.800832:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.800834:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.800835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.800836:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.800838:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.800840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.802317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.802324:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.802327:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.802327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.802330:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.802332:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.802334:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677472.802335:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.802336:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.802338:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.802339:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677472.802342:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.802343:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677472.802347:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.802348:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677472.802348:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.802350:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.802351:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.802352:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.802353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.802357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.802366:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.802368:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.802369:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.802369:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.802371:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.802373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677472.802377:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.802380:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.802381:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.802382:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.802385:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.802386:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.802388:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.802389:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.802390:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.802391:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.802394:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.802395:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.802397:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.802398:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.802399:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.802400:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.802401:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.802401:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.802402:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.802413:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.802415:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.802416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.802416:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.802418:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.802421:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.802431:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.802434:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.802435:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.802435:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.802436:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.802436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.802465:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.802467:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.802468:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.802470:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.802472:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.802473:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.802474:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.802476:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.802476:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.802477:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.802480:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.802482:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.802485:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.802486:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.802486:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.802488:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.802488:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.802489:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.802490:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.802502:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.802504:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.802505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.802506:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.802508:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.802510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677472.802980:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.802983:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.802984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.802985:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.802988:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.802989:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.802990:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.802992:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.802993:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.802994:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.802996:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.802998:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.802999:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.803000:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.803001:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.803002:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.803002:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.803003:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.803004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.803025:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.803026:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.803027:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.803028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.803029:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.803030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.803237:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.803239:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.803242:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.803242:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.803243:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.803245:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.803246:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.803248:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.803248:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.803250:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.803253:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.803255:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.803257:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.803258:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.803259:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.803260:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.803261:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.803262:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.803262:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.803284:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.803286:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.803287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.803288:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.803290:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.803292:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.803821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.803827:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:20.2:1619677472.803831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.803831:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.803834:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.803834:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.803835:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.803836:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677472.803837:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.803838:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.803839:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677472.803840:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.803842:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.803843:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.803844:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677472.803846:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.803847:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.803848:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677472.803849:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.803851:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677472.803851:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677472.803852:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.803852:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.803855:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.803857:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.803858:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.803859:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.803859:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677472.803860:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.803860:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.803861:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.803862:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.803863:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.803863:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.803863:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.803864:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.803864:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.803865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.803865:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677472.803866:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677472.803869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.803877:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:17.0:1619677472.803878:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677472.803879:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.803880:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.803881:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677472.803881:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.803882:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677472.803882:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.803883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.803884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:17.0:1619677472.803884:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.803887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.803971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.803977:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.803980:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.803981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.803982:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.803985:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.803986:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.803988:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.803989:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.803990:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.803991:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.803994:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.803995:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.803998:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.803999:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.803999:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.804001:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.804001:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.804002:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.804003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.804211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677472.804215:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.804217:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677472.804219:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.804221:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677472.804222:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.804224:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677472.804225:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.804227:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.804227:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.804229:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677472.804232:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.804234:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677472.804238:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.804238:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677472.804239:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677472.804241:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677472.804242:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.804242:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677472.804243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:13.0:1619677472.804255:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.804258:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.804259:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.804260:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.804262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.804263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:10.0:1619677472.804339:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.804342:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.804343:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.804345:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.804348:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.804349:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.804350:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.804352:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.804353:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.804354:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.804357:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.804359:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.804361:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.804362:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.804363:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.804364:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.804365:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.804366:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.804367:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.804368:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677472.804379:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.804381:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.804383:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677472.804383:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677472.804384:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677472.804384:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.804384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.804385:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677472.804386:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.804389:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.804396:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.804399:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.804400:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.804400:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.804401:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.804402:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.805410:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.805413:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.805414:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.805416:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.805418:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.805420:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677472.805421:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.805422:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.805423:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.805424:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677472.805427:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.805428:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677472.805431:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.805432:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677472.805432:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677472.805434:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677472.805435:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.805436:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.805436:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677472.805441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.805449:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.805452:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.805453:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.805455:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.805455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.805458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.805624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.805632:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.805634:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.805635:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.805637:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.805638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.805640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677472.806068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.806074:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.806077:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.806078:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.806080:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.806082:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.806084:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.806085:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.806086:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.806087:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.806088:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.806092:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.806094:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.806096:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.806097:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.806098:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.806099:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.806100:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.806101:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.806101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.0:1619677472.806107:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.806110:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.806113:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.806113:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:22.0:1619677472.806115:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:21.0:1619677472.806115:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.806117:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:22.0:1619677472.806118:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.806119:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.806119:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677472.806119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:21.0:1619677472.806121:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.806122:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.806122:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.806123:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677472.806127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.806128:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:21.0:1619677472.806128:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677472.806131:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677472.806131:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677472.806132:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.806132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.806133:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.806134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677472.806134:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.806135:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.806136:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.806138:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.806139:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.806140:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.806141:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.806142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.806152:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.806154:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.806155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.806156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677472.806348:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.806350:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.806351:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.806352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.806353:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.806356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.806515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.806523:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.806525:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.806526:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.806528:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.806528:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.806531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677472.807161:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.807164:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.807165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.807167:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.807170:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.807171:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.807173:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.807174:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.807175:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.807176:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.807179:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.807181:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.807183:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.807183:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.807184:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.807185:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.807186:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.807187:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.807187:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.807197:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.807200:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.807200:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.807201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.807202:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.807204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.807257:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.807260:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.807260:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.807262:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.807265:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.807266:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.807267:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.807269:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.807269:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.807270:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.807273:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.807274:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.807277:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.807278:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.807278:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.807280:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.807281:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.807282:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.807283:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.807284:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677472.807294:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677472.807298:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677472.807299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:23.0:1619677472.807300:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:10.0:1619677472.807300:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.807301:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677472.807301:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.807302:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677472.807302:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.807304:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.807307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.807432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.807437:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.807440:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.807442:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.807443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.807445:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.807446:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.807448:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.807449:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.807450:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.807451:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.807454:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.807456:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.807458:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.807459:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.807460:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.807461:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.807462:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.807463:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.807464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.807468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.807476:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.807478:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.807479:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.807481:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.807482:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.807483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677472.807701:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.807705:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677472.807706:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677472.807708:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.807711:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.807712:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.807713:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.807715:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.807715:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.807716:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.807719:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.807721:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.807723:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.807724:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.807725:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.807726:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.807727:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.807728:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (22)++
00000800:00000200:26.0:1619677472.807729:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:26.0:1619677472.807729:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:24.0:1619677472.807739:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.807741:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.807742:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.807742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.807743:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000400:00000200:24.0:1619677472.807744:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.807744:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000800:00000200:10.0:1619677472.807745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:24.0:1619677472.807746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.807746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:22.0:1619677472.807934:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677472.807937:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677472.807937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677472.807939:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.807943:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.807944:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677472.807945:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.807947:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.807948:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.807949:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677472.807952:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.807954:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677472.807957:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.807958:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677472.807958:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677472.807960:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677472.807961:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677472.807962:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677472.807962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:20.2:1619677472.808046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.808053:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.808056:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.808057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.808059:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.808062:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.808063:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.808065:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.808066:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.808067:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.808068:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.808072:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.808073:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.808077:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.808077:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.808078:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.808080:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.808080:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.808081:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.808082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.808103:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.808106:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.808107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.808108:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.808110:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.808111:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.808112:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.808114:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.808115:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.808116:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.808118:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.808120:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.808122:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.808122:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.808123:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.808125:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.808125:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.808126:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.808128:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.808128:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.808139:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.2:1619677472.808139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:24.0:1619677472.808140:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.808140:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.808142:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.808143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.808143:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:21.0:1619677472.808144:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:8.0:1619677472.808144:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.808144:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.808145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.808145:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677472.808147:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.808149:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.808149:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.808151:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.808153:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.808154:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.808155:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.808156:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.808157:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.808160:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.808162:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.808164:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.808165:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.808166:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.808167:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.808168:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.808168:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677472.808169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677472.808181:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.808183:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.808184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.808184:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.808187:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.808189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.808670:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.808672:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.808673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.808675:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.808678:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.808679:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.808681:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.808682:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.808683:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.808684:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.808687:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.808688:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.808692:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.808692:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.808693:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.808695:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.808695:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.808696:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.808696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677472.808709:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.808712:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.808713:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.808713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.808715:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.808718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.809124:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.809127:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.809128:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.809129:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.809132:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.809133:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.809135:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.809136:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.809137:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.809138:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.809141:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.809142:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.809145:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.809146:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.809146:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.809148:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.809148:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.809149:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.809149:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.809161:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.809163:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.809164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.809165:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.809167:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.809170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677472.809617:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946139b200] (21)++
00000800:00000200:20.0:1619677472.809620:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.24@o2ib3
00000400:00000200:20.0:1619677472.809622:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.2.24@o2ib3 : GET - for me
00000800:00000200:21.0:1619677472.809623:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946139b200] (22)--
00000400:00000200:20.0:1619677472.809625:0:19934:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.2.24@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:20.0:1619677472.809627:0:19934:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.2.24@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:20.0:1619677472.809629:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946139b200] (21)++
00000800:00000200:20.0:1619677472.809630:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946139b200] (22)--
00000400:00000200:20.0:1619677472.809632:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.2.24@o2ib3 local destination
00000400:00000200:20.0:1619677472.809636:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.2.24@o2ib3(10.51.2.24@o2ib3:10.51.2.24@o2ib3) : REPLY try# 0
00000800:00000200:20.0:1619677472.809637:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.2.24@o2ib3
00000800:00000200:20.0:1619677472.809639:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9197e5a60d00] -> 10.51.2.24@o2ib3 (2) version: 12
00000800:00000200:20.0:1619677472.809640:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91946139b200] (21)++
00000800:00000200:20.0:1619677472.809641:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91946139b200] (22)++
00000800:00000200:20.0:1619677472.809642:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91946139b200] (23)--
00000800:00000200:20.0:1619677472.809643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946139b200] (22)--
00000800:00000200:21.0:1619677472.809653:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946139b200] (22)++
00000800:00000200:21.0:1619677472.809655:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91946139b200] (23)--
00000400:00000200:21.0:1619677472.809656:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.809658:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.2.24@o2ib3: REPLY: OK
00000800:00000200:20.0:1619677472.809658:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946139b200] (22)--
00000800:00000200:21.0:1619677472.809660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946139b200] (21)--
00000800:00000200:8.2:1619677472.809941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.809949:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.809952:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.809953:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.809954:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.809955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.809957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.809964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.809973:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.809975:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.809976:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.809977:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.809978:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.809979:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677472.810033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.810039:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.810042:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.810043:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.810045:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.810048:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.810050:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677472.810051:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.810053:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.810054:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.810055:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677472.810058:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.810060:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677472.810063:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.810063:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677472.810064:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677472.810066:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677472.810067:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.810068:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.810068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.810079:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.810082:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.810083:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.810083:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.810085:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.810088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.810105:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.810107:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.810108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.810108:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.810109:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.810110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677472.810259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.810265:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.810269:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.810269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.810271:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.810274:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.810276:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.810277:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.810278:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.810279:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.810280:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.810283:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.810284:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.810287:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.810288:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.810288:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.810290:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.810291:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.810291:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.810291:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.810305:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.810307:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.810308:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.810308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.810310:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.810312:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.810396:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.810398:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677472.810399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677472.810401:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.810404:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.810406:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677472.810406:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.810408:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.810409:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.810410:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677472.810414:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.810415:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677472.810419:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.810420:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:6.0:1619677472.810421:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:6.0:1619677472.810422:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677472.810423:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.810424:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677472.810424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:12.0:1619677472.810438:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.810442:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.810443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.810446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677472.810446:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.810449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677472.810458:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677472.810461:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677472.810462:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.810462:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677472.810463:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.810464:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677472.811139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.811145:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.811147:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.811150:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.811150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.811152:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.811154:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.811155:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.811156:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.811157:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.811158:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.811162:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.811164:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.811166:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.811167:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.811167:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.811169:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.811169:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.811170:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.811172:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.811172:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.811175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.811178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.811185:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.811186:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.811186:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677472.811187:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.811187:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677472.811188:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.811188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.811189:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.811190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.811191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677472.811563:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.811565:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.811567:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677472.811568:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.811568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.811569:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.811570:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.811571:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.811572:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.811573:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.811576:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.811577:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.811579:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.811579:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.811580:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.811581:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.811582:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.811582:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.811583:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.811596:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.811598:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.811599:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.811600:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.811602:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.811605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.811613:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.811615:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.811616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.811617:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.811617:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.811618:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.811621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677472.811623:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.811626:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677472.811628:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.811629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677472.811630:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.811631:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.811633:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.811634:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.811635:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.811636:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.811640:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.811641:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.811643:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.811644:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.811645:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.811646:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.811647:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.811648:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677472.811648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.811653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.811661:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.811663:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.811664:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.811665:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.811666:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.811667:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.811669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.811676:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.811679:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.811680:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.811680:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.811681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.811682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.811766:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.811769:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.811771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.811772:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.811774:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.811776:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.811777:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.811779:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.811780:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.811781:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.811784:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.811786:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.811789:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.811790:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.811790:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.811792:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.811793:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.811793:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.811793:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.811807:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.811810:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.811810:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.811811:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.811813:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.811816:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.811889:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.811892:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.811893:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.811895:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.811898:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.811899:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.811900:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.811901:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.811902:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.811903:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.811906:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.811908:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.811910:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.811911:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.811911:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.811913:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.811913:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.811914:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.811914:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.811925:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.811927:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.811928:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.811928:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.811930:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.811933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.811949:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.811952:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.811953:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.811953:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.811954:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.811955:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.812021:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.812024:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.812025:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.812026:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.812028:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.812030:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.812031:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.812032:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.812033:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.812034:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.812037:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.812038:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.812041:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.812042:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.812042:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.812044:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.812044:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.812045:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.812045:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.812058:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.812061:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.812061:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.812062:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.812064:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.812067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.812404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.812410:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.812413:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.812414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.812416:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.812418:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.812420:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677472.812421:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.812423:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.812424:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.812425:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677472.812427:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.812429:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677472.812431:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.812432:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:2.0:1619677472.812433:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677472.812434:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677472.812435:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.812437:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.812438:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.812438:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.812444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677472.812449:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:18.0:1619677472.812451:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677472.812451:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.812452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.812452:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.812453:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.812453:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.812454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677472.812454:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.812457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.812470:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.812473:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.812473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.812474:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.812475:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.812475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.812626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.812632:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.812634:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.812636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.812637:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.812639:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.812640:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:2.0:1619677472.812642:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.812643:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.812644:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.812645:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677472.812647:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.812649:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677472.812651:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.812652:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:2.0:1619677472.812652:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:2.0:1619677472.812654:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:2.0:1619677472.812654:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.812655:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.812656:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.812661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.812669:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.812672:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.812673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.812673:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.812675:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.812678:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.812937:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.812940:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.812941:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.812942:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.812945:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.812946:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.812947:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.812949:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.812950:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.812951:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.812954:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.812956:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.812957:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.812958:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.812959:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.812960:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.812961:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.812961:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.812962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.812972:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.812973:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.812974:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.812975:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.812975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.812977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.813248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.813254:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.813257:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.813258:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.813260:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.813262:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.813263:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.813265:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.813266:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.813267:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.813268:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.813271:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.813273:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.813275:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.813275:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.813276:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.813278:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.813278:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.813279:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.813281:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.813281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.813286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.813292:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.813293:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.813298:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.813298:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.813299:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.813299:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.813300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.813300:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.813301:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.813302:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.813315:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.813318:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.813319:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.813319:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.813320:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.813320:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.813616:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.813619:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.813620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.813622:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.813624:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.813626:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677472.813627:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.813628:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.813629:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.813631:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677472.813634:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.813636:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677472.813638:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.813639:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:17.0:1619677472.813640:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677472.813641:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:17.0:1619677472.813642:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.813643:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.813643:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.813648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.813657:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.813659:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.813660:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.813662:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.813663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.813664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677472.813957:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.813961:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.813962:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.813964:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.813968:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.813969:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.813970:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.813972:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.813973:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.813974:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.813977:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.813979:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.813981:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677472.813982:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.813983:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.813984:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677472.813984:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:26.0:1619677472.813985:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677472.813985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677472.813986:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677472.813986:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677472.813987:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677472.813988:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.813989:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.813989:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677472.813989:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.813990:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.813991:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.813992:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.813993:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.813996:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.813997:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.813998:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.813999:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.814000:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.814001:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.814001:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677472.814002:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677472.814002:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.814002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677472.814002:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677472.814004:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.814005:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677472.814005:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.814006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.814006:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677472.814006:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.814008:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.814012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.814014:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.814016:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.814017:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.814018:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.814019:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.814020:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.814383:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.814386:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677472.814387:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677472.814388:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.814391:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.814393:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677472.814394:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.814396:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.814397:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.814398:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677472.814401:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.814402:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.814405:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.814406:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677472.814406:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677472.814408:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677472.814409:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.814410:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.814410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:22.0:1619677472.814423:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.814425:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.814426:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.814426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.814428:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.814431:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.814761:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.814764:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677472.814765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677472.814767:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.814768:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.814770:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677472.814771:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.814772:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.814773:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.814775:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.2:1619677472.814777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:8.0:1619677472.814778:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.814780:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677472.814780:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.814782:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677472.814783:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.814783:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:20.0:1619677472.814784:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.814784:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.814785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677472.814786:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677472.814786:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000400:00000200:20.0:1619677472.814787:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.814787:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677472.814787:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:20.0:1619677472.814788:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.814789:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.814791:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.814792:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.814793:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.814796:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.814798:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.814800:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.814801:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.814802:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.814803:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.814804:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.814805:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.814805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.814810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.814817:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.814819:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.814820:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.814821:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.814822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.814823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.814875:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.814878:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.814879:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.814879:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.814881:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.814885:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.815091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.815096:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.815099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.815099:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677472.815101:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.815103:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.815105:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677472.815106:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.815107:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.815108:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.815110:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677472.815112:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.815114:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677472.815117:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.815117:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677472.815118:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677472.815120:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677472.815120:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.815121:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.815122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677472.815133:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.815135:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.815136:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.815138:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.815139:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.815140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677472.815215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.815222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.815224:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677472.815227:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.115@o2ib7 : GET - for me
00000800:00000200:20.0:1619677472.815227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677472.815229:0:19935:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.115@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:21.0:1619677472.815231:0:19935:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.115@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:21.0:1619677472.815233:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.815233:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677472.815235:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677472.815239:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : REPLY try# 0
00000800:00000200:21.0:1619677472.815241:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.815244:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.815245:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.815245:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.815247:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.815247:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677472.815253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.815260:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.815263:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.815263:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.815265:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: REPLY: OK
00000800:00000200:21.0:1619677472.815266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.815267:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:23.0:1619677472.815302:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:8.2:1619677472.815385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.815393:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.815396:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.815397:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.815399:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.815399:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.815402:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.815834:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.815836:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.815839:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.815839:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.815841:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.815843:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.815844:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.815845:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.815846:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.815847:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.815851:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.815852:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.815855:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.815855:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.815856:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.815858:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.815859:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.815860:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.815861:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.815862:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677472.815872:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.815875:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.815876:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.815877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.815877:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:6.0:1619677472.815878:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.815879:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.815880:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.815880:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677472.815880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677472.816117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.816123:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.816126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.816126:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677472.816129:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.816131:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.816132:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.816134:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.816135:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.816136:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.816137:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.816140:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.816142:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.816145:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.816146:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.816146:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.816148:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.816148:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.816150:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677472.816151:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677472.816151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677472.816156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677472.816161:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677472.816162:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677472.816164:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677472.816164:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677472.816164:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.816165:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677472.816166:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.816166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.816168:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.816170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677472.816233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.816244:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.816247:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.816248:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.816249:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.816250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.816252:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.816353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677472.816356:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.816359:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.816360:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.816361:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.816361:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:26.0:1619677472.816364:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.816364:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.816364:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.816367:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:21.0:1619677472.816367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.816367:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677472.816368:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.816370:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.816370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:26.0:1619677472.816371:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.816372:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677472.816375:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.816377:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677472.816380:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.816380:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677472.816381:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677472.816383:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677472.816383:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.816384:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.816384:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677472.816399:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.816402:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.816402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.816403:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.816405:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:10.0:1619677472.816407:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677472.816408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.816409:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.816410:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.816412:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.816415:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.816417:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.816418:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.816419:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.816420:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.816421:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.816424:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.816425:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.816428:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.816429:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.816429:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.816431:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.816431:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.816431:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.816432:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.816444:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.816447:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.816448:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.816448:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.816450:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.816453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.817119:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.817122:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.817123:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.817125:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.817127:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.817128:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.817130:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.817131:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.817132:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.817133:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.817136:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.817137:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.817139:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.817140:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.817140:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.817142:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.817142:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.817143:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.817143:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.817153:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.817156:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.817157:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.817157:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.817158:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.817161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.817457:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.817459:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.817461:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.817462:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.817463:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.817465:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.817466:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.817468:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.817469:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.817470:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.817472:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.817474:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.817476:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.817477:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.817478:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.817479:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.817480:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.817480:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.817481:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.817493:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.817495:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.817496:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.817496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.817498:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.817500:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.817515:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.817518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.817518:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.817519:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677472.817520:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677472.817520:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.817520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.817523:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.817524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.817525:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.817528:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.817529:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.817531:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.817533:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.817533:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.817534:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.817537:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.817539:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.817542:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.817543:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.817543:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.817545:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.817545:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.817546:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.817547:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.817548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.817551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.817559:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.817560:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.817562:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677472.817562:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.817562:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677472.817563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677472.817563:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.817564:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677472.817565:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.817568:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.818579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.818582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.818584:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.818586:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.818587:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.818589:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.818591:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.818593:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.818595:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.818596:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.818596:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.0:1619677472.818598:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677472.818599:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.818600:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.818601:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677472.818601:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.818603:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.818604:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:8.0:1619677472.818604:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.818605:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.818606:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.818607:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677472.818607:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.818608:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.818608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677472.818608:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.818610:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677472.818612:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:8.0:1619677472.818616:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.818617:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.818618:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.818620:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:8.0:1619677472.818621:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.818622:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.818623:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.818623:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:21.0:1619677472.818625:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.818625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.818627:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.818627:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.818627:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.818628:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.818630:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.818630:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.818631:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.818632:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.818634:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.818642:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.818644:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677472.818645:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677472.818646:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.818646:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.818647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.818648:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677472.818648:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.818649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.818649:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.818651:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.818655:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.819300:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.819302:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.819303:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.819305:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.819307:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.819309:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.819310:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.819311:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.819312:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.819313:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.819316:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.819318:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.819321:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.819321:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.819322:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.819324:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.819324:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.819325:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.819326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.819339:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.819342:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.819343:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.819343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.819345:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.819348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.819721:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.819724:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.819725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.819726:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.819730:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.819731:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.819732:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.819734:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.819735:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.819735:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.819738:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.819740:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.819741:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.819742:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.819742:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.819743:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.819744:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.819744:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.819745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.819765:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.819767:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.819768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.819768:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.819769:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.819771:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.819819:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.819821:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.819823:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.819824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.819825:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.819827:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.819828:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.819829:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.819830:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.819831:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.819835:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.819836:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.819839:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.819840:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.819840:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.819842:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.819843:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.819844:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.819844:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.819865:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.819867:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.819868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.819869:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.819871:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.819874:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.819880:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.819883:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.819884:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.819884:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.819885:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.819886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.820735:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.820737:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.820738:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.820738:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.820739:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.820742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677472.820916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.820922:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.820925:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.820926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.820927:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.820930:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.820931:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.820933:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.820934:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.820935:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.820936:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.820938:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.820940:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.820943:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.820943:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.820944:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.820946:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.820947:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.820947:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.820948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.820959:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.820962:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.820963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.820963:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.820965:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.820967:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.821176:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.821179:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.821180:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.821182:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.821185:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.821186:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.821188:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.821189:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.821190:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.821191:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.821194:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.821196:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.821199:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.821200:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.821201:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.821202:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.821203:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.821204:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.821204:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.821216:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.821218:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.821219:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.821220:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.821222:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.821224:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.821234:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677472.821237:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.821238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.821238:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.821239:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.821240:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.821455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.0:1619677472.821460:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.821462:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:18.0:1619677472.821462:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:21.0:1619677472.821465:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.821465:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:18.0:1619677472.821465:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.821466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:21.0:1619677472.821467:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.821468:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677472.821469:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.821470:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.821471:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.821471:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.821472:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.821473:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.821474:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.821474:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.821475:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.821476:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.821478:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677472.821478:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.821481:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.821482:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.821484:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.821484:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.821485:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677472.821485:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.821487:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.821487:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.821488:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:18.0:1619677472.821488:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.821489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:18.0:1619677472.821489:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.821489:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.821491:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.821492:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.821493:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.821493:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677472.821501:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.821504:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.821505:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.821505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.821507:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.821509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677472.821736:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.821738:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.821740:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.821741:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.821742:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.821744:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.821745:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.821747:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.821748:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.821749:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.821752:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.821754:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.821757:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.821758:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.821759:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.821761:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.821761:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.821762:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.821763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677472.821854:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.821857:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.821858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.821860:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.821863:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.821864:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.821865:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.821866:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.821867:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.821868:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.821872:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.821874:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.821876:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.821877:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.821878:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.821879:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.821880:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.821880:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.821881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.821892:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:8.0:1619677472.821895:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000800:00000200:24.0:1619677472.821896:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:24.0:1619677472.821896:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:8.0:1619677472.821896:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.821897:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.821898:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.821899:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.821900:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.821901:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.821901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.821914:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.821916:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.821917:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.821917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.821918:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.821919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.822234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.822240:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.822242:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.822244:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.822245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.822247:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.822248:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.822250:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.822251:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.822252:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.822253:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.822256:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.822258:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.822260:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.822261:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.822261:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.822263:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.822264:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.822264:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.822265:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.822269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.822273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.822279:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677472.822280:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:2.0:1619677472.822282:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:20.0:1619677472.822283:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.822283:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.822283:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.822285:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.822286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.822286:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.822288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:2.0:1619677472.822288:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.822289:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.822291:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.822292:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.822297:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.822298:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.822301:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.822303:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.822305:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.822306:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.822306:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.822308:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.822309:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.822309:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.822310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.822323:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.822325:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.822326:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.822326:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.822327:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.822329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.822340:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.822342:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.822343:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.822344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.822344:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.822345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.823221:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.823224:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.823224:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.823226:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.823228:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.823229:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.823230:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.823232:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.823233:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.823234:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.823236:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.823238:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.823241:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.823242:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.823242:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.823244:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.823245:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.823245:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.823246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.823258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.823261:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.823261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.823262:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.823264:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.823267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.823856:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.823858:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.823859:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.823861:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.823863:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.823864:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677472.823866:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.823867:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.823868:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.823869:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677472.823872:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.823873:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677472.823876:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.823876:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677472.823877:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677472.823878:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677472.823879:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.823880:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.823880:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677472.824095:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.824100:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.824101:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.824104:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.824107:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.824110:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.824112:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.824114:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.824116:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.824117:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.824122:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.824125:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.824129:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.824130:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.824131:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.824133:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.824134:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.824136:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.824138:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.824138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677472.824150:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.824153:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677472.824154:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.824156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.824156:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.824157:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.824158:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.824160:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677472.824160:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.824165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.824278:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.824281:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.824282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.824284:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.824287:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.824289:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.824290:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.824292:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.824293:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.824298:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.824301:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.824302:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.824304:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.824305:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.824305:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.824307:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.824307:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.824309:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.824310:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.824311:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.824327:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.824329:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.824329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.824330:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.824331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.824331:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.824332:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.824332:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.824333:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.824334:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.824796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.824805:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.824807:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.824808:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.824810:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.824810:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.824813:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.825157:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.825163:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.825164:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677472.825169:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.825172:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.825175:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677472.825177:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677472.825178:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.825180:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.825181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677472.825181:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.825183:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.825184:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.825185:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.825185:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.825187:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.825188:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96512061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.825190:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.825192:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.825193:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677472.825193:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.825198:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.825198:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677472.825200:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677472.825202:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.825204:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.825204:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677472.825205:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.825205:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677472.825205:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677472.825208:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677472.825208:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677472.825209:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677472.825209:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:24.0:1619677472.825210:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.825210:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677472.825211:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677472.825213:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677472.825214:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:22.0:1619677472.825223:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677472.825223:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.825226:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:22.0:1619677472.825227:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.825228:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.825228:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:2.0:1619677472.825228:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.825229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677472.825229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.825231:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:22.0:1619677472.825233:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.825233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:2.0:1619677472.825233:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.825235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:2.0:1619677472.825236:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.825239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.825241:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.825243:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.825244:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.825244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.825245:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.825246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.825516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.825518:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.825521:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.825523:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.825524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.825525:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.825526:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.825528:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.825529:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.825530:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.825531:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.825535:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.825536:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.825539:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.825540:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.825541:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.825543:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.825543:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.825544:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.825545:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.825546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.825549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.825552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.825556:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.825557:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.825557:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.825558:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.825559:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.825559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.825559:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.825561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.825562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.825563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677472.825690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.825696:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.825699:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.825700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.825702:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.825704:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.825706:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.825707:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.825708:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.825710:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.825711:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.825714:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.825715:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.825718:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.825719:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.825720:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.825722:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.825722:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.825723:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.825723:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.825736:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.825739:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.825740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.825740:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.825742:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.825744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.826009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.826011:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.826012:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.826012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.826015:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.826018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.826072:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.826075:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.826076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.826078:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.826081:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.826082:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.826083:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.826085:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.826086:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.826087:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.826090:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.826091:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.826094:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.826095:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.826096:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.826097:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.826098:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.826098:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677472.826100:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.826100:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677472.826110:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677472.826113:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677472.826113:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.826114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.826114:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.826115:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.826116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677472.826116:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.826117:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.826119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677472.826741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.826750:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.826753:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.826754:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.826754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.826757:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.826759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.826761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.826769:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.826771:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.826772:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.826772:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.826773:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.826774:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677472.827090:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.827093:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.827094:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.827095:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.827098:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.827100:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.827101:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.827103:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.827104:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.827105:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.827108:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.827109:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.827111:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.827112:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.827112:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.827114:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.827114:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.827115:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.827115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.827127:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.827129:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.827130:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.827130:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.827132:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.827134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.827171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.827179:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.827182:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.827182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.827184:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.827187:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.827188:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677472.827190:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677472.827191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:18.0:1619677472.827191:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.827192:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.827194:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677472.827197:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:18.0:1619677472.827199:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.827199:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.827201:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677472.827201:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.827204:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.827205:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.827206:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:2.0:1619677472.827206:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:18.0:1619677472.827207:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:2.0:1619677472.827207:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.827209:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:2.0:1619677472.827209:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.827209:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.827211:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.827212:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.827213:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.827214:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:2.0:1619677472.827214:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.827215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.827216:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.827216:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.2:1619677472.827220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.827220:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.827220:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.827221:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677472.827223:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677472.827223:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.827224:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.827225:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:13.0:1619677472.827226:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:2.0:1619677472.827226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:13.0:1619677472.827227:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.827227:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:13.0:1619677472.827229:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.827229:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.827230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677472.827231:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.827232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677472.827232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677472.827236:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.827239:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677472.827239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677472.827240:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.827242:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.827245:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677472.827258:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677472.827261:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677472.827261:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677472.827262:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.827263:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677472.827263:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677472.827979:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.827981:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.827982:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.827984:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.827986:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.827987:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677472.827988:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.827989:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.827991:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.827991:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.827995:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.827996:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677472.827999:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.828000:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.828001:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.828002:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.828003:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.828004:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.828004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677472.828018:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.828021:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.828021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.828022:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.828024:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677472.828027:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677472.828453:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.828456:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.828457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.828458:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.828460:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.828462:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.828914:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.828916:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.828917:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.828919:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.828922:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.828923:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.828924:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.828925:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.828926:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.828927:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.828930:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.828931:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.828934:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.828935:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.828936:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.828937:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.828938:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.828939:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.828939:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.828951:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.828954:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.828955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.828955:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.828957:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.828960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.828987:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.2:1619677472.828987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677472.828989:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.828990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.828992:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.828993:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677472.828995:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.828996:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:24.0:1619677472.828997:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677472.828998:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677472.828999:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.828999:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.829000:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.829001:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.829001:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.829002:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:21.0:1619677472.829003:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.829005:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.829007:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677472.829007:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.829008:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.829009:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677472.829010:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:21.0:1619677472.829012:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.829013:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.829014:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.829015:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677472.829015:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677472.829016:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.829017:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.829017:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677472.829017:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.829018:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677472.829018:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.829018:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.829020:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.829021:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.829021:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.829022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.829022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.2:1619677472.829026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.829030:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.829033:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:20.0:1619677472.829034:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.829034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.829035:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.829036:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.829037:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.829037:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:20.0:1619677472.829039:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.829040:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.829040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677472.829041:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.829046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.829054:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.829056:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.829057:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.829058:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.829059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.829060:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.830121:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.830124:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.830125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.830127:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.830130:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.830131:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.830132:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.830134:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.830135:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.830136:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.830141:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.830143:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.830145:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.830146:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.830147:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.830148:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.830148:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.830149:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.830150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.830161:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.830163:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.830164:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.830164:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.830166:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.830167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.830175:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.830178:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.830178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.830179:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.830181:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.830183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.830195:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.830198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.830198:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.830199:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.830200:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.830200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.830519:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.830521:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.830522:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.830524:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.830527:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.830528:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.830530:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.830532:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.830533:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.830533:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.830537:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.830538:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.830541:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.830542:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.830542:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.830544:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.830544:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.830545:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.830545:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.830558:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.830561:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.830562:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.830562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:5.2:1619677472.830563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677472.830565:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.830567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677472.830570:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.830573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.830573:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677472.830576:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.830578:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.830579:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.830580:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.830582:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.830583:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.830584:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.830587:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.830589:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.830591:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.830592:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:20.2:1619677472.830593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:1.0:1619677472.830593:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.830595:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.830595:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.830596:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.830596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.830608:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.830611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.830611:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.830612:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.830615:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.830617:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.830626:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.830629:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.830629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.830630:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.830631:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.830632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:12.0:1619677472.830642:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.830645:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.830646:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.830649:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.830650:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.830651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:2.0:1619677472.830665:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:2.0:1619677472.830669:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.20@o2ib3
00000800:00000200:18.0:1619677472.830670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000400:00000200:2.0:1619677472.830671:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.20@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.830674:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.830676:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677472.830678:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.830679:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.830680:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.830681:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:2.0:1619677472.830685:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.830686:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677472.830689:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.830690:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:2.0:1619677472.830691:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:2.0:1619677472.830693:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:2.0:1619677472.830694:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:2.0:1619677472.830695:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:2.0:1619677472.830696:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)--
00000800:00000200:13.0:1619677472.830706:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.830709:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.830709:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.830711:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.830714:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.830716:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:18.0:1619677472.830727:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:12.0:1619677472.830727:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:18.0:1619677472.830730:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.20@o2ib3
00000800:00000200:2.0:1619677472.830731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:12.0:1619677472.830732:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677472.830733:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.20@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.830733:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.830734:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677472.830735:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:18.0:1619677472.830736:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.830736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:18.0:1619677472.830737:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677472.830738:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.830740:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.830741:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.830741:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677472.830745:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.830746:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.830747:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.830748:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677472.830748:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.830750:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.830751:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:18.0:1619677472.830752:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:18.0:1619677472.830752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)--
00000800:00000200:2.0:1619677472.830763:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.830766:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.830766:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.830767:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.830768:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.830770:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.830782:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:18.0:1619677472.830785:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.20@o2ib3
00000800:00000200:2.0:1619677472.830785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000400:00000200:18.0:1619677472.830787:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.20@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.830788:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.830789:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677472.830790:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.830791:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.830792:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.830793:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677472.830796:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.830797:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677472.830799:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.830800:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677472.830800:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.830802:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.830803:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:18.0:1619677472.830803:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:18.0:1619677472.830804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)--
00000800:00000200:8.2:1619677472.830809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.830817:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.830819:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.830820:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.830820:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:1.0:1619677472.830820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.830823:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.830823:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.20@o2ib3
00000800:00000200:18.0:1619677472.830824:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:17.0:1619677472.830825:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:2.0:1619677472.830825:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.20@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.830828:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.830830:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.830831:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.830832:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.830833:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.830834:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.830837:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.830839:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.830841:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.830842:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.830843:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.830845:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.830845:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:2.0:1619677472.830847:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906cbf6a00] (22)++
00000800:00000200:2.0:1619677472.830848:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906cbf6a00] (23)--
00000800:00000200:2.0:1619677472.830848:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:18.0:1619677472.830859:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)++
00000800:00000200:18.0:1619677472.830861:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906cbf6a00] (23)--
00000800:00000200:18.0:1619677472.830862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:18.0:1619677472.830863:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)--
00000800:00000200:2.0:1619677472.830956:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:2.0:1619677472.830959:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.20@o2ib3
00000800:00000200:18.0:1619677472.830960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000400:00000200:2.0:1619677472.830962:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.20@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.830964:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.830966:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:2.0:1619677472.830967:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.830969:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.830970:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.830971:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:2.0:1619677472.830974:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.830975:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:2.0:1619677472.830978:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.830979:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:2.0:1619677472.830979:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:2.0:1619677472.830981:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:2.0:1619677472.830982:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:2.0:1619677472.830982:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:2.0:1619677472.830983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)--
00000800:00000200:24.2:1619677472.830988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:10.0:1619677472.830994:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)++
00000800:00000200:10.0:1619677472.830997:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2fa00] (23)--
00000800:00000200:26.0:1619677472.830998:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (23)--
00000400:00000200:10.0:1619677472.830998:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.831000:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.51@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.831004:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (21)--
00000800:00000200:18.0:1619677472.831075:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:18.0:1619677472.831078:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.20@o2ib3
00000800:00000200:2.0:1619677472.831078:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000400:00000200:18.0:1619677472.831080:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.20@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.831083:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.831085:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.831086:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.831087:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.831088:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.831089:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.831092:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.831094:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.831096:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.831097:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.831097:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.831099:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.831099:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:18.0:1619677472.831100:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:18.0:1619677472.831101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)--
00000800:00000200:8.2:1619677472.831105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.831113:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:21.0:1619677472.831115:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677472.831115:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.20@o2ib3
00000800:00000200:18.0:1619677472.831116:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:21.0:1619677472.831118:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:2.0:1619677472.831118:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.20@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.831119:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.831120:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.831121:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.831121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:2.0:1619677472.831122:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677472.831123:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.831125:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.831126:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.831127:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677472.831129:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:2.0:1619677472.831131:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.831133:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677472.831136:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.831137:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677472.831137:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677472.831139:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677472.831139:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:2.0:1619677472.831140:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:2.0:1619677472.831141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)--
00000800:00000200:18.0:1619677472.831164:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:18.0:1619677472.831167:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.20@o2ib3
00000800:00000200:2.0:1619677472.831167:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000400:00000200:18.0:1619677472.831169:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.20@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.831172:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.831174:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677472.831175:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.831176:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.831177:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.831178:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.831181:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.831182:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677472.831184:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.831185:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677472.831185:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677472.831187:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677472.831187:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:18.0:1619677472.831189:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906cbf6a00] (22)++
00000800:00000200:18.0:1619677472.831190:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906cbf6a00] (23)--
00000800:00000200:18.0:1619677472.831191:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:6.0:1619677472.831200:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677472.831201:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)++
00000800:00000200:6.0:1619677472.831203:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:2.0:1619677472.831203:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906cbf6a00] (23)--
00000800:00000200:22.0:1619677472.831204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.831204:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.831204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:2.0:1619677472.831205:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)++
00000400:00000200:6.0:1619677472.831206:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.831207:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.20@o2ib3
00000800:00000200:18.0:1619677472.831208:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:6.0:1619677472.831209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:2.0:1619677472.831210:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.20@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.831213:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.831214:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.831215:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.831217:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.831218:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.831219:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.831222:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.831223:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.831226:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.831227:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.831227:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.831229:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.831230:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:2.0:1619677472.831230:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:2.0:1619677472.831231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)--
00000800:00000200:26.0:1619677472.831242:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.831244:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677472.831245:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677472.831246:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.831248:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.831251:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:18.0:1619677472.831266:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:18.0:1619677472.831268:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.20@o2ib3
00000800:00000200:2.0:1619677472.831269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (22)--
00000400:00000200:18.0:1619677472.831271:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.20@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.831274:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.831275:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677472.831276:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.831277:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.831279:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.831280:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677472.831283:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.831284:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677472.831287:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.831288:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677472.831288:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677472.831290:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677472.831291:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906cbf6a00] (21)++
00000800:00000200:18.0:1619677472.831292:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906cbf6a00] (22)--
00000800:00000200:18.0:1619677472.831292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906cbf6a00] (21)--
00000800:00000200:12.0:1619677472.831304:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.831307:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.831307:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.831309:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.831311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.831313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677472.831331:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.831334:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.831335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.831337:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.831339:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.831340:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.831341:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.831342:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.831343:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.831344:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.831347:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.831349:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.831352:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.831353:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.831354:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.831355:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.831356:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.831357:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.831358:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.831359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677472.831370:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.831371:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677472.831372:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677472.831373:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:18.0:1619677472.831373:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.831373:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.831374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.831375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677472.831376:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.831379:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.831861:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.831864:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.831865:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.831867:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.831869:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.831871:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.831872:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.831873:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.831874:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.831875:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.831878:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.831879:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.831881:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.831882:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.831882:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.831884:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.831884:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.831885:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.831885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.831896:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.831898:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.831899:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.831899:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.831901:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.831903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.832280:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.832282:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.832283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.832284:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.832286:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.832288:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.832446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.832449:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.832452:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.832454:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.832455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.832455:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.832457:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.832458:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.832459:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.832460:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.832461:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.832464:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.832466:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.832468:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.832469:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.832470:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.832471:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.832472:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.832473:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.832473:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.832478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.832486:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.832488:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.832489:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.832491:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.832491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.832493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.832588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.832595:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.832598:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.832599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.832601:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.832604:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.832605:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.832606:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.832608:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.832609:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.832610:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.832613:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.832615:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.832616:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.832617:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.832618:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.832620:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.832620:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.832621:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.832622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.832634:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.832636:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.832637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.832637:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.832639:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.832640:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.832654:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.832656:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.832657:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.832658:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.832658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.832659:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.833653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.833661:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.833664:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.833665:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.833666:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.833667:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.833669:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677472.834058:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.834061:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.834062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.834062:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.834064:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.834067:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.834248:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.834252:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.834253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.834256:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.834259:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.834261:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.2:1619677472.834262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:26.0:1619677472.834263:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.834266:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.834268:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.834269:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.834269:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.834272:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.834273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677472.834276:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677472.834276:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677472.834279:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677472.834279:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.834280:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.834281:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677472.834283:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677472.834283:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677472.834284:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:2.0:1619677472.834284:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677472.834285:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:2.0:1619677472.834285:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677472.834287:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.834288:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677472.834289:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.834290:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:2.0:1619677472.834291:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677472.834292:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.834293:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677472.834297:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.834298:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.834299:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677472.834301:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677472.834301:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.834301:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.834302:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.834303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677472.834304:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.834305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.834306:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.834308:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.834309:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677472.834311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.834312:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.834314:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.834315:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.834322:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.834325:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.834325:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.834326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.834327:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.834327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.834418:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.834421:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.834422:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.834423:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.834426:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.834428:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.834429:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.834430:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.834431:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.834432:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.834436:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.834437:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.834440:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.834441:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.834442:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.834444:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.834444:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.834445:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.834445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.834450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.834458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.834461:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.834462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.834462:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.834465:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.834467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.834662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.834670:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.834672:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.834673:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.834675:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.834675:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.834678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.834681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.834688:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.834691:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.834692:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.834693:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.834693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.834694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677472.834754:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.834757:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.834759:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.834759:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677472.834762:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.834766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677472.834794:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.834798:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.834800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677472.834802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:10.0:1619677472.834803:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.834807:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.834809:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677472.834810:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:10.0:1619677472.834811:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96512019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.834812:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.834813:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.834814:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677472.834815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.834816:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:10.0:1619677472.834816:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.834818:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.834824:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.834826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677472.834827:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.834831:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.834832:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.2:1619677472.834832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677472.834833:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.834835:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.834836:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.834837:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.834838:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677472.834839:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.2:1619677472.834841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677472.834842:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.834843:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.834843:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.834845:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.834848:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:17.0:1619677472.834848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.834850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:21.0:1619677472.834851:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.834852:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.834854:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.834854:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.834857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677472.834857:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.834860:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.834861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.834861:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.834862:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.834863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.834904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.834911:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.834914:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.834916:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.834917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.834918:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.834922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677472.834971:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.834974:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.834975:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.834975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.834978:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.834980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:13.0:1619677472.835026:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.835029:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.835030:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.835033:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.835035:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.835036:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677472.835074:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.835078:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.835078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.835080:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.835083:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.835084:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.835086:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.835087:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.835088:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.835089:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.835092:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.835094:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677472.835094:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677472.835097:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.835097:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:6.0:1619677472.835097:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.835098:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.835098:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677472.835098:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.835100:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:6.0:1619677472.835100:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.835101:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.835101:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.835102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:6.0:1619677472.835104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677472.835106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.835113:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.835116:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.835116:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.835118:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.835119:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.835121:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.835125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.835132:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.835135:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.835136:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.835136:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.835137:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.835138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677472.835270:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.835272:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.835273:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.835275:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.835278:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.835280:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.835281:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.835282:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.835283:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.835285:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.835288:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.835289:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.835292:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.835293:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.835298:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.835300:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.835301:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.835302:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.835302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.835314:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:5.2:1619677472.835316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677472.835317:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.835318:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.835318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.835320:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.835322:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677472.835323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677472.835325:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.835326:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.835328:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.835330:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.835332:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.835333:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.835334:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.835335:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.835336:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.835340:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.835341:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.835344:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.835345:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.835345:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.835347:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.835347:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.835349:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677472.835350:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677472.835350:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677472.835355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677472.835360:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677472.835361:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677472.835362:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.835363:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.835363:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:6.0:1619677472.835363:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.835364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.835365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677472.835365:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.835367:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677472.835488:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.835491:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.835492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.835493:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.835496:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.835498:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.835499:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.835500:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.835501:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.835502:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.835505:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.835507:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.835509:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.835509:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.835510:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.835511:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.835512:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.835512:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.835513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.835523:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.835524:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.835525:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.835526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.835527:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.835528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.836144:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.836146:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.836147:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.836148:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.836148:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:22.0:1619677472.836150:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.836151:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.836152:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:22.0:1619677472.836153:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677472.836154:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.836157:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.836158:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.836160:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.836161:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.836162:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.836163:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.836166:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.836167:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.836170:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.836171:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.836172:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.836174:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.836174:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.836175:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677472.836177:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677472.836177:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677472.836180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.836188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677472.836190:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:21.0:1619677472.836190:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.836191:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.836192:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677472.836193:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677472.836193:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.836194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.836194:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677472.836196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:12.0:1619677472.836438:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.836441:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.836443:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677472.836445:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.836446:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.836446:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677472.836448:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.836449:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.836450:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.836451:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677472.836454:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677472.836456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677472.836456:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677472.836458:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.836459:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677472.836459:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677472.836459:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677472.836461:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677472.836461:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:20.0:1619677472.836462:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:12.0:1619677472.836462:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.836463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:20.0:1619677472.836464:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.836465:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.836466:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.836468:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.836469:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96512004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.836470:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.836471:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.836472:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677472.836474:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:20.0:1619677472.836475:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.836477:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:20.0:1619677472.836477:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:26.0:1619677472.836478:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.836478:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:20.0:1619677472.836480:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677472.836481:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.836481:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.836482:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677472.836483:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.0:1619677472.836484:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.836484:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.836485:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.836486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.836490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.836498:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.836500:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.836501:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.836503:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.836503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.836505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.837064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677472.837067:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.837070:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:21.0:1619677472.837072:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677472.837072:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.837074:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.837075:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.837076:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96512003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.837078:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.837079:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.837080:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.837082:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.837084:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.837086:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.837086:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.837087:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.837088:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.837089:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.837089:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677472.837090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.837094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.837102:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.837104:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.837105:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.837107:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.837107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.837109:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677472.837119:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.837122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.837122:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677472.837124:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.837127:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.837128:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.837129:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96512002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.837131:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.837132:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.837133:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.837136:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.837138:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.837140:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.837141:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.837141:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.837143:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.837143:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.837144:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.837144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.837155:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.837157:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.837158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.837158:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.837160:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.837162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.837538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.837547:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.837550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.837550:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.837551:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.837554:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.837556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.837642:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.837645:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.837646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.837647:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.837650:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.837652:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.837653:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96512000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.837654:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.837655:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.837656:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.837659:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.837660:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.837663:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.837664:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.837665:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.837666:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.837667:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.837667:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.837668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.837682:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.837685:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.837686:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.837686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.837688:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.837692:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677472.837915:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.837918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.837918:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.837920:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.837921:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.837924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.838031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677472.838074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:2.0:1619677472.838079:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.838082:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.838083:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.838083:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.838085:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.838089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.839031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.839034:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.839037:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.839039:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.839040:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.839041:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.839043:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.839044:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.839047:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.839048:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.839050:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677472.839053:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677472.839053:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.839054:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677472.839056:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677472.839057:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.839057:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677472.839058:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:26.0:1619677472.839059:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.839059:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.839060:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:26.0:1619677472.839061:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.839061:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.839062:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:26.0:1619677472.839063:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677472.839063:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:26.0:1619677472.839064:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.839064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677472.839066:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.839067:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677472.839067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:26.0:1619677472.839068:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677472.839069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677472.839071:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.839073:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.839074:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:26.0:1619677472.839075:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.839076:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.839076:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677472.839076:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677472.839077:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.839077:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.839078:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677472.839078:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:26.0:1619677472.839079:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.839079:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677472.839079:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.839080:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677472.839080:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677472.839081:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677472.839081:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.839084:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.839090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677472.839093:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.839096:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.839097:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.839097:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677472.839097:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:22.0:1619677472.839099:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.839100:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.839101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.839101:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677472.839102:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677472.839102:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.839103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677472.839111:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.839113:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.839114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.839114:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.839115:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.839116:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.839820:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.839822:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.839823:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.839825:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.839828:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.839830:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.839831:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.839832:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.839833:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.839834:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.839838:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.839839:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.839842:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.839843:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.839843:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.839845:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.839845:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.839847:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.839848:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.839848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.839851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.839859:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.839862:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.839862:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.839863:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.839863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.839864:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.839864:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.839865:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677472.839865:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.839868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677472.840071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.840076:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.840079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.840079:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677472.840081:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.840083:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.840084:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.840085:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.840087:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.840088:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.840089:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.840092:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.840094:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.840097:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677472.840097:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.840097:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.840098:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.840100:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.840100:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677472.840100:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.840100:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.840101:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.840101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677472.840102:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.840105:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.840107:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.840108:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.840109:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.840110:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.840112:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.840115:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:10.0:1619677472.840115:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.840117:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677472.840117:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.840118:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.840120:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.840120:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.840121:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677472.840122:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.840122:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677472.840124:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677472.840125:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.840125:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.840126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.840128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.840130:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.840132:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.840132:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:22.0:1619677472.840133:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (24)++
00000800:00000200:22.0:1619677472.840134:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (25)--
00000800:00000200:6.0:1619677472.840134:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000800:00000200:22.0:1619677472.840135:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.840135:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.840136:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.840136:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.840139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.840240:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.840242:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.840243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.840244:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.840246:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.840248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.840260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.840268:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.840271:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.840272:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.840274:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.840277:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.840278:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677472.840280:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.840281:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.840282:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.840283:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677472.840287:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.840288:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677472.840291:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.840292:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677472.840292:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.840297:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.840298:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.840299:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.840299:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.840304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.840312:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.840315:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.840316:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.840316:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.840318:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.840321:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.840949:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.840951:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.840952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.840955:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.840958:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.840959:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677472.840960:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.840962:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.840964:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.840964:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677472.840968:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.840969:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677472.840972:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.840973:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677472.840974:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677472.840976:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677472.840976:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.840977:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.840977:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677472.840982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.840990:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.840992:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.840993:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.840994:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677472.840995:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.840995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677472.840996:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.840997:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677472.840997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.841000:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000400:00000200:8.0:1619677472.841004:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.841006:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.841007:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677472.841008:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000400:00000200:8.0:1619677472.841009:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.841010:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677472.841010:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.841011:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000400:00000200:8.0:1619677472.841011:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.841012:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.841012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:8.0:1619677472.841012:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.841013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:8.0:1619677472.841016:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.841017:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.841019:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.841019:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.841020:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.841021:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.841022:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.841022:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.841023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.841033:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.841035:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.841036:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.841037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.841038:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.841040:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.841058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.841061:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.841062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.841062:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.841063:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.841063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.841383:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.841386:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.841387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.841389:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.841391:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.841393:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.841394:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.841396:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.841397:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.841398:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.841401:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.841402:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.841405:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.841406:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.841406:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.841408:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.841408:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.841410:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.841411:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.841411:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.841415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.841422:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677472.841423:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.841424:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677472.841425:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677472.841425:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677472.841426:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.841426:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677472.841428:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.841430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.841430:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.841716:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.841718:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.841720:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.841721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.841723:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.841725:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.841726:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.841727:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.841728:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.841729:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.841733:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.841734:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.841737:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.841738:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.841738:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.841740:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.841741:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.841742:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.841742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677472.841862:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.841865:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.841866:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.841867:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.841870:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.841870:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:10.0:1619677472.841871:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:17.0:1619677472.841873:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:10.0:1619677472.841873:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677472.841873:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:10.0:1619677472.841875:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.841876:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.841877:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.841878:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677472.841879:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.841881:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.841881:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677472.841882:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677472.841882:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96511984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677472.841883:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:1.0:1619677472.841884:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677472.841885:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:1.0:1619677472.841885:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677472.841886:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677472.841886:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677472.841887:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.841887:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:6.0:1619677472.841887:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.841888:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.841889:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:6.0:1619677472.841889:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.841890:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.841890:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677472.841890:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.841891:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.841892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677472.841893:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677472.841896:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.841896:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677472.841897:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677472.841899:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677472.841899:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:24.0:1619677472.841901:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677472.841901:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677472.841902:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677472.841902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:24.0:1619677472.841904:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.841905:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.841905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.841907:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.841909:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677472.841912:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:17.0:1619677472.841913:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677472.841915:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:2.0:1619677472.841915:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:17.0:1619677472.841916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:2.0:1619677472.841916:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.841917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677472.841917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:2.0:1619677472.841918:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.841920:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677472.841922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.841923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.841923:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677472.841924:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.841925:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.841925:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.842021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.842028:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.842031:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.842031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.842033:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.842035:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.842037:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.842038:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.842040:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.842041:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.842042:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.842045:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.842047:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.842048:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.842049:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.842050:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.842052:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.842053:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.842054:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.842054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.842065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.842067:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.842068:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.842069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.842069:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.842071:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.842113:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.842116:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.842118:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.842119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.842121:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.842123:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.842124:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.842125:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.842126:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.842127:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.842130:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.842132:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.842134:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.842134:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.842135:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.842137:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.842138:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.842139:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.842140:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.842141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677472.842150:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677472.842151:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677472.842152:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.842153:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.842153:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.842154:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677472.842154:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.842155:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.842155:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.842157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.842216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.842222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.842224:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.842226:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.842227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.842229:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.842230:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.842231:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.842233:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.842233:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.842235:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.842237:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.842239:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.842241:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.842242:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.842243:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.842245:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.842245:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.842246:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.842246:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.842251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.842259:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.842261:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.842262:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.842263:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.842264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.842266:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.842270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.842278:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.842281:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.842281:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.842282:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.842283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.842284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.843321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.843329:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.843332:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.843333:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.843335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.843335:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.843337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677472.843436:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.843439:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.843440:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.843442:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.843444:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.843446:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677472.843447:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.843448:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.843449:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.843450:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677472.843453:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.843455:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677472.843458:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.843458:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677472.843459:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677472.843461:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677472.843462:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.843463:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.843463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677472.843486:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.843489:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.843490:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.843492:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.843493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.843495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677472.843505:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677472.843508:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.843509:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.843509:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677472.843510:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.843512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677472.843594:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.843597:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.843598:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.843600:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.843602:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.843604:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677472.843605:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.843607:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.843608:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.843609:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677472.843612:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.843614:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677472.843617:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.843617:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677472.843618:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677472.843620:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677472.843620:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.843622:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.843623:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.843623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677472.843626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.843634:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.843636:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.843637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677472.843637:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:1.0:1619677472.843638:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.843639:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677472.843640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:1.0:1619677472.843640:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.843641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:1.0:1619677472.843643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677472.843810:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.843813:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.843814:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.843815:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.843818:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.843820:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.843821:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.843822:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.843823:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.843824:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.843827:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.843828:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.843831:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.843832:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.843833:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.843834:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.843835:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.843836:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.843837:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.843837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677472.843848:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677472.843850:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677472.843851:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.843852:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.843852:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677472.843852:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.843853:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.843854:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677472.843854:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.843857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.843866:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.843869:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.843870:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.843871:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.843872:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.843872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.843891:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.843894:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.843895:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.843896:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.843898:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.843898:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:24.0:1619677472.843899:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.843901:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677472.843901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.843901:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:24.0:1619677472.843902:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.843902:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.843903:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.843905:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677472.843905:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.843908:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:24.0:1619677472.843909:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.843911:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.843914:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.843915:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.843915:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.843917:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.843918:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.843918:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.843919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.843923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.843931:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.843934:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.843934:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.843935:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.843937:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.843940:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.843941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.843948:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.843951:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.843952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.843952:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.843953:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.843954:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677472.844384:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.844387:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.844388:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.844389:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.844392:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.844394:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.844396:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.844397:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.844398:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.844399:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.844402:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.844404:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.844407:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.844407:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.844408:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.844410:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.844410:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.844411:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677472.844413:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.844417:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.844418:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.844421:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.844424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.844424:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677472.844425:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677472.844425:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677472.844426:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:10.0:1619677472.844426:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.844427:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677472.844427:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677472.844428:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:10.0:1619677472.844428:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677472.844428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.844429:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.844430:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96511972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.844432:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677472.844432:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.844433:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.844434:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677472.844436:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:10.0:1619677472.844438:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677472.844440:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677472.844442:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.844443:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677472.844444:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677472.844445:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.2:1619677472.844445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677472.844446:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.844446:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.844447:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677472.844454:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.844456:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.844457:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.844459:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.844460:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.844462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:13.0:1619677472.844469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.844472:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.844473:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.844474:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677472.844476:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.844477:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677472.844936:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.844938:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.844939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.844941:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.844943:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.844945:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.844946:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.844947:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.844948:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.844949:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.844952:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.844953:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.844955:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.844955:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.844956:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.844957:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.844957:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.844958:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.844959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.844969:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.844970:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.844971:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.844972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.844973:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.844974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.845126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.845135:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.845138:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.845138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.2:1619677472.845139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677472.845139:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.845141:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.845142:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.845144:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677472.845144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:20.0:1619677472.845146:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.845147:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.845148:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.845150:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.845151:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.845153:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.845154:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.845155:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.845158:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.845159:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.845162:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.845163:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.845163:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.845165:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.845166:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.845166:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.845167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.845171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.845178:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.845181:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.845182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.845182:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.845184:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.845186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677472.845298:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.845301:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.845302:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.845303:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.845306:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.845307:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677472.845308:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.845310:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.845311:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.845312:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677472.845315:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.845317:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677472.845320:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.845320:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677472.845321:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677472.845323:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677472.845323:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.845324:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.845324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677472.845329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.845337:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.845340:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.845340:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.845342:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.845343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.845345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677472.845439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.845445:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.845448:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.845448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.845450:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.845453:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.845454:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.845456:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.845457:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.845458:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.845459:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.845462:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.845463:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.845466:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.845466:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.845467:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.845469:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.845469:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.845470:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.845470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.845484:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.845488:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.845489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.845489:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.845491:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.845493:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.845527:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.845529:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.845530:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.845531:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.845533:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.845535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.845844:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.845847:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.845848:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.845849:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.845851:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.845852:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.845854:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.845855:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.845856:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.845857:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.845859:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.845861:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.845862:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.845863:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.845863:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.845864:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.845865:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.845866:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.845867:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.845868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.845878:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.845879:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.845880:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.845882:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.845882:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.845883:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.845883:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.845883:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.845884:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.845884:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677472.846793:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.846796:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.846798:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.846799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.846800:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.846801:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.846803:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.846804:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.846805:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.846806:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.846809:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.846811:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.846813:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.846814:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.846815:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.846816:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.846817:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.846818:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.846818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677472.846831:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.846833:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.846834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.846834:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.846836:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677472.846838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.847067:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.847070:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.847071:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.847073:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.847076:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.847077:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.847078:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.847079:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.847080:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.847081:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.847084:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.847085:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.847088:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.847089:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.847089:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.847091:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.847091:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.847092:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.847092:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.847103:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.847105:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.847106:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.847107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.847108:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.847110:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.847171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:18.0:1619677472.847222:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.847226:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.847228:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.847228:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.847231:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677472.847232:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.847232:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (23)++
00000800:00000200:2.0:1619677472.847234:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (24)--
00000800:00000200:2.0:1619677472.847235:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.847236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.847237:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677472.847534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.847542:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.847546:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.847547:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.847550:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.847553:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.847556:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.847558:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.847560:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.847562:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.847564:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.847569:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.847572:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.847576:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.847577:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.847578:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.847580:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.847581:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.847582:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.847583:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677472.847606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.847612:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.847614:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.847616:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.847617:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.847619:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.847620:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.847622:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.847623:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.847624:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.847625:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.847628:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.847630:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.847632:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.847633:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.847633:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.847635:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.847635:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.847641:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.847642:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.847643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.847650:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.847653:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.847654:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.847656:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.847656:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.847658:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.847686:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.847689:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.847689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.847690:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.847692:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.847695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.847705:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.847708:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.847709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.847709:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.847710:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.847711:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.847832:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677472.847834:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677472.847835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677472.847837:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.847840:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.847842:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677472.847844:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.847845:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.847846:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.847847:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677472.847850:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.847852:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677472.847855:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.847856:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677472.847856:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677472.847858:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677472.847858:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677472.847859:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677472.847859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677472.847864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.847872:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.847875:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.847876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.847876:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.847878:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.847880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.847977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.847984:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.847988:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.847990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.847993:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.847997:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.847999:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.848001:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.848004:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.848005:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.848007:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.848012:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.848015:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.848018:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.848019:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.848020:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.848022:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.848023:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.848025:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.848027:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.848028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.848033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.848040:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.848042:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.848042:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677472.848043:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.848043:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.848044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.848047:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.848047:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.848049:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.848049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.848063:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.848066:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.848068:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.848068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.848069:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.848070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.848571:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.848574:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.848574:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.848576:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.848579:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.848580:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.848582:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.848583:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.848584:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.848585:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.848588:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.848590:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.848593:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.848593:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.848594:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.848596:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.848596:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.848597:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.848597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.848610:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.848613:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.848614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.848614:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.848617:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.848620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.849068:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.849071:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.849072:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.849073:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.849076:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.849078:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677472.849079:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.849080:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.849081:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.849082:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677472.849085:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.849086:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677472.849089:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.849090:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.849090:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677472.849092:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.849092:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.849094:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677472.849095:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677472.849096:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677472.849107:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677472.849109:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677472.849109:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:26.0:1619677472.849111:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:18.0:1619677472.849111:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.849111:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.849112:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.849113:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677472.849113:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.849117:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677472.849195:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.849198:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.849199:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.849200:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.849203:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.849205:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.849206:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.849207:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.849208:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.849209:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.849212:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.849213:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.849216:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.849216:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.849217:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.849219:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.849219:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.849220:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.849220:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677472.849228:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.849231:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.849232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677472.849232:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:8.0:1619677472.849233:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677472.849235:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:8.0:1619677472.849235:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.849236:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.849236:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677472.849236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:8.0:1619677472.849237:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.849238:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.849239:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.849240:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677472.849241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:8.0:1619677472.849241:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.849244:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.849245:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.849248:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.849249:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.849249:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.849251:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.849251:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.849252:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677472.849252:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677472.849253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677472.849255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.849255:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.849256:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:8.2:1619677472.849257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677472.849257:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.849257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677472.849265:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.849268:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.849269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.849270:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.849272:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.849277:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.849278:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.849280:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.849280:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.849282:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.849285:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.849286:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.849287:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.849288:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.849290:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.849291:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.849300:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.849301:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1000] (21)++
00000800:00000200:17.0:1619677472.849302:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.849303:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.4.10@o2ib3
00000800:00000200:17.0:1619677472.849305:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677472.849305:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.849306:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.849307:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:1.0:1619677472.849307:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.849308:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677472.849309:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:1.0:1619677472.849310:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.849311:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.849312:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677472.849313:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:1.0:1619677472.849313:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.849314:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.849315:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:17.0:1619677472.849316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1000] (22)--
00000800:00000200:8.2:1619677472.849316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:1.0:1619677472.849316:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.849317:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.849319:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.849320:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.849321:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (23)++
00000800:00000200:1.0:1619677472.849322:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (24)--
00000800:00000200:1.0:1619677472.849323:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1000] (21)++
00000800:00000200:1.0:1619677472.849323:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1000] (22)--
00000800:00000200:21.0:1619677472.849324:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:1.0:1619677472.849324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1000] (21)--
00000800:00000200:21.0:1619677472.849327:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:21.0:1619677472.849327:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.849329:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.849330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:20.0:1619677472.849331:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000800:00000200:21.0:1619677472.849332:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (24)--
00000400:00000200:20.0:1619677472.849332:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.849333:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.849334:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.849334:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.849336:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)++
00000800:00000200:20.0:1619677472.849336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (24)--
00000800:00000200:21.0:1619677472.849337:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (24)--
00000800:00000200:21.0:1619677472.849337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.849338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.849404:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.849407:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.849408:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.849408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.849410:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.849413:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677472.849718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.849726:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.849729:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.849729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.849730:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.849732:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.849735:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677472.850238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677472.850243:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677472.850243:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.850246:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677472.850247:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677472.850247:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677472.850247:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677472.850250:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.850251:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.850252:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.850254:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.850254:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677472.850255:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677472.850256:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.850257:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.850257:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.850258:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.850259:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.850260:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.850260:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677472.850261:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677472.850265:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.850266:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.850268:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677472.850269:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677472.850270:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.850271:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.850271:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:24.0:1619677472.850272:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677472.850272:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677472.850273:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.850274:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677472.850274:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677472.850275:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677472.850275:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677472.850276:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677472.850276:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677472.850277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677472.850277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677472.850287:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:6.0:1619677472.850288:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:8.0:1619677472.850289:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.850290:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.850291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677472.850291:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.850292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:8.0:1619677472.850292:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:6.0:1619677472.850292:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.850300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677472.850301:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.850304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677472.850745:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.850747:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.850748:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.850749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.850750:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.850753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.850771:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.850774:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.850776:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.850776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.850782:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.850783:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.850785:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677472.850785:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:21.0:1619677472.850786:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.850787:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677472.850787:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:21.0:1619677472.850788:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677472.850788:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.850791:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.850792:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677472.850793:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.850795:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.850795:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677472.850796:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.850797:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.850798:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:17.0:1619677472.850798:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677472.850799:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:17.0:1619677472.850799:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677472.850800:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:17.0:1619677472.850800:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.0:1619677472.850801:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.850801:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.850802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:17.0:1619677472.850803:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.850804:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677472.850807:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.850807:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677472.850808:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677472.850810:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677472.850810:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.850811:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.850812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:22.0:1619677472.850814:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.850817:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.850818:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.850818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.850820:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.850822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677472.850825:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.850827:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.850828:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.850828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.850831:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.850833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.851163:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.851166:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.851167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.851168:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.851170:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.851171:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.851173:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.851174:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.851175:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.851176:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.851179:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.851181:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.851183:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.851183:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.851184:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.851185:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.851186:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.851186:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.851187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.851197:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.851199:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.851200:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.851201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.851202:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.851203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.851217:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.851219:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.851220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.851220:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.851221:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.851221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.851327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.851332:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.851335:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.851337:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.851338:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.851339:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.851341:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.851342:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.851344:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.851345:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.851346:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.851349:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.851351:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.851353:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.851354:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.851355:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.851357:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.851358:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.851359:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677472.851360:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.851361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.851364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677472.851366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.851371:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.851373:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677472.851373:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677472.851374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.851375:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677472.851375:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.851376:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.851376:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.851378:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.851381:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.851382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677472.851389:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.851392:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.851393:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.851393:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.851394:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.851395:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677472.851645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.851648:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.851649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.851649:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.851652:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.851658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.851659:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.851661:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.851662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.851662:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.851663:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.851664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.851806:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.851809:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.851810:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.851812:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.851815:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.851817:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.851818:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.851819:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.851820:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.851821:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.851824:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.851825:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.851827:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.851828:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.851828:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.851830:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.851830:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.851831:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.851831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.851841:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.851844:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.851844:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.851845:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.851846:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.851849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.852003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.852010:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.852013:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.852014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.852016:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.852018:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.852020:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677472.852021:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.852022:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.852024:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.852024:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677472.852027:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.852029:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.852031:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.852032:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677472.852033:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.852034:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.852035:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.852036:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.852037:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.852038:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.852041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677472.852043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677472.852048:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677472.852049:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.852051:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677472.852051:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677472.852052:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677472.852052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.852052:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.852053:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677472.852054:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677472.852057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.852059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.852067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.852069:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.852070:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.852070:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.852071:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.852072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.852716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.852722:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.852725:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.852726:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.852727:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.852730:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.852732:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677472.852733:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.852734:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.852735:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.852736:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677472.852739:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.852740:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.852742:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.852743:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.852743:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.852745:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.852746:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.852747:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677472.852748:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677472.852748:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.2:1619677472.852752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677472.852754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:21.0:1619677472.852760:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677472.852760:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677472.852762:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:21.0:1619677472.852763:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:2.0:1619677472.852763:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:21.0:1619677472.852764:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.852764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:21.0:1619677472.852765:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.852766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.852768:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677472.853321:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.853324:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.853325:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.853327:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.853329:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.853331:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.853332:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.853334:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.853335:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.853336:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.853339:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.853340:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.853344:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.853344:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.853345:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.853347:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.853347:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.853349:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.853350:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.853350:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677472.853362:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677472.853364:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677472.853365:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.853366:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.853366:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.853366:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677472.853366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.853368:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.853368:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.853371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.853418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.853421:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677472.853423:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677472.853424:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677472.853426:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677472.853427:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.853427:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.853427:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.853428:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.853430:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.853433:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:20.0:1619677472.853434:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.853436:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.853438:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.853439:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.853440:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.853441:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.853445:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.853447:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.2:1619677472.853447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.853449:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.853450:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.853451:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (23)++
00000800:00000200:20.0:1619677472.853453:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (24)--
00000800:00000200:20.0:1619677472.853453:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.853454:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:20.0:1619677472.853454:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.853455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677472.853456:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:21.0:1619677472.853457:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.853457:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000400:00000200:21.0:1619677472.853458:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.853458:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:20.0:1619677472.853459:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.853461:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.853461:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.853463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.853463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:22.0:1619677472.853701:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677472.853703:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677472.853704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677472.853706:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.853709:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.853710:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677472.853712:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.853714:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.853714:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.853716:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677472.853719:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.853721:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677472.853723:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.853724:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677472.853725:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677472.853727:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677472.853727:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677472.853728:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677472.853728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:26.0:1619677472.853740:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677472.853742:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677472.853743:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:18.0:1619677472.853744:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:10.0:1619677472.853744:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:18.0:1619677472.853745:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.853745:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:26.0:1619677472.853746:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.853748:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.853748:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677472.853750:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677472.853750:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:26.0:1619677472.853751:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.853753:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.853754:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.853755:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.853758:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.853760:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677472.853763:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.853764:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.853764:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.853766:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.853767:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.853768:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.853768:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677472.853780:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.853782:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.853783:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.853784:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.853786:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.853792:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.853794:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.853796:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.853797:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.853798:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.853799:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.853799:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:10.0:1619677472.854117:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.854119:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.854120:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.854122:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.854125:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.854126:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.854128:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.854129:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.854130:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.854131:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.854134:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.854135:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.854138:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.854139:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.854139:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.854141:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.854142:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.854143:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.854143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.854155:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.854158:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.854159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.854159:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.854161:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.854164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677472.854472:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.854475:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.854476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.854478:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.854480:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.854481:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.854483:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.854484:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.854485:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.854486:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.854489:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.854491:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.854494:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.854494:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.854495:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.854497:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.854497:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.854498:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.854498:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.854504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.854512:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.854514:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.854515:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.854517:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.854518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.854519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.854523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.854530:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.854532:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.854533:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.854534:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.854535:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.854536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.854679:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.854681:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.854684:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.854684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.854686:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.854687:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.854688:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.854690:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.854691:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.854692:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.854695:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.854697:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.854700:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.854701:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.854702:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.854703:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.854704:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.854705:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.854705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.854717:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.854719:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.854720:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.854720:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.854722:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.854725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.854830:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.854833:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.854833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.854835:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.854837:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.854839:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.854840:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.854841:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.854842:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.854843:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.854846:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.854848:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.854851:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.854852:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.854853:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.854854:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.854855:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.854856:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.854857:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.854857:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.854860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.854868:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677472.854870:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.854871:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.854872:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.854872:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677472.854872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.854873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677472.854874:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.854875:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677472.854878:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677472.854974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.854980:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.854983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.854983:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677472.854985:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.854987:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.854989:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.854990:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.854991:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.854992:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.854993:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.854996:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.854998:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.855001:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.855002:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.855002:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.855004:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.855004:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.855006:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677472.855006:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677472.855007:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677472.855012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.855017:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677472.855019:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677472.855019:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677472.855019:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.855020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.855021:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.855023:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.855023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.855025:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.855027:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677472.855101:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.855104:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677472.855105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677472.855106:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.855110:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.855111:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677472.855113:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.855114:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.855115:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.855116:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677472.855120:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.855122:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677472.855124:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.855125:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:24.0:1619677472.855125:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:24.0:1619677472.855127:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:24.0:1619677472.855127:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.855129:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677472.855129:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677472.855130:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:26.0:1619677472.855139:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.855142:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:8.0:1619677472.855142:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000400:00000200:26.0:1619677472.855143:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.855143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677472.855145:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.855145:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677472.855145:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677472.855146:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:26.0:1619677472.855147:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677472.855165:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.855168:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.855168:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677472.855169:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.855170:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.855170:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677472.855366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.855373:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.855376:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.855376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.855379:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.855381:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.855382:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.855384:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.855385:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.855386:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.855387:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.855390:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.855392:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.855394:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.855395:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.855396:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.855397:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.855398:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.855399:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.855399:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.855411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.855413:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.855414:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.855415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.855416:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.855417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677472.855552:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.855555:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.855556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.855556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.855558:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.855561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.855566:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.855569:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.855570:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.855570:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.855572:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.855575:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.855585:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.855588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.855588:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677472.855588:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.855589:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.855590:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.855936:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.855938:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.855939:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.855940:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.855943:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.855944:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.855945:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.855947:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.855948:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.855949:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.855951:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.855953:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.855956:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.855957:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.855957:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.855959:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.855959:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.855960:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.855960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.855973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.855976:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.855976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.855977:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.855979:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.855982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.856119:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.856122:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.856124:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.856125:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.856126:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.856128:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.856129:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.856131:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.856131:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.856133:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.856136:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.856137:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.856140:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.856141:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.856141:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.856143:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.856144:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.856145:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.856145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677472.857032:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.857035:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.857036:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.857038:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.857041:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.857042:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.857044:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.857045:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.857046:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.857047:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.857050:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.857052:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.857055:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.857055:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.857056:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.857058:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.857058:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.857059:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.857059:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.857073:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.857076:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.857077:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.857077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.857079:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.857082:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.857093:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.857096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.857096:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.857097:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.857098:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.857098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.857218:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.857221:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677472.857221:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000400:00000200:10.0:1619677472.857223:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677472.857226:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.857227:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677472.857228:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677472.857230:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677472.857230:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.857231:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.857231:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.857233:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.857234:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.857234:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677472.857234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.857236:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.857236:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677472.857239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.857239:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.857240:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677472.857241:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677472.857242:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677472.857243:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677472.857244:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677472.857244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:21.0:1619677472.857256:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.857259:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.857260:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.857262:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.857262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.857265:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677472.857279:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.857282:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.857283:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.857284:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.857284:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.857285:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677472.857315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677472.857318:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.857320:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677472.857322:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.857323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677472.857325:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.857326:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.857327:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.857329:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.857330:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.857331:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677472.857334:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.857336:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.857338:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.857339:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677472.857340:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677472.857341:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677472.857342:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677472.857342:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677472.857343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.857348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.857357:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.857360:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.857361:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.857361:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.857363:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.857365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.857445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.857454:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:20.2:1619677472.857456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677472.857456:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.857457:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.857457:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.857460:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.857462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677472.857463:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.857466:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.857467:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.857468:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.857471:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.857473:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677472.857474:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.857476:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.857477:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.857478:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677472.857482:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.857483:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677472.857487:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.857488:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677472.857488:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677472.857491:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677472.857492:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.857492:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.857493:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677472.857506:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.857508:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.857509:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.857511:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677472.857513:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.857514:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677472.857994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677472.858000:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.858003:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.858003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.858005:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.858008:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.858010:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677472.858011:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.858012:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.858013:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.858014:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677472.858017:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.858019:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677472.858022:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.858023:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677472.858023:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677472.858025:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677472.858026:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.858026:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.858027:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.0:1619677472.858038:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.858041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.858041:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.858042:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.858044:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.858047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.858189:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.858192:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.858193:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.858195:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.858198:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.858199:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.858200:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.858202:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.858203:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.858204:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.858207:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.858208:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.858211:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.858212:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.858213:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.858214:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.858215:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.858215:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.858216:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.858229:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.858232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.858232:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.858233:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.858235:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.858237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.858253:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.858256:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.858257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.858258:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.858261:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.858263:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.858264:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.858265:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.858266:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.858267:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.858270:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.858271:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.858273:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.858273:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.858274:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.858275:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.858276:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.858276:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.858277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.858287:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.858289:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.858289:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.858290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.858291:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.858293:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.858481:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.858483:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.858485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.858486:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.858489:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.858490:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.858492:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.858493:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.858494:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.858494:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.858498:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.858499:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.858502:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.858503:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.858503:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.858505:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.858505:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.858506:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.858506:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.858519:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.858522:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.858523:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.858523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.858525:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.858528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.858537:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.858540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.858540:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.858541:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.858542:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.858542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.858617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.858622:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.858625:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.858627:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.858627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.858630:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.858631:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.858632:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.858634:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.858635:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.858636:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.858639:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.858641:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.858643:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.858644:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.858644:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.858646:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.858647:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.858647:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.858648:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.858653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.858661:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.858663:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.858664:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.858666:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.858667:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.858668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677472.859104:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.859106:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.859107:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.859109:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.859112:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.859113:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677472.859115:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.859116:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.859117:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.859118:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677472.859121:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.859123:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.859124:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.859125:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677472.859126:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.859127:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.859127:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.859128:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.859129:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677472.859139:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.859140:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677472.859141:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.859142:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677472.859143:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.859145:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677472.859382:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.859385:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.859385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.859386:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.859388:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.859390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.859717:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.859720:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677472.859721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677472.859723:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.859725:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.859726:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677472.859727:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.859729:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.859730:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.859731:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677472.859734:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.859736:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677472.859739:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.859740:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677472.859740:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677472.859742:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677472.859742:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.859743:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677472.859743:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:5.2:1619677472.859753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.0:1619677472.859757:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.859759:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:1.0:1619677472.859759:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:21.0:1619677472.859760:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.859762:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.859762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.859762:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677472.859763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.859765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677472.859765:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.859767:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.859768:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.859770:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.859771:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.859772:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.859773:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.859777:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.859779:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.859781:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.859782:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.859783:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.859785:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.859785:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.859786:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.859786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.859799:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.859801:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.859802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.859802:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.859805:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.859807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677472.860146:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.860149:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.860150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.860152:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.860153:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.860155:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.860156:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.860157:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.860159:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.860159:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.860163:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.860165:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.860168:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.860169:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.860169:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.860171:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.860171:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.860172:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.860172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.860185:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.860188:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.860189:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.860190:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.860192:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.860195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.860408:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.860411:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.860411:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.860414:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.860416:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.860418:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.860419:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.860420:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.860421:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.860422:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.860425:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.860427:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.860429:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.860430:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.860431:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.860432:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:12.0:1619677472.860432:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:17.0:1619677472.860433:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.860434:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.860434:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:12.0:1619677472.860436:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.860439:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677472.860439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:13.0:1619677472.860440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.860443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.860445:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677472.860447:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:12.0:1619677472.860447:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677472.860450:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:12.0:1619677472.860450:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.860451:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.860451:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.860452:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:12.0:1619677472.860453:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.0:1619677472.860454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:12.0:1619677472.860459:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.860462:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:12.0:1619677472.860462:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677472.860467:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.860468:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:12.0:1619677472.860469:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:12.0:1619677472.860472:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:12.0:1619677472.860473:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.860475:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677472.860477:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677472.860478:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677472.860478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.860487:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.860489:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.860490:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677472.860491:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:1.0:1619677472.860491:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.860493:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.860494:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677472.860495:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.860496:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677472.860496:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677472.861056:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.861059:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.861059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.861060:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.861062:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.861066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.861068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.861069:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.861071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.861072:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.861073:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.861074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677472.861240:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.861242:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.861243:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.861245:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.861247:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.861249:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677472.861251:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.861253:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.861253:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.861254:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677472.861257:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.861259:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677472.861262:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.861263:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677472.861264:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677472.861266:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677472.861267:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.861267:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.861268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677472.861278:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.861281:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.861282:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.861284:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.861285:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.861287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677472.861833:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677472.861835:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677472.861835:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.861836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.861836:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.861838:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.861838:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:8.0:1619677472.861838:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:10.0:1619677472.861841:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.861841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:10.0:1619677472.861843:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.861844:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.861846:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.861847:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.861848:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.861849:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.861851:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.861853:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.861855:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.861856:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.861856:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.861858:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.861858:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.861859:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.861859:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.861869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.861871:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.861872:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.861873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.861874:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.861876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.862367:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.862369:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.862370:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.862371:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.862374:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.862376:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677472.862377:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.862378:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.862379:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.862380:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.862383:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.862385:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677472.862387:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.862388:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.862388:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.862390:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.862390:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.862391:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.862392:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677472.862405:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.862408:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.862409:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.862409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.862411:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677472.862414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.862747:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.862750:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.862751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.862752:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.862755:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.862756:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.862757:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.862758:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.862759:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.862760:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.862763:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.862765:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.862766:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.862767:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.862767:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.862768:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.862769:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.862770:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.862771:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.862772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.862782:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.862784:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.862784:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.862786:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.862786:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.862786:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.862787:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.862788:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.862788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.862789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677472.862958:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.862961:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.862962:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.862963:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.862966:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.862967:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.862968:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.862970:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.862971:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.862972:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.862975:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.862976:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.862979:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.862980:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.862980:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.862982:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.862982:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.862984:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.862985:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.862985:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677472.862996:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.862998:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677472.862999:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677472.863000:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677472.863000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.863000:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.863001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.863002:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677472.863002:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.863005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.863200:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.863203:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.863204:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.863204:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.863206:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.863209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.863260:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.863263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.863263:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677472.863266:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.863268:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.863270:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.863272:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.863273:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.863274:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.863275:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.863278:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.863279:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.863282:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.863283:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.863283:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.863285:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.863286:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.863286:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677472.863288:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.863288:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677472.863300:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677472.863301:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.863303:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677472.863303:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677472.863304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677472.863304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.863305:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.863306:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677472.863307:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.863309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677472.863402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.863404:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.863405:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.863406:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.863407:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.863409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.863423:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.863425:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.863426:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.863426:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.863427:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.863428:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.863661:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.863663:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.863664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.863665:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.863667:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.863668:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.863669:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.863670:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.863671:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.863672:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.863675:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.863676:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.863678:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.863679:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.863679:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.863680:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.863681:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.863681:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.863681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.863692:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.863693:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.863694:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.863695:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.863695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.863696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.863714:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.863716:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.863717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.863717:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.863718:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.863718:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.863788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.863795:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.863798:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.863799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.863800:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.863803:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677472.863804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:2.0:1619677472.863805:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.863807:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.863808:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.863809:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.863811:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677472.863811:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.863812:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677472.863815:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677472.863817:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677472.863817:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.863820:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.863820:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:18.0:1619677472.863821:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.863821:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:18.0:1619677472.863822:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677472.863823:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.863824:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.863826:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.863827:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.863828:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.863829:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.863830:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.863831:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.863831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.863833:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.863833:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.863834:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.863835:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.863836:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677472.863836:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.863836:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.863837:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:2.0:1619677472.863838:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.863839:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.863839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.863840:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.863840:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.863840:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.863841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.863845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.863853:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.863855:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.863856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.863856:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.863858:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:1.0:1619677472.863858:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:2.0:1619677472.863860:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677472.863861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677472.863862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.863862:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.863862:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.2:1619677472.863863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677472.863863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.863870:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.863873:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.863874:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.863874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.863875:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.863875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.864171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.864177:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.864179:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.864181:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.864182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.864183:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.864185:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.864186:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.864188:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.864188:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.864189:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.864192:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.864194:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.864196:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.864197:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.864198:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.864199:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.864200:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.864201:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.864202:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677472.864203:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677472.864205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.864209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.864215:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.864215:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677472.864216:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.864217:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.864217:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677472.864217:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.864218:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.864219:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.864220:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.864221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677472.864456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.864462:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.864464:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.864465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.864467:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.864470:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.864471:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.864473:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.864474:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.864475:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.864476:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.864480:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.864482:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.864484:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.864485:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.864486:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.864487:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.864488:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.864488:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.864489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.864502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.864505:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.864506:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.864506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.864508:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.864510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.864890:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.864893:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.864894:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.864894:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.864896:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.864899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.865588:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.865591:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.865592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.865593:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.865595:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.865597:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.865597:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.865599:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.865600:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.865601:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.865604:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.865605:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.865608:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.865609:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.865610:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.865612:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.865612:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.865613:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.865613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.865618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.865626:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.865629:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.865629:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.865630:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.865632:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.865635:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677472.866210:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.866215:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.866216:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.866219:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.866224:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.866226:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677472.866228:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.866231:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.866232:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.866234:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677472.866239:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.866242:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677472.866245:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.866247:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677472.866248:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677472.866250:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677472.866251:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.866252:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.866253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677472.866256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.866263:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.866266:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.866267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.866267:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.866269:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.866273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677472.866400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.866403:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.866405:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.866405:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.866406:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.866408:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.866409:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.866410:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.866411:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.866412:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.866416:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.866418:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.866421:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.866422:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.866422:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.866424:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.866425:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.866426:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.866427:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.866428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.866437:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.866439:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.866440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.866441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677472.866651:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.866656:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.866657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.866661:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.866665:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.866667:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.866669:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.866671:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.866673:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.866674:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.866679:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.866682:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.866685:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.866686:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.866687:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.866689:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677472.866689:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677472.866690:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.866691:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.866692:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677472.866692:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.866692:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.866693:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.866695:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.866699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677472.866702:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.866704:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.866705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.866706:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.866708:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.866710:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.867028:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.867030:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.867031:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.867032:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.867034:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.867036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.867045:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.867048:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.867048:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.867049:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.867050:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.867050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.867503:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.867506:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.867506:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677472.867508:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.867511:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.867512:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677472.867514:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.867515:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.867516:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.867517:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677472.867520:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.867521:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677472.867524:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.867525:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677472.867525:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677472.867527:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677472.867527:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.867528:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.867528:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:2.0:1619677472.867543:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.867546:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.867547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.867547:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.867549:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.867553:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.867927:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.867930:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.867931:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.867932:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.867936:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.867937:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.867938:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.867939:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.867940:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.867941:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.867943:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.867945:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.867946:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.867947:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.867947:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.867949:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.867949:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.867950:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.867950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.867961:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.867963:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.867963:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.867964:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.867965:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.867966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.868048:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.868051:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.868052:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.868054:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.868056:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.868058:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.868059:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.868060:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.868061:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.868062:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.868065:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.868066:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.868069:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.868070:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.868071:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.868072:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.868073:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.868073:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.868074:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.868086:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.868089:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.868090:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.868091:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.868093:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.868095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.868104:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.868107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.868107:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.868108:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.868108:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.868109:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677472.868355:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.868358:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.868359:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.868360:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.868363:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.868365:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.868366:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.868367:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.868368:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.868369:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.868372:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.868373:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.868376:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.868377:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.868377:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.868379:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.868379:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.868380:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.868381:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.868393:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.868395:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.868396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.868397:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.868399:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.868401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.868777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.868783:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.868786:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.868787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.868788:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.868791:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.868792:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.868793:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.868795:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.868796:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.868797:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.868801:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.868802:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.868805:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.868806:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.868807:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.868809:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.868809:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.868810:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.868811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677472.868844:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.868847:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.868849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.868851:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.868852:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.868853:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.868854:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.868856:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.868857:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.868857:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.868860:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.868862:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.868863:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.868864:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.868865:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.868865:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.868866:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.868867:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.868869:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.868869:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677472.868874:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.868877:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.868878:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.868878:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.868880:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:6.0:1619677472.868880:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.868881:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.868882:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.868883:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677472.868883:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.868884:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677472.868884:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.868885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.868886:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.868886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.868887:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677472.869029:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.869032:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.869032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.869034:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.869037:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.869038:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.869039:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.869041:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.869042:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.869043:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.869047:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.869049:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.869051:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.869052:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.869053:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.869054:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.869055:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.869056:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.869057:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.869058:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.869061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.869068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677472.869069:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.869070:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.869070:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677472.869071:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677472.869071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677472.869072:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.869074:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.869075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.869076:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.869080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.869088:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.869090:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.869091:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.869092:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.869093:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.869093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.869141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.869149:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.869151:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.869152:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.869153:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.869155:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.869158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677472.869234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.869239:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.869242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.869242:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677472.869244:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.869246:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.869248:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.869249:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.869250:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.869251:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.869252:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.869255:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.869257:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.869260:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.869261:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.869262:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.869263:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.869264:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.869264:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.869265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.869277:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.869281:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.869282:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.869282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.869284:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.869286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677472.869639:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.869641:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.869643:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.869644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.869646:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.869648:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.869649:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.869650:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.869652:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.869653:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.869656:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.869657:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.869660:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.869661:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.869662:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.869663:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.869664:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.869665:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.869666:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.869667:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.869675:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677472.869677:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.869678:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.869678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.869679:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.869680:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.869681:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.869681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.869683:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.869686:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.869949:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.869951:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.869952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.869954:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.869957:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.869958:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677472.869960:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.869961:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.869962:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.869963:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.869966:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.869968:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.869970:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.869971:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.869972:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.869974:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.869974:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.869975:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.869975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677472.869988:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.869990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.869991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.869991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.869994:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.869996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.870219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.870225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.870228:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.870229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.870231:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.870234:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.870235:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.870236:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.870238:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.870239:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.870240:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.870243:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.870245:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.870247:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.870247:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.870248:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.870250:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.870250:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.870251:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.870252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.870263:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.870265:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.870266:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.870266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.870268:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.870268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.870865:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.870868:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.870869:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.870871:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.870874:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.870876:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677472.870878:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.870879:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.870880:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.870882:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677472.870886:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.870888:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677472.870891:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.870892:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677472.870892:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677472.870894:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677472.870894:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.870896:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.870897:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.870898:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677472.870901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677472.870908:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677472.870909:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677472.870911:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677472.870911:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:6.0:1619677472.870912:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677472.870912:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:22.0:1619677472.870913:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.870913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677472.870913:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:1.0:1619677472.870913:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.870914:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.870915:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:1.0:1619677472.870916:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677472.870917:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.870920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677472.870922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.870924:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.870926:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.870926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.870927:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.870928:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.870928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.870957:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.870960:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.870960:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677472.870963:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677472.870965:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.870965:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677472.870967:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:24.0:1619677472.870968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.870968:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677472.870969:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.870970:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.870971:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.870971:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.870972:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677472.870972:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.870973:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.870974:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.870976:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677472.870976:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.870977:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677472.870978:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:8.0:1619677472.870978:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677472.870981:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677472.870981:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.870982:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.870982:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:8.0:1619677472.870983:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677472.870984:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.870984:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.870985:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.870986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.0:1619677472.870986:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.870986:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.870987:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.870989:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.870989:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.870990:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.870990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.870994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.870998:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.871001:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.871002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677472.871002:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:2.0:1619677472.871002:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.871004:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:17.0:1619677472.871005:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.871006:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.871006:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.871007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:17.0:1619677472.871008:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.871011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.871923:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.871926:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.871927:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.871928:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.871932:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.871933:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.871934:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.871935:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.871936:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.871937:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.871940:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.871942:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.871944:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.871944:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.871945:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.871946:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.871947:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.871948:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.871948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.871960:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.871962:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.871963:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.871963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.871964:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.871966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.872053:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.872056:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.872058:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.872059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.872061:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.872063:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.872064:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.872065:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.872066:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.872067:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.872070:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.872072:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.872074:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.872075:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.872075:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.872077:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.872078:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.872079:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.872079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677472.872091:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.872093:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.872094:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.872095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.872096:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.872098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.872360:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.872362:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.872363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.872364:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.872365:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.872368:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.872752:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.872755:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.872755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.872756:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.872758:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.872761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.872863:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.872866:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.872866:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677472.872868:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.872871:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.872872:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.872873:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.872875:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.872876:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.872876:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.872880:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.872881:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.872884:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.872885:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.872885:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.872887:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.872888:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.872888:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.872889:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.872900:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.872903:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.872904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.872904:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.872907:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.872909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.873183:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.873186:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.873187:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.873188:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.873191:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.873193:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.873194:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.873195:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.873196:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.873197:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.873200:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.873201:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.873204:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.873204:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.873205:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.873206:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.873207:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.873208:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.873209:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.873209:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677472.873220:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.873222:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677472.873222:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677472.873223:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.873223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.873225:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.873225:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.873225:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.873226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.873228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677472.873343:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.873346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.873346:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677472.873349:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.873351:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.873352:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677472.873353:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.873355:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.873356:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.873357:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677472.873359:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.873361:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.873363:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.873364:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.873364:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.873366:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.873366:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.873367:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.873367:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677472.873372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.873380:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.873382:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.873383:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.873384:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.873385:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.873386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.873391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.873398:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.873401:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677472.873401:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.873402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.873402:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.873403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677472.873492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.873497:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.873500:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.873501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.873502:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.873504:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.873505:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.873506:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.873507:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677472.873507:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.873508:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.873509:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677472.873510:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.873511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677472.873513:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.873514:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.873515:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677472.873517:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.873518:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:17.0:1619677472.873518:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.873519:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.873519:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:24.0:1619677472.873520:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.873521:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677472.873521:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.873521:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677472.873522:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.873523:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677472.873523:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677472.873524:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677472.873525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677472.873526:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.873528:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:5.2:1619677472.873529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677472.873530:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.873531:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.873531:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.873533:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.873533:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.873534:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.873534:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677472.873535:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677472.873536:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677472.873537:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677472.873538:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677472.873538:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.873539:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.873540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.873540:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.873542:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.873544:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:6.0:1619677472.873544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677472.873547:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.873547:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.873548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.873549:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.873551:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.873567:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.873569:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.873570:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.873571:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.873571:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.873571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:18.0:1619677472.873810:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.873813:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.873814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.873816:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.873818:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.873820:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677472.873821:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.873823:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.873824:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.873825:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677472.873828:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.873829:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.873833:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.873833:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.873834:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677472.873836:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677472.873836:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.873837:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677472.873838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:10.0:1619677472.873860:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.873863:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677472.873863:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677472.873864:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.873866:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.873869:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:22.0:1619677472.874803:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.874806:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.874807:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.874807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.874810:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.874812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.874820:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.874823:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.874824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.874825:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.874827:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.874829:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.874830:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.874831:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.874832:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.874833:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677472.874833:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:8.0:1619677472.874836:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.874837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.874837:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.874838:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677472.874838:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.874839:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.874839:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.874841:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.874841:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.874842:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.874844:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.874844:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.874845:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.874846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.874850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.874858:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.874861:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.874862:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.874862:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.874864:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.874867:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.874870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.874877:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.874880:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.874880:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.874881:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.874882:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.874882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.874953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.874959:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.874961:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.874962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.874964:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.874967:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.874969:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677472.874970:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.874972:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.874973:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.874974:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677472.874978:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.874980:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677472.874983:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.874984:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677472.874985:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677472.874987:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677472.874988:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.874990:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.874991:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.874991:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.874997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677472.875002:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677472.875003:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.875005:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:12.0:1619677472.875006:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677472.875007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677472.875007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:12.0:1619677472.875008:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.875010:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677472.875011:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.875014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677472.875104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.875111:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.875114:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.875115:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.875115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.875117:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.875120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.875192:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.875196:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.875196:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.875198:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.875201:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.875202:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677472.875204:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.875205:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.875206:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.875207:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677472.875210:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.875211:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677472.875213:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:26.0:1619677472.875214:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.875215:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677472.875215:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677472.875216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.875216:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:26.0:1619677472.875217:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677472.875217:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.875218:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000400:00000200:1.0:1619677472.875218:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677472.875219:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:1.0:1619677472.875220:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.875222:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.875223:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.875224:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.875225:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.875226:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.875229:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.875231:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:1.0:1619677472.875231:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:13.0:1619677472.875234:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:1.0:1619677472.875234:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.875234:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:13.0:1619677472.875235:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.875235:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:13.0:1619677472.875237:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.875237:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.875237:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:12.0:1619677472.875238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677472.875238:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.875239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:13.0:1619677472.875241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677472.875243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:12.0:1619677472.875248:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.0:1619677472.875251:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:12.0:1619677472.875251:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.875252:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.875253:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677472.875253:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677472.875254:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:12.0:1619677472.875254:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:21.0:1619677472.875255:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.875256:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.875258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.875258:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.875259:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.875260:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.875260:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.875262:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.875265:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.875266:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.875268:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.875269:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.875270:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.875271:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.875274:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.875276:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.875278:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.875279:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.875280:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.875282:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.875282:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.875283:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.875283:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.875298:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.875301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.875302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.875302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.875304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.875306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677472.875877:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.875880:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.875882:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.875882:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.875884:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.875886:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.875887:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.875889:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.875890:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.875891:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.875893:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.875895:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.875897:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.875898:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.875899:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.875900:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.875901:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.875902:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.875902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677472.875915:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.875917:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.875918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.875918:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.875920:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677472.875922:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.876136:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.876139:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.876140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.876142:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.876145:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.876146:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.876148:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.876149:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.876150:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.876151:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.876154:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.876156:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.876159:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.876159:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.876160:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.876162:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.876162:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.876163:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.876165:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.876165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.876168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.876176:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677472.876178:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677472.876179:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.876180:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.876180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.876181:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.876181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.876182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677472.876182:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.876185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677472.876414:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.876417:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.876418:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.876419:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.876422:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.876423:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677472.876425:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.876426:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.876427:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.876428:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677472.876430:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.876432:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677472.876434:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.876435:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677472.876435:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677472.876437:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677472.876438:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.876438:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.876439:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677472.876451:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.876454:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.876454:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.876456:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.876458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.876459:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677472.876753:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.876756:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.876757:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.876758:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.876761:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.876762:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.876764:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.876765:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.876766:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.876766:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.876769:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.876770:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.876772:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.876773:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.876773:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.876775:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.876775:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.876776:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.876776:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.876788:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.876790:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.876791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.876791:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.876792:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.876795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.876804:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.876807:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.876807:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.876808:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677472.876808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.876809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.876971:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.876974:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.876974:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.876975:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.876977:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.876980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.877262:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.877264:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.877265:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.877267:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.877270:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.877272:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677472.877274:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:10.0:1619677472.877274:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.877275:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677472.877276:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677472.877277:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.877278:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.877281:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.877282:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677472.877283:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.877284:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.877285:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677472.877286:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.877287:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.877288:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.877289:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677472.877292:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.877293:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:26.0:1619677472.877300:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.877300:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.877301:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677472.877302:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.877303:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.877304:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.877304:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677472.877305:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.877305:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677472.877305:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.877306:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677472.877308:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677472.877309:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.877310:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.877310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677472.877318:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.877321:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.877323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.877323:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.877325:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.877330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.877332:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677472.877334:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.877335:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.877335:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.877336:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.877337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677472.877755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.877761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.877764:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.877765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.877767:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.877770:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.877771:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.877772:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.877774:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.877775:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.877776:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.877779:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.877780:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.877783:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.877784:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.877784:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.877786:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.877786:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.877787:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.877787:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.877800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.877803:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.877804:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.877804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.877806:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.877808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677472.878057:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.878061:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.878062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.878064:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.878067:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.878068:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.878070:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.878071:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.878072:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.878073:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.878077:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.878078:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.878081:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.878082:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.878083:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.878084:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.878085:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.878085:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.878086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.878097:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.878100:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.878101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.878101:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.878104:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.878107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.878117:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.878119:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.878120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.878121:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.878121:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.878122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.878687:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.878690:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.878690:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.878691:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.878693:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.878696:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.878770:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.878774:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.878775:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.878776:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.878779:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.878780:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.878781:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.878783:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.878784:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.878785:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.878787:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.878789:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.878790:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.878791:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.878792:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.878793:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.878794:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.878794:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.878795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.878805:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.878807:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.878808:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.878808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.878810:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.878811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.879205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.879212:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.879215:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.879217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.879218:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.879221:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:2.0:1619677472.879222:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.879223:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:2.0:1619677472.879223:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.879224:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.879224:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.879225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:2.0:1619677472.879226:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.879227:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:2.0:1619677472.879228:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.879229:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.879230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:2.0:1619677472.879232:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.879235:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.879237:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.879237:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.879238:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.879240:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.879240:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.879241:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.879242:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.879243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.879249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.879255:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.879256:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.879256:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677472.879257:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.879257:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.879257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.879258:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.879259:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.879259:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.879260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.879600:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.879603:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.879605:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.879606:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.879607:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.879608:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.879610:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.879611:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.879612:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.879613:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.879615:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.879617:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.879619:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.879620:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.879621:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.879622:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.879623:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.879624:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.879625:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.879626:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677472.879637:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677472.879637:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677472.879639:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677472.879639:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:22.0:1619677472.879640:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.879640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.879640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677472.879641:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.879643:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.879646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.879756:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677472.879759:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677472.879760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677472.879762:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.879764:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.879766:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677472.879767:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.879769:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.879770:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.879771:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677472.879774:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.879776:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677472.879778:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.879779:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677472.879780:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677472.879781:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677472.879782:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677472.879783:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677472.879783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:2.0:1619677472.879795:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.879798:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.879799:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.879800:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.879801:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.879804:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.879813:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.879816:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.879817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.879818:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.879819:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.879819:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.880109:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.880112:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.880114:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.880114:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.880116:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.880117:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.880119:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.880120:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.880121:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.880122:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.880126:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.880127:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.880130:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.880131:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.880132:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.880133:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.880134:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.880135:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.880135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.880146:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.880149:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.880150:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.880150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.880152:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.880155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.880288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677472.880291:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.880297:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677472.880297:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677472.880299:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.880301:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.880303:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677472.880304:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.880305:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.880306:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.880307:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677472.880311:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.880312:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.880314:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.880315:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.880316:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.880318:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.880318:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.880319:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677472.880319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.880324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.880333:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.880335:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.880335:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.880337:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.880338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.880339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677472.880365:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.880367:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.880370:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.880370:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.880372:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.880373:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.880374:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.880375:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.880376:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.880377:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.880380:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.880382:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.880383:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.880384:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.880384:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.880386:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.880386:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.880387:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.880387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677472.880393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677472.880400:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677472.880400:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677472.880403:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:2.0:1619677472.880403:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677472.880404:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.880405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677472.880405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:2.0:1619677472.880406:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:22.0:1619677472.880407:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:2.0:1619677472.880409:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.880410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:2.0:1619677472.880410:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677472.880411:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.880413:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.880414:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.880415:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677472.880418:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.880420:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.880423:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.880424:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677472.880424:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.880426:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.880427:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.880428:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.880428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.880432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.880440:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.880443:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.880444:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.880444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.880447:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677472.880449:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677472.880736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677472.880742:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.880744:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677472.880745:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677472.880747:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.880749:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.880750:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677472.880751:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.880752:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.880753:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.880754:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677472.880758:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.880760:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677472.880763:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.880764:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677472.880764:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677472.880766:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:1.0:1619677472.880766:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677472.880767:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677472.880767:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.0:1619677472.880771:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.880774:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.880775:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.880777:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.880778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.880780:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.880781:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.880782:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.880783:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.880785:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.880788:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.880790:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.880793:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.880794:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.880794:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.880796:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.880796:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.880797:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.880797:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.880802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.880810:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)++
00000800:00000200:17.0:1619677472.880813:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (24)--
00000400:00000200:17.0:1619677472.880814:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.880814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.880816:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.880818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.880821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677472.880822:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:17.0:1619677472.880823:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.880825:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:17.0:1619677472.880825:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.880825:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.880826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:1.0:1619677472.880826:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:24.0:1619677472.880828:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.880828:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.880828:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.880830:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)++
00000800:00000200:1.0:1619677472.880830:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (24)--
00000400:00000200:24.0:1619677472.880831:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.880831:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (24)--
00000800:00000200:1.0:1619677472.880831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.880832:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:24.0:1619677472.880833:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.880834:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.880835:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.880837:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.880837:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.880840:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.880842:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.880844:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.880844:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.880845:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.880846:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.880847:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.880847:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.880848:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677472.880869:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.880871:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.880872:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.880872:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.880873:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.880875:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.880898:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.880901:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.880902:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.880904:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.880907:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.880908:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677472.880909:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.880910:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.880911:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.880912:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677472.880915:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.880916:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.880918:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.880919:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677472.880919:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.880921:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.880921:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.880922:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.880924:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.880924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.880935:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.880936:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677472.880936:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.880938:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:10.0:1619677472.880938:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.880939:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677472.880939:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.880940:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.880941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677472.880941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677472.881164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.881167:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.881168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.881168:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.881170:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.881173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.881183:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.881186:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.881186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.881187:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.881188:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.881188:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677472.881256:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.881259:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.881261:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.881261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.881263:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.881264:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.881266:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.881267:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.881268:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.881269:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.881272:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.881274:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.881277:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.881278:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.881278:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.881280:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.881281:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.881282:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.881282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.881292:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.881299:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.881300:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.881301:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.881303:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.881307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.881308:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.881310:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.881311:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.881311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.881312:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.881312:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.881500:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.881502:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.881503:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.881505:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.881507:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.881509:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.881510:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.881511:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.881512:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.881513:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.881516:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.881517:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.881520:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.881521:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.881521:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.881523:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.881523:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.881524:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.881524:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.881535:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.881537:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.881538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.881538:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.881540:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.881542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.882056:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.882059:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.882060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.882061:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.882063:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.882064:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.882065:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.882067:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.882067:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.882068:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.882071:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.882073:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.882075:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.882076:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.882077:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.882078:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.882079:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.882080:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.882080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.882093:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.882096:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.882097:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.882097:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.882099:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.882102:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.882150:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.882152:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.882153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.882155:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.882157:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677472.882157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677472.882159:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.882160:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.882161:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.882162:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.882163:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.882163:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.882166:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.882166:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677472.882168:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.882169:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677472.882170:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.882172:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.882174:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677472.882174:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677472.882175:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.882175:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:1.0:1619677472.882176:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677472.882177:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:1.0:1619677472.882177:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677472.882178:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677472.882178:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677472.882179:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677472.882179:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.882180:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.882181:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677472.882182:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.882184:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.882187:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.882188:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677472.882188:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (23)++
00000800:00000200:1.0:1619677472.882190:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (24)--
00000800:00000200:6.0:1619677472.882191:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:1.0:1619677472.882191:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.882191:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.882192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677472.882193:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677472.882195:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677472.882195:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000800:00000200:22.0:1619677472.882196:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:10.0:1619677472.882196:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677472.882196:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.882197:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.882197:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000400:00000200:22.0:1619677472.882197:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.882198:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677472.882199:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.882202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.882202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.882203:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677472.882448:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.882451:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.882451:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677472.882454:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.882457:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.882458:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677472.882459:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.882461:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.882462:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.882463:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677472.882466:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.882468:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677472.882470:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.882471:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677472.882472:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677472.882473:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677472.882474:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.882474:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.882475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677472.882480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.882489:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.882492:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677472.882492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.882493:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.882495:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.882498:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677472.882538:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.882541:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.882542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.882542:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.882544:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.882546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.882602:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.882605:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.882605:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.882608:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.882610:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.882611:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.882613:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.882614:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.882615:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.882616:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.882619:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.882621:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.882623:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.882624:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.882624:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.882626:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.882626:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.882627:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.882628:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.882632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.882640:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.882642:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.882643:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.882645:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.882646:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.882648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.882908:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.882910:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.882911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.882912:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.882914:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.882916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.882940:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.882943:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.882944:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.882945:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.882948:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.882950:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.882951:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.882952:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.882954:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.882954:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.882958:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.882959:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.882962:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.882963:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.882963:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.882965:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.882965:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.882967:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.882968:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.882968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.882981:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.882982:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.882983:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.882984:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.882985:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.882988:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.882989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.882989:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.882991:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.882994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.883173:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.883176:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.883177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.883178:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.883181:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.883182:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677472.883183:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.883185:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.883186:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.883187:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.883190:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.883192:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677472.883194:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.883195:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677472.883195:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677472.883198:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677472.883198:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.883200:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677472.883201:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677472.883202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:22.0:1619677472.883211:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.883214:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.883215:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.883215:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:6.0:1619677472.883215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.883217:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.883217:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677472.883218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677472.883218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:22.0:1619677472.883221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.883404:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.883407:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.883408:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.883410:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.883413:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.883414:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677472.883415:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.883416:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.883417:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.883418:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677472.883421:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.883423:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677472.883425:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.883425:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677472.883426:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677472.883427:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677472.883428:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.883428:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.883429:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.0:1619677472.884225:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.884227:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.884228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.884230:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.884233:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.884235:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.884236:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.884237:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.884238:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.884239:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.884242:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.884243:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.884244:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.884245:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.884246:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.884247:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.884247:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.884248:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.884250:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.884250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.884260:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.884261:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.884262:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.884263:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.884264:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.884264:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.884265:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.884265:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.884266:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.884266:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677472.884629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677472.884635:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.884637:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677472.884640:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.884640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677472.884642:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.884644:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.884645:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.884646:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.884648:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.884649:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.884652:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.884653:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.884656:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.884656:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.884657:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.884658:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.884659:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.884660:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:21.0:1619677472.884661:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677472.884662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677472.884665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.884668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677472.884675:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.884676:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.884676:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.884676:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:21.0:1619677472.884677:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.884677:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677472.884678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677472.884679:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.884680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677472.884681:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:6.0:1619677472.884972:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.884975:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.884976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.884977:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.884979:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.884981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.884991:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.884993:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.884994:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.884994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.884995:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.884996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.885307:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.885310:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677472.885311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677472.885312:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.885315:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.885317:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677472.885318:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.885320:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.885321:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.885322:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677472.885325:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.885326:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677472.885329:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.885329:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:24.0:1619677472.885330:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:24.0:1619677472.885331:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:24.0:1619677472.885332:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.885332:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677472.885333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:13.0:1619677472.885345:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.885348:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.885348:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.885350:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.885352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.885353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677472.886040:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.886042:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.886043:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.886043:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.886045:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.886047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.886090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.886097:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.886099:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.886101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677472.886102:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000400:00000200:2.0:1619677472.886102:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.886105:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:2.0:1619677472.886105:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.886106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:2.0:1619677472.886106:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.886108:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.886108:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.886109:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.886110:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.886111:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.886111:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.886112:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677472.886113:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.886115:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.886116:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.886117:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.886118:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.886118:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.886120:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.886121:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.886122:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:24.0:1619677472.886123:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.886124:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.886124:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.886124:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.886125:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.886126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677472.886127:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.886127:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677472.886128:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677472.886130:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677472.886130:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.886131:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677472.886133:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677472.886133:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:18.0:1619677472.886137:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.886139:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.886140:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.886141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.886142:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.886142:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:18.0:1619677472.886144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.886145:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:8.0:1619677472.886145:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:26.0:1619677472.886146:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677472.886146:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.886148:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.886148:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677472.886148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.886149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:10.0:1619677472.886150:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677472.886171:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677472.886174:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677472.886175:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.886175:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.886176:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.886177:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:17.0:1619677472.886285:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:20.2:1619677472.886287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677472.886288:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.886289:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.886292:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.886296:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677472.886300:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.886302:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677472.886303:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.886304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677472.886304:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.886305:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.886306:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.886309:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:17.0:1619677472.886310:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.886313:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.886313:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677472.886316:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677472.886316:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.886317:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677472.886317:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677472.886319:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:2.0:1619677472.886319:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:17.0:1619677472.886320:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.886320:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.886321:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:2.0:1619677472.886321:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.886323:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.886325:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.886326:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.886332:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677472.886333:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.886335:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:2.0:1619677472.886335:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677472.886336:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677472.886337:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.886339:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.886340:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.886341:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677472.886342:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:2.0:1619677472.886342:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.886345:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.886346:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.886347:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.886348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677472.886653:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.886656:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.886657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.886659:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.886661:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.886662:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.886664:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.886665:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.886666:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.886667:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.886670:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.886671:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.886673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677472.886674:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.886675:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677472.886675:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:26.0:1619677472.886676:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677472.886676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677472.886678:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.886678:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677472.886678:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677472.886679:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.886679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677472.886679:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.886681:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.886682:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.886683:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.886684:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.886685:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.886688:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.886689:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.886692:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.886692:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677472.886693:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.886693:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.886695:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.886695:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677472.886695:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.886696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.886696:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677472.886696:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.886697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677472.886698:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.886701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677472.886701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677472.886709:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.886711:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.886712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.886713:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.886715:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.886718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677472.886845:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.886847:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.886848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.886849:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.886851:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.886854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677472.886893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.886900:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.886903:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.886904:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.886904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.886907:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.886910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677472.887017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677472.887019:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.887022:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:1.0:1619677472.887022:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677472.887023:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.887024:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.887025:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.887026:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677472.887027:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.887028:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.887029:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.887030:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.887030:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.887032:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.887032:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.887033:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.887034:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.887035:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.887035:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.887037:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.887038:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.887040:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.887043:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.887045:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.887046:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.887047:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.887047:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.887048:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.887050:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.887050:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.887051:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.887051:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677472.887051:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.887051:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.887052:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677472.887054:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677472.887054:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.887055:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.887056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.887063:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:22.0:1619677472.887066:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (24)++
00000800:00000200:6.0:1619677472.887066:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000800:00000200:22.0:1619677472.887067:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000400:00000200:6.0:1619677472.887067:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.887068:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.887069:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677472.887070:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.887072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.887073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.887074:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.887076:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.887076:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.887077:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.887078:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.887078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677472.887192:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.887194:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.887196:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.887197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.887198:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.887200:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.887201:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.887202:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.887203:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.887204:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.887208:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.887210:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.887212:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.887213:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.887214:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.887216:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.887216:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.887217:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.887218:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.887219:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.887229:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677472.887229:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677472.887232:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:6.0:1619677472.887232:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.887233:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.887233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:6.0:1619677472.887233:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.887234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677472.887235:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677472.887238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.887845:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.887848:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.887848:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.887850:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.887854:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.887855:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.887856:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.887858:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.887859:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.887860:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.887863:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.887865:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.887868:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.887869:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.887870:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.887872:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.887872:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.887873:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.887873:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.887886:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.887889:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.887890:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.887890:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.887892:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.887895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.887902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.887905:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.887906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.887906:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.887907:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.887908:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677472.888706:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.888709:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.888710:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.888710:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.888713:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.888716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.888840:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.888843:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677472.888844:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677472.888844:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.888845:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677472.888847:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.888847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:24.0:1619677472.888848:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.888848:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.888850:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677472.888851:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.888852:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.888853:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.888854:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677472.888854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:24.0:1619677472.888855:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.888858:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.888860:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.888861:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.888862:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.888863:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.888864:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.888864:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.888865:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.888866:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.888876:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.888878:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.888879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.888879:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.888881:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.888882:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.889267:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.889269:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.889270:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.889271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.889272:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.889274:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.889287:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.889289:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.889290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.889290:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.889291:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.889291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.889829:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.889832:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.889833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.889834:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.889836:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.889837:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.889838:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.889840:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.889840:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.889841:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.889844:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.889846:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.889849:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.889850:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.889850:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.889852:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.889852:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.889853:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.889853:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.889867:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.889870:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.889871:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.889872:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.889873:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.889877:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.889886:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.889889:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.889890:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.889890:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.889891:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.889892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677472.890504:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.890507:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.890508:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.890510:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.890513:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.890514:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.890516:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.890517:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.890518:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.890519:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.890522:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.890524:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.890526:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.890527:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.890528:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.890529:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.890530:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.890530:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.890530:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.890549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.890557:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.890560:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.890560:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.890563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677472.890567:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.890569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.890607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:6.0:1619677472.890612:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.890614:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.890615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.890615:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677472.890616:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.890617:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677472.890618:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.890619:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.890619:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.890620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.890621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677472.890621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677472.891493:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.891496:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.891498:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.891499:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.891501:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.891503:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.891504:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677472.891505:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677472.891506:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.891507:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.891508:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.891508:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.891509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677472.891513:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677472.891513:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.891515:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.891516:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.891517:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677472.891518:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.891518:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.891519:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677472.891519:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.891520:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.891521:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.891521:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:10.0:1619677472.891521:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.891522:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000400:00000200:10.0:1619677472.891522:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677472.891523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:10.0:1619677472.891525:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.891527:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.891529:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.891530:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.891530:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.891532:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.891532:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.891533:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.891533:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.891536:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.891538:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.891539:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.891539:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.891541:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.891542:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677472.891543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.891544:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.891546:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:8.0:1619677472.891546:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677472.891547:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.891548:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.891549:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.891550:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.891551:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.891552:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.891554:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.891555:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.891556:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.891558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.891559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.891559:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.891560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.891561:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.891565:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.891565:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.891566:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.891568:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.891568:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.891569:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.891571:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.891571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.891574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.891582:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677472.891584:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677472.891585:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677472.891586:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677472.891586:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.891586:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.891587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.891588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677472.891588:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.891592:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.891593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.891600:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.891603:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.891604:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.891604:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.891605:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.891605:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.891652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.891655:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677472.891656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677472.891658:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.891661:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.891662:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677472.891663:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.891665:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.891666:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.891667:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677472.891670:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.891671:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677472.891673:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.891674:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677472.891674:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677472.891676:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677472.891677:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.891678:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677472.891678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:26.0:1619677472.891688:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.891690:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.891691:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.891691:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677472.891693:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.891695:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:5.2:1619677472.891745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.891751:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.891753:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.891754:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.891756:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.891758:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.891759:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.891760:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.891762:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.891762:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.891763:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.891766:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.891768:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.891770:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.891771:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.891772:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.891773:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.891774:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.891776:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677472.891777:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677472.891777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677472.891782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677472.891787:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677472.891788:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677472.891790:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677472.891790:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677472.891790:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.891791:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.891791:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.891792:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677472.891793:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.891795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677472.891826:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.891828:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.891829:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.891831:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.891834:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.891835:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.891836:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.891839:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.891840:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.891841:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.891844:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.891846:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.891849:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.891849:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.891850:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.891852:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.891852:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.891853:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.891853:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.891865:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.891867:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.891868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.891869:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.891871:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.891873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677472.892014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.892022:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.892025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.892025:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.892026:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.892028:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.892032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.892451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.892459:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.892461:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.892463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.892464:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.892467:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.892468:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677472.892470:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.892471:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.892472:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.892473:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677472.892477:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.892479:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677472.892482:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.892483:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677472.892484:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677472.892486:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677472.892487:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.892488:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.892488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677472.892501:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.892505:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.892506:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.892508:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677472.892509:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.892512:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677472.892688:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.892690:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.892691:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.892693:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.892696:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.892698:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.892699:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.892700:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.892701:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.892702:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.892705:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.892707:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.892710:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.892710:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.892711:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.892713:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.892713:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.892714:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.892715:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.892716:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677472.892726:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677472.892728:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677472.892729:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.892730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.892730:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677472.892730:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.892731:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.892732:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677472.892733:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.892735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.892743:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.892747:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.892747:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.892748:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.892749:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.892749:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.892955:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.892957:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.892958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.892958:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.892960:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.892963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.893042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.893048:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.893051:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.893052:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.893054:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.893057:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.893058:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.893060:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.893061:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.893062:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.893063:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.893067:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.893069:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.893071:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.893071:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.893072:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.893074:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.893074:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.893075:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.893076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.893087:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.893089:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.893090:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.893091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.893092:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.893093:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:12.0:1619677472.893252:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.893255:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677472.893257:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677472.893258:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.893260:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.893262:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677472.893263:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.893264:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.893265:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.893266:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677472.893270:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.893271:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677472.893274:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.893275:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:12.0:1619677472.893276:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:12.0:1619677472.893278:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:12.0:1619677472.893278:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.893279:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.893280:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677472.893284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.893293:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.893299:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.893300:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:20.2:1619677472.893301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:17.0:1619677472.893302:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.893304:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.893307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677472.893308:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.893311:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.893312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677472.893313:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.893315:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.893316:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.893318:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.893319:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.893320:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.893321:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.893325:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.893327:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.893330:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.893331:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.893331:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.893333:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.893334:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.893335:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677472.893335:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677472.893733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.893741:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.893744:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.893745:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.893745:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.893747:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.893750:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677472.894267:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.894270:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.894271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.894272:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.894274:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.894277:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677472.894287:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.894290:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677472.894290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.894291:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.894292:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.894292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.894359:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.894363:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.894364:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.894365:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.894368:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.894370:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.894371:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.894372:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.894373:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.894374:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.894377:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.894379:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.894382:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.894383:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.894383:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.894385:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.894385:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.894386:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.894386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.894400:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.894402:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.894403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.894404:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.894406:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.894409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.894895:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677472.894897:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677472.894897:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.894898:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.894898:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.894899:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:8.0:1619677472.894901:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:21.0:1619677472.894902:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.894902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677472.894903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:21.0:1619677472.894904:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.894905:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.894906:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.894908:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.894909:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.894910:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.894913:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.894915:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.894918:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.894919:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.894919:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.894921:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677472.894921:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.894922:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.894923:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.894926:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.894930:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.894930:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (24)++
00000800:00000200:8.0:1619677472.894930:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677472.894931:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (25)--
00000800:00000200:8.0:1619677472.894931:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677472.894932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.894932:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.894933:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.894934:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.894938:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.895203:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.895206:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.895207:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.895207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.895209:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.895212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.895518:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.895521:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.895522:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.895523:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.895526:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.895527:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.895528:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.895530:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.895531:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.895531:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.895535:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.895536:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.895539:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.895540:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.895541:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.895542:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.895543:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.895543:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.895544:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.895558:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.895561:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.895562:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.895562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.895564:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.895567:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.896126:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.896128:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.896129:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.896132:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.896135:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.896136:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.896137:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.896138:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.896139:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.896140:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.896143:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.896144:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.896147:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.896148:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.896148:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.896150:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.896150:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.896151:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.896152:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.896164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.896167:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.896168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.896169:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.896171:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.896173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.896185:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.896188:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.896188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.896189:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.896190:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.896190:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677472.896284:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.896287:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.896288:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.896290:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.896293:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.896299:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.896300:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.896301:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.896302:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.896303:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.896305:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.896307:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.896309:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.896310:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.896310:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.896312:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.896312:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.896313:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.896314:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.896325:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.896327:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.896328:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.896328:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.896330:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.896332:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677472.896477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.896483:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.896486:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.896487:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.896489:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.896491:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.896492:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.896493:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.896495:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.896496:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.896497:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.896500:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.896502:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.896504:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.896505:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.896506:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.896508:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.896508:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.896509:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.896509:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.896521:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.896524:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.896525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.896525:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.896527:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.896529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.896873:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.896876:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.896877:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.896877:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.896879:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.896882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.896924:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.896926:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677472.896928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677472.896929:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.896932:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.896934:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677472.896936:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.896936:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.896937:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677472.896938:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.896939:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.896940:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677472.896942:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.896944:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677472.896945:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.896946:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:8.0:1619677472.896946:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.896948:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.896949:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.896950:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.896950:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.896953:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.896954:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:8.0:1619677472.896954:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.896955:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677472.896956:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677472.896956:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.896958:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.896959:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.896959:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.0:1619677472.896959:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.896960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.896961:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.896961:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.896963:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.896964:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.896964:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.896964:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677472.896969:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:8.2:1619677472.896969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677472.896971:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.896972:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.896972:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677472.896974:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.896976:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:1.0:1619677472.896977:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.896980:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.896981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.896981:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.896983:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.896986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.897232:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.897235:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.897235:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.897237:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.897240:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.897241:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677472.897243:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.897244:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.897245:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.897246:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677472.897250:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.897251:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677472.897254:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.897255:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677472.897256:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677472.897258:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677472.897258:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.897259:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677472.897260:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677472.897261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:10.0:1619677472.897270:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.897272:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677472.897273:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:1.0:1619677472.897273:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:10.0:1619677472.897274:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.897275:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:10.0:1619677472.897276:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.897276:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.897277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:10.0:1619677472.897279:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677472.897332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.897338:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.897340:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.897342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.897344:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.897346:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.897348:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677472.897349:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.897352:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.897353:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.897354:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677472.897357:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.897359:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677472.897363:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.897363:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677472.897364:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677472.897366:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677472.897366:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.897368:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.897370:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.897370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677472.897376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.897382:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677472.897382:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677472.897384:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.897385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677472.897386:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:2.0:1619677472.897386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677472.897387:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.897390:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677472.897392:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.897393:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677472.897454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.897464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.897466:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.897467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.897468:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.897470:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.897473:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.897474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.897482:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.897484:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.897485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.897485:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.897486:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.897487:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.897517:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.897519:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.897520:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.897521:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.897524:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.897525:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.897527:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.897528:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.897529:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.897530:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.897533:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.897534:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.897537:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.897538:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.897539:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.897540:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.897541:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.897542:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.897542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.897555:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.897558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.897558:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.897559:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.897561:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.897563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.897571:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.897574:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.897574:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.897575:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.897576:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.897576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.897663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677472.897666:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.897669:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677472.897671:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.897672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677472.897674:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.897675:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.897677:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.897678:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.897679:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.897680:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.897683:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.897685:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.897688:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.897688:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.897689:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.897691:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.897691:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677472.897692:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677472.897692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677472.897704:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.897706:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.897707:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.897708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.897709:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.897711:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677472.897865:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.897868:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.897869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.897871:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.897874:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.897875:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.897876:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.897878:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.897879:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.897880:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.897885:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.897887:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.897890:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.897891:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.897891:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.897893:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.897893:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.897895:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.897896:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.897897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.897899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.897908:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677472.897909:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.897909:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.897910:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677472.897912:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677472.897912:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:21.0:1619677472.897913:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.897915:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.897916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.897917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677472.898137:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.898139:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.898140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.898142:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.898145:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.898146:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677472.898147:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.898149:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.898150:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.898151:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677472.898154:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.898155:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677472.898158:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.898159:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677472.898160:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677472.898161:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677472.898162:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.898162:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.898163:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677472.898167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677472.898175:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677472.898178:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.898179:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.898179:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.898181:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.898184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677472.898606:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.898609:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.898609:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.898610:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.898612:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.898614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.899462:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.899465:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.899465:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.899468:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.899470:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.899472:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677472.899474:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.899475:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.899476:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.899477:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.899480:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.899481:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.2:1619677472.899483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677472.899484:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.899485:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.899485:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.899487:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.899487:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.899488:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677472.899490:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677472.899490:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:1.0:1619677472.899492:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.899495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.899495:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.899496:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.899498:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.899500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677472.899503:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677472.899503:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677472.899505:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:18.0:1619677472.899506:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.899507:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.899507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677472.899507:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:26.0:1619677472.899508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677472.899509:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.899513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.900127:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.900130:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.900131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.900133:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.900135:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.900137:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.900138:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.900140:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.900141:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.900141:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.900144:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.900146:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.900147:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.900148:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.900148:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.900150:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.900150:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.900152:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.900153:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.900153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.900164:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.900165:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.900165:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.900167:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677472.900167:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.900168:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.900169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.900169:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.900170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.900170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.900394:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.900396:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.900397:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.900398:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.900399:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.900401:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.901052:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.901054:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.901055:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.901057:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.901059:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677472.901060:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.901062:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677472.901062:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.901063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677472.901063:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.901064:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.901065:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.901065:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.901066:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677472.901066:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.901068:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.901069:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.901070:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677472.901070:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.901071:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677472.901072:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677472.901072:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677472.901075:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.901076:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:8.0:1619677472.901076:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.901077:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677472.901077:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677472.901079:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.901079:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.901080:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.0:1619677472.901080:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.901081:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.901081:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677472.901081:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.901081:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.901083:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.901083:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.901084:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.901084:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.901089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677472.901092:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677472.901094:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677472.901095:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677472.901096:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.901097:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677472.901097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.901097:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:6.0:1619677472.901097:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.901098:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677472.901099:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.901100:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.901101:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.901101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677472.901102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:17.0:1619677472.901103:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.901106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.901299:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.901301:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.901302:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.901302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.901304:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.901305:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.901307:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:24.0:1619677472.901307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.901308:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.901310:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.901313:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.901315:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.901316:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.901317:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.901318:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.901319:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.901321:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.901323:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.901325:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.901326:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.901327:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.901328:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.901328:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.901329:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.901329:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.901340:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.901343:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.901344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.901344:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.901345:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.901347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.901358:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.901360:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.901361:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.901361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.901362:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.901362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.901449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.901456:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.901459:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.901460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.901462:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.901465:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.901466:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.901468:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.901469:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.901470:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.901471:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.901474:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.901476:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.901478:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.901478:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.901479:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.901481:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.901482:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.901483:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.901484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.901494:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.901496:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.901497:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.901497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.901499:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.901500:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.901514:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.901516:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.901517:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.901517:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.901517:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.901518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677472.901665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.901671:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.901674:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.901675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.901677:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.901679:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.901680:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.901682:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.901683:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.901684:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.901685:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.901688:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.901689:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.901692:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.901692:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.901693:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.901695:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.901696:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.901696:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.901697:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.901710:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.901713:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.901714:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.901714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.901716:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.901718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.902042:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.902045:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.902046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.902046:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.902048:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.902051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.902066:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.902069:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.902069:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.902071:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.902074:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.902076:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677472.902077:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.902078:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.902079:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.902080:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677472.902084:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.902085:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677472.902088:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.902089:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677472.902089:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677472.902091:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.902091:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.902092:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.902092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.902097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.902105:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.902107:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.902108:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.902109:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.902111:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.902114:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677472.902321:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.902324:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.902325:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.902326:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.902329:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.902330:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.902331:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.902333:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.902333:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.902334:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.902337:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.902339:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.902342:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.902343:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.902343:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.902345:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.902345:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.902346:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.902348:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.902348:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677472.902360:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.902361:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677472.902362:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.902363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.902364:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.902364:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677472.902364:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.902366:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677472.902366:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.902368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.902377:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.902381:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.902382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.902382:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.902383:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.902383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677472.902677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.902685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.902688:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.902689:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.902691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.902691:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.902694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677472.903346:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.903349:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.903350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.903352:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.903354:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.903356:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.903357:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.903359:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.903360:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.903361:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.903364:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.903366:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.903368:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.903369:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.903370:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.903371:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.903372:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.903373:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.903373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.903378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.903387:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.903389:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.903390:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.903391:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.903392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.903394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.903730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.903737:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.903740:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.903741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.903743:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.903745:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.903746:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677472.903748:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.903749:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.903750:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.903751:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677472.903755:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.903756:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677472.903760:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.903761:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677472.903762:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677472.903764:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677472.903764:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.903765:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.903766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677472.903780:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.903784:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.903785:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.903788:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.903790:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.903792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677472.903793:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.903793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.903794:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.903795:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.903796:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677472.904068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.904076:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.904079:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.904079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.904080:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.904081:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:17.0:1619677472.904082:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.904084:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.904085:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677472.904085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677472.904085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.904087:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.904089:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.904090:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677472.904091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677472.904092:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.904093:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.904094:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.904094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.904095:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.905802:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.905804:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.905805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.905807:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.905810:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.905811:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.905812:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.905813:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.905814:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.905815:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.905818:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.905819:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.905821:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.905822:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.905822:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.905824:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.905824:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.905825:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.905825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.905836:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.905838:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.905839:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.905840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.905840:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.905842:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.905854:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.905856:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.905857:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.905857:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.905858:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.905858:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:10.0:1619677472.905961:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.905964:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.905965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.905966:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.905970:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.905971:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.905972:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.905974:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.905974:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.905975:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.905978:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.905979:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.905982:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.905983:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.905984:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.905986:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.905986:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.905987:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.905987:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.906000:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.906003:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.906003:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.906004:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.906006:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.906009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.906017:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.906020:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.906021:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.906021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.906022:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.906023:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.906057:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.906060:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.906061:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.906063:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.906066:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.906067:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.906068:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.906069:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.906070:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.906071:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.906074:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.906075:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.906077:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.906077:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.906078:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.906079:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.906080:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.906081:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.906081:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.906092:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.906094:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.906095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.906095:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.906097:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.906099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677472.906130:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.906133:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.906134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.906134:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.906136:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.906139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677472.906144:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.906147:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.906149:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.906150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.906151:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.906153:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.906154:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.906155:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.906157:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.906158:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.906161:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.906163:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.906165:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.906166:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.906166:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.906168:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.906168:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.906169:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.906170:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.906182:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.906184:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.906185:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.906185:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.906187:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.906189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677472.906401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.906407:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.906410:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.906411:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.906413:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.906415:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.906416:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.906418:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.906419:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.906420:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.906421:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.906424:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.906426:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.906428:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.906429:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.906429:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.906431:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.906432:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.906432:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.906433:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.906446:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.906449:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.906450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.906450:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.906452:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.906454:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677472.907169:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.907172:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.907173:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.907175:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.907177:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.907179:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.907180:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.907181:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.907183:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.907184:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.907187:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.907188:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.907191:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.907192:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.907192:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.907194:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.907195:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.907195:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.907196:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.907207:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.907210:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.907211:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.907211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.907213:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.907216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.907225:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677472.907226:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677472.907229:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677472.907229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.907229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677472.907229:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.907230:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:24.0:1619677472.907231:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677472.907231:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.907232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:24.0:1619677472.907233:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.907235:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.907236:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.907237:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.907238:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.907239:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.907242:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.907244:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.907247:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.907248:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.907248:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.907250:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.907251:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.907252:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.907253:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.907254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677472.907264:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.907267:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677472.907267:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.907268:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.907269:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.907269:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.907270:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677472.907270:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.907271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.907274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.907656:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.907659:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.907659:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.907661:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.907664:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.907666:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677472.907667:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.907668:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.907669:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.907670:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677472.907673:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.907674:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677472.907676:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.907676:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677472.907677:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677472.907678:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.907679:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.907680:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.907681:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.907681:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.907692:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.907693:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677472.907694:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.907695:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.907696:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:10.0:1619677472.907696:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.907697:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.907698:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677472.907698:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677472.907699:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677472.907711:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677472.907714:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677472.907715:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.907715:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.907716:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.907716:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:22.0:1619677472.907791:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.907794:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.907795:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.907795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.907798:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.907800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.908292:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.908299:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.908301:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:24.0:1619677472.908303:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.908306:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.908307:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677472.908309:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.908310:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.908311:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.908312:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677472.908315:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.908317:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677472.908319:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.908320:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677472.908320:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677472.908322:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677472.908323:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.908323:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.908324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:21.0:1619677472.909122:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.909125:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.909127:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.909128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.909129:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.909131:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.909132:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.909134:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.909135:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.909136:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.909139:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.909140:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.909142:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.909143:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.909144:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.909146:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.909146:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.909148:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.909149:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.909149:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.909158:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.909160:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:6.0:1619677472.909160:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677472.909161:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.909162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677472.909162:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.909163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.909163:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.909165:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677472.909167:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.909829:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.909832:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.909833:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.909833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.909835:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.909838:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677472.909848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.909857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.909860:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.909861:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.909861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.2:1619677472.909864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:17.0:1619677472.909864:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.909867:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677472.909871:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.909874:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.909875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.909877:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.909879:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.909881:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677472.909882:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.909884:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.909884:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.909885:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677472.909888:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.909890:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677472.909892:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.909893:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677472.909894:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.909895:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.909896:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.909897:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.909898:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.909901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.909909:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.909912:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.909913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.909913:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.909915:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677472.909917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.910502:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.910505:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.910506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.910507:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.910510:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.910511:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.910513:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.910514:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.910515:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.910516:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.910519:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.910520:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.910522:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.910523:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.910524:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.910525:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.910525:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.910526:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.910526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.910537:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.910539:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.910539:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.910540:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.910541:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.910543:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677472.910651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.910657:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.910660:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.910661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.910662:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.910665:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.910666:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.910667:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.910669:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.910670:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.910671:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.910674:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.910676:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.910678:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.910679:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.910680:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.910681:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.910682:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.910683:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677472.910684:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677472.910684:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677472.910689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.910694:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677472.910696:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677472.910696:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677472.910697:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677472.910698:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677472.910698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.910699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.910699:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.910701:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.910703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677472.910720:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.910722:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.910723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.910725:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.910727:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.910729:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.910730:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.910731:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.910732:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.910733:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.910737:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.910738:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.910741:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.910742:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.910743:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.910744:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.910745:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.910746:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.910746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.910757:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.910760:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.910761:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.910761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.910763:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.910766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.910857:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.910859:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.910860:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.910862:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.910865:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677472.910865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:26.0:1619677472.910866:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.910867:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.910868:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.910869:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.910870:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.910871:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677472.910873:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.910874:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.910874:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.910875:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677472.910877:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677472.910877:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677472.910878:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.910879:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:2.0:1619677472.910880:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.910881:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.910881:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677472.910881:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677472.910882:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.910882:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677472.910883:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.910884:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.910885:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.910886:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.910889:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.910891:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.910893:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.910894:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677472.910894:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.910895:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677472.910896:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:2.0:1619677472.910896:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677472.910897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.910897:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.910897:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.910898:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.910899:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:6.0:1619677472.910900:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.910900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677472.910902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.910906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.910912:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.910913:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.910913:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.910913:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:22.0:1619677472.910914:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.910914:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:18.0:1619677472.910914:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.910914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.910915:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.910915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.910916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677472.910917:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:18.0:1619677472.910917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.910918:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677472.910918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.910919:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.910920:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.910920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.910920:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677472.910921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:20.0:1619677472.910922:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.910923:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.910925:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.910926:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.910927:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.910928:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.910931:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.910933:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.910935:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.910936:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.910937:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.910939:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.910939:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.910940:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.910941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.910951:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.910954:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.910954:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.910955:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.910957:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.910960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.911111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.911119:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.911121:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.911122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.911123:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.911125:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.911128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.911424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.911426:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.911427:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.911428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.911429:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.911432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677472.911556:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.911559:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.911561:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.911562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.911564:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.911566:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.911567:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.911569:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.911570:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.911571:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.911574:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.911576:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.911579:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.911579:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.911580:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.911582:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.911582:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.911584:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.911584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677472.911862:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.911865:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.911866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.911867:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.911869:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.911871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.911948:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.911951:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.911952:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.911954:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.911957:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.911958:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.911959:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.911960:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.911962:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.911962:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.911966:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.911967:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.911970:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.911971:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.911971:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.911973:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.911973:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.911974:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.911975:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.911976:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677472.911986:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.911989:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677472.911989:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.911989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.911990:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.911991:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.911992:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677472.911992:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.911993:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.911995:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.913008:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.913011:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.913012:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.913013:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.913015:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.913016:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.913018:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.913019:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.913020:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.913021:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.913024:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.913025:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.913028:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.913029:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.913029:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.913031:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.913031:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.913032:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.913032:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.913046:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.913049:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.913050:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.913051:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.913052:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.913056:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.913101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.913110:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.913112:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.913113:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.913115:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.913116:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.913118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.913124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.913132:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.913134:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.913135:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.913136:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.913137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.913137:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.913290:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.913295:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677472.913297:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:24.0:1619677472.913300:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.913304:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.913306:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677472.913307:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.913308:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.913309:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.913310:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677472.913314:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.913315:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677472.913317:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.913318:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677472.913319:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677472.913320:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677472.913321:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677472.913322:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677472.913323:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677472.913323:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:10.0:1619677472.913334:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.913337:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677472.913337:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.913337:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:26.0:1619677472.913338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677472.913339:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.913339:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677472.913340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677472.913341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:10.0:1619677472.913342:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677472.913455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677472.913459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677472.913459:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677472.913462:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.913465:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.913467:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677472.913469:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.913471:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.913472:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.913473:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677472.913477:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.913479:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677472.913482:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.913483:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677472.913484:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677472.913486:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677472.913487:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.913489:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677472.913490:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677472.913490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:26.0:1619677472.913499:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.913502:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:22.0:1619677472.913502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:10.0:1619677472.913502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.913503:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.913504:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000400:00000200:26.0:1619677472.913505:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.913505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677472.913506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:26.0:1619677472.913508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677472.913521:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.913524:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.913524:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677472.913525:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.913526:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.913527:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.0:1619677472.913646:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.913649:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:20.0:1619677472.913652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.913653:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.913657:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.913659:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.913661:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.913664:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.913665:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.913667:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.913672:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.913675:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.913679:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.913680:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.913681:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.913683:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.913684:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.913685:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.913686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677472.913698:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.913700:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.913701:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.913701:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.913703:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.913706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.913867:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.913870:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.913871:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.913871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.913873:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.913876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.913884:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.913886:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.913887:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.913888:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.913889:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.913889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677472.913993:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.913998:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.913999:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677472.914003:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.914006:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.914008:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677472.914010:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.914013:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.914014:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.914016:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677472.914021:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.914024:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677472.914028:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.914030:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677472.914031:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677472.914033:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677472.914034:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.914035:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.914036:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677472.914046:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.914048:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.914049:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.914050:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.914052:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.914054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.914066:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.914069:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.914069:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.914070:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.914071:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.914071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.914787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.914794:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.914796:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.914797:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.914799:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.914801:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.914803:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.914804:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.914806:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.914807:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.914808:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.914811:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.914813:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.914816:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.914817:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.914818:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.914820:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.914820:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.914821:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.914822:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677472.915144:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.915146:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.915147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.915149:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.915149:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:26.0:1619677472.915152:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.915152:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:26.0:1619677472.915154:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.915155:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.915155:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.915156:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.915158:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.915158:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.915159:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.915160:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.915161:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.915161:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000400:00000200:20.0:1619677472.915162:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.915163:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677472.915164:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000400:00000200:20.0:1619677472.915165:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.915166:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.915167:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677472.915167:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.915170:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.915171:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.915171:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677472.915172:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677472.915173:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96511695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.915174:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677472.915175:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.915175:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.915176:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677472.915177:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.915178:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.915178:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.915180:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.915180:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.915181:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:20.0:1619677472.915181:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:10.0:1619677472.915181:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.915182:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677472.915182:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.915182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:10.0:1619677472.915183:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677472.915184:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.915185:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.915186:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.915187:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677472.915187:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.915188:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677472.915188:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677472.915188:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677472.915190:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677472.915191:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677472.915191:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677472.915192:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:22.0:1619677472.915196:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.2:1619677472.915197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677472.915197:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.915198:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.915199:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.915199:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.915199:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.915201:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:6.0:1619677472.915201:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.915204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.915205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.915205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.915206:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:6.0:1619677472.915206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677472.915209:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.915210:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.915210:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677472.915212:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.915215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.915221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677472.915228:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.915231:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677472.915232:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.915232:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.915233:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677472.915234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677472.915237:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.915240:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.915241:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.915241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.915242:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.915243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677472.915479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.915485:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.915488:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.915488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.915491:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.915493:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.915494:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.915496:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.915497:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.915498:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.915499:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.915502:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.915504:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.915507:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.915508:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.915508:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.915510:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.915510:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.915511:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.915511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.915525:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.915527:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.915528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.915528:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.915530:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.915533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.915643:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.915646:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.915647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.915649:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.915652:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.915654:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.915655:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.915656:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.915657:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.915659:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.915662:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.915664:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.915667:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.915668:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.915668:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.915670:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.915670:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.915671:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.915672:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677472.915674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.915681:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:22.0:1619677472.915683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677472.915684:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.915685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.915685:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.915686:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.915687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.915687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:1.0:1619677472.915688:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677472.915690:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.915691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677472.915693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677472.915694:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.915696:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.915698:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.915699:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.915700:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.915701:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.915701:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677472.915955:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.915958:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.915959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.915959:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.915961:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.915964:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.916823:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.916826:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.916827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.916829:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.916832:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.916833:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.916834:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.916836:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.916837:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.916837:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.916840:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.916842:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.916845:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.916845:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.916846:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.916848:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.916848:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.916849:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.916849:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.916863:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.916865:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.916866:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.916866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.916869:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.916871:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.916874:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677472.916875:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677472.916875:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.916876:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677472.916876:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:24.0:1619677472.916879:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.916879:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.916880:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:22.0:1619677472.916880:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.916880:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.916880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.916881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:24.0:1619677472.916882:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.916883:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.916884:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.916885:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.916887:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.916889:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.916890:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.916891:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.916892:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.916893:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.916894:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.916894:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.916895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.916965:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.916967:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.916968:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.916969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.916969:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.916970:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.917154:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:8.2:1619677472.917154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677472.917157:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.917158:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.917160:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.917163:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.917163:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:26.0:1619677472.917164:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677472.917165:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.917166:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677472.917166:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.917166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:26.0:1619677472.917167:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.917167:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.917169:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677472.917169:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677472.917171:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.917172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.917173:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677472.917176:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.917177:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677472.917177:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677472.917179:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677472.917180:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.917180:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.917181:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677472.917193:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.917196:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.917198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.917198:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.917200:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.917203:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677472.918012:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.918015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.918015:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.918016:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.918018:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.918021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.918194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677472.918200:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.918203:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677472.918205:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.918205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677472.918207:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.918209:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677472.918210:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.918211:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.918212:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.918213:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677472.918217:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.918219:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677472.918221:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.918222:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677472.918223:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677472.918225:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677472.918225:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677472.918226:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677472.918227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677472.918232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.918240:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.918243:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.918244:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.918244:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.918246:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.918248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677472.918944:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.918947:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.918948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.918949:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.918951:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.918952:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.918953:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.918955:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.918956:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.918957:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.918960:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.918961:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.918964:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.918965:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.918966:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.918967:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.918968:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.918969:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.918971:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.918971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.918983:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.918985:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677472.918985:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:18.0:1619677472.918986:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.918986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.918987:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.918988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.918988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677472.918989:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.918992:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.919001:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.919004:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.919004:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.919005:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.919005:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.919006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.919393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.919399:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.919402:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.919402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.919404:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.919406:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.919407:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.919408:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.919409:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.919410:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.919411:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.919414:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.919416:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.919418:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.919418:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.919419:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.919420:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.919421:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.919422:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.919422:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.919434:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.919436:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.919437:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.919437:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.919438:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.919440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.919785:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.919788:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.919788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.919789:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.919791:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.919794:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677472.919820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.919826:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.919829:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.919829:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.919831:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.919833:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.919835:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.919836:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.919838:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.919839:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.919840:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.919844:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.919845:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.919848:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.919849:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.919849:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.919851:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.919852:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.919852:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.919853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.919865:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.919867:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.919868:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.919868:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.919870:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.919872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677472.920000:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.920003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.920003:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677472.920005:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.920008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.920009:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677472.920010:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.920012:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.920013:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.920014:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677472.920016:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.920018:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677472.920020:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.920021:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677472.920021:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677472.920023:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677472.920023:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.920024:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677472.920024:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677472.920036:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.920038:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.920040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.920040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.920042:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.920044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.920076:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.920079:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.920080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.920082:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.920085:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.920086:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.920087:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.920089:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.920089:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.920090:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.920093:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.920095:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.920098:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.920099:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.920099:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.920101:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.920101:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.920103:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677472.920104:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.920104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677472.920114:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677472.920116:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677472.920117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.920117:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677472.920118:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677472.920118:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.920119:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.920120:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677472.920120:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.920123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.920721:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.2:1619677472.920721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:26.0:1619677472.920724:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.920725:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.920726:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677472.920728:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677472.920729:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.920730:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677472.920731:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:26.0:1619677472.920732:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677472.920732:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677472.920734:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.920735:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.920736:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.920736:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.920737:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.920739:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677472.920740:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677472.920740:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.920741:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677472.920742:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677472.920742:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677472.920744:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677472.920744:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677472.920746:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.920746:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:18.0:1619677472.920747:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.920748:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.920748:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.920749:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.920749:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677472.920749:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677472.920753:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.920753:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677472.920754:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677472.920756:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677472.920757:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.920758:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.920759:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.920759:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677472.920763:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.920765:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.2:1619677472.920765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677472.920766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.920767:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.920769:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.920770:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677472.920771:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:12.0:1619677472.920771:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677472.920772:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.920773:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.920774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677472.920775:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.920776:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.920779:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:6.0:1619677472.920780:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:13.0:1619677472.920782:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.920783:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677472.920783:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.920784:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.920785:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.920786:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.920787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677472.921594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.921602:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.921605:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.921606:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.921608:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.921609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.921611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.921615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.921622:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.921625:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.921626:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.921627:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.921627:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.921628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677472.921719:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.921722:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.921723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.921725:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.921728:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.921730:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.921731:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.921732:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.921733:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.921734:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.921737:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.921739:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.921741:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.921742:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.921743:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.921744:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.921745:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.921745:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.921747:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.921747:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677472.921759:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.921760:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.921762:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677472.921762:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677472.921763:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.921763:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.921763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.921764:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.921766:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.921769:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.921804:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.921807:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.921808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.921808:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.921810:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.921813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.921944:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.921947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.921947:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.921949:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.921952:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.921954:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.921955:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.921956:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.921957:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.921958:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.921961:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.921962:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.921964:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.921964:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.921965:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.921966:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.921967:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.921967:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.921968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.921978:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.921980:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.921980:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.921981:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.921982:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.921983:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677472.922505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.922514:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.922516:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.922517:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.922518:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.922520:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.922523:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.922642:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.922644:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.922645:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.922647:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.922649:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.922650:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.922651:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.922653:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.922654:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.922654:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.922657:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.922659:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.922661:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.922662:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.922663:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.922664:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.922665:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.922666:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.922666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.922681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.922690:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.922692:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.922693:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.922695:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.922695:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.922697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677472.923470:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.923473:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.923473:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.923474:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.923475:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.923478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.923617:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.923621:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.923622:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.923622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.923624:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.923627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.923788:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.923791:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677472.923792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677472.923794:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.923797:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.923798:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677472.923800:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.923802:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.923803:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.923804:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677472.923807:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.923809:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677472.923812:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.923813:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677472.923814:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677472.923816:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677472.923817:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.923817:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677472.923818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:24.0:1619677472.923971:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.923973:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.923974:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.923974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.923976:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.923978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677472.924613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677472.924617:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.924620:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677472.924620:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677472.924620:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.924622:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.924623:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:8.0:1619677472.924623:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.924624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677472.924625:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677472.924627:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.924627:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.924629:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.924630:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.924630:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.924631:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:8.0:1619677472.924631:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677472.924633:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.924634:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.924635:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.924635:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677472.924636:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677472.924637:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:17.0:1619677472.924640:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.924641:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.924642:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677472.924642:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.924642:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677472.924644:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.924644:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.924644:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677472.924645:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:8.0:1619677472.924645:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677472.924646:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677472.924646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677472.924648:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.924648:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.924649:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.924650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677472.924660:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677472.924663:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:18.0:1619677472.924663:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.924664:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.924664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677472.924666:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:18.0:1619677472.924666:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:22.0:1619677472.924667:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.924667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.924669:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.924670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.924672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.924681:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.924684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.924684:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.924685:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.924686:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.924686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677472.924757:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.924759:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.924760:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.924760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.924762:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.924765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.924774:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.924776:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.924777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.924777:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.924778:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.924779:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.924828:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.924830:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.924831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.924833:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.924836:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.924837:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.924838:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.924839:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.924840:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.924841:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.924844:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.924846:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.924849:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.924849:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.924850:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.924852:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.924852:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.924853:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.924853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.924865:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.924868:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.924868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.924869:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.924871:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.924873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677472.925514:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.925517:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.925518:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.925519:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.925523:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.925524:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.925525:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.925526:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.925527:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.925528:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.925531:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.925532:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.925534:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.925535:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.925535:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.925537:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.925537:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.925538:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.925538:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.925549:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.925551:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.925552:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.925553:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.925554:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.925557:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677472.925654:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.925657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.925657:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.925658:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.925660:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.925663:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.925672:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.925674:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.925675:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.925675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.925676:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.925677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.926479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.926485:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.926488:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.926489:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.926491:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.926493:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.926495:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677472.926496:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.926498:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.926499:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.926500:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677472.926503:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.926504:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.926507:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.926507:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677472.926508:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.926510:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.926511:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.926511:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.926512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.926517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.926525:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.926528:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.926529:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.926529:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.926531:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677472.926533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.926551:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.926554:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.926555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.926557:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.926559:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.926561:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.926562:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.926563:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.926564:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.926565:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677472.926565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:26.0:1619677472.926568:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.926569:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677472.926571:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677472.926572:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.926573:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.926573:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677472.926574:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:26.0:1619677472.926575:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677472.926575:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677472.926576:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.926576:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.926577:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677472.926577:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.926580:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.926581:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.926582:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.926583:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.926584:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.926585:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.926588:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.926589:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677472.926590:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677472.926591:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.926592:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.926593:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677472.926593:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.926594:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677472.926594:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:6.0:1619677472.926594:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.926595:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.926596:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.926597:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:6.0:1619677472.926597:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.926600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677472.926608:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.926610:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.926611:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.926612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.926613:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.926615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.927303:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.927305:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.927306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.927308:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.927311:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.927312:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.927313:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.927314:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.927315:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.927316:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.927319:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.927321:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.927322:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.927323:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.927324:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.927325:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.927325:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.927326:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.927327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.927337:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.927339:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.927340:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.927341:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.927341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.927343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677472.927552:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.927555:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.927556:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.927558:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.927561:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.927563:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677472.927564:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.927565:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.927566:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.927568:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677472.927570:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.927572:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677472.927574:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.927575:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677472.927575:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677472.927576:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.927577:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.927577:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.927578:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.0:1619677472.927584:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.927587:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.927587:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.927588:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:24.0:1619677472.927588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.927589:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.927590:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.927591:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.927591:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:8.0:1619677472.927591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677472.927593:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.927594:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677472.927607:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.927610:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.927611:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.927611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.927612:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.927612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.927748:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.927751:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.927751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.927752:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.927754:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.927756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.927988:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.927991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677472.927991:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000400:00000200:10.0:1619677472.927993:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.927995:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.927996:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:10.0:1619677472.927998:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.927999:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.928000:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.928001:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:10.0:1619677472.928004:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.928005:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677472.928007:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.928008:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:10.0:1619677472.928008:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677472.928010:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677472.928010:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677472.928011:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (22)++
00000800:00000200:10.0:1619677472.928013:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:10.0:1619677472.928013:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:8.0:1619677472.928022:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.928024:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:26.0:1619677472.928025:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000800:00000200:24.0:1619677472.928025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.928025:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.928027:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000400:00000200:8.0:1619677472.928027:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.928028:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677472.928029:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:8.0:1619677472.928029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677472.928257:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.928260:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.928262:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.928263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.928265:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.928266:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677472.928267:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.928270:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.928271:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.928273:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677472.928276:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.928277:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:20.0:1619677472.928278:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677472.928279:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677472.928280:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:20.0:1619677472.928281:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.928282:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677472.928282:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:10.0:1619677472.928283:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.928284:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.928284:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.928285:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:10.0:1619677472.928285:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.928286:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:10.0:1619677472.928287:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677472.928288:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.928289:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.928290:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.928291:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.0:1619677472.928299:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:10.0:1619677472.928300:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.928301:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.0:1619677472.928302:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.928303:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.928304:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677472.928305:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.928305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677472.928305:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677472.928306:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677472.928308:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677472.928308:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677472.928308:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677472.928309:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677472.928310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:13.0:1619677472.928324:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677472.928326:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677472.928327:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.928329:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.928331:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677472.928332:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:26.0:1619677472.928562:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.928565:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677472.928565:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000800:00000200:10.0:1619677472.928567:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000400:00000200:26.0:1619677472.928569:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.928570:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.928572:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.928574:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677472.928576:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.928577:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.928578:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.928579:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677472.928580:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.928582:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.928583:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677472.928583:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96511648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677472.928585:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677472.928585:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.928586:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.928586:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.928589:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.928590:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677472.928591:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677472.928591:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.928592:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677472.928592:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.928593:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677472.928594:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (23)++
00000800:00000200:24.0:1619677472.928594:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677472.928595:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (24)--
00000400:00000200:24.0:1619677472.928595:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.928595:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.928596:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (23)--
00000800:00000200:10.0:1619677472.928596:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677472.928596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.928597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:24.0:1619677472.928597:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.928597:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677472.928599:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.928599:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.928599:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677472.928600:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677472.928601:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:6.0:1619677472.928613:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.928615:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.928616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.928617:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.928619:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.928622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.928659:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.928662:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.928662:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.928664:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.928667:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.928668:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677472.928669:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.928670:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.928671:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.928672:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677472.928674:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.928676:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.928677:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.928677:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677472.928678:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.928679:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.928679:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.928680:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.928681:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677472.928691:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.928693:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677472.928694:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677472.928694:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.928696:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.928697:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677472.928820:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.928822:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677472.928823:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677472.928824:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.928826:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.928827:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.928828:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.928830:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.928830:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.928831:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.928834:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.928835:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.928838:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.928839:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.928839:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.928841:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.928841:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.928842:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (22)++
00000800:00000200:26.0:1619677472.928844:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:26.0:1619677472.928844:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:22.0:1619677472.928854:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.928856:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677472.928857:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000800:00000200:6.0:1619677472.928857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.928858:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.928859:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000400:00000200:22.0:1619677472.928860:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.928860:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677472.928861:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:22.0:1619677472.928863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.928969:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.928972:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677472.928973:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677472.928975:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.928978:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.928979:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677472.928980:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.928981:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677472.928981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677472.928982:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.928983:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677472.928985:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.928987:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677472.928987:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677472.928988:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.928988:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677472.928989:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677472.928989:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677472.928990:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.928990:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:1.0:1619677472.928990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677472.928991:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677472.928992:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000400:00000200:17.0:1619677472.928992:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.928994:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.928996:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.928997:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.928998:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.928999:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.929000:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.929003:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.929005:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.929008:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.929009:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.929009:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.929011:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.929011:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.929013:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677472.929014:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677472.929014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677472.929018:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:5.2:1619677472.929019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677472.929020:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677472.929021:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.929022:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677472.929023:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.929025:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677472.929025:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677472.929025:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677472.929028:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677472.929028:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677472.929028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677472.929029:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.929029:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.929030:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677472.929031:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.929033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.929198:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.929200:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.929201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.929203:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.929205:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.929206:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.929207:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.929207:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:26.0:1619677472.929208:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.929209:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677472.929209:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:26.0:1619677472.929210:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677472.929210:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.929210:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.929213:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677472.929214:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.929215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.929216:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.929219:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.929220:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.929220:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.929222:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.929222:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.929223:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.929224:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.929235:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.929238:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.929239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.929240:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.929242:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.929244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.929252:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677472.929255:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.929256:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.929256:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.929257:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.929258:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677472.929354:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.929356:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.929357:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.929358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.929359:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.929361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.929391:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677472.929394:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677472.929395:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677472.929396:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.929399:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.929401:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.929402:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.929403:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.929404:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.929405:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.929408:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.929410:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.929413:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.929414:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.929414:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.929416:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.929417:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677472.929417:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677472.929418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:8.2:1619677472.929422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.929431:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.929433:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.929434:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.929436:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.929437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.929439:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677472.929769:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.929772:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.929773:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.929773:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.929775:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.929781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.929782:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.929784:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.929785:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.929785:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.929786:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.929787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677472.929849:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.929852:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.929852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.929854:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.929857:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.929858:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.929859:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.929861:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.929862:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.929863:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.929866:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.929868:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.929871:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.929872:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.929872:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.929874:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.929874:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.929875:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.929876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.929882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.929890:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.929892:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.929893:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.929894:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.929895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.929897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677472.930014:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.930017:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.930018:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.930020:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.930024:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.930025:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.930027:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.930029:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.930031:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.930032:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.930036:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.930039:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.930042:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.930043:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.930043:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.930045:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.930046:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.930046:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.930047:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.930058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.930060:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677472.930060:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677472.930061:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.930061:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.930062:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.930063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:8.0:1619677472.930063:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677472.930064:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.930065:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:6.0:1619677472.930066:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.930068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.930076:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.930079:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.930080:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.930080:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.930081:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.930081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.930147:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677472.930150:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677472.930151:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677472.930153:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.930156:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.930157:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.930158:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.930160:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.930161:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.930161:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.930164:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.930166:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.930168:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.930169:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.930169:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.930171:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.930171:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677472.930172:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677472.930172:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:8.0:1619677472.930182:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.930185:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.930186:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.930188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.930188:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.930189:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.930190:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.930190:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.930191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.930192:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.930194:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.930195:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.930196:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.930198:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.930198:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.930199:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.930202:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.930204:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.930207:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.930208:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.930208:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.930210:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.930211:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.930212:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.930213:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.930213:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.930225:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.930226:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.930228:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677472.930228:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677472.930229:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.930229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677472.930229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.930231:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.930232:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.930235:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.930245:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.930248:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.930249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.930249:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.930250:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.930250:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.930275:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.930277:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677472.930278:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677472.930280:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.930282:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.930283:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677472.930284:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.930286:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.930287:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.930288:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677472.930290:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.930292:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.930293:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.930294:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677472.930300:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.930301:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.930301:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677472.930303:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (22)++
00000800:00000200:26.0:1619677472.930304:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:26.0:1619677472.930304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677472.930314:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.930315:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677472.930316:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.930317:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677472.930318:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.930319:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677472.930333:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.930336:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.930336:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.930337:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677472.930337:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677472.930337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.930364:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000800:00000200:10.0:1619677472.930365:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000800:00000200:10.0:1619677472.930366:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677472.930367:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:20.2:1619677472.930481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677472.930484:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.930486:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677472.930488:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.930489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677472.930490:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.930492:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.930493:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.930494:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.930496:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.930496:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.930500:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.930502:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.930504:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.930505:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.930505:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.930507:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.930508:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677472.930508:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677472.930509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677472.930514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.930522:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.930524:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.930525:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.930527:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.930528:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.930529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677472.930742:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.930745:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.930746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.930747:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.930749:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.930751:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.930752:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.930753:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.930754:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.930755:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.930758:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.930760:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.930763:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.930763:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.930764:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.930766:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.930766:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.930767:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.930768:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.930769:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677472.930780:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677472.930781:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677472.930783:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677472.930783:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.930783:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677472.930784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.930785:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.930785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677472.930787:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.930789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677472.930835:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.930837:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677472.930840:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.930840:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.930842:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.930843:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677472.930844:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.930846:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.930847:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.930848:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677472.930850:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.930852:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.930854:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.930855:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.930856:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.930857:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.930858:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.930858:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.930859:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677472.930863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.930870:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.930872:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.930873:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.930875:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.930876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.930877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.930881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.930889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.930892:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.930893:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.930893:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.930894:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.930895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677472.931578:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.931581:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.931582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.931582:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.931585:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.931587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.932384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.932392:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.932395:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.932396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.932398:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.932400:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.932402:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677472.932403:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.932405:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.932406:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.932408:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677472.932412:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.932414:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.932417:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.932418:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677472.932419:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677472.932421:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677472.932421:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.932422:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.932423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677472.932514:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.932517:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.932518:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.932520:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677472.932522:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.932524:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677472.932525:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.932526:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.932527:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.932528:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677472.932531:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.932533:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677472.932536:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.932537:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677472.932538:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677472.932539:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677472.932540:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.932541:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.932541:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.2:1619677472.932545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.2:1619677472.932546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.932551:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677472.932553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677472.932554:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.932555:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677472.932556:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.932557:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.932557:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.932559:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.932560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:2.0:1619677472.932560:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.932562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:2.0:1619677472.932562:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.932563:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.932564:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.932565:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.932566:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.932570:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.932572:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.932573:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.932574:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.932575:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.932576:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.932577:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.932578:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.932579:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.932590:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.932592:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.932593:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.932593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.932595:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.932596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.932625:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.932629:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.932630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.932632:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.932634:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.932636:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.932637:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.932638:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.932639:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.932640:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.932643:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.932645:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.932646:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.932647:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.932648:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.932649:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.932650:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.932650:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.932650:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.932661:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.932662:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.932663:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.932664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.932665:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.932667:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.933704:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.933706:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.933707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.933708:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.933710:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.933713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.933721:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.933723:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.933724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.933724:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.933726:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.933726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677472.933756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.933762:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.933765:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.933766:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.933768:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.933770:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.933772:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.933773:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.933775:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.933776:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.933777:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.933780:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.933782:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.933784:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.933785:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.933786:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.933787:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.933788:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.933788:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.933789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.933801:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.933803:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.933804:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.933804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.933806:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.933808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677472.933872:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677472.933875:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677472.933876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:2.0:1619677472.933878:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.933880:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.933882:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:2.0:1619677472.933883:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.933884:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.933885:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.933886:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:2.0:1619677472.933890:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.933892:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:2.0:1619677472.933896:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.933896:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:2.0:1619677472.933897:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:2.0:1619677472.933899:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:2.0:1619677472.933900:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677472.933901:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677472.933902:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677472.933902:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:24.2:1619677472.933906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:18.0:1619677472.933913:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:10.0:1619677472.933914:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)++
00000800:00000200:18.0:1619677472.933916:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677472.933916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:26.0:1619677472.933917:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (23)--
00000800:00000200:10.0:1619677472.933917:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2fa00] (23)--
00000800:00000200:2.0:1619677472.933917:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000400:00000200:10.0:1619677472.933918:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.933919:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677472.933920:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:10.0:1619677472.933921:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.51@o2ib7: PUT: OK
00000400:00000200:2.0:1619677472.933922:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.933926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (21)--
00000400:00000200:2.0:1619677472.933927:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.933928:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:24.2:1619677472.933929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (20)++
00000400:00000200:2.0:1619677472.933929:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677472.933930:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.933930:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:2.0:1619677472.933930:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.933931:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677472.933932:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:2.0:1619677472.933932:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.933934:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.933934:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.933934:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.933937:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.933937:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.933937:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677472.933939:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677472.933939:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677472.933941:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677472.933941:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.933942:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677472.933942:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:26.0:1619677472.933943:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.933944:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677472.933944:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.933944:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000400:00000200:26.0:1619677472.933945:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.933945:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677472.933946:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000400:00000200:10.0:1619677472.933947:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677472.933948:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96511619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.933949:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677472.933950:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.933951:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677472.933952:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.933952:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677472.933954:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.933955:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:10.0:1619677472.933956:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677472.933957:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:10.0:1619677472.933958:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.933959:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.933959:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.933959:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677472.933960:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.933960:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:18.0:1619677472.933960:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.933960:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.933960:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677472.933961:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.933962:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.933962:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.933962:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.933963:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000400:00000200:18.0:1619677472.933963:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.933963:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000400:00000200:2.0:1619677472.933963:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.933964:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.933964:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.933965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677472.933965:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:10.0:1619677472.933965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677472.933966:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (21)++
00000400:00000200:18.0:1619677472.933966:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96511618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677472.933967:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677472.933967:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:26.0:1619677472.933968:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.51@o2ib7
00000400:00000200:18.0:1619677472.933968:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677472.933969:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (21)++
00000400:00000200:18.0:1619677472.933969:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677472.933970:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (22)--
00000400:00000200:18.0:1619677472.933970:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677472.933971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (21)--
00000800:00000200:26.0:1619677472.933972:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.933973:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:2.0:1619677472.933973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:26.0:1619677472.933974:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.933974:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:2.0:1619677472.933974:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:18.0:1619677472.933975:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.933975:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:26.0:1619677472.933976:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.933977:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677472.933977:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677472.933978:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677472.933980:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.933981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677472.933981:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.933981:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000400:00000200:6.0:1619677472.933982:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.933983:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:6.0:1619677472.933984:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.933985:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:18.0:1619677472.933986:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677472.933986:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:18.0:1619677472.933987:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (24)--
00000800:00000200:6.0:1619677472.933987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677472.933987:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677472.933988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677472.933989:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:2.0:1619677472.933990:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.933992:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.933993:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:2.0:1619677472.933995:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.933996:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.933997:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.933998:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677472.933999:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.934001:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:2.0:1619677472.934001:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677472.934002:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.934003:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:12.0:1619677472.934004:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.934005:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.934006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:2.0:1619677472.934006:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:2.0:1619677472.934006:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:12.0:1619677472.934007:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:2.0:1619677472.934008:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:2.0:1619677472.934009:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677472.934010:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677472.934011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677472.934012:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:18.0:1619677472.934013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677472.934013:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:2.0:1619677472.934016:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.934018:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677472.934019:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:2.0:1619677472.934020:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677472.934021:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677472.934022:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677472.934023:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:2.0:1619677472.934023:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677472.934024:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677472.934024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:8.2:1619677472.934024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:2.0:1619677472.934024:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.934025:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:13.0:1619677472.934026:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:2.0:1619677472.934028:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.934030:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.934032:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:2.0:1619677472.934033:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.934033:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:2.0:1619677472.934034:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.934035:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677472.934036:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.934036:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.934036:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:18.0:1619677472.934037:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677472.934037:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (22)++
00000400:00000200:17.0:1619677472.934038:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.934039:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:2.0:1619677472.934039:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (24)++
00000800:00000200:2.0:1619677472.934040:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000800:00000200:17.0:1619677472.934041:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677472.934041:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000400:00000200:18.0:1619677472.934042:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677472.934042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677472.934042:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000800:00000200:2.0:1619677472.934043:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:18.0:1619677472.934045:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.934046:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.934046:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.934047:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.934049:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.934050:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.934050:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677472.934050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677472.934051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.934053:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:10.0:1619677472.934053:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.934054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:21.0:1619677472.934054:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.934054:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677472.934056:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.934056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:18.0:1619677472.934057:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677472.934057:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.934057:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.934059:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.934059:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:2.0:1619677472.934059:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:18.0:1619677472.934060:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677472.934061:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.934061:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.934062:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:10.0:1619677472.934062:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677472.934063:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.934064:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.934065:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.934066:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.934067:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677472.934068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677472.934068:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (24)++
00000800:00000200:18.0:1619677472.934069:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000400:00000200:10.0:1619677472.934069:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.934070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000400:00000200:10.0:1619677472.934070:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677472.934072:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000800:00000200:18.0:1619677472.934072:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (24)--
00000400:00000200:2.0:1619677472.934072:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.934073:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677472.934074:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:10.0:1619677472.934074:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.934075:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.934076:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.934076:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.934077:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677472.934078:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.934080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.934080:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.934081:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:10.0:1619677472.934081:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.934081:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.934082:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:2.0:1619677472.934082:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677472.934083:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:10.0:1619677472.934083:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:18.0:1619677472.934085:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.934085:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677472.934085:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.934086:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677472.934086:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000400:00000200:18.0:1619677472.934087:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.934087:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677472.934087:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (24)--
00000800:00000200:2.0:1619677472.934087:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:10.0:1619677472.934088:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677472.934089:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677472.934089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:18.0:1619677472.934090:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96511612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677472.934090:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677472.934091:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:18.0:1619677472.934092:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.934093:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.934094:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677472.934096:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.934097:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.934098:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:2.0:1619677472.934098:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.934099:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:18.0:1619677472.934099:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677472.934099:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677472.934100:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96511611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.934101:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.934102:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.934102:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:8.0:1619677472.934102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:2.0:1619677472.934102:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.934103:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.934103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:2.0:1619677472.934103:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.934104:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677472.934105:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677472.934106:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.934108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677472.934108:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.934109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677472.934109:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:18.0:1619677472.934110:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:2.0:1619677472.934110:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.934112:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677472.934113:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:18.0:1619677472.934114:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000800:00000200:18.0:1619677472.934115:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (24)--
00000800:00000200:2.0:1619677472.934115:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.934115:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677472.934116:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677472.934116:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.934117:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:18.0:1619677472.934118:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:2.0:1619677472.934118:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.934119:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000400:00000200:18.0:1619677472.934120:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.934121:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (24)++
00000800:00000200:2.0:1619677472.934122:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000400:00000200:18.0:1619677472.934123:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.934123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000800:00000200:2.0:1619677472.934123:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000800:00000200:8.2:1619677472.934124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.934124:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:18.0:1619677472.934125:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:6.0:1619677472.934125:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:18.0:1619677472.934126:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96511610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.934128:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677472.934128:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.934129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:18.0:1619677472.934129:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.934129:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677472.934129:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.934132:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:2.0:1619677472.934132:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.934132:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:18.0:1619677472.934134:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677472.934134:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.934135:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677472.934135:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677472.934136:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677472.934136:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:2.0:1619677472.934136:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677472.934136:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.934138:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96511609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.934139:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677472.934140:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.934141:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.934142:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677472.934144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677472.934145:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.934146:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:8.2:1619677472.934146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000400:00000200:2.0:1619677472.934146:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.934147:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.934147:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677472.934149:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677472.934150:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (24)++
00000800:00000200:2.0:1619677472.934150:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.934151:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000800:00000200:2.0:1619677472.934151:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.934152:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000800:00000200:18.0:1619677472.934152:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000800:00000200:2.0:1619677472.934152:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677472.934153:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (24)--
00000800:00000200:18.0:1619677472.934154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:17.0:1619677472.934154:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.934154:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677472.934154:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (22)++
00000800:00000200:18.0:1619677472.934155:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677472.934155:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677472.934155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677472.934156:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:17.0:1619677472.934157:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.934157:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000400:00000200:18.0:1619677472.934158:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.934158:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:1.0:1619677472.934158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.934159:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.934159:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.934160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.934160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:2.0:1619677472.934160:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677472.934161:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:18.0:1619677472.934161:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.934162:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.934163:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677472.934164:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:18.0:1619677472.934164:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96511608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.934165:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.934165:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677472.934165:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:18.0:1619677472.934166:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677472.934167:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:26.0:1619677472.934168:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.934168:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.934169:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:2.0:1619677472.934169:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677472.934170:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.934171:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96511607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.934173:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.934173:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.934173:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677472.934173:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.934174:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.934175:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:2.0:1619677472.934175:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677472.934180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:18.0:1619677472.934180:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677472.934180:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.934181:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.934181:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677472.934182:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:2.0:1619677472.934182:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.934183:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.934184:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000800:00000200:18.0:1619677472.934185:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (24)--
00000800:00000200:2.0:1619677472.934185:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.934186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677472.934186:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.934187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677472.934187:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677472.934189:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:8.2:1619677472.934189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.934189:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.934190:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000800:00000200:2.0:1619677472.934190:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (24)++
00000400:00000200:18.0:1619677472.934191:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.934192:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000800:00000200:2.0:1619677472.934192:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000400:00000200:18.0:1619677472.934193:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.934193:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000800:00000200:2.0:1619677472.934194:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000400:00000200:18.0:1619677472.934195:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.0:1619677472.934196:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:18.0:1619677472.934196:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96511606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.934197:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.934198:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.934199:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.934201:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:2.0:1619677472.934201:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.934202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000400:00000200:2.0:1619677472.934202:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.934203:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (22)--
00000400:00000200:18.0:1619677472.934203:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.934204:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:21.0:1619677472.934204:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.934204:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:2.0:1619677472.934204:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:18.0:1619677472.934205:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.934205:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.934206:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:2.0:1619677472.934206:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96511605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.934207:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:2.0:1619677472.934208:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.934209:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.934210:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677472.934213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:18.0:1619677472.934213:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.934214:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677472.934215:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.934215:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:2.0:1619677472.934216:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.934217:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677472.934217:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (24)++
00000800:00000200:18.0:1619677472.934218:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000800:00000200:18.0:1619677472.934219:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000800:00000200:2.0:1619677472.934219:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677472.934220:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000800:00000200:18.0:1619677472.934222:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:8.2:1619677472.934222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.934222:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.934223:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:2.0:1619677472.934223:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677472.934225:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:18.0:1619677472.934226:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677472.934226:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (24)++
00000800:00000200:2.0:1619677472.934226:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (25)--
00000800:00000200:2.0:1619677472.934227:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (24)--
00000800:00000200:2.0:1619677472.934228:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (23)++
00000800:00000200:2.0:1619677472.934228:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (24)--
00000800:00000200:2.0:1619677472.934229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:20.0:1619677472.934230:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:18.0:1619677472.934230:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.934230:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000400:00000200:18.0:1619677472.934231:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677472.934231:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:20.0:1619677472.934232:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.934233:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.934233:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96511603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.934235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.934235:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.934235:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.934237:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677472.934238:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:18.0:1619677472.934238:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677472.934240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:2.0:1619677472.934240:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677472.934241:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.934242:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.934242:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.934242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.934244:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:2.0:1619677472.934244:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.934245:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677472.934245:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677472.934246:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96511602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.934248:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.934249:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677472.934250:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:2.0:1619677472.934250:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677472.934252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677472.934253:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.934253:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.934254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677472.934254:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:8.0:1619677472.934254:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.934254:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.934255:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (22)++
00000400:00000200:8.0:1619677472.934256:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.934257:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677472.934258:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (23)--
00000800:00000200:18.0:1619677472.934258:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (23)++
00000800:00000200:8.0:1619677472.934258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677472.934259:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (24)--
00000800:00000200:2.0:1619677472.934259:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.934260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677472.934260:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677472.934261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:6.0:1619677472.934261:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:2.0:1619677472.934261:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.934263:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677472.934264:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677472.934264:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:22.0:1619677472.934265:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:6.0:1619677472.934265:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (23)++
00000800:00000200:2.0:1619677472.934265:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (22)++
00000800:00000200:22.0:1619677472.934266:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000800:00000200:2.0:1619677472.934266:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677472.934266:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:22.0:1619677472.934267:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.934267:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (25)--
00000800:00000200:6.0:1619677472.934267:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.934269:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.934269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677472.934269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.934271:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677472.934277:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:1.0:1619677472.934277:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677472.934279:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677472.934280:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:1.0:1619677472.934280:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.934281:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.934281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000400:00000200:1.0:1619677472.934281:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.934283:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.934286:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.935150:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.935153:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.935153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.935155:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.935158:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.935160:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.935161:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.935163:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.935163:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.935164:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.935167:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.935169:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.935171:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.935172:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.935173:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.935174:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.935175:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.935175:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.935176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.935187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.935190:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.935191:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.935191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.935192:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.935194:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.935198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677472.935205:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677472.935206:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677472.935207:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.935208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.935208:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677472.935209:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.935210:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677472.935210:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.935210:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677472.935211:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.935211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.935212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:18.0:1619677472.935299:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.935302:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677472.935302:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677472.935305:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.935307:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.935309:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677472.935310:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.935312:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.935313:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.935314:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677472.935318:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.935320:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677472.935324:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.935325:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677472.935326:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677472.935328:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677472.935328:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677472.935329:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677472.935330:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677472.935331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:10.0:1619677472.935341:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:2.0:1619677472.935343:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:10.0:1619677472.935344:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677472.935345:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677472.935345:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.935345:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677472.935346:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:10.0:1619677472.935347:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.935347:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:10.0:1619677472.935350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:24.0:1619677472.935489:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.935492:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677472.935492:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.935493:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.935494:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.935497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677472.935780:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.935783:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.935783:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.935784:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.935786:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.935788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677472.936542:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.936545:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.936545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.936547:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.936550:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.936551:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.936553:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.936554:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.936555:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.936556:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.936560:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.936562:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.936564:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.936565:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.936566:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.936567:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.936568:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.936569:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677472.936570:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.936570:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.936574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.936581:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677472.936582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677472.936583:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677472.936584:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677472.936584:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677472.936585:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.936585:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677472.936587:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.936588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.936589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.936972:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677472.936975:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677472.936975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677472.936978:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.936981:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.936982:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677472.936983:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.936985:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.936986:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.936987:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677472.936990:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.936991:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677472.936994:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.936995:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677472.936995:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677472.936997:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677472.936998:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677472.936999:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677472.937000:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677472.937001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677472.937011:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677472.937012:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:18.0:1619677472.937013:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:6.0:1619677472.937013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677472.937014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:18.0:1619677472.937015:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.937016:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.937016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.937018:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.937021:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677472.937029:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.937032:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.937033:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.937033:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.937034:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.937035:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677472.937183:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.937185:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.937188:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.937188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.937190:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.937191:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.937193:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.937194:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.937195:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.937196:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.937199:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.937201:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.937204:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.937204:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.937205:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.937207:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.937207:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.937209:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.937210:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.937210:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.937219:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677472.937221:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677472.937221:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.937222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677472.937223:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677472.937223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:22.0:1619677472.937224:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.937224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.937226:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.937228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.937237:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.937240:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.937241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.937243:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.937245:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.937246:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.937248:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.937249:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.937250:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.937251:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.937254:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.937256:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.937258:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.937259:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.937259:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.937260:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.937261:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.937262:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.937262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.937272:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.937274:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.937274:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.937275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.937276:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.937277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.937295:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.937299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.937302:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.937303:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.937303:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.937304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677472.937542:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.937545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.937545:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.937546:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.937548:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.937550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.937559:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.937562:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.937563:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.937563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.937564:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.937565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.937757:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.937759:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.937760:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.937762:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.937765:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.937766:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.937767:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.937769:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.937770:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.937770:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.937774:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.937775:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.937778:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.937779:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.937779:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.937781:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.937781:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.937782:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.937782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.937794:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.937796:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.937797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.937798:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.937800:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.937803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677472.938110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677472.938117:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677472.938120:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677472.938120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677472.938123:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677472.938125:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.938127:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677472.938129:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.938130:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.938131:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.938132:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677472.938135:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677472.938137:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677472.938140:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.938141:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677472.938142:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677472.938143:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677472.938144:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677472.938145:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677472.938146:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677472.938146:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:5.2:1619677472.938153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:24.0:1619677472.938156:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677472.938158:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:24.0:1619677472.938159:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.938160:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.938160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677472.938160:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677472.938161:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:24.0:1619677472.938162:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:1.0:1619677472.938162:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:24.0:1619677472.938165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677472.938522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.938528:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.938531:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.938531:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.938533:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.938536:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.938538:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.938539:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.938540:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.938541:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.938542:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.938545:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.938547:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.938549:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.938550:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.938550:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.938552:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.938552:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.938553:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.938553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.938567:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.938570:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.938571:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.938571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.938573:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.938575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.938753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.938760:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.938763:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.938764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.938766:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.938768:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.938770:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677472.938771:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.938773:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.938774:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.938775:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677472.938778:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.938779:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677472.938782:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.938783:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677472.938784:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677472.938786:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677472.938786:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.938787:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.938788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677472.938799:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.938802:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.938803:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.938805:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677472.938807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.938807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.0:1619677472.938819:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.938822:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677472.938823:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.938825:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677472.938825:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.938828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677472.938872:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.938875:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.938876:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.938878:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.938881:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.938882:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.938884:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.938885:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.938886:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.938887:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.938890:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.938892:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.938895:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.938896:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.938896:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.938898:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.938899:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.938900:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677472.938901:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.938902:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677472.938911:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.938914:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677472.938915:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677472.938915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.938915:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.938917:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.938917:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677472.938918:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.938919:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.938921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.939773:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.939776:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.939776:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.939777:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.939779:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.939782:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.939858:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.939860:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.939861:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.939863:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.939866:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.939867:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.939869:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.939870:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.939871:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.939871:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.939874:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.939876:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.939878:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.939879:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.939880:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.939881:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.939881:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.939883:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.939884:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.939885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.939894:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.939896:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677472.939897:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677472.939897:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.939897:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.939899:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677472.939899:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.939900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.939901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.939901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.940354:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.940357:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.940358:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.940359:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.940361:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.940362:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677472.940363:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.940364:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.940365:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.940366:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677472.940368:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.940370:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677472.940372:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.940372:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677472.940372:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677472.940374:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677472.940374:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.940375:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677472.940376:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677472.940377:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.940388:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677472.940389:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.940390:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.940391:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:26.0:1619677472.940392:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.940392:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677472.940392:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677472.940393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.940394:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677472.940394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677472.940789:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.940791:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677472.940792:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677472.940792:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.940794:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.940796:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677472.940809:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677472.940812:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677472.940813:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.940813:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.940814:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677472.940814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677472.941271:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.941274:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.941275:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.941276:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677472.941278:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.941279:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.941280:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.941281:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.941282:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.941283:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.941286:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677472.941288:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.941290:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.941291:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.941292:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.941299:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.941300:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:8.2:1619677472.941300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.941301:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.941301:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677472.941309:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.941311:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.941312:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.941314:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.941316:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.941317:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677472.941332:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.941334:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.941335:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.941335:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677472.941337:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.941339:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.2:1619677472.941352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.941360:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.941363:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.941364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.941365:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.941367:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.941370:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677472.941469:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.941472:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.941473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.941474:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.941477:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.941478:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.941480:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.941481:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.941482:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.941483:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.941486:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.941488:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.941490:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.941491:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.941492:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.941493:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.941494:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.941495:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.941495:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.941507:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.941510:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.941511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.941511:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.941513:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.941516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.941547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.941553:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.941556:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.941557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.941559:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.941562:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.941563:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.941565:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.941566:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.941567:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.941568:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.941572:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.941575:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.941577:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.941578:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.941578:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.941580:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.941580:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.941582:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.941583:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.941583:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.941588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.941594:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.941595:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.941595:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677472.941596:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.941596:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.941597:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.941598:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.941598:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.941600:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.941600:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677472.941606:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.941609:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.941610:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.941610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.941612:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.941615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.941625:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.941628:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.941629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.941629:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.941630:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.941631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677472.942149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.942157:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.942159:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.942160:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.942162:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.942163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.942166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:13.0:1619677472.942225:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.942229:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.942233:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677472.942233:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.942236:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.942239:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677472.942241:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.942243:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.942244:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.942246:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677472.942251:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.942253:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677472.942256:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.942257:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677472.942257:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677472.942259:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677472.942260:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.942260:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677472.942261:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:26.0:1619677472.942271:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.942274:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.942275:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.942275:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677472.942277:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.942279:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.0:1619677472.942546:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.942549:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.942551:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.942552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.942553:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.942555:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.942556:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.942557:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.942558:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.942559:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.942562:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.942564:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.942567:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.942568:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.942568:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.942570:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.942571:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.942571:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.942572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.942584:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.942586:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.942587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.942588:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.942590:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.942593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677472.943328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.943334:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.943336:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.943337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.943339:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.943341:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.943343:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.943344:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.943345:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.943346:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.943347:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.943350:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.943352:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.943354:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.943355:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.943355:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.943357:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.943358:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.943359:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.943359:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.943372:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.943374:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.943375:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.943375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.943378:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.943380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.943393:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.943396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.943396:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.943397:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.943398:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.943398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677472.943410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.943418:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.943421:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.943421:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.943423:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.943424:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.943426:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.943428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.943435:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.943438:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.943439:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.943439:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.943440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.943441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677472.943528:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.943531:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.943531:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677472.943533:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.943536:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.943538:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677472.943539:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.943540:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.943541:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.943542:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677472.943546:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.943548:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677472.943550:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.943551:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677472.943552:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677472.943553:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677472.943554:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.943555:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.943555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677472.943560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.943567:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.943570:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.943571:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.943571:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.943574:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.943577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.943608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.943615:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.943618:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.943619:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.943621:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.943623:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.943624:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677472.943626:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677472.943626:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.943627:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677472.943629:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.943629:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677472.943629:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.943630:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677472.943633:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.943634:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677472.943636:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.943637:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677472.943638:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.943639:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677472.943639:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.943640:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000400:00000200:26.0:1619677472.943641:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677472.943641:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:26.0:1619677472.943642:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.943643:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677472.943643:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.943643:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.943644:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.943645:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:26.0:1619677472.943646:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.943646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677472.943647:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.2:1619677472.943648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677472.943650:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.943651:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:20.2:1619677472.943651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677472.943652:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.943654:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.943654:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.943655:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.943656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677472.943656:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677472.943657:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677472.943658:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677472.943659:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.943660:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677472.943660:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.943660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677472.943660:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.943662:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677472.943664:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677472.943667:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.943670:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.943671:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.943671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.943673:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.943676:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.943899:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.943901:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.943903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.943903:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.943905:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.943908:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.944124:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.944126:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677472.944127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677472.944129:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.944131:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.944133:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677472.944134:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.944135:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.944136:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.944137:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677472.944140:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.944142:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677472.944144:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.944145:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677472.944145:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677472.944147:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677472.944148:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677472.944148:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677472.944149:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:10.0:1619677472.944159:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.944161:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677472.944162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677472.944162:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.944164:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.944166:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:1.0:1619677472.944393:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.944396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.944396:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.944399:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.944400:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.944402:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.944403:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.944404:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.944406:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.944406:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.944410:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.944412:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.944415:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.944416:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.944416:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.944418:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.944418:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.944419:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.944420:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.944425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.944433:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.944435:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.944436:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.944438:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.944439:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.944441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677472.944614:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.944616:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.944617:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.944617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.944619:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.944622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.944630:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.944632:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.944633:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.944633:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.944634:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.944635:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.945105:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.945107:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.945108:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.945110:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.945112:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.945113:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.945115:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.945116:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.945117:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.945118:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.945121:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.945122:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.945125:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.945125:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.945126:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.945128:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.945128:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.945129:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.945129:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.945139:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.945141:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.945142:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.945143:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.945144:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.945146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.945670:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.945673:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.945674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.945675:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.945677:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.945679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.945701:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677472.945704:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677472.945705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677472.945706:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.945708:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.945710:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677472.945711:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.945712:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.945713:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.945714:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677472.945717:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.945719:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677472.945721:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.945722:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:8.0:1619677472.945723:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:8.0:1619677472.945725:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:8.0:1619677472.945725:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677472.945726:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677472.945726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:18.0:1619677472.945739:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.945742:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677472.945743:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.945743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.945745:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.945749:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677472.945755:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.945758:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677472.945759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677472.945760:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.945763:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.945765:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677472.945766:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.945767:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.945768:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.945769:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.945772:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.945773:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677472.945776:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.945777:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677472.945777:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677472.945779:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677472.945779:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.945780:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677472.945780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:6.0:1619677472.945792:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.945795:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.945796:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.945796:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.945799:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.945801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.946102:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.946105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.946105:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.946107:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.946110:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.946112:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.946113:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.946114:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.946115:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.946116:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.946119:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.946120:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.946123:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.946123:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.946124:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.946125:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.946126:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.946127:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.946127:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.946139:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.946142:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.946143:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.946143:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.946145:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.946148:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.946164:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.946167:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.946167:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.946168:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.946169:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.946169:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.946193:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.946196:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677472.946198:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.946199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677472.946201:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.946202:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677472.946204:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.946205:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.946206:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.946207:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677472.946211:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.946213:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677472.946216:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.946216:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677472.946217:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677472.946219:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677472.946220:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677472.946221:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.946222:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.946223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.946233:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.946235:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.946236:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.946237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677472.946361:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.946364:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677472.946364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.946367:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.946370:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.946371:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677472.946373:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.946374:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.946375:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.946376:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.946379:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.946381:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677472.946384:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.946385:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677472.946385:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.946387:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677472.946387:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677472.946388:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677472.946389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677472.946402:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.946405:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.946406:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.946406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.946408:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.946411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.946420:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.946423:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.946424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.946424:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.946425:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.946426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.946551:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.946554:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:1.0:1619677472.946554:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:8.0:1619677472.946555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677472.946557:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.946557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.946557:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677472.946560:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.946561:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.946562:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.946563:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677472.946564:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.946564:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677472.946565:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.946568:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.946568:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.946570:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.946570:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.946571:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677472.946572:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677472.946577:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677472.946577:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.946579:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677472.946580:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677472.946582:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.946582:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677472.946583:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:24.0:1619677472.946584:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.946585:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677472.946585:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677472.946585:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677472.946585:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:24.0:1619677472.946587:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677472.946587:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:24.0:1619677472.946588:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:1.0:1619677472.946588:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677472.946588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:24.0:1619677472.946589:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677472.946590:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677472.946591:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.2:1619677472.946593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677472.946599:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677472.946601:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:21.0:1619677472.946602:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:17.0:1619677472.946602:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.946603:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:21.0:1619677472.946604:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:8.0:1619677472.946604:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000400:00000200:21.0:1619677472.946605:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.946606:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000400:00000200:21.0:1619677472.946607:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.946607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677472.946607:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677472.946608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:2.0:1619677472.946609:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:21.0:1619677472.946610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:2.0:1619677472.946612:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.946613:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.946613:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.946615:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.946617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.0:1619677472.946659:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.946662:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.946664:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.946665:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.946667:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.946669:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.946670:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.946671:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.946672:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.946674:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.946677:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.946679:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.946682:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.946683:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.946683:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.946685:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.946685:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.946686:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.946687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.946697:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.946700:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.946701:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.946701:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.946703:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677472.946705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.947011:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.947014:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.947014:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.947016:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.947018:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.947020:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.947021:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.947022:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.947023:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.947024:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.947027:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.947028:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.947030:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.947031:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.947032:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.947033:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.947034:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.947035:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.947036:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.947037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.947047:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.947048:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.947049:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.947050:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.947051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.947052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.947052:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.947053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.947053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.947054:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677472.947476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.947483:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.947486:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.947486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.947488:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.947491:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.947492:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.947493:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.947495:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.947495:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.947497:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.947500:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.947501:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.947503:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.947504:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.947505:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.947507:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.947508:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.947509:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.947509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.947521:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.947523:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.947523:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.947524:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.947525:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.947526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.947827:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.947830:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.947830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.947831:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.947833:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.947836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.947994:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.947997:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.947998:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.948000:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.948003:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.948004:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.948005:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.948007:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.948008:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.948009:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.948012:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.948014:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.948016:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.948017:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.948018:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.948019:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.948020:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.948021:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.948021:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.948033:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.948035:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.948036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.948036:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.948039:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.948042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677472.948115:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.948118:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.948119:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.948119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.948121:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.948125:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.948136:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.948139:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.948140:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.948141:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.948144:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.948146:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.948147:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.948148:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.948149:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.948150:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.948153:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.948155:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.948157:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.948158:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.948159:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.948160:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.948161:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.948161:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.948162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.948174:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.948176:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.948177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.948178:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.948180:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.948182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677472.948600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.948608:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.948611:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.948612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.948612:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.948614:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.948618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.948618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677472.948626:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.948629:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.948630:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.948630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.948631:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.948631:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677472.948667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.948672:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.948674:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.948675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.948676:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.948678:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.948680:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.948681:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.948682:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.948683:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.948684:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.948687:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.948689:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.948691:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.948692:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.948693:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.948695:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.948695:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.948696:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677472.948697:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677472.948697:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677472.948702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.948707:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677472.948709:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677472.948710:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.948711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.948721:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.948723:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.948724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.948724:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.948726:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.948728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677472.949230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.949238:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.949241:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[1] from 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677472.949243:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.115@o2ib7 : PUT - routed
00000800:00000200:20.0:1619677472.949244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.949245:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677472.949640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.949643:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.949644:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.949644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.949646:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.949649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677472.949874:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.949876:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677472.949877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677472.949879:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.949881:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.949883:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677472.949884:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.949886:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.949887:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.949888:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677472.949890:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.949893:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.949895:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.949896:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.949897:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.949898:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.949899:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677472.949899:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677472.949901:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677472.949901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.2:1619677472.949905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.949913:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.949915:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.949915:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:17.0:1619677472.949916:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.949916:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677472.949917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677472.949918:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.949918:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.2:1619677472.949919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:1.0:1619677472.949919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677472.949921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677472.949925:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.949928:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.949929:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.949931:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.949933:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.949935:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.949936:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.949938:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.949940:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.949941:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.949943:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.949945:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.949946:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.949946:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677472.949948:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.949948:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677472.949951:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.949951:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.949952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.949952:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.949953:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.949956:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.949957:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.949957:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.949958:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677472.950201:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.950204:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.950205:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.950207:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677472.950209:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.950211:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.950212:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.950213:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.950214:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.950215:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.950218:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677472.950220:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.950222:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.950223:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.950224:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.950226:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.950226:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.950228:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677472.950229:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677472.950229:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677472.950232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.950241:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677472.950243:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677472.950244:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677472.950245:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677472.950245:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677472.950245:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.950246:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:26.0:1619677472.950247:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677472.950248:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.950249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677472.950264:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.950268:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677472.950269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677472.950270:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.950272:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.950274:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677472.950275:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.950276:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.950277:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.950278:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677472.950281:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.950283:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677472.950285:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.950286:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677472.950287:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677472.950288:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677472.950289:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.950289:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677472.950290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677472.950294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.950302:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.950305:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.950306:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.950307:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.950308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.950310:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.950703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.950710:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.950713:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[2] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.950716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.950716:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.115@o2ib7 : PUT - routed
00000800:00000200:20.0:1619677472.950718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.951216:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.951219:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.951220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.951222:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.951224:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.951225:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.951227:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.951229:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.951230:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.951231:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.951233:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.951234:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:8.0:1619677472.951236:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677472.951237:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677472.951238:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.951239:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.951240:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677472.951241:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.951241:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.951242:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:24.0:1619677472.951243:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677472.951243:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:24.0:1619677472.951244:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.951245:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677472.951245:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.951246:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.951247:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.951247:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:24.0:1619677472.951248:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677472.951249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677472.951250:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677472.951253:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.951253:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677472.951254:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677472.951255:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677472.951256:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677472.951257:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677472.951257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:18.0:1619677472.951260:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.951263:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.951264:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.951264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.951266:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.951270:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.951275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677472.951279:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.951282:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.951283:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677472.951283:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.951284:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.951284:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677472.951285:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:2.0:1619677472.951285:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:21.0:1619677472.951286:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.951288:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677472.951288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.951291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677472.951684:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.951687:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.951688:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.951689:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.951691:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.951693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.951702:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.951704:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.951705:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.951705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.951706:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.951707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.951813:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677472.951816:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677472.951817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677472.951818:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677472.951822:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677472.951822:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.951823:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677472.951825:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.951825:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:8.0:1619677472.951825:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.951826:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.951827:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.951828:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.951828:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.951831:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.951831:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677472.951832:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.951833:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677472.951833:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.951834:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.951835:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.951836:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.951836:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.951837:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677472.951837:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:10.0:1619677472.951839:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.951839:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677472.951839:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677472.951840:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:10.0:1619677472.951841:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677472.951841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:10.0:1619677472.951844:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.951845:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.951845:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.2:1619677472.951845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.951847:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.951848:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.951849:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.951850:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.951851:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677472.951852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.951855:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.951857:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.951858:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.951859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677472.951861:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677472.951862:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677472.951863:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.951864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.951865:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:6.0:1619677472.951865:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.951867:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.951867:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677472.951867:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.951868:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.951870:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.951904:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.951906:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.951907:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.951907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.951909:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.951911:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.952339:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.952341:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.952342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.952344:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.952346:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.952347:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.952348:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.952349:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.952350:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.952351:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.952354:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.952356:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.952359:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.952359:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.952360:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.952362:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.952362:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.952363:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.952363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.952377:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.952379:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.952380:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.952380:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.952382:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.952386:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.952951:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.952954:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.952955:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.952956:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.952960:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.952961:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.952962:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.952963:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.952964:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.952965:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.952968:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.952970:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.952972:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.952973:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.952973:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.952975:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.952975:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.952976:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.952977:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.952989:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.952992:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.952993:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.952993:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.952996:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.952998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.953058:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.953061:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.953062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.953063:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.953066:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.953068:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.953069:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.953070:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.953071:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.953072:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.953075:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.953076:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.953079:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.953079:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.953080:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.953082:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.953082:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.953083:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.953083:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.953095:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.953098:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.953098:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.953099:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.953101:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.953104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.953112:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.953115:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.953116:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.953116:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.953117:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.953118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.953379:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.953382:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.953383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.953383:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.953386:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.953389:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677472.953754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.953763:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.953767:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.953769:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.953772:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.953775:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.953777:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.953779:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.953781:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.953783:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.953785:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.953790:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.953793:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.953796:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.953798:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.953799:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.953801:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.953802:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.953803:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.953804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.953814:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.953817:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.953818:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.953818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.953821:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.953824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.954087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.954093:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.954096:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.954098:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.954099:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.954102:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.954103:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.954105:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.954106:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.954107:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.954108:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.954111:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.954113:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.954115:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.954115:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.954116:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.954118:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.954119:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.954120:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.954120:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.954131:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.954133:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.954134:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.954134:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.954135:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.954137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677472.954512:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.954515:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677472.954516:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677472.954518:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.954521:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.954522:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677472.954524:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.954525:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.954526:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.954527:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677472.954530:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.954532:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677472.954535:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.954536:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677472.954537:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677472.954538:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677472.954539:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.954540:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677472.954540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:17.0:1619677472.954936:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.954939:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.954939:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.954942:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.954944:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.954945:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.954947:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.954948:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.954949:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.954950:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.954953:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.954955:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.954958:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.954959:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.954959:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.954961:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.954962:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.954962:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.954963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.954968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.954976:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.954978:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.954979:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.954981:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.954982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.954983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.954986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.954994:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.954996:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.954997:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.954998:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677472.954998:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.954999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677472.955463:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.955466:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.955466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.955467:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.955469:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.955471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.955482:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.955485:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.955486:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.955486:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.955487:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.955488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.955793:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677472.955795:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677472.955796:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677472.955798:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677472.955801:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.955802:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677472.955804:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677472.955805:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677472.955806:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677472.955807:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677472.955811:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677472.955813:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677472.955816:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.955817:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677472.955818:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677472.955819:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677472.955820:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677472.955820:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677472.955821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677472.955826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.955835:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.955838:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.955839:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.955839:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.955841:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.2:1619677472.955843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:1.0:1619677472.955844:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677472.955847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:2.0:1619677472.955849:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.955852:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.955853:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677472.955855:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:2.0:1619677472.955855:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677472.955857:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677472.955857:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.955858:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.955859:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:2.0:1619677472.955859:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677472.955860:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677472.955860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:2.0:1619677472.955860:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.955861:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.955862:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.955863:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677472.955867:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.955869:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.955871:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.955871:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677472.955872:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.955874:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.955874:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.955875:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.955876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.955880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.955887:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.955889:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.955890:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.955891:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.955892:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677472.955895:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.955915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.955922:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.955925:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.955925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.955926:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.955928:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.955930:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.956083:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.956086:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.956087:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.956089:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.956092:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.956093:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.956094:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.956096:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.956096:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.956097:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.956100:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.956102:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.956104:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.956105:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.956105:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.956107:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.956107:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.956108:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.956109:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.956119:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.956121:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.956122:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.956123:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.956124:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.956126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.956548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677472.956554:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.956556:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.956559:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.956559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.956561:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.956563:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677472.956564:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.956565:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.956566:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.956568:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677472.956571:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.956573:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677472.956575:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.956575:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677472.956576:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677472.956578:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677472.956579:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.956580:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677472.956581:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677472.956581:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677472.956588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.956594:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677472.956595:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677472.956596:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.956596:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000400:00000200:21.0:1619677472.956597:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.956597:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677472.956598:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677472.956599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.956600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677472.956601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:20.0:1619677472.956614:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677472.956616:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677472.956617:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677472.956618:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677472.956619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.956620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677472.956660:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.956662:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.956663:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.956665:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.956668:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.956669:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.956670:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.956671:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.956672:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.956673:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.956676:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.956677:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.956679:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.956680:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.956680:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.956682:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.956682:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.956683:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.956683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.956693:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.956695:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.956696:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.956697:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.956697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.956699:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.956713:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.956715:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.956716:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.956716:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.956717:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.956717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677472.957180:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.957183:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.957184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.957184:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.957186:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.957189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677472.957542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.957551:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.957554:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.957555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.957555:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.957557:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.957560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.957561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.957568:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.957570:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.957571:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.957572:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.957573:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.957573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.957650:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.957653:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.957654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.957656:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.957657:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.957659:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.957660:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.957662:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.957663:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.957664:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.957667:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.957668:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.957671:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.957672:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.957673:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.957674:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.957675:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.957677:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.957678:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.957678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.957689:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.957691:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.957692:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.957693:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.957693:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677472.957693:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.957694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.957695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.957696:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.957699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677472.957753:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.957756:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.957757:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.957759:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.957761:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.957763:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.957764:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.957765:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.957766:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.957767:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.957770:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.957772:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.957774:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.957775:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.957776:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.957777:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.957778:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.957778:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.957779:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.957791:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.957794:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.957795:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.957795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.957797:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.957800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.957928:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.957931:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.957932:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.957934:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.957937:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.957938:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.957940:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.957941:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.957942:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.957943:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.957946:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.957948:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.957950:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.957951:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.957951:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.957953:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.957953:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.957954:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.957956:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.957956:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677472.957966:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.957968:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677472.957969:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677472.957970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.957970:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.957971:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677472.957972:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677472.957972:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.957973:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.957975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.958625:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.958627:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677472.958628:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677472.958630:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.958633:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.958635:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677472.958636:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.958638:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.958639:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.958640:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677472.958643:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.958645:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677472.958647:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.958648:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677472.958649:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677472.958651:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677472.958651:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677472.958651:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.958652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677472.958657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.2:1619677472.958665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677472.958666:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.958668:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.958669:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.958671:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.958672:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677472.958673:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:21.0:1619677472.958675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677472.958675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.958676:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.958678:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.958678:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.958680:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677472.958680:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.958681:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.958683:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:17.0:1619677472.958684:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677472.958684:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.958686:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.958688:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.958689:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.958691:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.958692:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.958693:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.958696:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.958698:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.958701:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.958702:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.958703:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.958704:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.958705:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.958706:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.958706:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.958720:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.958722:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.958723:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.958723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.958725:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.958728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:12.0:1619677472.958946:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.958950:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:13.0:1619677472.958954:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677472.958954:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677472.958957:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.958960:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677472.958962:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677472.958964:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677472.958966:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677472.958968:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677472.958973:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677472.958976:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677472.958980:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677472.958981:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677472.958982:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677472.958985:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677472.958986:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677472.958987:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.958988:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677472.958999:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.959002:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.959003:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.959004:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.959006:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.959010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677472.959231:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.959234:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.959235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.959236:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.959238:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.959241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677472.960077:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.960080:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.960082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.960082:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677472.960084:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.960086:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.960087:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.960088:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.960090:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.960091:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.960094:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.960096:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.960098:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.960099:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.960100:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.960102:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.960102:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.960103:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.960104:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.960117:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.960119:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.960120:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.960120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.960122:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.960128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.960129:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.960131:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.960132:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.960132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.960133:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.960133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677472.960154:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.960157:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677472.960158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677472.960160:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.960162:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.960164:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.960165:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.960166:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.960167:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.960168:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.960172:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.960174:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.960177:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.960177:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.960178:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.960180:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.960180:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.960181:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.960181:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.960186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.960195:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.960197:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.960198:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.960199:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.960200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.960202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677472.960958:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.960961:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677472.960962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:20.2:1619677472.960962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:6.0:1619677472.960963:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.960966:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.960968:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677472.960969:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677472.960970:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677472.960971:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.960972:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677472.960973:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:6.0:1619677472.960973:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677472.960973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.960977:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:6.0:1619677472.960979:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677472.960980:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.960981:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677472.960981:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:18.0:1619677472.960983:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677472.960984:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677472.960985:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.960985:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677472.960985:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677472.960986:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677472.960987:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677472.960988:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677472.960988:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.960989:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000400:00000200:18.0:1619677472.960990:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677472.960990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:18.0:1619677472.960992:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677472.960995:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.960996:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677472.960996:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677472.960998:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677472.960999:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.961000:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.961000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.961001:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.961003:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.961004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.961005:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.961007:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.961010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:13.0:1619677472.961011:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.961014:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.961015:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.961016:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677472.961018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.961019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677472.961023:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.961026:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.961027:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.961027:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.961028:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.961029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677472.961184:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.961187:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.961188:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.961188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.961191:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.961193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.961275:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.961278:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.961279:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.961281:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.961284:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.961285:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.961287:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.961288:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.961289:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.961290:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.961293:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.961299:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.961301:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.961302:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.961303:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.961304:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.961304:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.961306:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.961307:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.961307:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.961317:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.961319:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.961320:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.961321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.961322:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677472.961322:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.961324:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:8.0:1619677472.961324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.961325:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.961326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.961763:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.961766:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.961767:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.961768:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.961770:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.961771:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.961772:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.961774:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.961774:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.961775:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.961778:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.961779:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.961780:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.961781:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.961782:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.961783:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.961783:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.961784:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.961784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.961794:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.961796:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.961797:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.961798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.961798:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.961800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677472.961892:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.961895:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677472.961896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677472.961898:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.961901:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.961902:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677472.961904:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.961905:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.961906:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.961907:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677472.961910:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.961912:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.961915:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.961916:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677472.961916:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677472.961918:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677472.961919:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677472.961920:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677472.961920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:6.0:1619677472.961932:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.961934:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.961935:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.961935:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.961938:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.961940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.962109:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.962112:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.962113:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.962113:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.962115:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.962117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.962257:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.962259:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.962260:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677472.962262:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.962264:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.962265:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.962266:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.962267:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.962268:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.962269:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.962272:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.962274:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.962277:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.962277:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.962278:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.962280:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.962280:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.962281:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.962281:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.962293:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.962299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.962301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.962302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.962304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.962307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.962316:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677472.962319:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.962320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.962320:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.962321:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.962322:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677472.962422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.962429:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.962431:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.962432:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.962434:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.962437:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.962438:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.962440:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.962441:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.962442:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.962443:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.962446:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.962448:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.962450:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.962451:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.962451:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.962453:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.962454:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.962455:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.962455:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.962467:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.962469:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.962470:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.962471:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.962472:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.962473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.962490:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.962492:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.962493:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.962493:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.962494:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.962494:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.962664:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.962667:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.962667:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.962670:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.962672:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.962674:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.962675:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.962676:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.962677:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.962678:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.962681:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.962683:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.962686:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.962686:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.962687:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.962689:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.962689:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.962690:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.962690:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.962704:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.962707:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.962707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.962708:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.962710:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.962713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.963131:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.963134:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.963135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.963136:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.963138:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.963139:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.963141:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.963142:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.963143:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.963144:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.963147:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.963148:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.963151:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.963152:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.963152:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.963154:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.963154:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.963155:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.963155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.963168:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.963171:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.963172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.963172:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.963175:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.963178:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.963334:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.963337:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.963338:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.963338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.963340:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.963342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.963383:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.963386:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677472.963386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.963389:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.963391:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.963393:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677472.963394:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.963395:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.963396:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.963397:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677472.963400:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.963402:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677472.963405:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.963405:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677472.963406:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677472.963408:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677472.963409:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.963410:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677472.963410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:5.2:1619677472.963412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.963418:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.963420:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.963421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677472.963422:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:1.0:1619677472.963422:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.963424:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.963425:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:1.0:1619677472.963425:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677472.963426:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.963426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:1.0:1619677472.963427:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.963428:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:1.0:1619677472.963429:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.963430:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.963431:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677472.963432:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:1.0:1619677472.963435:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.963437:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.963439:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.963440:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.963440:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677472.963442:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677472.963442:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.963442:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.963443:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677472.963444:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677472.963444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677472.963445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.963445:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.963446:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.963447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677472.963456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.963459:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.963460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.963460:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.963462:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.963464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.963696:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.963698:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.963699:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.963699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.963702:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.963704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.963717:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.963719:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.963720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.963721:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.963722:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.963722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677472.964474:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.964476:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.964479:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.964480:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.964480:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.964482:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.964483:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.964485:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.964486:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.964487:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.964490:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.964492:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.964494:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.964495:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.964496:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.964498:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.964498:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.964499:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.964501:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.964501:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677472.964512:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.964515:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.964516:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.964516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.964518:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.964520:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677472.964523:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677472.964525:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677472.964526:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.964527:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:10.0:1619677472.964716:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.964719:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.964720:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.964722:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.964725:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.964726:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677472.964728:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.964729:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.964730:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.964731:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677472.964734:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.964736:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677472.964739:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.964739:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677472.964740:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677472.964742:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677472.964742:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.964743:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.964743:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677472.964757:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677472.964759:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677472.964760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677472.964761:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.964763:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.964766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677472.964857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.964866:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.964868:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.964869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.964870:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.964872:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.964875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.965112:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.965114:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.965115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.965115:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.965117:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.965119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.965467:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.965470:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.965471:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.965471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.965472:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.965475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677472.965488:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.965490:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.965491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.965491:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.965492:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.965493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677472.965669:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.965673:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.965673:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.965674:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.965676:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.965679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677472.965920:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.965923:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.965925:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.965926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.965927:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.965928:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.965930:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.965931:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.965932:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.965933:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.965937:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.965938:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.965941:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.965942:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.965943:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.965945:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.965945:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.965946:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.965947:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677472.965968:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.965971:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.965972:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.965974:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.965977:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.965978:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.965980:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.965981:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.965982:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.965983:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.965985:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.965987:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.965989:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.965990:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.965990:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.965992:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.965992:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.965993:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.965994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.966004:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.966006:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.966007:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.966007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.966009:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.966011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677472.966070:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.966073:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.966073:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.966076:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.966078:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.966079:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.966081:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.966082:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.966083:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.966084:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.966088:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.966089:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.966092:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.966093:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.966093:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.966095:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.966095:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.966097:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.966098:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.966098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.966101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.966109:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677472.966110:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.966110:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.966111:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677472.966112:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677472.966112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677472.966113:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.966115:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.966116:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.966118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677472.966408:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677472.966411:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677472.966412:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:10.0:1619677472.966414:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.966416:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.966418:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.966419:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.966420:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.966421:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.966422:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.966425:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.966427:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.966430:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.966431:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.966432:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.966434:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.966434:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677472.966435:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677472.966435:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677472.966440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.966448:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.966451:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.966451:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.966453:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.966454:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.966456:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.966463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677472.966470:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.966473:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677472.966474:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.966475:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677472.966475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.966476:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677472.966850:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.966852:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.966854:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.966855:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.966857:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.966859:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.966860:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.966861:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.966862:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.966863:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.966867:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.966868:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.966871:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.966871:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.966872:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.966873:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.966874:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.966875:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.966875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.966889:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.966891:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.966892:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.966892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.966894:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.966896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.967185:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.967188:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.967189:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.967191:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.967194:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.967196:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.967197:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.967198:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.967199:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.967200:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.967204:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.967206:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.967208:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.967209:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.967210:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.967211:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.967212:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.967213:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.967213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.967225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.967228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.967228:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677472.967229:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.967231:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.967234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677472.967247:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.967251:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.967252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.967253:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.967257:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.967258:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.967259:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.967260:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.967261:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.967262:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.967265:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.967266:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.967268:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.967268:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.967269:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.967270:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.967271:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.967271:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.967272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.967283:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.967285:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.967285:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.967287:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.967287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.967289:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677472.967396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.967399:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.967400:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.967400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.967402:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.967405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.967414:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.967417:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.967418:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.967418:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.967419:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.967420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.967441:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.967443:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.967444:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.967444:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.967447:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.967449:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.967492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.967499:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.967502:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.967503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.967505:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677472.967506:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677472.967507:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.967508:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.967509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677472.967509:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.967510:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.967512:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.967512:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.967514:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.967515:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.967515:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677472.967517:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.967519:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.967519:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677472.967520:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.967521:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.967522:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.967522:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.967525:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677472.967526:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.967526:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677472.967527:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677472.967527:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677472.967529:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.967529:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.967530:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:10.0:1619677472.967531:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.967531:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.967532:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677472.967532:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.967532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677472.967534:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.967534:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.967535:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.967536:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677472.967537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.967543:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.967545:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.967546:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677472.967547:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677472.967547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677472.967550:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.967551:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.967551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.967553:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.967556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677472.967638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.967644:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.967647:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.967648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.967651:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.967653:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.967654:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.967656:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.967657:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.967658:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.967659:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.967662:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.967664:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.967666:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.967667:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.967668:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.967669:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.967670:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.967671:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677472.967672:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677472.967673:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677472.967678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677472.967682:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677472.967683:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677472.967685:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677472.967685:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.967686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.967686:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677472.967686:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677472.967688:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677472.967688:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.967690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677472.968530:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.968533:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.968535:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.968536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.968538:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.968539:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.968541:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.968542:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.968543:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.968544:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.968547:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.968549:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.968551:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.968552:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.968552:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.968554:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.968554:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.968555:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.968556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.968579:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.968581:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.968582:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.968582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.968585:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677472.968587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.968655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.968661:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.968663:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.968665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.968667:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.968670:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.968671:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.968672:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.968674:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.968675:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.968676:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.968679:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.968681:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.968683:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.968683:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.968684:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.968686:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.968686:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.968687:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.968688:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677472.968689:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677472.968695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.968701:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.968701:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677472.968702:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677472.968702:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677472.968703:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.968703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.968704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.968705:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.968705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.968707:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.969150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.969155:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.969158:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.969159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.969161:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.969164:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.969165:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677472.969167:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.969168:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.969169:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.969170:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677472.969174:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.969175:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677472.969179:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.969180:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677472.969180:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677472.969182:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677472.969183:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.969184:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.969184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:26.0:1619677472.969195:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677472.969198:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.969199:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.969199:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677472.969201:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.969204:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677472.969229:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677472.969231:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677472.969232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677472.969232:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.969233:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677472.969233:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:24.0:1619677472.969410:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.969413:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.969414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.969415:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.969417:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.969419:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677472.969420:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.969421:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.969422:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.969423:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677472.969426:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.969428:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677472.969431:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.969431:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677472.969432:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677472.969434:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677472.969434:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.969435:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.969436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677472.969449:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.969452:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.969453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.969453:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.969455:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.969458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.969520:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.969523:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.969524:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.969524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.969526:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.969529:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.969538:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.969541:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.969542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.969542:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.969543:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.969544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.969996:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.969999:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.969999:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.970001:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.970004:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.970006:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677472.970007:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.970008:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.970009:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.970010:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677472.970013:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.970015:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677472.970018:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.970019:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677472.970019:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677472.970021:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677472.970022:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.970023:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.970024:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.970025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.970035:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677472.970038:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.970038:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.970039:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.970039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.970040:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.970041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677472.970041:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.970042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.970045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677472.970484:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.970487:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677472.970489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.970489:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.970491:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.970493:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.970494:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.970495:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.970497:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.970497:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.970500:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.970502:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.970505:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.970506:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.970506:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.970508:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.970508:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.970509:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.970509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.970521:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.970524:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.970525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.970525:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.970527:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.970530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.970541:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.970544:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.970544:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.970545:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.970545:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.970546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.970686:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.970688:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.970690:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.970691:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.970694:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.970695:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.970696:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.970697:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.970698:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.970699:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.2:1619677472.970699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:26.0:1619677472.970702:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.970703:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.970706:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.970706:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.970707:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677472.970708:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677472.970709:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.970709:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.970711:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:1.0:1619677472.970711:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:26.0:1619677472.970712:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:17.0:1619677472.970712:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.970712:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.970713:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677472.970714:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.970718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.970720:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.970723:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.970724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.970724:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.970726:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677472.970726:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.970728:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.970728:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.970728:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677472.970729:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.971534:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.971536:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.971537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.971538:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.971540:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.971542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.971887:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.971889:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.971890:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.971892:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.971894:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.971895:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.971897:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.971898:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.971899:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.971900:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677472.971902:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:10.0:1619677472.971903:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.971904:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:10.0:1619677472.971905:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677472.971905:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677472.971907:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.971908:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677472.971909:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.971909:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.971910:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:17.0:1619677472.971911:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:10.0:1619677472.971911:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:17.0:1619677472.971912:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677472.971912:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.971913:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.971913:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:17.0:1619677472.971914:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.971915:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.971916:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.2:1619677472.971918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:17.0:1619677472.971919:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.971921:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677472.971924:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.971925:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.971926:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:17.0:1619677472.971926:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677472.971927:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677472.971928:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677472.971928:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:21.0:1619677472.971929:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:17.0:1619677472.971929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:21.0:1619677472.971930:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.971931:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.971932:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.971935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677472.971977:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.971980:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.971981:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.971983:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.971985:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.971987:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.971988:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.971989:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.971990:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.971991:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.971994:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.971996:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.971998:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.971999:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.972000:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.972002:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.972002:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.972003:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.972003:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.972016:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.972018:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.972019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.972020:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.972022:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.972024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677472.972038:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677472.972041:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.972042:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.972042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.972043:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677472.972044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677472.972102:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.972105:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.972107:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.972108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.972109:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.972110:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.972111:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.972113:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.972114:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.972114:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.972117:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.972119:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.972122:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.972122:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.972123:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.972125:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.972125:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.972127:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.972128:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.972129:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677472.972137:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677472.972139:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677472.972140:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.972141:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.972141:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:8.0:1619677472.972141:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.972142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:24.0:1619677472.972143:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677472.972143:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677472.972145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.972365:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.972367:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.972368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.972368:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.972370:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.972372:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.0:1619677472.972373:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.972375:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.972376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.972378:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.972381:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.972382:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.972383:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677472.972384:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677472.972385:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.972385:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.972386:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677472.972387:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.972388:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.972388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.972389:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.972389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:10.0:1619677472.972389:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.972391:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.972394:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.972395:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.972395:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.972397:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.972398:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.972398:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.972399:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677472.972403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.972411:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.972414:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.972415:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.972417:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.972417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.972420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677472.972485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.972490:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.972493:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.972494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.972496:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.972499:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.972501:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.972502:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.972503:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.972504:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.972505:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.972508:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.972510:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.972513:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.972514:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.972514:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.972516:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.972516:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.972517:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.972518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.972530:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.972533:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.972534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.972534:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.972536:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.972538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.972821:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.972824:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.972825:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.972827:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.972830:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.972831:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.972833:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.972834:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.972835:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.972836:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.972840:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.972841:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.972844:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.972845:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.972846:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.972847:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.972848:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.972848:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.972849:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.972871:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.972873:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.972874:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.972874:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.972877:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.972880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677472.973031:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.973034:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.973036:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.973037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.973038:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.973040:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.973041:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.973042:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.973044:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.973044:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.973048:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.973050:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.973052:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.973053:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.973053:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.973055:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.973055:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.973056:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.973057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677472.973079:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.973082:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.973082:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.973083:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.973085:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677472.973087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.973434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677472.973440:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.973443:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.973444:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.973446:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.973449:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.973450:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677472.973452:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.973453:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.973454:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.973455:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677472.973458:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.973460:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677472.973463:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.973463:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677472.973464:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677472.973466:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.973467:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.973468:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677472.973468:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677472.973473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.973481:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.973484:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.973485:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.973485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.973487:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677472.973491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.973493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677472.973500:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.973503:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.973503:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677472.973504:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.973505:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677472.973505:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677472.973697:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.973700:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.973700:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.973701:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.973703:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.973709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.973711:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.973713:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.973714:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.973714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.973716:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.973716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677472.973796:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.973799:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.973801:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.973801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.973803:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.973805:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.973806:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.973807:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.973808:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.973809:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.973812:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.973814:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.973817:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.973817:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677472.973818:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.973820:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.973820:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.973821:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.973822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.973843:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.973846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.973846:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.973847:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.973849:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.973852:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.974164:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.974167:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.974167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.974169:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.974171:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.974172:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.974173:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.974175:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.974176:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.974176:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.974179:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.974181:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.974182:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.974183:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.974183:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.974185:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.974185:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.974186:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.974186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677472.974207:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.974209:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.974210:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.974211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.974211:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.974213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.974423:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.974425:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677472.974427:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.974427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.974430:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.974431:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.974432:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.974433:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.974434:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.974435:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.974437:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.974438:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.974440:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.974440:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.974441:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.974442:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.974442:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.974442:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.974443:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.974454:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.974455:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.974456:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.974457:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.974458:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.974459:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.974474:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.974476:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.974477:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.974477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.974478:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.974478:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677472.975167:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.975170:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.975170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.975172:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.975174:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.975176:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.975177:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.975178:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.975179:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.975180:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.975184:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.975186:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.975188:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.975189:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.975190:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.975191:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.975192:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.975192:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.975193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.975197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677472.975204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:20.0:1619677472.975209:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:18.0:1619677472.975211:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:20.0:1619677472.975212:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.975212:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.975214:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.975214:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677472.975214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:21.0:1619677472.975215:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.975217:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:18.0:1619677472.975217:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.975220:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.975222:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677472.975223:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.975225:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.975225:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.975227:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677472.975230:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.975232:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677472.975234:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.975235:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677472.975236:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.975237:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.975238:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677472.975239:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677472.975240:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677472.975241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.2:1619677472.975244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677472.975246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:1.0:1619677472.975251:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677472.975252:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677472.975254:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:1.0:1619677472.975254:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.975255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677472.975255:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:1.0:1619677472.975255:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.975257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:1.0:1619677472.975257:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.975260:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677472.975383:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.975386:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.975387:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.975388:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.975392:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.975393:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.975394:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.975396:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.975397:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.975398:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.975400:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.975402:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.975404:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.975405:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.975406:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.975407:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.975408:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.975409:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.975409:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.975420:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.975423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677472.975423:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.975423:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.975425:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.975428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.975654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.975661:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.975664:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.975665:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.975666:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.975669:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.975670:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.975672:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.975673:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.975674:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.975675:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.975678:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.975680:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.975682:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.975683:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.975683:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.975685:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.975686:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.975687:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.975687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.975698:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.975700:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.975701:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.975702:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.975703:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.975704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677472.975719:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.975721:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.975722:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.975722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.975723:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.975723:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677472.975814:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.975817:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677472.975817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677472.975819:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677472.975821:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.975822:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.975823:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.975824:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.975826:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.975827:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.975829:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677472.975831:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.975832:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.975833:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.975834:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.975835:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.975835:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677472.975836:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677472.975836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677472.975846:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.975848:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.975848:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.975849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.975850:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.975852:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677472.975885:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.975888:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.975889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.975889:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.975892:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.975894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.976303:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.976306:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.976307:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.976308:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.976311:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.976313:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.976314:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.976315:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.976316:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.976317:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.976320:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.976322:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.976325:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.976326:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.976326:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.976328:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.976328:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.976329:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677472.976331:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.976331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677472.976343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677472.976345:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677472.976346:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.976346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.976347:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.976347:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.976348:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677472.976348:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677472.976349:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.976352:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677472.976367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.976376:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.976379:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.976380:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.976380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.976382:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.976385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.976517:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.976520:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.976522:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.976523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.976525:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.976526:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.976528:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.976529:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.976530:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.976531:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.976534:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.976536:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.976539:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.976540:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.976541:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.976543:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.976543:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.976544:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.976545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677472.976826:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.976829:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.976830:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.976832:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.976833:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.976835:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.976836:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.976837:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.976838:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.976839:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.976842:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.976843:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.976846:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.976847:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.976848:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.976849:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.976850:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.976850:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.976851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677472.976863:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.976866:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.976867:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.976868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.976870:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.976873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677472.977234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.977243:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.977245:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.977246:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.977248:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.977248:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.977251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677472.977345:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.977347:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.977348:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.977351:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.977354:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.977355:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677472.977357:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.977358:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.977359:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.977360:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677472.977363:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.977364:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677472.977367:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.977368:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677472.977368:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677472.977370:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677472.977370:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.977371:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677472.977372:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677472.977372:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.977386:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677472.977386:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677472.977387:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677472.977388:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.977389:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677472.977389:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.977390:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.977392:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.977394:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.977396:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677472.977396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677472.977397:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)++
00000800:00000200:13.0:1619677472.977398:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677472.977398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)++
00000800:00000200:12.0:1619677472.977399:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677472.977400:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.977401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677472.977540:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.977542:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.977543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.977544:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.977546:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.977548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.977702:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.977705:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.977706:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.977708:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.977710:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.977712:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.977713:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.977714:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.977715:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.977716:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.977719:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.977720:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.977723:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.977724:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.977724:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.977726:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.977726:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.977727:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.977728:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.977740:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.977742:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.977743:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.977744:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.977746:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.977748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677472.977764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677472.977770:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.977773:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677472.977774:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677472.977776:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.977778:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.977779:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677472.977781:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.977782:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.977783:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.977784:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677472.977787:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.977789:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677472.977791:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.977792:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677472.977792:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677472.977794:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677472.977795:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677472.977795:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.977796:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.977807:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.977810:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.977811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.977811:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.977813:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.977815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677472.978052:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.978054:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677472.978056:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.978057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677472.978059:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.978061:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.978062:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.978063:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.978064:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.978065:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.978069:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.978070:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.978072:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.978073:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.978074:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.978076:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.978076:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677472.978077:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.978078:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677472.978089:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.978092:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.978093:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.978093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.978095:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.978097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.978108:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.978111:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.978112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.978112:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.978113:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.978114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677472.978139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.978147:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.978150:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.978151:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.978151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.978153:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.978156:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677472.978876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.978883:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.978886:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.978886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.978888:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.978890:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.978892:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677472.978893:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.978894:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.978896:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.978897:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677472.978900:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.978901:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677472.978904:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.978905:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677472.978906:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677472.978908:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677472.978909:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.978910:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.978910:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677472.978922:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.978925:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.978926:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.978928:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677472.978929:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.978930:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677472.979668:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.979671:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.979672:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.979672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.979674:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.979677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.979815:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.979818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.979818:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677472.979820:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.979824:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.979825:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.979826:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.979827:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.979828:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.979829:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.979832:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.979833:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.979835:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.979836:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.979836:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.979837:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.979838:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.979838:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.979839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677472.979849:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.979851:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.979851:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.979852:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.979853:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.979854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677472.980124:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.980127:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.980128:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.980130:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.980133:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.980134:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.980135:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.980136:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.980137:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.980138:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.980140:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.980142:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.980144:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.980144:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.980145:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.980146:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.980147:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.980148:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.980149:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.980149:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.980158:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.980160:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.980161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.980161:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.980162:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677472.980163:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.980164:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.980165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677472.980165:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.980165:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.980848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.980857:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.980860:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.980861:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.980863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.980863:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.980866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677472.980868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.980875:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.980878:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677472.980879:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.980880:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.980880:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677472.980881:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677472.981053:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.981056:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677472.981057:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677472.981058:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.981061:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.981063:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.981064:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.981065:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.981066:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.981067:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.981070:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.981072:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.981075:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.981076:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.981076:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.981078:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.981078:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.981079:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.981079:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677472.981084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.981091:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.981094:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.981095:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.981096:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.981097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.981100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677472.981143:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.981146:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.981147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.981148:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.981151:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.981153:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.981154:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.981155:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.981156:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.981157:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.981160:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.981162:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.981164:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.981165:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.981166:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.981167:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:20.2:1619677472.981167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:26.0:1619677472.981168:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.981168:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.981169:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677472.981173:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677472.981176:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.2.6@o2ib3
00000800:00000200:18.0:1619677472.981177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:2.0:1619677472.981179:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.981182:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.981183:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:2.0:1619677472.981184:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:6.0:1619677472.981185:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:2.0:1619677472.981185:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677472.981186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:2.0:1619677472.981186:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.981187:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.981187:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.981189:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:2.0:1619677472.981189:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677472.981192:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.981193:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.981194:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677472.981194:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677472.981196:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677472.981197:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.981198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.981198:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:2.0:1619677472.981198:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.981198:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677472.981199:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677472.981199:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677472.981201:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677472.981202:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677472.981203:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:2.0:1619677472.981203:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:20.2:1619677472.981206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:18.0:1619677472.981212:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:24.0:1619677472.981213:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:18.0:1619677472.981215:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:2.0:1619677472.981215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000800:00000200:24.0:1619677472.981216:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.981217:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.981217:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677472.981217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.981219:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:18.0:1619677472.981219:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.981221:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677472.981222:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.981223:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.981224:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.981225:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677472.981227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:18.0:1619677472.981229:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.981231:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677472.981234:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.981235:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:18.0:1619677472.981235:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:18.0:1619677472.981237:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:18.0:1619677472.981237:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677472.981238:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:18.0:1619677472.981239:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:12.0:1619677472.981251:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.981254:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.981255:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677472.981258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:13.0:1619677472.981258:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677472.981258:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.981262:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:2.0:1619677472.981264:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677472.981267:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:18.0:1619677472.981268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:2.0:1619677472.981270:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.981273:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.981274:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:2.0:1619677472.981276:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.981277:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.981278:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.981279:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:2.0:1619677472.981282:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.981283:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677472.981286:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.981286:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:2.0:1619677472.981287:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:2.0:1619677472.981289:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:2.0:1619677472.981289:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677472.981290:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:2.0:1619677472.981290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:8.2:1619677472.981296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677472.981304:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677472.981306:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677472.981307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677472.981307:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.981309:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.981312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677472.981411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:18.0:1619677472.981417:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677472.981420:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:2.0:1619677472.981421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:18.0:1619677472.981423:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.981425:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.981426:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677472.981428:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.981430:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.981431:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.981432:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:18.0:1619677472.981435:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.981438:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.981440:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.981440:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677472.981441:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677472.981442:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.981443:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677472.981444:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919078fb5400] (22)++
00000800:00000200:18.0:1619677472.981445:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (23)--
00000800:00000200:18.0:1619677472.981446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000800:00000200:20.2:1619677472.981452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677472.981457:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.981458:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.981459:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (22)++
00000400:00000200:2.0:1619677472.981459:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.981460:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919078fb5400] (23)--
00000800:00000200:18.0:1619677472.981461:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:2.0:1619677472.981461:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.981462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677472.981463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:2.0:1619677472.981464:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677472.981466:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.981469:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.2:1619677472.981469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000400:00000200:22.0:1619677472.981470:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.981470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.981472:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.981475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677472.981475:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677472.981478:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:2.0:1619677472.981479:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:18.0:1619677472.981480:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.981482:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.981484:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677472.981485:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.981486:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.981487:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.981488:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677472.981492:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.981493:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677472.981496:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.981497:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:18.0:1619677472.981498:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677472.981499:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:18.0:1619677472.981500:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677472.981501:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:18.0:1619677472.981501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:8.2:1619677472.981506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.981513:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.981516:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.981517:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.981518:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.981519:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.981521:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.981530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.981537:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.981539:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.981540:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.981541:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.981542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.981542:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677472.981544:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:2.0:1619677472.981550:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677472.981552:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000800:00000200:18.0:1619677472.981554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000400:00000200:2.0:1619677472.981555:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.981558:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.981559:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677472.981561:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.981562:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.981563:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.981564:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677472.981567:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.981569:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677472.981572:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.981573:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677472.981573:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677472.981575:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677472.981576:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677472.981577:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:2.0:1619677472.981577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:10.0:1619677472.981588:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.981591:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677472.981592:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677472.981592:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.981594:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.981597:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677472.981609:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.981611:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677472.981612:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.981612:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.981613:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677472.981613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:8.0:1619677472.981958:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.981961:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.981961:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.981962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.981964:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.981966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677472.982210:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.982213:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677472.982214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677472.982216:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.982219:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.982221:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677472.982222:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.982224:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.982225:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.982226:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677472.982229:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.982231:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677472.982234:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.982235:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:6.0:1619677472.982235:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677472.982238:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677472.982238:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.982240:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677472.982241:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677472.982241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677472.982252:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677472.982254:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677472.982254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677472.982255:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:20.2:1619677472.982463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677472.982468:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.982471:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.982473:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.982473:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.982476:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.982477:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677472.982479:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.982480:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.982481:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.982482:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677472.982486:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.982487:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677472.982490:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.982491:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677472.982492:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677472.982494:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677472.982494:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.982495:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677472.982495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:10.0:1619677472.982508:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677472.982510:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677472.982511:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677472.982512:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.982514:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.982517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:5.2:1619677472.982539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677472.982543:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677472.982545:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677472.982546:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.982547:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.982549:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677472.982549:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.982549:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677472.982552:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.982552:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.982554:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677472.982554:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.982555:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.982556:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.982557:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.982557:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.982559:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.982559:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.982560:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.982561:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.982562:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.982565:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.982567:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677472.982570:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.982571:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.982571:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.982572:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677472.982572:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.982574:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.982575:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.982575:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677472.982575:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.982575:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.982576:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.982578:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677472.982578:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.982578:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677472.982579:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:17.0:1619677472.982579:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.982580:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677472.982581:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677472.982583:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677472.982584:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.982587:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.982588:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.982589:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96511406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.982590:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677472.982590:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:26.0:1619677472.982591:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677472.982591:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:26.0:1619677472.982592:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.982592:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.982592:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677472.982593:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.982593:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.982595:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677472.982595:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:26.0:1619677472.982597:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.982599:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.982602:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.982602:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.982603:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.982603:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.982604:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.982604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.982605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677472.982606:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.982607:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.982608:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677472.982609:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.982610:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:6.0:1619677472.982620:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.982622:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.982623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.982623:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:6.0:1619677472.982623:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.982625:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677472.982626:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677472.982626:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:6.0:1619677472.982626:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.982628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.982644:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.982646:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.982648:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.982648:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.982648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.982649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.982782:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.982785:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.982786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.982788:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.982790:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.982791:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.982793:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.982794:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.982795:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.982796:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.982800:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.982801:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.982804:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.982805:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.982806:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.982807:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.982808:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.982809:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.982810:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.982810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677472.982822:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677472.982823:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677472.982825:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677472.982825:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677472.982826:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.982826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677472.982826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677472.982827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677472.982828:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.982832:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677472.982927:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.982929:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.982930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.982930:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.982932:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.982934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677472.982945:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.982947:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.982948:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.982948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.982949:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.982950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677472.983251:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.983253:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.983255:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.983256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.983258:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.983260:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.983261:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.983262:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.983263:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.983264:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.983267:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.983269:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.983272:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.983272:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.983273:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.983275:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.983275:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.983277:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677472.983278:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677472.983279:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677472.983287:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677472.983289:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677472.983290:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.983291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.983291:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:8.0:1619677472.983291:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.983292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677472.983292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677472.983298:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.983301:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.983493:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.983496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.983496:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.983499:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.983501:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.983503:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.983504:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.983505:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.983506:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.983507:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.983510:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.983512:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.983514:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.983515:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.983516:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.983517:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.983518:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.983518:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.983519:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.983524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.983532:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677472.983534:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677472.983535:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.983537:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.983537:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.983540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.983678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677472.983687:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.983690:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677472.983691:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.983693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.983693:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.983693:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677472.983696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677472.983696:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.983697:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677472.983699:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.983702:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.983704:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677472.983705:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.983706:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.983708:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.983708:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677472.983711:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.983713:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677472.983715:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.983716:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677472.983717:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677472.983718:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677472.983719:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.983719:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.983720:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677472.983732:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.983735:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.983735:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.983737:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.983739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.983740:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677472.983760:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.983764:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.983765:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.983765:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.983767:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.983770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.984112:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.984115:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:21.0:1619677472.984116:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677472.984116:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.984117:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.984118:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:24.0:1619677472.984120:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.984121:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677472.984121:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.984121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:24.0:1619677472.984123:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.984124:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.984124:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.984125:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677472.984126:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.984127:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677472.984127:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.984129:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.984130:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.984131:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677472.984134:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677472.984136:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.984137:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677472.984138:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677472.984139:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.984140:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.984141:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677472.984141:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.984142:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677472.984142:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677472.984142:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677472.984143:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.984144:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677472.984144:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677472.984145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677472.984145:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.984145:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677472.984146:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677472.984150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677472.984157:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677472.984158:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677472.984159:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677472.984160:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:8.0:1619677472.984160:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.984161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677472.984161:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.984163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677472.984163:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:8.0:1619677472.984163:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.984166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677472.984166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.984384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677472.984390:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.984393:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677472.984394:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.984396:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.984399:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.984400:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677472.984402:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.984403:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.984405:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.984406:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677472.984409:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.984411:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.984413:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.984414:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677472.984414:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.984417:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.984417:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.984418:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677472.984419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677472.984431:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.984433:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.984434:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.984435:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.984436:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.984438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677472.984822:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.984825:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.984826:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.984827:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.984830:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.984831:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.984833:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.984834:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.984835:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.984836:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677472.984839:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.984840:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677472.984842:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.984843:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.984843:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677472.984845:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677472.984845:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.984846:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.984846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.984857:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.984860:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.984861:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.984861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.984863:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.984865:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677472.985507:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.985509:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677472.985510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677472.985510:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677472.985512:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677472.985513:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.985514:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.985514:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.985514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.985517:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677472.985517:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677472.985518:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677472.985520:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677472.985520:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.985521:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.985522:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.985525:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.985527:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.985530:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.985531:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677472.985531:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677472.985533:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677472.985533:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.985534:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.985534:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.985546:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.985549:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.985550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.985550:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.985552:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.2:1619677472.985555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:6.0:1619677472.985555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677472.985561:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677472.985563:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677472.985565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677472.985566:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677472.985568:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.985570:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677472.985571:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.985573:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.985574:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.985575:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677472.985579:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677472.985580:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677472.985584:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.985585:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677472.985585:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677472.985586:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677472.985587:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677472.985588:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677472.985589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.985589:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677472.985589:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677472.985590:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.985590:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677472.985590:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677472.985591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.986677:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.986679:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.986680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.986682:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.986684:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.986685:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.986686:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.986688:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.986689:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.986689:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.986692:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.986694:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.986695:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.986696:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.986697:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.986698:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.986698:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.986699:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.986700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677472.986710:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.986712:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.986713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.986713:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.986715:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.986716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.987251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677472.987257:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.987259:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677472.987261:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.987262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677472.987264:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.987265:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677472.987267:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.987268:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.987269:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.987270:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677472.987273:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.987275:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.987278:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.987278:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677472.987279:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.987280:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677472.987281:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677472.987282:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677472.987282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:22.0:1619677472.987295:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.987299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.987301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.987302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.987304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.987305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677472.987384:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.987387:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.987387:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.987389:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.987392:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.987393:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.987394:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.987396:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.987397:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.987397:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.987401:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.987402:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.987405:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.987406:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.987407:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.987408:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.987409:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.987409:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.987410:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.987423:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.987426:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.987427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.987427:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.987430:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.987433:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.987443:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.987446:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.987447:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.987447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.987448:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.987449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.987544:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.987546:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677472.987547:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.987547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.987549:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.987551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677472.987594:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.987597:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.987598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.987598:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.987601:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.987603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677472.987830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.987839:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.987844:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.987845:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.987849:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.987852:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.987854:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.987856:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.987858:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.987860:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.987861:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.987866:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.987869:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.987873:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.987874:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.987876:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.987878:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.987878:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.987880:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677472.987882:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677472.987882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677472.987888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677472.987891:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.987893:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.987894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.987895:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.987896:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677472.987897:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.987899:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677472.987900:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677472.987901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677472.987903:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677472.988527:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.988529:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.988530:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.988530:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.988532:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.988534:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.988766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:18.0:1619677472.988773:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677472.988776:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:2.0:1619677472.988776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:18.0:1619677472.988779:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.988781:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.988783:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677472.988784:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.988785:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.988786:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.988788:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677472.988791:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.988793:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677472.988796:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.988796:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677472.988797:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677472.988799:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677472.988799:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677472.988801:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907cbe3400] (22)++
00000800:00000200:18.0:1619677472.988802:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (23)--
00000800:00000200:18.0:1619677472.988802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:20.2:1619677472.988808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677472.988813:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)++
00000800:00000200:20.0:1619677472.988815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677472.988815:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907cbe3400] (23)--
00000800:00000200:2.0:1619677472.988816:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:20.0:1619677472.988817:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:2.0:1619677472.988817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000400:00000200:20.0:1619677472.988818:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.988820:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.988821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677472.988822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677472.988966:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.988968:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.988969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.988971:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.988973:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.988974:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677472.988975:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.988977:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.988978:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.988979:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677472.988982:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.988984:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677472.988986:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.988987:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677472.988988:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677472.988990:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677472.988990:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.988991:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.988991:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677472.989004:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.989007:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.989008:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.989008:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.989010:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.989013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677472.989070:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677472.989072:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.989073:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.989074:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.989075:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677472.989075:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677472.989698:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.989700:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.989701:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.989702:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.989704:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.989706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677472.990033:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.990035:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677472.990036:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.990037:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677472.990038:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.990040:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.990041:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.990043:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.990044:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.990045:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.990046:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.990047:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.990048:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677472.990050:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677472.990051:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.990052:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677472.990053:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.990053:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.990054:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.990054:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677472.990058:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.990058:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.990059:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.990059:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.990060:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677472.990062:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.990062:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677472.990063:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677472.990063:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677472.990063:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677472.990064:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677472.990064:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677472.990065:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677472.990065:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677472.990066:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677472.990066:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677472.990067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.990067:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677472.990068:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677472.990069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677472.990070:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677472.990072:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677472.990072:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.990074:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.990076:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677472.990077:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96511378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.990078:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.990078:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677472.990078:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (23)++
00000400:00000200:26.0:1619677472.990079:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:21.0:1619677472.990080:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (24)--
00000400:00000200:21.0:1619677472.990081:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.990083:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677472.990083:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.990083:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (23)++
00000800:00000200:20.0:1619677472.990084:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (24)--
00000400:00000200:20.0:1619677472.990085:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677472.990086:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:20.0:1619677472.990087:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.990092:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677472.990093:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.990093:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677472.990094:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677472.990094:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677472.990094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677472.990095:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677472.990096:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677472.990096:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677472.990096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677472.990097:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677472.990097:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677472.990097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677472.990098:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677472.990099:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.990100:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677472.990108:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.990110:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677472.990111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677472.990111:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677472.990112:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677472.990113:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.990114:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.990115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.990116:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677472.990116:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677472.990127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.990131:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.990133:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677472.990134:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.990134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.990135:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677472.990136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677472.990136:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.990139:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.990140:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.990140:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.990142:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677472.990144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.990147:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.990148:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.990150:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.990151:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.990152:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677472.990152:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.990153:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.990342:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.990344:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677472.990345:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677472.990347:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.990350:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.990351:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.990352:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.990354:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.990355:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.990355:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677472.990359:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.990360:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677472.990363:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.990364:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.990364:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677472.990366:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677472.990366:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.990367:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.990368:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677472.990381:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.990383:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.990384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.990385:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.990387:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.990390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677472.990401:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677472.990403:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677472.990404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.990404:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.990405:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677472.990406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677472.990525:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.990528:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.990530:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.990531:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.990532:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.990534:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677472.990535:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.990537:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.990537:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.990539:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677472.990542:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.990544:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677472.990546:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.990547:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677472.990548:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677472.990549:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677472.990550:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.990551:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.990551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677472.990562:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.990565:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.990566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.990566:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.990568:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677472.990570:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677472.990872:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.990875:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.990876:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.990877:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.990879:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.990881:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.990882:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.990884:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.990885:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.990886:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.990888:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.990890:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.990893:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.990894:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.990894:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.990896:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.990897:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.990898:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677472.990899:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677472.990900:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677472.990902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.990911:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677472.990913:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677472.990914:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.990915:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677472.990915:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677472.990915:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.990916:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.990917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677472.990917:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.990920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.990950:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.990952:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677472.990954:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677472.990955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677472.990957:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.990959:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677472.990960:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.990961:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.990962:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.990963:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677472.990967:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677472.990968:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677472.990971:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.990972:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677472.990973:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677472.990974:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677472.990975:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677472.990976:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677472.990977:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677472.990978:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.990987:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677472.990989:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677472.990990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677472.990990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677472.991017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.991024:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.991027:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.991028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.991030:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.991032:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.991034:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.991035:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.991036:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.991037:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.991038:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.991041:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.991043:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.991044:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.991045:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.991046:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.991047:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.991048:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.991049:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677472.991049:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.991061:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677472.991063:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677472.991064:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677472.991064:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.991065:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.991067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677472.991124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.991130:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.991133:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.991133:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.991135:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.991137:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.991138:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677472.991139:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.991140:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.991141:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.991142:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677472.991144:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.991146:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677472.991148:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.991148:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677472.991149:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677472.991150:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677472.991151:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.991152:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677472.991153:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677472.991153:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677472.991156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677472.991159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677472.991164:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677472.991164:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677472.991166:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677472.991166:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677472.991167:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677472.991167:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:17.0:1619677472.991168:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.991169:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.991170:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677472.991172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:13.0:1619677472.991663:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.991665:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677472.991667:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677472.991670:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.991670:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677472.991672:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677472.991673:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677472.991674:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677472.991675:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677472.991676:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677472.991679:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677472.991681:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677472.991684:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677472.991685:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677472.991685:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677472.991687:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677472.991688:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677472.991689:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677472.991690:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677472.991691:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:18.0:1619677472.991703:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.991705:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:12.0:1619677472.991706:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:2.0:1619677472.991706:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677472.991707:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677472.991708:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:18.0:1619677472.991709:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:12.0:1619677472.991709:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677472.991710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677472.991712:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677472.991812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.991814:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.991815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.991816:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.991818:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.991821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677472.992115:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.992118:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.992119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.992120:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.992124:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.992125:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677472.992127:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.992128:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.992129:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.992130:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677472.992132:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.992134:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.992136:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.992137:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.992137:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.992139:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677472.992139:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.992140:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.992140:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677472.992144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.992150:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677472.992152:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677472.992152:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.992154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677472.992154:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677472.992156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677472.992168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677472.992174:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.992176:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677472.992177:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.992177:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677472.992177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677472.992178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677472.992197:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.992200:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677472.992201:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677472.992203:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.992205:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.992207:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677472.992208:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.992210:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.992211:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.992212:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677472.992215:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.992217:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677472.992219:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.992220:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677472.992220:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677472.992222:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677472.992223:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677472.992223:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677472.992224:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677472.992228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.992235:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.992238:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.992239:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.992239:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.992241:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677472.992244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677472.992314:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.992317:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677472.992317:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677472.992319:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.992321:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.992323:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677472.992324:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.992325:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.992326:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.992327:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677472.992330:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.992331:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677472.992334:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.992334:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677472.992335:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.992337:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677472.992337:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.992338:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677472.992338:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677472.992343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.992351:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.992354:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.992354:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.992356:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.992358:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677472.992359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.992408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.992416:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.992419:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.992420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.992420:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.992422:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.992425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677472.992594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.992599:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.992601:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.992602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.992604:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.992606:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.992607:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.992609:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.992610:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.992611:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.992612:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.992615:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.992617:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.992620:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.992620:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.992621:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.992623:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.992623:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.992624:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.992624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677472.992637:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677472.992640:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.992641:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.992641:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.992643:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.2:1619677472.992645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:6.0:1619677472.992645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:12.0:1619677472.992690:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677472.992693:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677472.992693:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.992695:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.992697:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677472.992698:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:10.0:1619677472.992702:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.992705:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.992706:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.992708:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.992710:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.992712:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677472.992713:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.992714:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.992715:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.992716:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677472.992718:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.992720:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677472.992723:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.992724:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677472.992724:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677472.992726:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677472.992726:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.992727:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.992728:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677472.992740:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.992742:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.992743:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.992744:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.992745:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.992749:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.992758:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677472.992760:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.992761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.992761:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.992763:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.992763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.993526:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.993529:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.993530:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.993530:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.993533:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.993535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.993884:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677472.993887:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677472.993887:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677472.993889:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677472.993893:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.993895:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677472.993897:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677472.993898:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677472.993899:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677472.993900:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677472.993904:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677472.993906:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677472.993908:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677472.993909:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:6.0:1619677472.993910:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677472.993912:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677472.993913:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677472.993913:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677472.993914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.2:1619677472.993919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.993927:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.993930:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.993931:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.993931:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.993933:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.993936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.993937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.993945:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.993947:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.993948:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.993948:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.993949:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.993950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677472.994527:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (21)++
00000800:00000200:21.0:1619677472.994530:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.1.52@o2ib3
00000400:00000200:21.0:1619677472.994532:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.994533:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (22)++
00000800:00000200:20.0:1619677472.994535:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000400:00000200:21.0:1619677472.994536:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.994537:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.994538:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:21.0:1619677472.994539:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.994541:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.994542:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.994543:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:20.0:1619677472.994545:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.994547:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677472.994547:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.994548:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677472.994549:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000400:00000200:20.0:1619677472.994550:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.994551:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.994552:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.994555:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677472.994555:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.994556:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:21.0:1619677472.994557:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:20.0:1619677472.994558:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677472.994559:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:21.0:1619677472.994560:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (23)++
00000800:00000200:20.0:1619677472.994560:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.994561:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (24)--
00000800:00000200:21.0:1619677472.994561:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (23)--
00000800:00000200:20.0:1619677472.994561:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.994562:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677472.994563:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000800:00000200:20.0:1619677472.994563:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.994564:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (21)++
00000800:00000200:20.0:1619677472.994565:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (22)--
00000800:00000200:24.2:1619677472.994566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:20.0:1619677472.994566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (21)--
00000800:00000200:26.0:1619677472.994574:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)++
00000800:00000200:26.0:1619677472.994577:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2fa00] (23)--
00000800:00000200:10.0:1619677472.994577:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)--
00000400:00000200:26.0:1619677472.994578:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677472.994578:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:26.0:1619677472.994580:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.51@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.994580:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.994581:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677472.994581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.994583:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.994584:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (21)--
00000800:00000200:22.0:1619677472.994587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677472.994588:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (21)++
00000800:00000200:21.0:1619677472.994590:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000400:00000200:21.0:1619677472.994592:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.994593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000400:00000200:21.0:1619677472.994595:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.994596:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:21.0:1619677472.994598:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.994599:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.994600:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.994601:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:21.0:1619677472.994605:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.994606:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:21.0:1619677472.994609:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.994610:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:21.0:1619677472.994611:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.0:1619677472.994613:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:21.0:1619677472.994613:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (21)++
00000800:00000200:21.0:1619677472.994614:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (22)--
00000800:00000200:21.0:1619677472.994615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (21)--
00000800:00000200:9.2:1619677472.994618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677472.994625:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677472.994627:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677472.994628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677472.994629:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.994631:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677472.994634:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.0:1619677472.994724:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (21)++
00000800:00000200:20.0:1619677472.994726:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000400:00000200:20.0:1619677472.994728:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677472.994729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000400:00000200:20.0:1619677472.994731:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.994732:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677472.994733:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.994735:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.994735:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.994736:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677472.994739:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.994741:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677472.994744:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.994745:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:20.0:1619677472.994745:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677472.994747:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677472.994747:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (21)++
00000800:00000200:20.0:1619677472.994749:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e855c000] (22)++
00000800:00000200:20.0:1619677472.994750:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (23)--
00000800:00000200:20.0:1619677472.994751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000800:00000200:9.2:1619677472.994753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677472.994759:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:21.0:1619677472.994761:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (22)++
00000800:00000200:26.0:1619677472.994762:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677472.994763:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.994763:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e855c000] (23)--
00000800:00000200:10.0:1619677472.994763:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677472.994764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000400:00000200:26.0:1619677472.994765:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.994765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (21)--
00000800:00000200:26.0:1619677472.994768:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677472.994793:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.994795:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.994796:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.994798:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.994800:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.994801:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.994803:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.994804:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.994805:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.994806:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.994809:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.994811:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.994813:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.994814:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.994815:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.994816:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.994816:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.994817:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677472.994817:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677472.994822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677472.994828:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677472.994830:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677472.994831:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677472.994832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677472.994833:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (21)++
00000400:00000200:8.0:1619677472.994833:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.994835:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000800:00000200:8.0:1619677472.994835:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:21.0:1619677472.994837:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677472.994838:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000400:00000200:21.0:1619677472.994840:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.994842:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677472.994843:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.994844:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.994845:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.994846:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677472.994849:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.994851:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677472.994854:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.994854:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677472.994855:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677472.994857:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.994857:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (21)++
00000800:00000200:21.0:1619677472.994858:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (22)--
00000800:00000200:21.0:1619677472.994859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (21)--
00000800:00000200:8.2:1619677472.994863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.994871:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.994874:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677472.994875:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677472.994877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677472.994877:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.994878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (21)++
00000800:00000200:21.0:1619677472.994879:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000800:00000200:20.0:1619677472.994879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.994880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000400:00000200:21.0:1619677472.994882:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677472.994884:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.994885:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677472.994887:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677472.994888:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677472.994889:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677472.994890:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677472.994893:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677472.994894:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677472.994896:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677472.994897:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677472.994897:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677472.994899:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677472.994900:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (21)++
00000800:00000200:21.0:1619677472.994900:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (22)--
00000800:00000200:21.0:1619677472.994901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (21)--
00000800:00000200:9.2:1619677472.994905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.994913:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.994916:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677472.994917:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677472.994917:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677472.994919:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e855c000] (21)++
00000400:00000200:18.0:1619677472.994919:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.994921:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.52@o2ib3
00000800:00000200:18.0:1619677472.994922:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677472.994924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (22)--
00000400:00000200:20.0:1619677472.994924:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.52@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677472.994926:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.994928:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677472.994929:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.994930:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.994931:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.994932:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677472.994935:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677472.994936:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677472.994939:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.994940:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:20.0:1619677472.994940:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677472.994942:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.0:1619677472.994943:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e855c000] (21)++
00000800:00000200:20.0:1619677472.994943:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e855c000] (22)--
00000800:00000200:20.0:1619677472.994944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e855c000] (21)--
00000800:00000200:8.2:1619677472.994948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.994957:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.994959:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.994960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.994960:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.994963:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.994965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677472.995185:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.995187:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677472.995188:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677472.995190:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677472.995192:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.995194:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677472.995195:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677472.995197:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677472.995198:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677472.995199:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677472.995202:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677472.995204:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677472.995207:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677472.995208:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677472.995208:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677472.995210:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677472.995211:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677472.995211:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.995212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677472.995216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.995224:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677472.995226:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677472.995227:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677472.995227:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677472.995229:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677472.995232:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677472.995291:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677472.995296:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.995299:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677472.995301:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.995304:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.995305:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677472.995307:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.995308:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.995310:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.995311:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677472.995313:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.995314:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677472.995315:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677472.995316:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677472.995317:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677472.995319:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.995320:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677472.995321:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.995322:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677472.995322:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677472.995322:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.995323:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.995324:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677472.995325:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:10.0:1619677472.995325:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677472.995326:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677472.995327:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:10.0:1619677472.995328:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677472.995329:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677472.995329:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677472.995330:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.995330:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677472.995332:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.995333:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677472.995334:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677472.995336:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677472.995336:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677472.995337:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677472.995338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677472.995339:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677472.995340:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677472.995340:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677472.995341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677472.995342:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677472.995343:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677472.995344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677472.995344:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.995346:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.995349:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:6.0:1619677472.995349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:13.0:1619677472.995352:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.995353:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.995355:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677472.995356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.995357:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677472.995445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677472.995451:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.995454:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677472.995456:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677472.995457:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677472.995460:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.995462:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677472.995463:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677472.995464:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677472.995466:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677472.995467:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677472.995470:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677472.995472:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677472.995476:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677472.995477:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677472.995477:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677472.995479:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677472.995480:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677472.995481:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677472.995482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:12.0:1619677472.995492:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677472.995495:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677472.995496:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677472.995497:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677472.995498:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677472.995500:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677472.995512:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677472.995515:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677472.995515:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.995516:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677472.995517:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.995518:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677472.995844:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677472.995847:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.995847:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677472.995849:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.995852:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.995853:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677472.995854:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.995856:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.995857:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.995858:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677472.995861:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.995863:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677472.995866:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.995867:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677472.995868:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677472.995870:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677472.995870:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677472.995871:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677472.995871:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677472.995876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677472.995885:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677472.995887:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677472.995888:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677472.995890:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677472.995895:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677472.995898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677472.995938:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.995940:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677472.995941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677472.995942:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.995944:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.995947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677472.996484:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.996487:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.996488:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.996490:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.996492:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.996494:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.996495:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.996496:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.996497:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.996498:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.996500:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.996502:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.996504:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.996504:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.996505:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.996506:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.996507:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.996507:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677472.996508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677472.996513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.996519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.996521:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.996521:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.996522:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677472.996523:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.996524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677472.996679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677472.996686:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.996688:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677472.996689:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677472.996691:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677472.996694:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.996695:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677472.996697:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.996698:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.996699:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.996700:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677472.996703:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677472.996705:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677472.996708:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677472.996709:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677472.996710:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677472.996711:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677472.996712:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677472.996713:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677472.996713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677472.996725:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677472.996728:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677472.996729:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677472.996731:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677472.996732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677472.996734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677472.996996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.997004:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.997007:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.997008:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677472.997008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.997011:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.997013:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677472.997215:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.997218:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677472.997219:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677472.997220:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677472.997222:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677472.997223:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677472.997224:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677472.997226:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677472.997227:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677472.997228:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677472.997231:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677472.997233:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677472.997234:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677472.997235:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677472.997235:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677472.997236:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.997236:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677472.997238:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.997239:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.997239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677472.997242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677472.997250:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677472.997251:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677472.997251:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677472.997253:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677472.997254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677472.997254:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677472.997254:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677472.997255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677472.997256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677472.997256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677472.997498:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.997501:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677472.997501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677472.997503:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677472.997506:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677472.997507:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677472.997508:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677472.997510:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677472.997510:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677472.997511:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677472.997514:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677472.997516:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677472.997519:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677472.997519:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677472.997520:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677472.997522:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677472.997522:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677472.997523:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.997523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677472.997537:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.997540:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.997541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.997541:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.997543:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.997546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677472.997561:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677472.997564:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677472.997565:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.997565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.997566:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677472.997567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677472.997678:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677472.997681:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677472.997681:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677472.997682:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.997684:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677472.997687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677472.997756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677472.997762:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.997765:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677472.997766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677472.997768:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677472.997770:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677472.997771:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677472.997773:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677472.997774:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677472.997775:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677472.997776:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677472.997779:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677472.997781:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677472.997784:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677472.997784:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677472.997785:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677472.997787:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677472.997787:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677472.997788:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677472.997788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677472.997800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677472.997803:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677472.997803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677472.997804:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677472.997806:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677472.997808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677472.998046:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.998048:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677472.998050:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677472.998051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677472.998052:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677472.998054:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677472.998055:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677472.998056:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677472.998057:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677472.998058:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677472.998061:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677472.998063:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677472.998065:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677472.998066:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677472.998066:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677472.998068:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677472.998069:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677472.998070:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677472.998070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677472.998081:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677472.998084:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677472.998085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677472.998085:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677472.998087:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677472.998089:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677472.998568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677472.998609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.998618:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.998621:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677472.998622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677472.998622:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677472.998624:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677472.998627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.998630:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.998632:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677472.998634:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677472.998635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.998635:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677472.998636:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677472.998637:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677472.999213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677472.999221:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.999224:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677472.999225:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677472.999226:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.999228:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677472.999231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677472.999232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677472.999239:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677472.999242:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677472.999242:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677472.999242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.999244:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677472.999244:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677472.999375:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677472.999378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677472.999378:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677472.999380:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677472.999382:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677472.999384:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677472.999385:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677472.999386:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677472.999387:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677472.999388:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677472.999391:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677472.999393:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677472.999395:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677472.999396:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677472.999397:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677472.999398:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677472.999399:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677472.999400:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677472.999401:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677472.999402:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677472.999405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677472.999413:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677472.999415:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677472.999415:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677472.999416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677472.999417:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677472.999417:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:21.0:1619677472.999418:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677472.999419:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677472.999420:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677472.999422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677472.999427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677472.999435:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677472.999437:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677472.999438:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677472.999439:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677472.999440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677472.999440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:10.0:1619677472.999488:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.999491:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677472.999492:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677472.999494:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677472.999497:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677472.999498:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677472.999499:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677472.999501:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677472.999502:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677472.999503:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677472.999506:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677472.999508:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677472.999510:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677472.999511:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677472.999512:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677472.999513:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677472.999514:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677472.999515:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677472.999516:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677472.999517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677472.999519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677472.999525:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677472.999528:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677472.999529:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677472.999529:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677472.999529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677472.999530:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677472.999531:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677472.999532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677472.999533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677472.999533:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677472.999714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677472.999720:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677472.999723:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677472.999724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677472.999726:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677472.999729:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677472.999730:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677472.999732:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677472.999733:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677472.999734:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677472.999735:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677472.999738:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677472.999739:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677472.999741:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677472.999742:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677472.999742:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.999743:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677472.999745:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:18.0:1619677472.999745:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.999745:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677472.999745:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677472.999746:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.999747:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:22.0:1619677472.999748:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.999748:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.999748:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677472.999751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677472.999751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677472.999753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677472.999759:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.999761:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677472.999761:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677472.999761:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677472.999762:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677472.999762:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677472.999763:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677472.999764:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.999765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677472.999765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677472.999781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677472.999788:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677472.999791:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677472.999791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.999792:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677472.999792:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677472.999793:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677473.000038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.000045:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.000048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.000048:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677473.000050:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.113@o2ib7 : GET - for me
00000400:00000200:1.0:1619677473.000053:0:19931:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.113@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1619677473.000055:0:19931:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.113@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:1.0:1619677473.000057:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.000058:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.000059:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677473.000063:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : REPLY try# 0
00000800:00000200:1.0:1619677473.000064:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.000066:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.000067:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.000067:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.000069:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.000069:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.2:1619677473.000086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.000092:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.000094:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.000095:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.000095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.000097:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: REPLY: OK
00000800:00000200:17.0:1619677473.000098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.000098:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.000101:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.000104:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.000104:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677473.000105:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.000107:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.000107:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.000108:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.000110:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.000111:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.000113:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.000114:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.000116:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.000117:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677473.000118:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.000119:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.000120:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.000121:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.000122:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.000123:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.000124:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.000127:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.000128:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.000128:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.000129:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.000129:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.000131:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.000132:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.000133:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.000133:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.000134:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.000134:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.000135:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.000135:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.000136:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.000136:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.000137:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.000137:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677473.000137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.000138:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677473.000145:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.000148:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677473.000148:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.000149:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.000149:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.000150:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.000151:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:2.0:1619677473.000151:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:10.0:1619677473.000152:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:6.0:1619677473.000152:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.000153:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.000153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:6.0:1619677473.000154:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.000154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:6.0:1619677473.000156:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.000158:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.000171:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.000173:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.000174:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.000174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.000175:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.000176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.000829:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.000832:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.000832:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.000834:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.000837:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.000839:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677473.000840:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.000841:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.000842:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.000843:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677473.000846:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.000847:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.000850:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.000851:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677473.000852:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.000853:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.000854:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.000855:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.000855:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.000861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.000869:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.000872:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.000872:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.000873:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.000875:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.000878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677473.000922:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb6400] (21)++
00000800:00000200:10.0:1619677473.000925:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.58@o2ib3
00000800:00000200:26.0:1619677473.000926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb6400] (22)--
00000400:00000200:10.0:1619677473.000928:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.53@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.58@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.000931:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.000933:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.53@o2ib7 local destination
00000400:00000200:10.0:1619677473.000935:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.000936:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.000937:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.000938:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.53@o2ib7
00000400:00000200:10.0:1619677473.000941:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.53@o2ib7(10.0.10.53@o2ib7:10.0.10.53@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.000943:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 520 bytes in 1 frags to 12345-10.0.10.53@o2ib7
00000800:00000200:10.0:1619677473.000944:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a879baa300] -> 10.0.10.53@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.000945:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a86cfd9a00] (20)++
00000800:00000200:10.0:1619677473.000946:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86cfd9a00] (21)++
00000800:00000200:10.0:1619677473.000948:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a86cfd9a00] (22)--
00000800:00000200:10.0:1619677473.000949:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb6400] (21)++
00000800:00000200:10.0:1619677473.000949:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb6400] (22)--
00000800:00000200:10.0:1619677473.000950:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb6400] (21)--
00000800:00000200:9.2:1619677473.000960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86cfd9a00] (21)++
00000800:00000200:6.0:1619677473.001245:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.001248:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.001249:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.001250:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.001251:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.001254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.001265:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.001267:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.001268:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.001268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.001270:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.001270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.001338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.001346:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.001349:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.001350:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.001350:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.001352:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.001354:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.001474:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.001476:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.001478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.001479:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.001480:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.001481:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.001482:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.001484:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.001485:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.001486:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.001488:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.001490:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.001491:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.001492:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.001492:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.001493:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.001494:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.001494:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.001495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.001499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.001505:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.001507:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.001507:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.001508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.001509:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.001510:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677473.001818:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.001820:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.001822:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.001823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.001824:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.001826:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.001827:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.001828:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.001829:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.001830:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.001834:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.001835:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.001838:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.001838:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.001839:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.001841:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.001841:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.001842:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.001842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.001855:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.001857:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.001858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.001858:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.001860:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.001862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.002201:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.002204:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.002205:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.002206:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.002208:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677473.002208:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:24.0:1619677473.002209:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.002211:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.002212:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.002213:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.002214:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.002216:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:24.0:1619677473.002217:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.002218:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.002219:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:24.0:1619677473.002220:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.002220:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:21.0:1619677473.002220:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.002221:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:21.0:1619677473.002221:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.002222:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.002222:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677473.002222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677473.002223:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.002223:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677473.002225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.002227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.002233:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.002235:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.002236:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.002237:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.002238:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.002239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.002249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.002255:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.002257:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.002258:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.002258:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.002258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.002259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.002437:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.002440:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.002442:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.002443:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.002445:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.002447:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.002448:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.002450:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.002451:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.002451:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.002454:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.002456:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.002459:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.002460:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.002460:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.002462:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.002462:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.002463:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.002463:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.002477:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.002480:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.002481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.002482:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.002484:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.002487:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.002578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.002585:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.002588:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.002589:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.002592:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.002594:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.002595:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.002597:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.002598:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.002599:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.002600:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.002603:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.002604:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.002607:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.002607:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.002608:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.002610:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.002610:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.002611:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.002611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.002623:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.002626:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.002627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.002627:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.002629:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.002631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.002957:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.002960:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.002961:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.002963:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.002965:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.002966:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.002968:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.002969:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.002970:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.002971:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.002974:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.002976:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.002979:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.002980:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.002981:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.002982:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.002983:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.002983:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.002984:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.002989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.002998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.003001:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.003003:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.003003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.003005:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.003008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.003013:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.003015:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.003016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.003018:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.003021:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.003023:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.003024:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.003025:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.003026:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.003027:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.003030:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.003031:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.003034:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.003035:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.003036:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.003038:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.003038:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.003039:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.003041:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.003041:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677473.003044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.003053:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677473.003055:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:20.0:1619677473.003055:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.003056:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.003057:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.003057:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.003058:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:20.0:1619677473.003058:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.003059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.003061:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.003064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.003072:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.003074:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.003075:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.003076:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.003076:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.003077:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.003384:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.003387:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.003388:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.003389:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.003391:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.003394:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.003611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.003617:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.003621:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.003621:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677473.003623:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.003626:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.003627:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.003628:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.003630:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.003631:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.003632:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.003636:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.003637:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.003640:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.003641:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.003642:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.003644:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.003645:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.003646:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.003646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.003774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.003783:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.003785:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.003786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.003787:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.003789:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.003792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677473.004010:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.004012:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.004014:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.004015:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.004017:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.004018:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.004019:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.004021:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.004022:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.004023:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.004026:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.004028:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.004030:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.004031:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.004032:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.004033:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.004034:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.004035:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.004036:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.004037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.004046:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677473.004046:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.004048:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.004049:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677473.004049:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.004050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.004050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677473.004050:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.004052:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.004054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.004063:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.004066:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.004066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.004067:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.004068:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.004069:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.004211:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.004214:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.004215:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.004217:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.004220:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.004221:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.004222:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.004223:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.004225:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.004226:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.004229:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.004231:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.004233:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.004234:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.004234:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.004236:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.004236:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.004237:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.004237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.004242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.004248:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.004250:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.004251:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.004252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.004253:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.004256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.004261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.004267:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.004269:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.004270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.004270:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.004271:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.004271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.004509:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.004512:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.004513:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.004514:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.004517:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.004519:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.004520:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.004521:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.004522:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.004523:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.004526:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.004527:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.004530:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.004531:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.004532:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.004533:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.004534:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.004535:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.004535:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.004548:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.004551:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.004552:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.004552:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.004554:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.004557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.005124:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.005126:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.005127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.005128:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.005130:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.005133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.005758:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.005761:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.005763:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.005764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.005765:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.005767:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.005768:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.005769:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.005770:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.005771:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.005774:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.005776:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.005779:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.005779:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.005780:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.005782:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.005782:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.005783:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.005784:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.005788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.005796:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.005798:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.005799:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.005799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.005801:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.005804:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.006335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.006342:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.006344:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.006345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.006347:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.006350:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.006351:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.006353:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.006354:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.006355:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.006356:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.006359:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.006361:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.006363:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.006364:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.006364:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.006366:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.006367:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.006368:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.006368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.006373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.006380:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.006382:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.006383:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.006384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.006385:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.006386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677473.006888:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.006891:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.006892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.006893:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.006896:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.006897:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.006898:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.006899:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.006900:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.006901:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.006904:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.006905:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.006907:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.006907:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.006908:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.006909:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.006910:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.006911:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.006912:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.006912:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.006915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.006923:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.006924:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.006924:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.006926:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.006927:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.006928:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.006928:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.006928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.006929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.006929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.007232:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.007235:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.007236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.007236:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.007239:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.007241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.007291:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.007299:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.007303:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.007306:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.007309:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.007310:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.007312:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.007313:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.007314:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.007315:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.007317:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.007319:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.007321:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.007322:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.007323:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.007324:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.007325:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.007326:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.007326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.007341:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.007343:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.007344:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.007346:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.007348:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.007348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677473.007362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.007368:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.007371:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.007373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.007374:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.007376:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.007378:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.007380:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.007381:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.007383:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.007384:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.007387:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.007389:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.007392:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.007393:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.007394:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.007395:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.007396:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.007397:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.007398:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.007399:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.007404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.007409:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.007409:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.007411:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.007411:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.007412:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677473.007413:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.007413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.007413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.007415:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.007418:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.007656:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.007659:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.007660:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.007661:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.007663:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.007664:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.007665:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.007666:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.007668:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.007668:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.007671:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.007673:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.007675:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.007675:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.007676:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.007677:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.007677:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.007678:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.007679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.007683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.007688:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.007690:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.007691:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.007692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.007693:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.007694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.007821:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.007823:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.007824:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.007826:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.007828:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.007830:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.007831:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.007832:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.007833:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.007834:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.007838:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.007840:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.007842:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07079a800] (21)++
00000800:00000200:26.0:1619677473.007844:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.007844:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.49@o2ib3
00000800:00000200:26.0:1619677473.007845:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.007846:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677473.007846:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.49@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.007848:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.007849:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677473.007849:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.007850:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.007850:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677473.007851:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677473.007852:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07079a800] (22)--
00000400:00000200:10.0:1619677473.007852:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.007854:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.007854:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.007855:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.007858:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.007860:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.007861:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.007862:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.007863:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.007864:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.007864:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.007865:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.007865:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:6.0:1619677473.007865:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.007866:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07079a800] (21)++
00000800:00000200:10.0:1619677473.007866:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07079a800] (22)--
00000800:00000200:10.0:1619677473.007867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07079a800] (21)--
00000400:00000200:6.0:1619677473.007867:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.007870:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.007880:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.007882:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.007883:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.007885:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.007885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.007889:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.007904:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07079a800] (21)++
00000800:00000200:26.0:1619677473.007907:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.49@o2ib3
00000800:00000200:10.0:1619677473.007907:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07079a800] (22)--
00000400:00000200:26.0:1619677473.007909:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.49@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.007912:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.007913:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.007914:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.007916:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.007917:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.007917:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.007920:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.007922:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.007923:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.007924:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.007924:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.007926:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.007926:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07079a800] (21)++
00000800:00000200:26.0:1619677473.007927:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07079a800] (22)--
00000800:00000200:26.0:1619677473.007927:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07079a800] (21)--
00000800:00000200:9.2:1619677473.007956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.007962:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.007964:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.007964:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.007965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.007966:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.007969:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.007970:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.007972:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.007974:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.007974:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.007976:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.007978:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677473.007979:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.007980:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.007980:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.007981:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677473.007984:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.007985:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.007987:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.007988:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677473.007988:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677473.007990:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677473.007990:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.007991:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.007992:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.2:1619677473.008309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.008317:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.008319:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.008321:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.008322:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.008324:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.008326:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.008327:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.008329:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.008330:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.008331:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.008334:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.008336:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.008338:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.008339:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.008339:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.008341:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.008342:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.008343:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.008343:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.008348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.008357:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.008359:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.008361:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.008361:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.008362:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.008365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.008689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.008695:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.008698:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.008699:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.008699:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.008700:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.008703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.008728:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.008731:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.008732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.008733:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.008734:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.008736:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.008737:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.008738:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.008739:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.008740:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.008743:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.008744:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.008747:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.008748:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.008748:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.008750:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.008751:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.008751:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.008751:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677473.008752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677473.008755:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:9.2:1619677473.008756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677473.008758:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677473.008760:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.008761:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677473.008762:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.008764:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:2.0:1619677473.008764:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:20.0:1619677473.008765:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.008766:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677473.008766:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:20.0:1619677473.008767:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.008768:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677473.008768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.008768:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.008770:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.008771:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.008773:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677473.008773:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677473.008776:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.008776:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.008777:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.008779:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.008779:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677473.008780:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677473.008781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.008785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.008793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.008795:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.008795:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.008797:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.008798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.008799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.008933:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.008935:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.008936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.008937:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.008939:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.008941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.008952:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.008955:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.008956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.008956:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.008957:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.008957:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.008966:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.008969:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.008970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.008972:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.008974:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.008976:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.008977:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.008978:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.008979:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.008980:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.008983:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.008985:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.008987:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.008988:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.008988:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.008990:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.008991:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.008992:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.008993:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.008994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.008995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.009001:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.009003:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.009004:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.009005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.009006:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.009006:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.009008:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677473.009009:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.009009:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.009010:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677473.009346:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.009349:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.009350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.009351:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.009354:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.009355:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.009356:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.009357:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.009358:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.009359:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.009362:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.009363:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.009366:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.009367:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.009367:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.009369:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.009369:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.009370:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.009371:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.009384:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.009386:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.009387:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.009388:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.009390:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.009392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.009392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:6.0:1619677473.009400:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.009404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.009404:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.009405:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.009406:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.009406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.009435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:9.2:1619677473.009442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.009448:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.009450:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.009451:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.009451:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.009453:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.009456:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.009465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.009471:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.009474:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.009474:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.009474:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.009475:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.009476:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.009487:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.009489:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.009490:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.009491:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.009494:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.009495:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.009496:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.009497:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.009498:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.009499:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.009502:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.009503:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.009506:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.009507:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.009507:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.009509:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.009509:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.009510:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.009510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677473.009522:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.009525:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.009526:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.009526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.009528:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.009531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.010267:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.010270:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.010272:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.010274:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.010274:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.010276:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.010277:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.010278:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.010279:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.010280:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.010283:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.010285:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.010287:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.010288:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.010289:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.010291:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.010291:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.010292:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.010292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.010299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.010306:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.010309:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.010310:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.010310:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.010312:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.010314:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.010315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.010321:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.010324:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.010324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.010325:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.010325:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.010326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677473.010398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.010407:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.010410:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.010411:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.010412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.010414:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.010417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677473.010576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.010582:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.010585:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.010586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.010586:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.010588:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.010590:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.011024:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feaea7000] (21)++
00000800:00000200:24.0:1619677473.011027:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.39@o2ib3
00000800:00000200:8.0:1619677473.011028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feaea7000] (22)--
00000400:00000200:24.0:1619677473.011029:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.39@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.011032:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.011034:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.011035:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.011036:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.011037:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.011037:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.011040:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.011042:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 520 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.011045:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.011045:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.011046:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.011048:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.011048:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feaea7000] (21)++
00000800:00000200:24.0:1619677473.011049:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feaea7000] (22)--
00000800:00000200:24.0:1619677473.011049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feaea7000] (21)--
00000800:00000200:9.2:1619677473.011054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.011062:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.011065:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.011066:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.011066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.011068:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.011072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.011075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.011082:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.011084:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.011085:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.011085:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.011086:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.011087:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.011573:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.011576:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.011577:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.011579:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.011582:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.011584:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677473.011585:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.011586:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.011587:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.011588:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677473.011591:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.011592:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677473.011595:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.011595:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:26.0:1619677473.011596:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677473.011597:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677473.011598:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.011599:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.011599:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:13.0:1619677473.011612:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.011614:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.011615:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.011616:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.011618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.011619:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:6.0:1619677473.011640:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.011642:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.011643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.011644:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.011646:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.011649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.011874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.011881:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.011884:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.011885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.011885:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.011886:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.011889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.012554:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.012556:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.012557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.012559:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.012561:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.012562:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.012563:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.012564:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.012565:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.012566:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.012569:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.012570:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.012573:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.012574:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.012575:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.012576:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.012577:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.012578:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.012578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.012583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.012591:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.012594:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.012595:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.012595:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.012597:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.012600:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.012603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.012610:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.012613:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.012614:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.012614:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.012615:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.012616:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677473.012640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.012645:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.012647:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.012648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.012650:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.012652:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.012653:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.012654:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.012656:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.012657:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.012658:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.012661:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.012662:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677473.012663:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.012665:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.012665:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.012666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677473.012667:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.012667:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677473.012668:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.012669:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.012669:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.012670:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.012670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677473.012671:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.012672:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.012673:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.012676:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.012677:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.012678:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.012681:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.012683:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.012684:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.012685:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.012686:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677473.012686:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:22.0:1619677473.012687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.012687:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.012688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.012689:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.012689:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677473.012690:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.012690:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.012690:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.012692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.012702:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.012704:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.012705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.012705:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.012707:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.012710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.012881:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.012884:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.012885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.012887:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.012889:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.012891:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.012892:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.012893:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.012895:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.012895:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.012899:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.012900:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.012903:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.012904:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.012904:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.012906:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.012906:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.012907:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.012908:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.012912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.012921:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.012923:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.012924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.012925:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.012927:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.012930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677473.013015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.013023:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.013026:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.013027:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.013029:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.013030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.013032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.013037:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.013040:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.013041:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.013043:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.013046:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.013047:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.013048:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.013049:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.013050:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.013051:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.013054:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.013056:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.013059:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.013059:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.013060:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.013061:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.013062:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.2:1619677473.013063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677473.013063:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.013064:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.013064:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:2.0:1619677473.013068:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.013071:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.013072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:13.0:1619677473.013074:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:2.0:1619677473.013074:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.013076:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.013077:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:13.0:1619677473.013077:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.013078:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.013078:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677473.013079:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:13.0:1619677473.013079:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.013079:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.013080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.013081:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677473.013081:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:2.0:1619677473.013081:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677473.013082:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:2.0:1619677473.013082:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.013083:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.013086:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.013088:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.013089:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.013090:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.013091:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.013092:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.013093:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.013094:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.013095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.013098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.013106:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.013108:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.013109:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.013109:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.013111:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.013112:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677473.013251:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.013254:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.013255:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.013257:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.013260:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.013262:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.013263:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.013264:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.013265:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.013266:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.013269:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.013270:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.013273:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.013274:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.013275:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.013276:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.013277:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.013278:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.013279:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.013280:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677473.013282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.013290:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.013292:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677473.013293:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.013293:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:8.0:1619677473.013298:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.013301:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.013301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677473.013303:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.013305:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.013308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.013719:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.013722:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.013723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.013724:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.013726:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.013729:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.013730:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.013732:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.013733:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.013733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.013734:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.013735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.013786:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.013789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.013789:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677473.013791:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.013794:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.013795:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677473.013797:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.013798:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.013799:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.013800:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677473.013803:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.013805:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677473.013808:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.013809:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:6.0:1619677473.013810:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:6.0:1619677473.013812:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677473.013812:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.013813:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.013813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:12.0:1619677473.013826:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.013829:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.013830:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.013831:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.013832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.013834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.013847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.013850:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.013851:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.013851:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.013852:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.013854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677473.013907:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.013910:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.013910:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.013913:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.013915:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.013916:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.013918:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.013919:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.013920:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.013921:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.013924:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.013926:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.013928:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.013929:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.013930:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.013932:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.013932:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.013933:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.013933:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.013938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.013946:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.013949:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.013950:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.013952:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.013952:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.013954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.014184:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.014187:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.014188:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.014190:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.014192:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.014194:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.014195:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.014196:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.014197:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.014198:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.014202:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.014204:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.014206:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.014206:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.014207:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.014207:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.014208:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:26.0:1619677473.014209:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.014210:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:26.0:1619677473.014211:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.014211:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.014212:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.014212:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:26.0:1619677473.014213:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677473.014213:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.014214:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.014215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:26.0:1619677473.014216:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.014216:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.014217:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.014220:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.014220:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:26.0:1619677473.014222:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677473.014223:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.014223:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677473.014224:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677473.014225:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.014225:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:24.0:1619677473.014225:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.014225:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.014226:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.014227:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:24.0:1619677473.014227:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.014228:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677473.014228:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.014230:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:8.0:1619677473.014230:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677473.014231:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:8.0:1619677473.014231:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.014232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677473.014233:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.014234:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.014235:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.014236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.014237:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677473.014238:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.014239:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677473.014239:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.014240:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.014240:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.014241:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:22.0:1619677473.014241:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:24.0:1619677473.014242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.014243:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.014244:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677473.014244:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.0:1619677473.014244:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677473.014245:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.014245:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677473.014245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.014246:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:8.0:1619677473.014246:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:22.0:1619677473.014247:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.014247:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.014248:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677473.014248:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.014248:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.014249:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:22.0:1619677473.014250:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.014250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677473.014254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.014263:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.014266:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.014267:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.014268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.014269:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.014273:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.014451:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.014454:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.014455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.014456:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.014459:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.014461:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.014462:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.014463:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.014464:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.014465:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.014468:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.014469:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.014471:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.014472:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.014473:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.014474:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.014475:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.014475:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.014476:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677473.014489:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.014491:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.014492:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.014493:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.014495:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.014496:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677473.014561:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.014564:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.014566:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.014567:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.014570:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.014571:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.014572:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.014574:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.014575:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.014576:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.014579:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.014580:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.014583:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.014584:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.014584:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.014586:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.014586:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.014587:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.014587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.014592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.014600:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.014603:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.014604:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.014604:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.014606:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.014609:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.015158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.015164:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.015167:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.015168:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.015170:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.015172:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.015173:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.015174:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.015176:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.015177:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.015178:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.015181:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.015183:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.015186:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.015187:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.015188:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.015190:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.015190:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.015191:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.015192:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.015193:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.015198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677473.015203:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677473.015204:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677473.015205:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677473.015206:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:13.0:1619677473.015206:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.015207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.015207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677473.015208:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.015209:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.015211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.015400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.015407:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.015409:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.015410:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.015410:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.015412:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.015415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677473.015541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.015549:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.015552:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.015552:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.015554:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.015555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.015558:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.015794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.015797:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.015800:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:21.0:1619677473.015802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.015802:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.015804:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.015806:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677473.015807:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.015808:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.015809:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.015810:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677473.015813:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.015815:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677473.015818:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.015819:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677473.015820:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677473.015822:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677473.015822:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.015823:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.015824:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:12.0:1619677473.015835:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.015837:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.015838:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.015840:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.015842:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.015843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677473.015855:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677473.015858:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677473.015858:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.015859:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677473.015860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.015860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:5.2:1619677473.016246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f74b400] (20)++
00000800:00000200:13.0:1619677473.016254:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f74b400] (21)++
00000800:00000200:13.0:1619677473.016257:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.12@o2ib3
00000400:00000200:13.0:1619677473.016259:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.2.12@o2ib3 : GET - for me
00000800:00000200:12.0:1619677473.016260:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f74b400] (22)--
00000400:00000200:13.0:1619677473.016262:0:19942:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.2.12@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:13.0:1619677473.016264:0:19942:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.2.12@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:13.0:1619677473.016266:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f74b400] (21)++
00000800:00000200:13.0:1619677473.016267:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f74b400] (22)--
00000400:00000200:13.0:1619677473.016269:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.2.12@o2ib3 local destination
00000800:00000200:13.0:1619677473.016270:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f74b400] (21)--
00000800:00000200:8.2:1619677473.016600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.016608:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.016611:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.016612:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.016613:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.016615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.016617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.016619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.016627:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.016630:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.016631:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.016631:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.016632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.016633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.016903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.016909:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.016912:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.016913:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.016913:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.016915:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.016917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.017402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.017405:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.017408:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677473.017410:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.017411:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.017414:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.017415:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677473.017417:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.017418:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.017419:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.017420:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677473.017423:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.017425:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677473.017428:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.017428:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677473.017429:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677473.017431:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677473.017431:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.017433:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677473.017434:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677473.017434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677473.017439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.017445:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:12.0:1619677473.017445:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:21.0:1619677473.017447:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677473.017448:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:12.0:1619677473.017448:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:21.0:1619677473.017449:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:12.0:1619677473.017449:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.017450:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.017452:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.017453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:5.2:1619677473.017454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.017460:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677473.017463:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677473.017463:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.017464:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677473.017466:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.017467:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677473.017467:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.017470:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.017470:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.017471:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.017472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.017472:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.017473:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.017475:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.017475:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.017476:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.017477:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.017477:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.017478:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.017480:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.017481:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.017483:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.017486:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.017487:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.017487:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.017488:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.017488:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.017489:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.017490:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677473.017491:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.017491:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677473.017492:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.017492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677473.017493:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.017494:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.017495:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.017495:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.017496:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.017506:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.017508:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.017509:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:6.0:1619677473.017509:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.017511:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.017511:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.017511:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.017513:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.017514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.017515:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.017516:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.017517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.017823:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.017825:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.017826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.017827:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.017830:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.017832:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.017833:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.017834:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.017835:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.017836:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.017838:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.017840:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.017841:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.017842:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.017843:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.017844:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677473.017845:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.017845:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.017845:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.017846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.017848:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.017849:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.017849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:9.2:1619677473.017850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:22.0:1619677473.017851:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.017854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.017856:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.017858:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.017859:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.017859:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.017861:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.017863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.018657:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.018660:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.018661:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.018663:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.018666:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.018667:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.018668:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.018670:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.018671:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.018671:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.018674:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.018676:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.018679:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.018680:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.018680:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.018682:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.018683:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.018683:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.018684:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.018689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.018698:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.018700:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.018701:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.018703:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.018703:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.018706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.018884:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.018887:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.018888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.018889:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.018892:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.018893:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.018895:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.018896:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.018897:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.018898:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.018901:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.018902:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.018904:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.018905:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.018905:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.018907:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.018907:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.018908:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.018910:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.018910:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.018913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.018919:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.018922:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677473.018923:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677473.018923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.018923:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.018924:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677473.018924:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.018925:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.018926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.018927:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.019073:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.019075:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.019076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.019078:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.019080:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.019081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677473.019082:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.019083:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.019084:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.019085:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.019086:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677473.019087:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:10.0:1619677473.019088:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.019089:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.019090:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.019090:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677473.019090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.019092:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.019093:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.019094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.019094:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.019094:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.019096:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.019096:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.019097:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.019098:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677473.019099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.019105:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.019107:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.019108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.019108:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.019109:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.019109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.019111:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.019113:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.019114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.019115:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.019117:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.019119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.019129:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.019132:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.019132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.019133:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.019134:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.019135:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.019399:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.019402:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.019403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.019404:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.019406:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.019408:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.019409:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.019410:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.019411:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.019412:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.019415:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.019417:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.019419:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.019420:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.019421:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.019422:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.019423:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.019423:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.019424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.019428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.019436:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.019439:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.019440:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.019441:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.019443:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.019446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.019501:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.019504:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.019505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.019506:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.019508:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.019510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:13.0:1619677473.019710:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.019712:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.019715:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.019717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.019718:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.019719:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677473.019721:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.019722:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.019723:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.019724:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677473.019727:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.019729:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677473.019731:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.019732:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677473.019732:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677473.019734:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677473.019734:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.019735:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.019735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.019739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.019747:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.019749:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.019750:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.019750:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.019752:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.019754:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.020154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.020161:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.020163:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.020164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.020164:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.020166:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.020168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.020623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.020629:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677473.020629:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.020632:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:18.0:1619677473.020633:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677473.020633:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:2.0:1619677473.020634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677473.020636:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.020637:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677473.020638:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.020640:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.020640:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.020641:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677473.020641:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.020643:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.020643:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.020645:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.020645:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.020646:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.020646:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.020648:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677473.020651:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677473.020653:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677473.020653:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.020655:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677473.020656:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.020656:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.020657:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.020658:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.020658:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.020659:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:18.0:1619677473.020660:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.020660:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677473.020660:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677473.020661:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677473.020661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677473.020662:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.020663:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.020663:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.020664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.020665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.020674:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.020675:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.020675:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000400:00000200:17.0:1619677473.020676:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.020676:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677473.020677:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.020678:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.020679:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.020679:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677473.020681:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.020708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.020716:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.020719:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.020720:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.020721:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.020722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.020725:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677473.020845:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.020848:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.020849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.020850:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.020853:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.020855:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.020856:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.020858:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.020859:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.020860:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.020863:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.020865:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.020868:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.020869:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.020869:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.020871:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.020871:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.020872:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.020874:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.020874:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677473.020877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.020885:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677473.020887:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677473.020888:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.020889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.020889:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677473.020889:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.020890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.020890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677473.020891:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.020894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.020896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.020903:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.020906:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.020907:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.020907:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.020908:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.020908:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.021602:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.021605:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.021606:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.021606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.021608:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.021611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.021619:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.021622:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.021623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.021623:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.021625:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.021625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.021631:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.021634:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.021635:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.021637:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.021638:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.021640:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677473.021641:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.021643:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.021644:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.021645:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677473.021648:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.021650:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677473.021653:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.021654:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677473.021655:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677473.021657:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677473.021657:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.021658:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.021658:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.021663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.021670:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.021673:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.021674:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.021674:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.021677:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.021679:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.021812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.021819:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.021821:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.021822:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.021822:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.021824:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.021827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.021915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.021922:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.021925:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.021926:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.021928:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.021931:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.021932:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.021933:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.021935:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.021936:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.021937:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.021940:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.021942:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.021944:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.021944:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.021945:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.021947:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.021948:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.021948:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.021949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.021953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.021961:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.021963:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.021963:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.021964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.021965:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.021966:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.021976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.021983:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.021985:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.021986:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.021987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.021987:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.021987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677473.022148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.022154:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.022157:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.022157:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.022159:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.022162:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.022164:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.022165:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.022166:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.022167:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.022168:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.022171:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.022173:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.022175:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.022176:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.022177:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.022178:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.022179:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.022180:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.022180:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.022192:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.022194:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.022195:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.022195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.022197:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.022199:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.022334:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f783000] (21)++
00000800:00000200:21.0:1619677473.022336:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.4@o2ib3
00000400:00000200:21.0:1619677473.022338:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.4@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.022339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000400:00000200:21.0:1619677473.022342:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.022343:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677473.022345:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.022346:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.022347:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.022348:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677473.022352:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.022354:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.022356:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.022357:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677473.022357:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.022359:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.022360:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946f783000] (21)++
00000800:00000200:21.0:1619677473.022361:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946f783000] (22)--
00000800:00000200:21.0:1619677473.022361:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (21)--
00000800:00000200:20.0:1619677473.022373:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.022375:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f783000] (21)++
00000800:00000200:20.0:1619677473.022375:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.022376:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.4@o2ib3
00000400:00000200:20.0:1619677473.022376:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.022378:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:21.0:1619677473.022379:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.4@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.022381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677473.022382:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.022382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.022383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000400:00000200:21.0:1619677473.022384:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:21.0:1619677473.022385:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.022387:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.022388:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.022388:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677473.022392:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.022393:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.0:1619677473.022396:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.022397:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677473.022397:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677473.022399:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677473.022400:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946f783000] (21)++
00000800:00000200:21.0:1619677473.022401:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946f783000] (22)--
00000800:00000200:21.0:1619677473.022401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (21)--
00000800:00000200:12.0:1619677473.022413:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.022416:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.022417:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.022418:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.022420:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.022421:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.0:1619677473.022463:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f783000] (21)++
00000800:00000200:20.0:1619677473.022465:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.4@o2ib3
00000400:00000200:20.0:1619677473.022467:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.4@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.022468:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000400:00000200:20.0:1619677473.022470:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.022472:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677473.022474:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.022475:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.022476:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.022477:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677473.022480:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.022482:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677473.022485:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.022486:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.0:1619677473.022487:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677473.022489:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677473.022489:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946f783000] (21)++
00000800:00000200:20.0:1619677473.022491:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91946f783000] (22)++
00000800:00000200:20.0:1619677473.022492:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946f783000] (23)--
00000800:00000200:20.0:1619677473.022493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000800:00000200:9.2:1619677473.022494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.022501:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677473.022503:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f783000] (22)++
00000800:00000200:24.0:1619677473.022504:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:21.0:1619677473.022505:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91946f783000] (23)--
00000800:00000200:8.0:1619677473.022505:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.022506:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.022506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000800:00000200:21.0:1619677473.022507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (21)--
00000400:00000200:24.0:1619677473.022508:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.022511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.022615:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.022618:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.022619:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.022621:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.022624:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.022625:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.022626:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.022628:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.022629:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.022630:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.022633:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.022635:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.022637:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.022638:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.022639:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.022641:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.022641:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.022642:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.022642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.022655:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.022658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.022658:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.022659:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.022661:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.022664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.022714:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f783000] (21)++
00000800:00000200:21.0:1619677473.022716:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.4@o2ib3
00000400:00000200:21.0:1619677473.022719:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.4@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.022719:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000400:00000200:21.0:1619677473.022721:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.022723:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:21.0:1619677473.022724:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.022726:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.022726:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.022728:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:21.0:1619677473.022731:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.022733:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.0:1619677473.022735:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.022736:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:21.0:1619677473.022737:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677473.022738:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:21.0:1619677473.022739:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946f783000] (21)++
00000800:00000200:21.0:1619677473.022740:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946f783000] (22)--
00000800:00000200:21.0:1619677473.022740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (21)--
00000800:00000200:9.2:1619677473.022744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.022752:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.022755:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.022756:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.022756:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.022758:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.022761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.0:1619677473.022784:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f783000] (21)++
00000800:00000200:20.0:1619677473.022786:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.4@o2ib3
00000400:00000200:20.0:1619677473.022788:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.4@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.022789:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000400:00000200:20.0:1619677473.022790:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.022792:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677473.022793:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.022795:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.022796:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.022797:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677473.022800:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.022802:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677473.022805:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.022805:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.0:1619677473.022806:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677473.022808:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677473.022808:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946f783000] (21)++
00000800:00000200:20.0:1619677473.022809:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946f783000] (22)--
00000800:00000200:20.0:1619677473.022809:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (21)--
00000800:00000200:9.2:1619677473.022813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.022820:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.022822:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.022823:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.022824:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.022826:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.022828:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.022833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.022839:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.022841:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.022842:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.022842:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.022844:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.022844:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677473.022894:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f783000] (21)++
00000800:00000200:21.0:1619677473.022897:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.4@o2ib3
00000400:00000200:21.0:1619677473.022899:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.4@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.022899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000400:00000200:21.0:1619677473.022902:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.022904:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:21.0:1619677473.022905:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.022906:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.022907:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.022908:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:21.0:1619677473.022911:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.022913:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.0:1619677473.022916:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.022916:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:21.0:1619677473.022917:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677473.022918:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677473.022919:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946f783000] (21)++
00000800:00000200:21.0:1619677473.022920:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946f783000] (22)--
00000800:00000200:21.0:1619677473.022920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (21)--
00000800:00000200:20.0:1619677473.022930:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f783000] (21)++
00000800:00000200:22.0:1619677473.022932:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:20.0:1619677473.022933:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.4@o2ib3
00000800:00000200:22.0:1619677473.022934:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.022935:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.022935:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.4@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.022936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:20.0:1619677473.022937:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.022938:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.022939:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.0:1619677473.022940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000400:00000200:20.0:1619677473.022940:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.022941:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.022942:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.022943:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677473.022947:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.022950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.022950:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.022951:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.022952:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.022953:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.022954:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.022955:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946f783000] (21)++
00000800:00000200:20.0:1619677473.022956:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91946f783000] (22)++
00000800:00000200:20.0:1619677473.022957:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946f783000] (23)--
00000800:00000200:20.0:1619677473.022958:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000800:00000200:8.2:1619677473.022960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.022968:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.022969:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.022970:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f783000] (22)++
00000400:00000200:20.0:1619677473.022970:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.022971:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91946f783000] (23)--
00000800:00000200:21.0:1619677473.022971:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000400:00000200:20.0:1619677473.022972:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.022973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.022974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.022974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677473.022975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (21)--
00000800:00000200:8.0:1619677473.022981:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.022983:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.022984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.022984:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.022987:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.022989:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.023227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:24.0:1619677473.023230:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677473.023230:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:24.0:1619677473.023233:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677473.023233:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:8.0:1619677473.023233:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677473.023235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677473.023236:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.023237:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.023238:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.023239:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677473.023240:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.023241:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.023241:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.023243:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.023243:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.023244:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.023245:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.023245:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677473.023246:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.023247:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.023249:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.023252:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.023252:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.023254:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.023254:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677473.023255:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.023255:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.023256:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677473.023257:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.023257:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:24.0:1619677473.023258:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.023258:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.023258:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677473.023259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:24.0:1619677473.023260:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.023260:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.023261:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.023262:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.023263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:9.2:1619677473.023265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.023271:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677473.023271:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.023273:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677473.023274:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:8.0:1619677473.023274:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.023275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677473.023275:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.023277:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677473.023278:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.023278:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.023281:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.023285:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677473.023285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.023287:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.023289:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.023290:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.023290:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.023290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.023291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.023338:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.023341:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.023342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.023342:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.023345:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.023348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.023434:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.023437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.023437:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.023439:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.023442:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.023444:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.023445:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.023446:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.023447:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.023448:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.023451:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.023453:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.023456:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.023456:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.023457:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.023459:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.023459:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.023459:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.023460:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.023472:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.023475:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.023476:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.023476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.023478:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.023481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.023638:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.023641:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.023642:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.023643:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.023646:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.023647:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.023648:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.023650:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.023651:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.023651:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.023654:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.023655:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.023657:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.023658:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.023658:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.023660:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.023660:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.023661:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.023661:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.023665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.023672:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.023675:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.023676:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.023676:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.023677:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.023680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.023781:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.023784:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.023785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.023786:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.023788:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.023790:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.023791:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.023792:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.023793:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.023794:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.023797:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.023798:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.023801:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.023801:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.023802:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.023803:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.023804:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.023804:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.023805:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.023810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.023818:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.023820:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.023821:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.023823:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.023824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.023826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.025430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.025441:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677473.025445:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.025446:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.025448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.025448:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.025448:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.025449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.025450:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.025452:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.025455:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:6.0:1619677473.025455:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.025459:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.025499:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677473.025502:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:24.0:1619677473.025505:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677473.025506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.025506:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:8.0:1619677473.025507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677473.025511:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.025512:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.025513:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.025515:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.025515:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677473.025517:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.025518:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677473.025520:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.025521:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.025522:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.025523:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677473.025525:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.025527:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.025528:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.025529:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.025531:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:24.0:1619677473.025535:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.025539:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.025540:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.025542:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:24.0:1619677473.025543:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.025543:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:24.0:1619677473.025544:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.025546:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.025546:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677473.025547:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:24.0:1619677473.025548:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.025548:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:24.0:1619677473.025549:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677473.025549:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:24.0:1619677473.025550:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.025551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.025552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.2:1619677473.025555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677473.025563:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.025567:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:17.0:1619677473.025567:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:26.0:1619677473.025568:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.025568:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.025571:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:26.0:1619677473.025572:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:17.0:1619677473.025573:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.025573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677473.025576:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:17.0:1619677473.025577:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.025582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.025986:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.025990:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.025991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.025994:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:20.2:1619677473.025996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:10.0:1619677473.025999:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.026001:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.026003:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677473.026005:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677473.026006:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.026007:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.026009:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.026010:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.026011:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677473.026014:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.026017:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677473.026020:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677473.026021:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.026022:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:2.0:1619677473.026022:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.026023:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:2.0:1619677473.026024:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677473.026026:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.026026:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:2.0:1619677473.026026:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.026028:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:2.0:1619677473.026029:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.026030:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.026031:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677473.026031:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677473.026032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:2.0:1619677473.026032:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.026039:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.026043:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.026043:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.026046:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:26.0:1619677473.026047:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:18.0:1619677473.026047:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.026047:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.026049:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677473.026050:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:2.0:1619677473.026050:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677473.026051:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:18.0:1619677473.026051:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:26.0:1619677473.026052:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677473.026053:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.026055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.026055:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.026056:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.026057:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.026058:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677473.026059:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677473.026060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.026060:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.026061:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.026062:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.026063:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677473.026070:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.026074:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.026076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.026076:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.026079:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.026082:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677473.026558:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.026561:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.026562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.026565:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.026568:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.026569:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677473.026571:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.026572:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.026573:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.026574:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677473.026578:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.026580:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677473.026583:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.026584:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677473.026585:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677473.026587:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.026587:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.026588:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.026588:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.026593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.026600:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.026603:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.026604:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.026604:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.026606:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.026609:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:5.2:1619677473.026898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.026904:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.026907:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.026908:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.026910:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.026912:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.026914:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.026915:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.026916:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.026917:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.026918:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.026921:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.026923:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.026925:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.026926:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.026926:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.026928:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.026929:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.026930:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.026931:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.026931:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.026936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.026941:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.026942:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.026944:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.026944:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.026945:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.026945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.026946:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.026946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.026948:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.026950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.026961:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.026964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.026964:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.026965:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.026966:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.026966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.027051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.027054:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.027055:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.027057:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.027059:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.027061:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.027063:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.027064:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.027065:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.027066:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.027069:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.027071:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.027075:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.027076:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.027076:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.027078:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.027079:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.027080:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.027080:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.027085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.027093:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.027096:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.027097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.027097:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.027099:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.027102:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.027448:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.027450:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.027451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.027452:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.027454:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.027458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.027459:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.027461:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.027462:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.027462:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.027463:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.027464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.027538:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.027540:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.027543:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.027543:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.027545:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.027547:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.027548:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.027550:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.027551:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.027552:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:9.2:1619677473.027553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:21.0:1619677473.027555:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.027557:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.027560:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677473.027560:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.027560:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.027561:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677473.027563:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:21.0:1619677473.027563:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:24.0:1619677473.027564:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.027564:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.027564:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.0:1619677473.027564:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677473.027565:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:24.0:1619677473.027566:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.027569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.027572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.027578:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.027580:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.027581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.027582:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.027582:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.027583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.027609:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.027612:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.027613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.027614:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.027617:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.027618:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.027620:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.027621:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.027622:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.027623:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.027626:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.027628:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.027630:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.027631:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.027632:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.027634:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.027634:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.027635:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.027635:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.027648:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.027650:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.027651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.027652:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.027654:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.027657:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.027898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.027905:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.027908:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.027909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.027911:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.027913:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.027914:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.027916:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.027917:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.027918:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.027919:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.027922:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.027924:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.027926:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.027927:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.027927:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.027929:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.027930:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.027931:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677473.027932:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.027932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.027935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.027938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.027944:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.027945:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.027946:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677473.027946:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.027947:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.027947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.027948:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.027948:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.027949:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677473.027949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677473.028002:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.028005:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677473.028006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677473.028007:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.028010:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.028012:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677473.028013:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.028014:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.028015:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.028016:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677473.028019:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.028021:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677473.028023:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.028024:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677473.028025:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677473.028026:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677473.028027:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.028028:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677473.028028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677473.028032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.028038:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.028040:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.028041:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.028042:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.028043:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.028046:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.028058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677473.028065:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.028067:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677473.028068:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.028068:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.028069:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.028069:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.0:1619677473.028086:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.028089:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.028090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.028091:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.028093:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.028094:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.028095:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.028096:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.028097:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.028098:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.028100:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.028102:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.028103:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.028104:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.028104:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.028105:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.028105:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.028107:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.028108:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.028108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.028110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.028119:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.028120:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.028120:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.028122:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.028122:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.028123:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.028123:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.028124:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.028125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.028125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.028129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.028136:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.028138:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.028139:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.028139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.028140:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.028141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.028245:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.028248:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677473.028249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677473.028251:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.028254:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.028254:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:22.0:1619677473.028255:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677473.028257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677473.028257:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.028257:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:22.0:1619677473.028259:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.028261:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.028262:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.028263:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.028265:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.028266:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.028267:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677473.028268:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.028269:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.028270:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.028271:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.028271:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:10.0:1619677473.028271:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.028272:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:10.0:1619677473.028272:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.028274:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:10.0:1619677473.028275:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.028277:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.028277:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:10.0:1619677473.028277:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.028278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:10.0:1619677473.028280:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677473.028280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.028281:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.028281:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.028283:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.028284:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.028284:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.028285:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677473.028287:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.028290:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.028291:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.028292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677473.028297:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.028300:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:2.0:1619677473.028300:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.028301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.028301:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.028303:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.028303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677473.028306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.028344:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.028347:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.028349:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.028350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.028352:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.028353:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.028355:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.028356:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.028357:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.028358:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.028361:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.028363:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.028365:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.028366:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.028367:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.028368:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.028369:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.028370:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.028370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.028382:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.028385:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.028386:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.028386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.028388:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.028391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.028851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.028858:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.028860:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.028863:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.028863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.028866:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.028867:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677473.028868:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.028871:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.028872:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.028873:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677473.028878:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.028880:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.028883:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.028883:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.028884:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677473.028884:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.028886:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.028886:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:26.0:1619677473.028887:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:21.0:1619677473.028888:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:10.0:1619677473.028888:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677473.028889:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000400:00000200:26.0:1619677473.028890:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.028890:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677473.028892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:26.0:1619677473.028893:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.028894:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.028895:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96511195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677473.028895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000400:00000200:26.0:1619677473.028896:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.028897:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677473.028897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:26.0:1619677473.028898:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.028902:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.028902:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677473.028903:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.0:1619677473.028903:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.028903:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.028903:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000800:00000200:20.0:1619677473.028904:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000400:00000200:21.0:1619677473.028905:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.028905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:1.0:1619677473.028905:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677473.028906:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.028907:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.028907:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677473.028907:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.028908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677473.028908:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:26.0:1619677473.028909:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.028909:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677473.028909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:17.0:1619677473.028909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.028909:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.028910:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.028911:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:1.0:1619677473.028911:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.028912:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.028914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677473.028915:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677473.028921:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.028924:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.028925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.028925:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.028925:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.028927:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.028927:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677473.028927:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.028928:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.028929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.029452:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.029455:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.029456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.029456:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.029458:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.029461:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.030275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.030283:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.030286:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.030287:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.030287:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.030290:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.030292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.030296:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.030301:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.030303:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.030303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.030304:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.030305:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.030305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.031243:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.031246:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.031247:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.031247:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.031249:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.031252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.031260:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.031262:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.031263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.031263:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.031264:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.031265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.031664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.031670:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.031673:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.031674:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.031675:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.031677:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.031679:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.031680:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.031681:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.031682:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.031683:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.031686:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.031688:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.031691:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.031692:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.031692:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.031694:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.031694:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.031695:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.031695:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.031709:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.031712:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.031713:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.031713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.031715:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.031717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.031792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.031799:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.031802:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.031803:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.031805:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.031807:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.031808:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.031810:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.031811:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.031812:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.031813:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.031816:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.031818:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.031822:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.031822:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.031823:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.031825:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.031826:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.031827:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.031828:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:13.0:1619677473.031839:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.031842:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.031843:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.031845:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.031846:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.031847:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.031959:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.031961:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.031962:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.031964:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.031967:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.031968:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.031969:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.031970:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.031971:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.031972:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.031975:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.031977:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.031980:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.031980:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.031981:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.031983:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.031983:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.031984:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.031984:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.031995:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.031997:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.031998:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.031999:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.032001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.032002:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.032018:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.032021:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.032021:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.032022:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.032023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.032024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677473.032653:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.032655:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.032657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.032658:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.032660:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.032662:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.032663:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.032664:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.032665:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.032666:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.032669:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.032670:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.032673:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.032674:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.032675:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.032676:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.032677:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.032678:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.032678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.032683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.032692:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.032695:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.032696:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.032696:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.032699:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.032702:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.032717:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.032720:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.032721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.032723:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : REPLY - routed
00000800:00000200:26.0:1619677473.032725:0:19941:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:26.0:1619677473.032730:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.032731:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.032732:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.032732:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.032745:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.032747:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.032748:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.032954:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.032957:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.032957:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.032959:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.032960:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.032961:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.032963:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.032964:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.032965:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.032965:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.032968:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : REPLY try# 0
00000800:00000200:26.0:1619677473.032970:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.032972:0:19941:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:26.0:1619677473.032985:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.032986:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.032986:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.032988:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.032988:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.032989:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.032989:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.032994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.033002:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.033004:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.033005:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:9.2:1619677473.033028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.033035:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.033037:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.033038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.033043:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.033057:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.033057:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.033227:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.033230:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.033231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.033232:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.033234:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.033236:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.033237:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.033238:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.033239:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.033240:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.033243:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.033244:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677473.033244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.033247:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.033248:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.033248:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.033250:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.033251:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.033251:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677473.033252:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.033252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677473.033256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.033257:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.033258:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.033260:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: REPLY: OK
00000800:00000200:18.0:1619677473.033263:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677473.033264:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.033267:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.033268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.033269:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.033271:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.033274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.033316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:2.0:1619677473.033322:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677473.033325:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:18.0:1619677473.033326:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:2.0:1619677473.033328:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.033331:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.033333:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677473.033334:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.033335:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.033336:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.033338:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677473.033341:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.033343:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 600 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677473.033345:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.033346:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677473.033347:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677473.033348:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677473.033349:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677473.033350:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:2.0:1619677473.033350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:21.0:1619677473.033364:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.033366:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.033367:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.033368:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.033369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.033371:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.033382:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.033385:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.033386:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.033387:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.033387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.033388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.033450:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.033453:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.033454:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.033454:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.033456:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.033459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.033606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.033613:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.033616:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.033616:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.033619:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.033621:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.033623:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.033625:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.033626:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.033627:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.033628:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.033631:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.033633:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.033636:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.033637:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.033638:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.033640:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.033640:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.033642:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.033642:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677473.033652:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677473.033653:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677473.033654:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.033655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677473.033656:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.033656:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.033657:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677473.033658:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:26.0:1619677473.033660:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.033660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.033661:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677473.033661:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677473.033662:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.033663:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.033663:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677473.033664:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.033664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677473.033665:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.033666:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.033667:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.033670:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.033672:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:8.0:1619677473.033673:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.033674:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677473.033674:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:26.0:1619677473.033675:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:8.0:1619677473.033675:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.033676:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.033677:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.033677:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.033677:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.033678:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.033678:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677473.033678:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677473.033679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677473.033681:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677473.033683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.033683:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.033685:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.033686:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.033687:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.033688:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.033689:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.033689:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.033690:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.033691:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.033691:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.033691:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.033692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.033693:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.033694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.033696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.033698:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.033699:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.033699:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.033700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.033701:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.033702:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.034194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.034197:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.034200:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.034202:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.034203:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.034204:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.034206:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677473.034208:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.034209:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.034210:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.034211:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677473.034214:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.034216:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677473.034219:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.034220:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:20.0:1619677473.034221:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.0:1619677473.034222:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:20.0:1619677473.034223:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.034224:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.034225:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677473.034228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.034235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.034238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677473.034238:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.034239:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.034241:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.034244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.0:1619677473.034359:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.034361:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.034364:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.034364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.034366:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.034367:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.034369:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.034370:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.034371:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.034372:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.034376:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.034378:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.034380:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.034381:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.034382:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.034383:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.034384:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.034386:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.034387:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.034387:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677473.034391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.034396:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.034398:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.034399:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677473.034399:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.034400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677473.034401:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.034402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.034403:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.034405:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.034407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.034435:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.034437:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.034438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.034440:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : REPLY - routed
00000800:00000200:26.0:1619677473.034442:0:19941:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 900855
00000800:00000200:26.0:1619677473.034446:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.034447:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.034447:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.034448:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.034460:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.034462:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.034463:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677473.034510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.034519:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.034522:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.034522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.034523:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.034525:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.034528:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.034627:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.034630:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.034631:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.034632:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.034633:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.034635:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.034636:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.034637:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.034638:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.034639:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.034643:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : REPLY try# 0
00000800:00000200:26.0:1619677473.034644:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 900855 bytes in 256 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.034646:0:19941:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 900855
00000800:00000200:26.0:1619677473.034650:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.034651:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.034651:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.034653:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.034653:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.034654:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.034655:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:13.0:1619677473.034667:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.034669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.034670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.034693:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.034696:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.034700:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)++
00000800:00000200:12.0:1619677473.034700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.034712:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.034713:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.034827:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.034830:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.034830:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.034832:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.034834:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.034836:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.034837:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.034838:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.034839:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.034840:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.034843:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.034844:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.034847:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.034848:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.034848:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.034850:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.034850:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.034851:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.034852:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.034853:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.034865:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.034867:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.034868:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.034869:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.034897:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.034900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.034900:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.034901:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.034903:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.034906:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:12.0:1619677473.034957:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.034961:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.034962:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.034964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677473.034964:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: REPLY: OK
00000800:00000200:12.0:1619677473.034966:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677473.035252:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.035255:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.035256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.035257:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.035259:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.035261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.035401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.035403:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.035406:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677473.035409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.035409:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.035411:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.035413:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677473.035414:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.035416:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.035417:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.035418:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.035421:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.035422:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.035426:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.035426:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677473.035427:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677473.035429:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677473.035429:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.035430:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.035431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.2:1619677473.035454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.035460:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.035463:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.035464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.035466:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.035469:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.035470:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.035472:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.035473:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.035474:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.035475:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.035478:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.035480:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.035482:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.035483:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.035484:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.035485:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.035486:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.035487:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.035487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.035492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.035499:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.035501:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.035501:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.035502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.035503:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.035503:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677473.035505:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.035506:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.035506:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.035509:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : REPLY - routed
00000800:00000200:10.0:1619677473.035511:0:19940:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 902835
00000800:00000200:9.2:1619677473.035513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677473.035514:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.035516:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.035516:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.035517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677473.035520:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.035523:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.035524:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.035524:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.035525:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.035526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.035530:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.035532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.035533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677473.035611:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.035614:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677473.035615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677473.035617:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.035619:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.035620:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.035621:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.035622:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.035623:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.035624:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.2:1619677473.035624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:18.0:1619677473.035627:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.035628:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.035630:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.035631:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.035631:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.035633:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.035633:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:1.0:1619677473.035633:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677473.035634:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677473.035635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:1.0:1619677473.035636:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.035637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.035637:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.035639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:1.0:1619677473.035639:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.035642:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677473.035646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.035648:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.035649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.035649:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.035651:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.035652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.035701:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.035703:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.035704:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.035705:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.035706:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.035708:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677473.035709:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.035710:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.035712:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.035712:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677473.035716:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : REPLY try# 0
00000800:00000200:10.0:1619677473.035718:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 902835 bytes in 256 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677473.035719:0:19940:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 902835
00000800:00000200:10.0:1619677473.035731:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.035732:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677473.035733:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677473.035734:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677473.035735:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.035735:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.035736:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.035740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.035748:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.035751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.035752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:9.2:1619677473.035781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.035788:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.035791:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.035792:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.035796:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.035808:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.035809:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:9.2:1619677473.036052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.036060:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.036063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.036064:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.036065:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.036067:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: REPLY: OK
00000800:00000200:18.0:1619677473.036070:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677473.036523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.036529:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.036532:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.036533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.036535:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.036537:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.036539:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.036540:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.036541:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.036542:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.036543:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.036546:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.036548:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.036550:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.036551:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.036552:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.036553:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.036554:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.036555:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.036555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.036568:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.036571:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.036572:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.036572:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.036574:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.036576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.036588:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.036591:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.036592:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.036592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.036593:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.036594:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677473.036896:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.036899:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677473.036900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677473.036902:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.036904:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.036906:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677473.036907:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.036908:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.036909:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.036910:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677473.036913:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.036914:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677473.036918:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.036918:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677473.036919:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677473.036921:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677473.036921:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.036922:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.036923:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677473.036934:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.036937:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.036938:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.036940:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.036941:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.036942:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:26.0:1619677473.037375:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.037378:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.037378:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.037381:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.037383:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.037385:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.037386:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.037387:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.037389:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.037389:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.037393:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.037395:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.037398:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.037399:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.037399:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.037401:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.037401:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.037402:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.037403:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.037416:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.037419:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.037420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.037420:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.037422:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.037429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.037431:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.037432:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.037433:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.037434:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.037435:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.037435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.037516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.037519:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.037520:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.037520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.037522:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.037525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.037540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.037546:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.037548:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.037550:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.037551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.037553:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.037555:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677473.037556:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.037558:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.037559:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.037560:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677473.037562:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.037564:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.037566:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.037566:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.037567:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.037568:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.037569:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.037570:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.037570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.037575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.037583:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.037585:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.037586:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.037588:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.037589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.037590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677473.038041:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.038043:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677473.038045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677473.038046:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.038048:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.038049:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677473.038051:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.038052:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.038053:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.038054:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.038057:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.038059:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.038061:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.038062:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677473.038062:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677473.038064:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677473.038064:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.038065:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677473.038067:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677473.038067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.038080:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677473.038081:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677473.038082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.038084:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:20.2:1619677473.038094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.038099:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.038102:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.038104:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.038105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.038107:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.038108:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.038109:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.038111:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.038112:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.038113:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.038117:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.038118:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.038121:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.038121:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.038122:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.038124:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.038125:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.038126:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.038127:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.038128:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.038130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677473.038133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.038136:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.038138:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677473.038138:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677473.038139:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.038139:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677473.038141:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.038142:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.038143:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.038144:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.038147:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.038153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.2:1619677473.038157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:1.0:1619677473.038160:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.038163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:2.0:1619677473.038163:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677473.038163:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.038164:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.038165:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.038165:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677473.038166:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.038167:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.038169:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.038171:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.038173:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.038174:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.038176:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.038177:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.038178:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.038182:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.038183:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.038187:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.038187:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.038188:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.038190:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.038191:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.038192:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.038193:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.038194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.038199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.038205:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.038207:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.038207:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.038208:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677473.038365:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.038367:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.038368:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.038370:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.038373:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.038375:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.038376:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.038377:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.038378:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.038379:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.038382:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.038383:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.038386:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.038387:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.038388:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.038389:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.038390:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.038390:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.038391:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.038404:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.038407:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.038408:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.038408:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.038410:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.038413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.038588:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.038591:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.038592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.038593:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.038596:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.038598:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.038599:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.038600:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.038601:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.038602:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.038605:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.038606:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.038609:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.038610:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.038610:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.038612:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.038612:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.038613:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.038613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.038618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.038626:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.038629:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.038630:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.038630:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.038632:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.038636:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.038779:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.038781:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.038782:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.038784:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.038787:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.038788:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.038789:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.038790:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.038791:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.038792:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.038795:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.038797:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.038799:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.038800:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.038801:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.038803:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.038803:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.038804:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.038804:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.038809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.038816:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.038819:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.038820:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.038820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.038822:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.038826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677473.038923:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.038926:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.038926:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.038928:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.038931:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.038933:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677473.038934:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.038935:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.038936:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.038937:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677473.038941:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.038943:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677473.038946:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.038946:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677473.038947:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677473.038949:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677473.038949:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.038951:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677473.038952:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677473.038952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.038963:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:9.2:1619677473.038965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.038965:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.038965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.038966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677473.038971:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.038974:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.038975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.038975:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.038977:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.038980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677473.039151:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.039153:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.039155:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.039156:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.039158:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.039159:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.039161:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.039162:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.039163:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.039164:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.039168:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.039169:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.039172:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.039173:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.039173:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.039175:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.039176:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.039177:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.039177:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.039181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.039187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.039190:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.039191:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.039191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.039193:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.039196:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.039199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.039205:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.039208:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.039209:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.039209:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.039210:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.039211:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.039288:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.039291:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.039291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.039292:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.039300:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.039303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.039398:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee354600] (21)++
00000800:00000200:24.0:1619677473.039401:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.13.19@o2ib3
00000800:00000200:8.0:1619677473.039402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee354600] (22)--
00000400:00000200:24.0:1619677473.039404:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.039407:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.039408:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:24.0:1619677473.039410:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.039411:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.039412:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.039413:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:24.0:1619677473.039416:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.039417:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.039419:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.039420:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.039420:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.039422:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.039422:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee354600] (21)++
00000800:00000200:24.0:1619677473.039423:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee354600] (22)--
00000800:00000200:24.0:1619677473.039424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee354600] (21)--
00000800:00000200:24.2:1619677473.039428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.039431:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.039433:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.039434:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.039435:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.039435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.039437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.0:1619677473.039574:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.039576:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.039577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.039578:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.039580:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.039582:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.039583:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.039584:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.039584:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.039585:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.039588:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.039589:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.039590:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.039591:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.039591:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.039592:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.039593:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.039594:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.039595:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.039595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.039598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.039606:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.039607:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.039608:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.039609:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.039609:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:24.0:1619677473.039610:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.039610:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.039611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.039611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.039612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677473.040977:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.040979:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.040981:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.040982:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.040985:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.040987:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.040988:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.040989:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.040990:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.040991:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.040994:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.040996:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.040998:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.040999:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.041000:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.041001:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.041002:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.041002:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.041003:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.041015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.041018:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.041019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.041020:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.041022:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.041024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.041392:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.041394:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.041395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.041396:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.041398:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.041401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.041410:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.041413:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.041414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.041414:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.041415:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.041416:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.041913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.041918:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.041921:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.041923:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.041924:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.041926:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.041928:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.041929:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.041930:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.041931:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.041932:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.041935:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.041937:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.041940:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.041940:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.041941:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.041943:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.041943:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.041944:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.041945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.041949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.041958:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.041960:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.041961:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.041962:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.041963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.041964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.042279:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.042281:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.042282:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.042284:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.042287:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.042288:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.042289:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.042291:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.042292:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.042293:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.042302:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.042303:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.042306:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.042308:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.042308:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.042310:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.042310:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.042311:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.042312:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.042312:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.042325:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.042326:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.042328:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.042328:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.042329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677473.042329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.042330:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.042330:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.042332:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.042334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.043227:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.043230:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.043231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.043231:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.043234:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.043236:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.043236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.043239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.043239:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.043241:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.043245:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.043246:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.043247:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.043248:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.043249:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.043250:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.043253:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.043254:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.043256:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.043257:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.043258:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.043259:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.043259:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.043260:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.043261:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.043265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.043272:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.043274:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.043275:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.043276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.043277:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.043279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.043532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.043539:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.043541:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.043542:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.043542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.043543:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.043546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.043552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.043558:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.043560:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.043561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.043561:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.043562:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.043563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.043658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.043665:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.043668:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.043670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.043671:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.043674:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.043676:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.043678:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.043679:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.043680:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.043681:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.043685:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.043687:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.043689:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.043690:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.043691:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.043693:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.043693:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.043694:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.043694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.043699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.043709:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.043713:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.043715:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.043715:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.043718:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.043721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.043836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.043843:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.043846:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.043847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.043849:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.102@o2ib7 : GET - for me
00000400:00000200:18.0:1619677473.043852:0:19933:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.102@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:18.0:1619677473.043854:0:19933:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.102@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:18.0:1619677473.043855:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.043856:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.043858:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:18.0:1619677473.043861:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : REPLY try# 0
00000800:00000200:18.0:1619677473.043862:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.043863:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.043864:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.043864:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.043866:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.043866:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:9.2:1619677473.043872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.043879:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.043881:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.043882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.043882:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.043883:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: REPLY: OK
00000800:00000200:2.0:1619677473.043885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677473.043892:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.043895:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677473.043895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677473.043897:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.043899:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.043901:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.043902:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.043903:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.043904:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.043905:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.043908:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.043910:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.043913:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.043913:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.043914:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.043916:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.043916:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.043917:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677473.043918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.043922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.043930:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.043932:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.043933:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.043934:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.043936:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.043937:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677473.043939:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.043940:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.043941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.043943:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.043946:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.043947:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.043948:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.043950:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.043950:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.043951:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.043954:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.043956:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.043959:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.043960:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.043961:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.043962:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.043963:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.043963:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.043964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.043969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.043977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.043980:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.043981:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.043983:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.043983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.043986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.043987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.043995:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.043998:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.043999:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.044000:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.044000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.044001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.044220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.044223:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.044225:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677473.044228:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.044228:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.044229:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.044231:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.044232:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.044233:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.044235:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.044236:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.044240:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.044242:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677473.044243:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677473.044244:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.044245:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.044246:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677473.044246:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.044247:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677473.044247:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.044248:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.044248:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.044249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:8.0:1619677473.044249:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.044251:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677473.044253:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:8.0:1619677473.044257:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.044258:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.044260:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.044261:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.044261:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677473.044263:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:8.0:1619677473.044264:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.044266:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677473.044266:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:8.0:1619677473.044268:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.044268:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:1.0:1619677473.044268:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.044269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677473.044269:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.044270:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.044271:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.044271:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677473.044271:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.044272:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677473.044275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677473.044276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.044282:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.044284:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.044285:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.044285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.044287:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.044288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.044667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.044674:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.044676:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.044677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.044677:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.044679:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.044681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.044797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.044803:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.044806:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.044806:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.044808:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.044811:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.044812:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.044814:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.044815:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.044816:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.044817:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.044820:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.044822:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.044824:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.044825:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.044826:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.044827:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.044828:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.044828:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.044829:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.044842:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.044845:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.044846:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.044846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.044848:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.044850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677473.045070:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.045073:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.045074:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.045075:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.045078:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.045079:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677473.045081:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.045082:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.045083:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.045084:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677473.045086:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.045088:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677473.045091:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.045091:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:1.0:1619677473.045092:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677473.045094:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677473.045094:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.045095:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.045095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:12.0:1619677473.045107:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.045110:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.045110:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.045112:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.045114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.045115:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677473.045201:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.045204:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.045205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.045207:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.045208:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.045210:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.045211:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.045212:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.045213:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.045214:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.045218:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.045219:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.045222:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.045223:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.045224:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.045225:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.045226:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.045226:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.045227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.045231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.045239:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.045242:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.045243:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.045243:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.045245:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.045249:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.045252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.045259:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.045262:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.045263:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.045263:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677473.045264:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677473.045265:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.045265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.045267:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.045268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.045268:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.045270:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.045273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.045404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.045410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.045412:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.045415:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.045415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.045417:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.045419:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:21.0:1619677473.045420:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.045422:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.045423:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.045424:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677473.045427:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.045429:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.0:1619677473.045432:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.045433:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677473.045433:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677473.045435:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677473.045436:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.045436:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677473.045437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:13.0:1619677473.045448:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.045451:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.045452:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.045453:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.045455:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.045456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677473.045467:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677473.045469:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677473.045470:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.045471:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677473.045471:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.045473:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:10.0:1619677473.045758:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.045761:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.045762:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.045764:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.045767:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.045768:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.045769:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.045771:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.045772:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.045773:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.045776:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.045777:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.045780:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.045781:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.045782:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.045783:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.045784:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.045784:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.045785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.045797:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.045800:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.045801:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.045801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.045803:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.045806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.046176:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.046179:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.046181:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.046182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.046184:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.046186:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677473.046187:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.046188:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.046189:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.046190:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677473.046194:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.046196:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.046198:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.046199:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677473.046200:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.046201:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.046202:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.046202:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.046203:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:21.0:1619677473.046214:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.046217:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.046217:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.046219:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.046220:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.046221:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.046721:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.046724:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.046725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.046726:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.046729:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.046731:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.046732:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.046734:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.046735:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.046736:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.046739:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.046741:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.046744:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.046745:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.046745:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.046747:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.046748:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.046748:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.046749:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.046753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.046762:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.046765:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.046765:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.046766:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.046768:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.046771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.046772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.046780:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.046783:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.046783:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.046784:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.046785:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.046785:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677473.046999:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.047002:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.047002:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.047004:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.047007:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.047008:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.047009:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.047011:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.047012:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.047013:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.047017:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.047018:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.047021:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.047022:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.047023:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.047024:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.047025:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.047025:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.047025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.047031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.047038:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.047041:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.047042:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.047043:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.047044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.047046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.047203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.047211:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.047214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.047214:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.047215:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.047217:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.047220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.047266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.047269:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.047270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.047270:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.047273:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.047275:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.047288:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.047291:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.047292:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.047292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.047298:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.047299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.047672:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.047675:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.047676:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.047678:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.047681:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.047682:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.047683:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.047685:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.047686:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.047687:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.047689:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.047691:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.047694:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.047695:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.047695:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.047697:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.047697:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.047698:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.047698:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.047711:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.047714:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.047715:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.047715:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.047717:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.047720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.048105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677473.048111:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.048114:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.048115:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.048117:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.048120:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.048122:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677473.048123:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.048124:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.048125:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.048126:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677473.048129:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.048131:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677473.048134:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.048135:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677473.048136:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677473.048137:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677473.048138:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.048139:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.048140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677473.048153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.048161:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.048163:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.048165:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.048165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.048167:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.048170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.048599:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.048601:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.048602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.048604:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.048607:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.048608:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.048610:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.048611:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.048612:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.048613:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.048616:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.048617:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.048620:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.048620:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.048621:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.048623:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.048623:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.048624:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.048624:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.048639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.048646:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.048648:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.048649:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.048649:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.048651:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.048653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.048828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.048834:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.048836:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.048837:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.048837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.048839:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.048841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.048845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.048851:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.048853:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.048854:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.048854:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.048855:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.048856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.049490:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.049493:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677473.049493:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677473.049495:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.049498:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.049499:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677473.049500:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.049502:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.049503:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.049503:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677473.049506:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.049508:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.049511:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.049512:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677473.049512:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.049514:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.049514:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.049515:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677473.049515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677473.049521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.049529:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.049532:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.049533:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.049533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.049536:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.049539:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677473.049544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.049549:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.049553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.049553:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.049555:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.049557:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.049559:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.049560:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.049561:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.049563:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.049563:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.049567:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.049569:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677473.049569:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.049571:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677473.049571:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677473.049572:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.049572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.049572:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.049573:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677473.049574:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.049575:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.049575:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.049577:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.049577:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.049578:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.049579:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.049583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.049588:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.049589:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.049590:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.049591:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:6.0:1619677473.049591:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.049592:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.049592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.049593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677473.049593:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.049595:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.049609:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.049612:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.049613:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.049613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.049614:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.049615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.049845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.049852:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.049854:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.049855:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.049855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.049857:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.049859:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.049998:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.050000:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.050001:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.050003:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.050005:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.050007:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.050008:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.050009:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.050010:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.050011:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.050013:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.050015:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.050016:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.050017:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.050017:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.050018:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.050019:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.050020:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.050022:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.050022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.050024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.050032:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.050033:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.050034:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.050035:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.050036:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.050037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.050037:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.050038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.050038:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.050039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677473.050559:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.050562:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.050563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.050564:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.050567:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.050569:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.050570:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.050571:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.050572:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.050573:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.050576:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.050577:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.050580:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.050581:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.050582:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.050583:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.050584:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.050584:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.050585:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.050598:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.050601:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.050601:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.050602:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.050604:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.050607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.050717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.050724:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.050727:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.050727:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.050730:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.050733:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.050734:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.050735:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.050737:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.050738:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.050739:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.050742:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.050744:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.050746:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.050746:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.050747:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.050749:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.050749:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.050750:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.050751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.050755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.050762:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.050764:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.050765:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.050766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.050767:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.050768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.050906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.050912:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.050915:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.050915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.050917:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.050918:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.050920:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.050921:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.050922:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.050923:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.050924:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.050927:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.050928:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.050931:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.050932:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.050932:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.050934:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.050935:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.050935:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.050936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677473.050996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.051003:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.051006:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.051007:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.051007:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.051009:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.051011:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.051275:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.051278:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.051278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.051279:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.051281:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.051284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.051376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677473.051383:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677473.051386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677473.051386:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000400:00000200:1.0:1619677473.051389:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.051391:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.051393:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677473.051394:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.051395:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.051396:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.051397:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677473.051400:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.051401:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.051403:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.051404:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.051405:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.051407:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.051407:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.051408:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677473.051409:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.2:1619677473.051413:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677473.051416:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.051419:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.051420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677473.051420:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:8.0:1619677473.051421:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.051422:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.051423:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.051423:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.051424:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:8.0:1619677473.051425:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677473.051426:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.051427:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.051428:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:8.0:1619677473.051428:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.051429:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.051430:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.051434:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.051435:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.051438:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.051439:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.051439:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.051441:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.051441:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.2:1619677473.051442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:8.0:1619677473.051442:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.051443:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.051447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677473.051448:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.051451:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.051453:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.051454:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.051455:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.051455:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:21.0:1619677473.051457:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677473.051457:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:21.0:1619677473.051458:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.051458:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.051459:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.051460:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.051461:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.051461:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:21.0:1619677473.051462:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677473.051464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:21.0:1619677473.051466:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.051468:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.051470:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.051471:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.051471:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.051473:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.051473:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.051474:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.051475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.051479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.051487:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.051489:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.051490:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.051492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.051492:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.051494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677473.051663:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.051666:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.051666:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.051669:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.051671:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.051673:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.051674:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.051676:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.051677:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.051678:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.051681:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.051683:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.051686:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.051687:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.051687:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.051689:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.051690:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.051691:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.051692:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.051693:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.051696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.051704:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.051706:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677473.051707:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.051708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.051708:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:2.0:1619677473.051708:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.051709:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.051710:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677473.051710:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.051714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677473.051729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.051735:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.051738:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.051740:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.051740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.051742:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.051744:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:21.0:1619677473.051745:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.051748:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.051749:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.051750:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677473.051753:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.051755:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.0:1619677473.051758:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.051759:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677473.051760:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677473.051761:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677473.051762:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.051763:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677473.051764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:13.0:1619677473.051777:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.051780:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.051782:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.051784:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.051785:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.051787:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.0:1619677473.052108:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.052111:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.052113:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.052114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.052116:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.052118:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.052119:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.052120:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.052121:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.052122:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.052126:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.052128:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.052130:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.052131:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.052131:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.052133:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.052133:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.052134:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.052135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.052147:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.052149:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.052150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.052150:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.052152:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.052155:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.052162:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.052165:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.052166:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.052166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.052167:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.052168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.052531:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.052534:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677473.052535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677473.052537:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.052540:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.052541:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677473.052542:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.052544:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.052545:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.052546:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677473.052549:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.052551:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677473.052553:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.052554:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677473.052555:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677473.052557:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677473.052558:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.052558:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.052558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.2:1619677473.052564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.052572:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.052575:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.052576:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.052576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.052578:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.052580:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.052584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.052592:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.052594:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.052595:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.052595:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.052596:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.052597:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677473.052612:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.052615:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.052616:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.052618:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.052621:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.052622:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.052624:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.052625:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.052626:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.052627:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.052630:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.052631:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.052634:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.052635:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.052636:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.052637:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.052638:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.052639:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.052640:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.052640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.052652:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.052653:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.052655:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677473.052655:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.052656:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.052656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.052657:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.052657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.052658:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.052661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.052710:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.052713:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.052714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.052715:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.052718:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.052720:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.052721:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.052722:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.052723:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.052724:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.052727:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.052729:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.052731:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.052732:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.052733:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.052734:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.052735:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.052736:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.052736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.052741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.052749:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.052752:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.052753:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.052753:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677473.052756:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:18.0:1619677473.052756:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.052759:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:18.0:1619677473.052759:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.052760:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.052762:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.052765:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.052766:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.052767:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.052768:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.052769:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.052770:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.052773:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.052774:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.052776:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.052777:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.052777:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.052779:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.052779:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.052780:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.052780:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.052784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.052791:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.052792:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.052793:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.052794:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.052795:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.052796:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.052956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.052962:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.052965:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.052966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.052966:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.052968:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.052970:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.053416:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.053419:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.053420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.053420:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.053422:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.053425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.053523:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.053526:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.053526:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.053528:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.053530:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.053531:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.053532:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.053534:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.053535:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.053536:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.053539:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.053540:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.053542:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.053543:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.053543:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.053545:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.053545:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.053546:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.053547:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.053551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.053557:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.053559:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.053560:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.053561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.053562:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.053564:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.053570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.053575:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.053578:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.053579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.053579:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.053580:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.053580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677473.053766:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.053769:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.053769:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.053772:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.053774:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.053776:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.053777:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.053779:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.053780:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.053781:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.053786:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.053789:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.053792:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.053793:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.053793:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.053795:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.053795:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.053796:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.053797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.053801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677473.053805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.053813:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:21.0:1619677473.053814:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.053816:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677473.053816:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:21.0:1619677473.053817:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.053817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.053817:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.053819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677473.053820:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.053820:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.053823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677473.053824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.053825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.053833:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.053835:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.053836:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.053837:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.053838:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.053839:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677473.054062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.054068:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.054070:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.054071:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.054072:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.054073:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.054076:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.054083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.054089:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.054092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.054092:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.054093:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.054094:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.054094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.054383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.054389:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.054392:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.054392:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.054394:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.054396:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.054398:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.054399:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.054401:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.054402:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.054403:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.054405:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.054407:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.054410:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.054411:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.054412:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.054414:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.054414:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.054415:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.054416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.054429:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.054431:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.054432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.054433:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.054435:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.054437:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.054853:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.054856:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.054857:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.054858:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.054861:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.054862:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.054863:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.054864:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.054865:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.054866:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.054869:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.054871:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.054872:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.054873:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.054874:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.054875:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.054876:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.054876:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.054877:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.054881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.054887:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.054889:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.054889:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.054890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.054891:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.054893:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.054960:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.054963:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.054964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.054965:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.054968:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.054969:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.054971:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.054972:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.054973:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.054973:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677473.054974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:10.0:1619677473.054976:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.054978:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.054981:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.054981:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:2.0:1619677473.054981:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677473.054982:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677473.054983:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:10.0:1619677473.054984:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.054984:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677473.054985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:10.0:1619677473.054985:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.054986:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:2.0:1619677473.054986:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.054987:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677473.054989:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.054991:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677473.054992:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.054994:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.054995:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.054996:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677473.054997:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.054999:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:2.0:1619677473.054999:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.055000:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:2.0:1619677473.055000:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677473.055001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.055001:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.055002:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.055002:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.055003:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.055004:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.055004:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.055004:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677473.055005:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677473.055007:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677473.055007:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677473.055008:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.055009:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677473.055010:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677473.055011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677473.055013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677473.055015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677473.055019:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:18.0:1619677473.055021:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:10.0:1619677473.055022:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.055023:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:18.0:1619677473.055023:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:10.0:1619677473.055023:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.055024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.055025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:10.0:1619677473.055025:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.055029:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.055037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.055043:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.055045:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.055046:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.055046:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.055047:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.055048:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677473.055200:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.055203:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.055204:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.055205:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.055207:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.055209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.055733:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.055736:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677473.055737:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677473.055739:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.055741:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.055742:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.055743:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.055744:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.055745:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.055746:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.055749:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.055751:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.055752:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.055752:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.055753:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.055754:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.055754:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.055756:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677473.055757:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677473.055757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:9.2:1619677473.055760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.055769:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.055770:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.055770:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.055772:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.055773:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677473.055773:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677473.055774:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677473.055774:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.055775:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.0:1619677473.055775:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677473.055862:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.055865:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.055867:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.055868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.055869:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.055870:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.055871:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.055873:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.055873:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.055875:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.055878:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.055880:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.055882:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.055883:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.055883:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.055885:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.055886:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.055886:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.055887:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.055900:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.055902:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.055903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.055903:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.055905:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.055907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.055915:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.055917:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.055918:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.055918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.055919:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.055920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.056093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.056099:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.056102:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.056103:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.056105:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.056108:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.056110:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.056111:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.056113:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.056113:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.056114:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.056118:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.056119:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.056122:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.056123:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.056124:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.056126:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.056127:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.056128:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.056128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.056139:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.056142:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.056143:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.056144:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.056146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.056147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.056234:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.056237:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.056238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.056240:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.056243:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.056244:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.056245:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.056246:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.056247:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.056248:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.056251:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.056252:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.056255:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.056256:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.056256:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.056258:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.056258:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.056259:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.056260:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.056270:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.056272:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.056273:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.056274:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.056276:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.056277:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.056291:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.056298:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.056301:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.056302:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.056302:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.056303:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.057284:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.057287:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.057288:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.057288:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.057291:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.057302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.057338:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.057341:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.057342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.057343:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.057345:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.057346:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.057347:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.057348:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.057349:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.057350:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677473.057351:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)++
00000400:00000200:8.0:1619677473.057353:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.057354:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.12@o2ib3
00000800:00000200:17.0:1619677473.057355:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000800:00000200:8.0:1619677473.057355:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:1.0:1619677473.057357:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.12@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.057358:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.057359:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.057359:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:1.0:1619677473.057359:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.057361:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.057361:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677473.057361:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.0:1619677473.057362:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:1.0:1619677473.057362:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.057363:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.057364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677473.057364:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.057365:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.057366:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:9.2:1619677473.057368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:1.0:1619677473.057370:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.057373:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 560 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.057376:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.057376:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:1.0:1619677473.057376:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.057377:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677473.057378:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.057379:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:18.0:1619677473.057379:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:1.0:1619677473.057379:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.057380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677473.057380:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:1.0:1619677473.057380:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4200] (21)++
00000400:00000200:18.0:1619677473.057381:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.057381:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677473.057381:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b4200] (22)++
00000800:00000200:1.0:1619677473.057382:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4200] (23)--
00000400:00000200:18.0:1619677473.057383:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.057383:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000800:00000200:9.2:1619677473.057385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677473.057387:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677473.057392:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (24)++
00000800:00000200:17.0:1619677473.057393:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)++
00000800:00000200:24.0:1619677473.057395:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000800:00000200:17.0:1619677473.057395:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b4200] (23)--
00000400:00000200:24.0:1619677473.057396:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.057396:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (22)--
00000800:00000200:8.0:1619677473.057396:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (24)--
00000800:00000200:1.0:1619677473.057397:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4200] (21)--
00000400:00000200:24.0:1619677473.057398:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.057398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.057403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.057404:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.057406:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.057407:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.057407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.057408:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.057411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.057507:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.057510:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.057511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.057512:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.057515:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.057516:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.057518:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.057519:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.057520:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.057521:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.057524:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.057525:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.057528:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.057529:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.057530:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.057531:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.057532:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.057532:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.057533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.057545:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.057548:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.057548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.057549:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.057551:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.057554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.058332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.058338:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.058340:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.058342:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.058343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.058345:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.058346:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.058347:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.058349:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.058350:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.058351:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.058354:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.058356:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.058359:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.058360:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.058360:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.058362:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.058363:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.058364:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.058364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.058380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.058388:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.058391:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.058392:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.058392:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.058394:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.058397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677473.058480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.058485:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677473.058487:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:1.0:1619677473.058488:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.058489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677473.058491:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677473.058491:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:1.0:1619677473.058491:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.058494:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.058494:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.058495:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.058497:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.058497:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.058498:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.058499:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.058500:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.058500:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.058501:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.058502:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.058502:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:10.0:1619677473.058504:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.058505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:10.0:1619677473.058505:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.058505:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:1.0:1619677473.058506:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.058510:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.058510:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.058511:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.058512:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677473.058513:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.058513:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.058514:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677473.058514:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677473.058515:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:1.0:1619677473.058515:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.058516:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96511077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.058516:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.058516:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677473.058517:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.058517:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677473.058517:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677473.058518:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:6.0:1619677473.058519:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.058520:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.058520:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:6.0:1619677473.058520:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.058521:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:6.0:1619677473.058521:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677473.058522:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000400:00000200:6.0:1619677473.058525:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677473.058527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677473.058528:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677473.058531:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.058531:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.058532:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.058532:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:22.0:1619677473.058533:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.058533:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:22.0:1619677473.058535:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.058535:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.058536:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:6.0:1619677473.058537:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.058538:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.058538:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:2.0:1619677473.058538:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:22.0:1619677473.058539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.058539:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:18.0:1619677473.058540:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677473.058540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:2.0:1619677473.058540:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.058542:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.058545:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677473.058552:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677473.058554:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677473.058555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.058556:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.058556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:22.0:1619677473.058558:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.058558:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.058558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.058559:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.058560:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.058585:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.058587:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.058588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.058590:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.058593:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.058594:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.058595:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.058597:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.058598:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.058599:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.058601:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.058603:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.058606:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.058607:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.058608:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.058610:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.058611:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.058612:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.058613:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.058614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.058614:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.058615:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.058615:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.058617:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677473.058618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:10.0:1619677473.058620:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.058621:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.058622:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.058623:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.058624:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.058624:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677473.058628:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:10.0:1619677473.058629:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.058630:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677473.058631:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:20.0:1619677473.058631:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.058631:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.058632:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.058632:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.058633:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.0:1619677473.058634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.058634:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.058634:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.058635:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.058636:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677473.058637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.058638:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.058638:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.058639:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.058640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.058645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.058651:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.058652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.058653:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.058654:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.058655:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.058655:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.058656:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.058659:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.058659:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.058660:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.058661:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:8.0:1619677473.058661:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.058662:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.058664:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.058665:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.058666:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.058670:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.058671:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.058674:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.058675:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.058675:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.058677:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.058677:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.058678:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.058678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.058710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.058718:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.058721:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.058721:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.058722:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.058724:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.058727:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.058730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.058738:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.058740:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.058741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.058742:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.058743:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.058743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677473.058888:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677473.058889:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:21.0:1619677473.058892:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:24.0:1619677473.058893:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677473.058893:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:21.0:1619677473.058894:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.058894:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:24.0:1619677473.058897:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.058897:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.058899:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.058899:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.058900:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.058901:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.058902:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.058902:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.058903:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.058904:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.058905:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.058907:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.058908:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677473.058909:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.058911:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.058912:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.058914:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677473.058914:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.058914:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.058915:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.058917:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.058917:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.058917:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677473.058918:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.058918:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677473.058919:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677473.058920:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.058921:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:21.0:1619677473.058921:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.058921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677473.058922:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677473.058922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.058927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677473.058930:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677473.058931:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.058933:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.058933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.058934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.058934:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:18.0:1619677473.058935:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:6.0:1619677473.058935:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.058936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.058937:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.058938:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.058939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.058940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.058940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:18.0:1619677473.058942:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.058945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677473.059237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.059245:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.059248:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.059248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.059249:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.059251:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.059254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.059255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.059263:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.059265:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.059266:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.059267:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.059268:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.059269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.059400:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.059403:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.059403:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.059404:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.059406:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.059409:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.059418:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.059420:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.059421:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.059421:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.059422:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.059423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.059783:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.059786:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.059787:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.059789:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.059792:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.059794:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.059795:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.059797:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.059798:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.059799:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.059802:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.059804:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.059807:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.059808:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.059809:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.059810:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.059811:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.059812:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.059812:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677473.059815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677473.059824:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.059824:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:22.0:1619677473.059827:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.059827:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:22.0:1619677473.059828:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.059828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.059828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.059829:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.059831:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677473.059832:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.059835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.059835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677473.060051:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.060054:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.060056:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.060057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.060058:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.060060:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.060061:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.060062:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.060063:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.060064:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.060068:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.060069:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.060071:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.060072:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.060073:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.060074:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.060075:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.060076:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.060076:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.060080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.060088:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.060091:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.060091:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.060093:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.060094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.060095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677473.060266:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.060269:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.060270:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.060272:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.060275:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.060276:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.060278:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.060279:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.060280:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.060281:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.060284:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.060285:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.060287:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.060288:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.060289:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.060290:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.060290:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.060291:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.060291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.060297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.060303:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.060305:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.060306:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.060307:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.060307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.060309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677473.060951:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.060954:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.060956:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.060957:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.060958:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.060959:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.060961:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.060962:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.060963:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.060964:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.060968:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.060969:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.060972:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.060972:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.060973:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.060975:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.060975:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.060976:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.060976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:1.0:1619677473.060983:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.060986:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.060987:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:6.0:1619677473.060988:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677473.060988:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.060991:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:1.0:1619677473.060991:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.060992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.060992:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.060992:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677473.060994:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:1.0:1619677473.060995:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.060996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:1.0:1619677473.060996:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.060997:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.060998:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.061001:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.061003:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.061006:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.061007:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.061007:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.061009:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.061010:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.061011:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.061012:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.061013:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.061016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.061023:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677473.061025:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677473.061025:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.061025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.061026:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677473.061027:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.061028:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.061030:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.061030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.061032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.061333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.061336:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.061339:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677473.061341:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.061342:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.061344:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.061345:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677473.061347:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.061348:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.061349:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.061350:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677473.061354:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.061355:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677473.061359:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.061359:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677473.061360:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677473.061362:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677473.061362:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.061363:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.061364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:20.2:1619677473.061369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.061376:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677473.061379:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:2.0:1619677473.061380:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.061381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677473.061382:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.061383:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.061383:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:13.0:1619677473.061386:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.061386:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.061387:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:12.0:1619677473.061388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:2.0:1619677473.061388:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677473.061390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:2.0:1619677473.061390:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.061391:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.061392:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.061396:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.061397:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.061400:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.061401:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.061401:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.061403:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.061404:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.061406:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.061407:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.061407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.061410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.061412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.061419:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677473.061419:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677473.061421:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.061422:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677473.061422:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.061423:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.061423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677473.061423:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.061425:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.061428:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.061553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.061556:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.061557:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.061557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.061559:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.061562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677473.062179:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:18.0:1619677473.062182:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.5.5@o2ib3
00000800:00000200:2.0:1619677473.062183:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:18.0:1619677473.062185:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : REPLY - routed
00000800:00000200:18.0:1619677473.062188:0:19933:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 354801
00000800:00000200:18.0:1619677473.062192:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (21)++
00000800:00000200:18.0:1619677473.062193:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (22)++
00000800:00000200:18.0:1619677473.062194:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677473.062195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677473.062204:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677473.062206:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677473.062207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677473.062250:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677473.062252:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677473.062253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677473.062254:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (22)--
00000400:00000200:2.0:1619677473.062255:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.062257:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677473.062258:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.062259:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.062260:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.062261:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677473.062264:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : REPLY try# 0
00000800:00000200:2.0:1619677473.062266:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 354801 bytes in 256 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.062267:0:19932:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 354801
00000800:00000200:2.0:1619677473.062272:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.062273:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677473.062273:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677473.062274:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677473.062275:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677473.062276:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677473.062276:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:6.0:1619677473.062288:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.062291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.062292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.062313:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.062316:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[2] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.062317:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.062319:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.062326:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.062327:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.062375:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.062378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.062378:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.062379:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.062380:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: REPLY: OK
00000800:00000200:6.0:1619677473.062383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.062421:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.062424:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.062425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.062427:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.062430:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.062431:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.062433:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.062434:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.062435:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.062436:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.062439:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.062440:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677473.062442:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:10.0:1619677473.062443:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.062444:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677473.062445:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.5.5@o2ib3
00000800:00000200:10.0:1619677473.062445:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677473.062445:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:18.0:1619677473.062447:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : REPLY - routed
00000800:00000200:10.0:1619677473.062447:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.062447:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.062448:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677473.062449:0:19933:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 350101
00000800:00000200:10.0:1619677473.062450:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.062450:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677473.062451:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (21)++
00000800:00000200:18.0:1619677473.062452:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (22)++
00000800:00000200:18.0:1619677473.062453:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (23)--
00000800:00000200:18.0:1619677473.062454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:22.0:1619677473.062459:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.062462:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.062463:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677473.062463:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.062463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.062465:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677473.062465:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.062465:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:26.0:1619677473.062466:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.062467:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677473.062467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:22.0:1619677473.062468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677473.062468:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677473.062504:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677473.062506:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677473.062507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677473.062508:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (23)--
00000400:00000200:2.0:1619677473.062509:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.062510:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.062512:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.062513:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.062514:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.062515:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.062518:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:2.0:1619677473.062520:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 350101 bytes in 256 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.062522:0:19932:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 350101
00000800:00000200:2.0:1619677473.062528:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.062529:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.062529:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.062531:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.062532:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677473.062533:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000800:00000200:2.0:1619677473.062533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:8.0:1619677473.062556:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.062559:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677473.062560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677473.062562:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.062563:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.062565:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677473.062566:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96511056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.062568:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.062569:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.062570:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677473.062573:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.062574:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677473.062577:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.062578:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677473.062579:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677473.062580:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677473.062581:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.062582:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.062582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677473.062586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.062595:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.062598:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.062599:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.062599:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.062601:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.062604:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677473.062610:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.062613:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.062615:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.062615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.062618:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.062619:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.062621:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.062627:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.062628:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.062629:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.062632:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.062634:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.062637:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.062637:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.062638:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.062640:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.062640:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.062641:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.062642:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.062646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.062652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.062654:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.062655:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.062656:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.062657:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.062661:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.062663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.2:1619677473.062664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677473.062671:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.062672:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.062674:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:8.0:1619677473.062674:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.062675:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.062675:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000400:00000200:20.0:1619677473.062676:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.062676:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.062677:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.062677:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.062677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677473.062680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677473.063423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.063429:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.063432:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:24.0:1619677473.063433:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:17.0:1619677473.063433:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.063435:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.063436:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677473.063437:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:1.0:1619677473.063437:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.063439:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677473.063439:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677473.063441:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.063441:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.063442:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.063443:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.063443:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.063445:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.063445:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677473.063447:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.063448:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.063449:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.063450:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.063452:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677473.063454:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677473.063455:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.063456:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.063456:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.063456:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.063458:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677473.063459:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.063459:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.063459:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677473.063460:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.063460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677473.063461:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.063463:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.063463:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.063464:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677473.063466:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677473.063466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:10.0:1619677473.063471:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.063474:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.063474:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:6.0:1619677473.063475:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.063477:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677473.063477:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.063477:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.063478:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677473.063478:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000400:00000200:6.0:1619677473.063478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.063479:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.063480:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.063480:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000400:00000200:6.0:1619677473.063480:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.063481:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.063481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:10.0:1619677473.063482:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.0:1619677473.063482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:10.0:1619677473.063483:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.063483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.063484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.063485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677473.063485:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.063485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677473.063486:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.063487:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.063490:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.063492:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.063494:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.063495:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.063496:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.063497:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.063497:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.063498:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.063499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.063503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.063509:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.063511:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.063512:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.063513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.063514:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.063516:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.063520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.063526:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.063529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.063529:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.063530:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.063531:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.063531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.063619:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.063622:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.063623:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.063624:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.063627:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.063628:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.063629:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.063630:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.063631:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.063632:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.063635:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.063637:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.063639:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.063640:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.063641:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.063643:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.063643:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.063644:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.063644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.063649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.063657:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.063660:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.063661:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.063661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.063663:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.063666:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.063770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.063776:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.063777:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677473.063778:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677473.063779:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677473.063779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677473.063780:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:22.0:1619677473.063780:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.063780:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.063782:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.063783:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.063787:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.063788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.063790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.063792:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.063792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.063793:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.063794:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.063795:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.063871:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.063874:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.063874:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.063876:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.063879:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.063880:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.063882:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.063883:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.063884:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.063885:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.063888:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.063890:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.063893:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.063893:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.063894:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.063895:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.063896:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.063897:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.063897:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.063902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.063911:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.063913:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.063914:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.063916:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.063916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.063919:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.063920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.063928:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.063931:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.063932:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.063932:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.063933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.063934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677473.064519:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.064522:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.064524:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.064525:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.064527:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.064528:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.064530:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.064531:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.064532:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.064533:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.064536:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.064538:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.064541:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.064541:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.064542:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.064544:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.064544:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.064545:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.064546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.064563:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.064565:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.064566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.064566:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.064568:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.064570:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.064651:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.064654:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.064655:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.064657:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.064660:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.064661:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.064663:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.064664:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.064665:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.064666:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.064669:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.064671:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.064674:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.064674:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.064675:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.064677:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.064677:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.064678:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.064678:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.064690:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.064693:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.064694:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.064694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.064696:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.064699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.064913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.064918:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.064921:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.064923:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.064924:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.064926:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.064927:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677473.064929:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.064930:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.064931:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.064932:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677473.064935:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.064937:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 600 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.064939:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.064940:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677473.064940:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.064942:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.064942:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.064944:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677473.064945:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677473.064946:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677473.064951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.064958:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.064959:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.064959:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000400:00000200:21.0:1619677473.064960:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.064960:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000400:00000200:21.0:1619677473.064961:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.064961:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.064962:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.064963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.064964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:26.0:1619677473.065014:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.065017:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.065018:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.065020:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.065023:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.065025:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677473.065026:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.065027:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.065028:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.065029:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677473.065032:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.065033:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677473.065036:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.065036:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677473.065037:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677473.065039:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677473.065039:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.065040:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.065040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:24.0:1619677473.065068:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.065071:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.065072:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.065074:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.065077:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.065079:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.065080:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.065081:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.065082:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.065083:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.065086:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.065087:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.065089:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.065089:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.065090:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.065091:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.065092:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.065093:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.065094:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.065095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.065097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.065105:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.065106:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.065107:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.065109:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677473.065109:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.065110:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.065110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.065111:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.065111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.065112:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677473.065147:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.065150:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.065150:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.065153:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.065155:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.065157:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677473.065158:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.065159:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.065160:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.065161:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.065165:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.065166:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.065169:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.065169:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.065170:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.065172:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.065172:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.065173:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.065173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677473.065185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.065188:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.065189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.065189:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.065191:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.065193:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.065218:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.065221:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.065223:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.065224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.065225:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.065226:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.065228:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.065231:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.065232:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.065234:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.065240:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.065242:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.065245:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.065246:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.065247:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.065249:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.065250:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.065252:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.065253:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.065254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.2:1619677473.065255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677473.065262:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677473.065263:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.065264:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:1.0:1619677473.065264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:22.0:1619677473.065265:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677473.065265:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:22.0:1619677473.065266:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.065266:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:1.0:1619677473.065266:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:6.0:1619677473.065267:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.065268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.065268:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.065269:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:1.0:1619677473.065270:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.065271:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.065274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.065628:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.065631:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.065632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.065632:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.065635:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.065637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.065646:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.065649:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.065649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.065650:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.065650:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.065651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677473.065755:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.065758:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.065759:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.065761:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.065763:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.065765:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.065766:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.065768:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.065769:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.065770:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.065774:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.065776:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.065778:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.065779:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.065779:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.065781:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.065781:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.065782:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.065782:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.065788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.065796:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.065798:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.065799:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.065801:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.065803:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.065804:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.065826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.065832:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.065835:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.065836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.065838:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.065841:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.065842:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.065843:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.065845:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.065846:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.065847:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.065850:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.065852:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.065854:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.065855:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.065855:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.065857:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.065857:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.065858:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.065859:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.065863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.065870:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.065873:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.065873:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.065874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.065875:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.065876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677473.066088:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.066091:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.066093:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.066094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.066095:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.066097:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.066098:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.066100:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.066100:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.066102:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.066104:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.066106:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.066108:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.066109:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.066110:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.066111:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.066112:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.066113:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.066113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.066125:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.066127:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.066128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.066128:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.066130:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.066132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.066432:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.066435:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677473.066436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677473.066438:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.066441:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.066442:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.066443:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.066445:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.066446:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.066447:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.066450:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.066452:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.066453:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.066454:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.066454:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.066455:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.066456:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.066456:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.066457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677473.066462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.066468:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.066469:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.066470:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.066471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.066472:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.066473:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.067237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.067240:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.067242:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677473.067244:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.067245:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.067247:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.067248:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677473.067249:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.067251:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.067252:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.067253:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677473.067256:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.067258:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677473.067260:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.067261:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:20.0:1619677473.067262:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.0:1619677473.067264:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:20.0:1619677473.067264:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.067265:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.067266:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:9.2:1619677473.067270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.067278:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.067280:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.067281:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.067282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.067283:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.067286:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.067294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.067301:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.067304:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.067305:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.067305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.067306:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.067307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:10.0:1619677473.067389:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.067392:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.067393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.067395:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.067397:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.067399:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.067400:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.067401:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.067403:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.067403:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.067407:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.067409:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.067412:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.067413:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.067414:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.067416:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.067416:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.067417:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.067417:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.067430:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.067433:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.067434:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.067434:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.067436:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.067440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.067814:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.067817:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.067818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.067818:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.067821:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.067823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.068115:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.068117:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677473.068119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677473.068120:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.068123:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.068124:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.068125:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.068126:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.068128:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.068128:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.068131:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.068133:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.068135:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.068135:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.068136:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.068137:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.068137:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.068138:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.068138:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677473.068142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.068149:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.068150:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.068151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.068152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.068153:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.068154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.068213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.068220:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.068222:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.068223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.068225:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.068227:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.068229:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.068230:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.068231:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.068233:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.068233:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.068237:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.068239:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.068242:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.068242:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.068243:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.068245:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.068245:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.068247:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.068248:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.068248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.068253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.068258:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.068258:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.068261:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.068261:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677473.068262:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.068262:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677473.068262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.068263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.068265:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.068267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.068283:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.068285:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.068286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.068287:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.068288:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.068288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.068429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.068435:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.068438:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.068439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.068440:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.068443:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.068445:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.068446:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.068447:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.068448:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.068450:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.068453:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.068455:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.068458:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.068459:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.068459:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.068461:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.068462:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.068463:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.068463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677473.068509:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.068511:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.068512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.068514:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.068516:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.068518:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.068519:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.068520:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.068521:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.068522:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.068525:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.068527:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.068530:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.068530:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.068531:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.068533:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.068533:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.068534:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.068535:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.068548:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.068551:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.068552:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.068554:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.068556:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.2:1619677473.068556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:12.0:1619677473.068558:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.0:1619677473.068564:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.068567:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.068568:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.068570:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.068570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.068573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.068702:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.068704:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.068705:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.068707:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.068710:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.068711:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.068712:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.068714:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.068715:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.068716:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.068718:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.068720:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.068722:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.068723:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.068723:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.068725:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.068725:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.068726:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.068727:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.068730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.068736:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.068739:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.068740:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.068740:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.068742:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.068744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.068806:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.068809:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.068811:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.068811:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.068813:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.068815:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.068816:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.068817:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.068818:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.068819:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.068823:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.068825:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.068827:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.068828:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.068828:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.068830:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.068830:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.068831:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.068832:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.068843:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.068846:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.068847:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.068847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.068849:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.068851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.068860:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.068863:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.068864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.068864:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.068865:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.068865:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.069465:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.069468:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.069469:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.069469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.069472:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.069474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.069532:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.069535:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.069536:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.069538:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.069541:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.069542:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.069543:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96511020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.069545:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.069545:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.069546:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.069549:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.069551:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.069554:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.069554:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.069555:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.069557:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.069557:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.069558:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.069558:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.069579:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.069581:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.069582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.069583:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.069585:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.069588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.070041:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.070044:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.070045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.070046:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.070049:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.070050:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.070051:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.070052:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.070053:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.070054:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.070057:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.070058:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.070060:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.070060:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.070061:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.070062:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.070063:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.070063:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.070064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.070068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.070073:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.070075:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.070076:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.070077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.070078:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.070079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.070090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.070096:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.070098:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.070099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.070099:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.070099:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.070100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677473.070807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.070818:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.070821:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.070822:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.070824:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.070824:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.070827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677473.071132:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.071134:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.071136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.071137:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.071139:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.071140:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.071142:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.071143:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.071144:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.071145:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.071148:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.071150:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.071153:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.071154:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.071154:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.071156:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.071156:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.071157:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.071158:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.071163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.071172:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.071174:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.071175:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.071176:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.071178:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.071181:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.071189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.071196:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.071199:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.071200:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.071200:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.071201:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.071202:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.071581:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.071583:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.071584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.071585:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.071587:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.071589:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.072274:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.072277:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.072278:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.072279:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.072282:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.072284:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.072285:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.072286:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.072287:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.072288:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.072291:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.072293:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.072300:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.072301:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.072301:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.072303:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.072303:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.072305:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.072306:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.072306:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677473.072317:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.072320:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.072320:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.072321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.072321:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.072323:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.072323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677473.072323:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.072325:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.072327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.072579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.072585:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.072588:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.072590:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.072591:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:1.0:1619677473.072591:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.072592:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.072594:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.072595:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.072596:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.072597:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.072599:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.072600:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.072601:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.072603:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.072604:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.072605:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677473.072606:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96511013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.072607:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677473.072607:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.072608:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.072609:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.072612:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.072612:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.072613:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.072614:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.072614:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.072616:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.072616:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.072617:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.072617:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.072618:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.072618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677473.072619:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.072619:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.072621:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.072621:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.072622:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.072623:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.072627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677473.072629:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.072631:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.072632:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.072632:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.072634:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.072635:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:22.0:1619677473.072636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.072637:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.072638:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.072640:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.072641:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.072643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.072647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.072654:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.072657:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.072658:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.072658:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677473.072659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.072660:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677473.072787:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.072790:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677473.072791:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677473.072793:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.072796:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.072797:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.072798:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.072800:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.072801:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.072802:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.072805:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.072807:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.072810:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.072811:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.072811:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.072813:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.072814:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.072815:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677473.072816:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677473.072817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677473.072819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.072828:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.072829:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677473.072831:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:18.0:1619677473.072831:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.072832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677473.072832:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.072832:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.072834:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677473.072834:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.072838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677473.073192:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.073194:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.073197:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.073197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.073199:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.073201:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.073202:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.073203:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.073204:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.073205:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.073209:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.073211:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.073213:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.073214:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.073214:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.073216:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.073216:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.073218:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:8.2:1619677473.073222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.073223:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677473.073225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.073226:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.073228:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.073228:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.073230:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.073230:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.073231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677473.073232:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677473.073234:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677473.073235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.073236:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677473.073258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.073265:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.073267:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.073268:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.073268:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.073270:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.073273:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.073448:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.073451:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.073451:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.073453:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.073456:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.073457:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.073458:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.073460:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.073460:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.073461:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.073464:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.073466:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.073469:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.073469:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.073470:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.073472:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.073472:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.073473:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.073473:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.073487:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.073489:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.073490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.073490:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.073493:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.073495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.073506:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.073509:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.073510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.073511:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.073512:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.073512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.074132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.074138:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.074141:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.074142:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.074144:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.074147:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.074148:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.074150:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.074151:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.074152:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.074153:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.074156:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.074157:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.074160:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.074161:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.074162:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.074164:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.074165:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.074165:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.074166:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.074177:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.074180:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.074181:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.074183:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.074185:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.074185:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.074420:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.074422:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.074423:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.074425:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.074428:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.074430:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.074431:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.074432:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.074433:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.074434:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.074437:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.074439:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.074442:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.074442:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.074443:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.074445:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.074446:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.074446:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.074448:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.074448:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677473.074460:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.074462:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.074462:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.074463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.074464:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677473.074464:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.074465:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.074466:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.074466:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.074469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.074967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.074975:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.074978:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.074979:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.074979:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.074981:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.074985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677473.075156:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.075158:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.075161:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.075161:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.075162:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.075164:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.075165:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.075166:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.075167:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.075168:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.075171:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.075173:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.075175:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.075176:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.075177:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.075179:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.075179:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.075180:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.075180:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.075185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.075192:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.075194:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.075195:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.075196:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.075198:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.075201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677473.075481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.075489:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.075492:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.075493:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.075495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.075495:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.075498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677473.075514:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.075516:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.075517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.075519:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.075522:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.075523:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.075525:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96511002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.075526:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.075527:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.075528:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.075531:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.075533:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.075536:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.075536:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.075537:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.075539:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.075539:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.075540:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.075540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.075545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.075553:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.075556:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.075557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.075557:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.075559:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.075562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.075564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.075572:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.075574:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.075575:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.075575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.075576:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.075576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.076706:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.076709:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.076710:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.076712:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.076714:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.076715:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.076717:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96511001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.076718:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.076719:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.076719:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.076722:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.076724:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.076726:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.076726:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.076727:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.076728:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.076729:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.076730:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.076731:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.076732:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677473.076743:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677473.076745:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677473.076746:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.076746:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.076747:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:12.0:1619677473.076748:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:10.0:1619677473.076748:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.076749:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.076750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.076751:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677473.077388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.077393:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.077396:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.077397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.077398:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.077401:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.077402:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.077404:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96511000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.077405:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.077406:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.077407:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.077411:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.077412:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.077415:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.077416:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.077416:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.077418:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.077418:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.077419:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.077419:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.077433:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.077435:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.077436:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.077436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.077438:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.077440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.077450:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.077453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.077453:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.077454:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.077455:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.077455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.077694:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.077696:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.077697:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.077699:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.077702:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.077703:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.077705:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.077706:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.077707:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.077708:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.077711:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.077713:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.077716:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.077717:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.077718:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.077719:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.077720:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.077720:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.077721:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677473.077726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.077732:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:22.0:1619677473.077733:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.077734:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:22.0:1619677473.077736:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.077736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.077737:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.077737:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.077737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:22.0:1619677473.077739:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:21.0:1619677473.077739:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.077741:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:22.0:1619677473.077742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:21.0:1619677473.077743:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.077744:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.077745:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.077746:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677473.077749:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.077751:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.077754:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.077754:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.077755:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.077756:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.077757:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.077758:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:21.0:1619677473.077759:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677473.077760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677473.077763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677473.077766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.077773:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.077774:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.077774:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:21.0:1619677473.077775:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.077775:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000400:00000200:21.0:1619677473.077776:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.077776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.077778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.077779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677473.077780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.077782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.077788:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.077791:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.077792:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.077793:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.077794:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.077795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677473.077909:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.077912:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.077914:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.077914:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.077916:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.077918:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.077919:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.077920:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.077921:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.077922:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.077925:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.077927:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.077930:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.077930:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.077931:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.077933:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.077933:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.077934:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.077935:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.077936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.077946:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.077948:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.077949:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.077950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.078019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.078026:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.078029:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.078029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.078030:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.078032:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.078034:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.078197:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.078200:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.078201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.078203:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.078205:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.078207:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.078208:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.078209:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.078210:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.078211:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.078214:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.078216:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.078218:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.078219:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.078219:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.078220:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.078221:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.078222:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.078223:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.078224:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.078236:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.078238:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.078239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.078240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.078315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.078322:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.078324:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.078324:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.078325:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.078326:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.078327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.078345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.078351:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.078353:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.078354:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.078354:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.078355:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.078355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:10.0:1619677473.078544:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.078547:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.078547:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.078549:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.078552:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.078553:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.078555:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.078556:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.078557:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.078559:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.078562:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.078564:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.078566:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.078567:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.078567:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.078568:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.078569:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.078570:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.078570:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.078574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.078581:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.078583:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.078584:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.078584:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.078586:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.078588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.078677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.078683:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.078685:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.078686:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.078686:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.078688:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.078690:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.079005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.079012:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.079014:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.079016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.079017:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.079019:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.079021:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.079022:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.079024:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.079024:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.079026:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.079029:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.079031:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.079033:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.079034:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.079035:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.079036:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.079037:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.079038:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.079038:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.079042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.079051:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.079053:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.079054:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.079054:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.079056:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.079059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677473.079178:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.079181:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677473.079181:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677473.079183:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.079185:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.079187:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677473.079188:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.079189:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.079190:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.079191:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677473.079194:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.079195:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677473.079198:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.079199:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677473.079199:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677473.079200:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677473.079201:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.079202:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677473.079202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:6.0:1619677473.079214:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.079217:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.079218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.079218:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.079220:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.079222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.079316:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.079319:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.079320:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.079322:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.079324:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.079326:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.079327:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.079329:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.079330:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.079331:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.079334:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.079335:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.079338:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.079339:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.079340:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.079341:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.079342:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.079342:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.079343:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.079356:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.079359:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.079359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.079360:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.079362:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.079365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677473.079631:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.079634:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.079635:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.079637:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.079639:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.079641:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.079642:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.079643:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.079644:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.079646:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.079649:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.079651:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.079653:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.079654:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.079655:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:12.0:1619677473.079657:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a4637ec200] (21)++
00000800:00000200:1.0:1619677473.079657:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.079657:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.079658:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.079658:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:12.0:1619677473.079661:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.35@o2ib3
00000400:00000200:12.0:1619677473.079664:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.1.35@o2ib3 : GET - for me
00000800:00000200:8.2:1619677473.079664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:13.0:1619677473.079665:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a4637ec200] (22)--
00000400:00000200:12.0:1619677473.079669:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.1.35@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:21.0:1619677473.079672:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:12.0:1619677473.079672:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.1.35@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677473.079674:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a4637ec200] (21)++
00000800:00000200:21.0:1619677473.079675:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.079675:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.079676:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a4637ec200] (22)--
00000800:00000200:20.0:1619677473.079677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677473.079678:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:12.0:1619677473.079679:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.1.35@o2ib3 local destination
00000800:00000200:21.0:1619677473.079681:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:12.0:1619677473.079681:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a4637ec200] (21)--
00000800:00000200:20.2:1619677473.079682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.079685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.079688:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677473.079690:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.079691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.079692:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.079694:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677473.079695:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.079698:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.079698:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.079700:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677473.079703:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.079705:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677473.079709:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.079709:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677473.079710:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677473.079712:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677473.079712:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.079713:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.079714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:13.0:1619677473.079725:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.079728:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.079728:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.079730:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.079732:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.079734:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:9.2:1619677473.079762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.079769:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.079771:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.079772:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.079773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.079774:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.079777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.080471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.080476:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:24.0:1619677473.080478:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677473.080479:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.080480:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:24.0:1619677473.080480:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.080481:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.080482:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.080482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.080483:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.080485:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:10.0:1619677473.080485:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.080486:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677473.080487:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.080489:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.080489:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677473.080489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:10.0:1619677473.080490:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677473.080493:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.080495:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677473.080495:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.080496:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.080497:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.080498:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.0:1619677473.080498:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.080499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.080499:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.080499:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.0:1619677473.080499:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677473.080500:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.0:1619677473.080500:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.080500:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.080501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.080505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.080511:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.080513:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.080513:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.080514:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.080515:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.080517:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677473.081012:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.081015:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677473.081016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677473.081016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677473.081018:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.081021:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.081022:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:6.0:1619677473.081022:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:24.0:1619677473.081024:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:6.0:1619677473.081024:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.081025:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.081025:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.081026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:6.0:1619677473.081027:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.081028:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.081028:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677473.081030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677473.081033:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.081035:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677473.081038:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.081039:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677473.081039:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677473.081041:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677473.081042:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.081043:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.081044:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677473.081048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.081056:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.081059:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.081060:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.081060:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.081062:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.081065:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677473.081637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.081643:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.081646:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.081646:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677473.081648:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677473.081648:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.081650:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.081651:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.081652:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677473.081652:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.081653:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.081655:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.081655:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.081656:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.081657:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.081657:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677473.081659:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.081660:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96510980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.081661:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.081662:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.081664:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.081664:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677473.081665:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.081667:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.081667:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.081668:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:8.0:1619677473.081669:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.081670:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.081670:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677473.081671:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.081671:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.081672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.0:1619677473.081674:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.081675:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.081676:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.081678:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.081678:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.081679:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.081679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.081683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677473.081684:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.081687:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.081688:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.081688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.081690:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.081692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.081692:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.081694:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.081695:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.081696:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.081698:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.081701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677473.082191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.082197:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.082200:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.082201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.082203:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.082206:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.082207:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.082208:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.082209:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.082211:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.082212:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.082215:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.082217:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.082219:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.082219:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.082220:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.082222:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.082222:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.082224:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.082225:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.082226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.082228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.082231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.082237:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.082238:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.082238:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677473.082239:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.082239:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.082240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.082241:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.082241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.082242:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.082242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.082517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.082524:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.082526:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.082527:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.082527:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.082529:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.082532:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677473.082556:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.082559:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.082560:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.082561:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.082563:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.082564:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.082565:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.082566:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.082567:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.082568:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.082571:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.082573:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.082576:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.082576:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.082577:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.082579:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.082579:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.082580:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.082581:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.082582:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677473.082593:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.082595:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.082595:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.082596:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.082596:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677473.082596:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.082597:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.082598:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.082598:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.082601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.082911:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.082914:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.082916:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.082917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.082919:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.082921:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.082922:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.082923:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.082924:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.082925:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.082929:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.082931:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.082934:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.082935:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.082935:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.082938:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.082938:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.082939:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.082939:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677473.083320:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.083323:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.083324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.083326:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.083328:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.083329:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.083331:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.083332:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.083333:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.083334:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.083336:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.083338:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.083339:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.083340:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.083341:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.083342:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.083342:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.083343:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.083343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.083348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.083354:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.083356:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.083357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.083357:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.083358:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.083360:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.083371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.083377:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.083379:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.083380:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.083380:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.083380:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.083381:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.083606:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.083608:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.083609:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.083611:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.083614:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.083615:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.083616:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.083617:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.083618:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.083619:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.083622:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.083623:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.083625:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.083626:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.083627:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.083628:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.083629:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.083629:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.083630:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.083634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.083640:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.083642:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.083643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.083643:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.083645:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.083646:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677473.083647:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.083648:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.083649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.083650:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.083653:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.083654:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.083655:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.083657:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.083657:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.083658:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.083661:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.083662:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.083665:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.083666:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.083666:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.083668:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.083668:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.083669:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.083669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.083681:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.083684:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.083685:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.083685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.083687:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.083690:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.083842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.083848:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.083851:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.083853:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.083854:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.083856:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.083858:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.083859:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.083860:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.083861:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.083862:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.083866:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.083868:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.083871:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.083872:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.083872:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.083874:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.083875:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.083876:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.083877:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.083878:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.083883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.083889:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.083891:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.083891:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.083892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677473.083935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.083941:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.083943:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.083944:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.083945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.083947:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.083949:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.084298:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.084301:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.084302:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.084304:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.084307:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.084308:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.084309:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.084310:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.084311:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.084312:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.084315:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.084317:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.084320:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.084320:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.084321:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.084323:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.084323:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.084324:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.084324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.084336:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.084339:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.084339:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.084341:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.084343:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.084344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677473.084346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.084355:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.084357:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.084358:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:12.0:1619677473.084358:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:1.0:1619677473.084359:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.084361:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.084362:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.084363:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.084363:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.084364:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677473.084365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:13.0:1619677473.084366:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.085557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.085564:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.085566:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.085567:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.085567:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.085569:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.085572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677473.085835:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.085838:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.085839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.085840:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.085843:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.085844:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.085845:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.085847:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.085848:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.085849:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.085852:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.085853:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.085856:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.085856:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.085857:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.085859:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.085859:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.085861:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677473.085862:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.085862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.085865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.085872:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677473.085873:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677473.085875:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677473.085876:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677473.085876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677473.085877:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.085877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677473.085879:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.085879:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.085882:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677473.085909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.085915:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.085918:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.085919:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.085919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.085921:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.085923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.086053:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.086056:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.086057:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.086059:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.086061:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.086062:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677473.086063:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.086065:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.086065:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.086066:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677473.086069:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.086071:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.086072:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.086073:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.086073:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.086075:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.086075:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.086076:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.086076:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.086080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.086086:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.086088:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.086089:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.086090:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.086090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.086092:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:5.2:1619677473.086454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.086460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.086462:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.086463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.086465:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.086467:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.086469:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.086470:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.086471:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.086472:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.086473:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.086477:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.086479:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.086481:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.086482:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.086482:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.086484:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.086485:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.086486:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.086487:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.086487:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.086493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.086499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.086499:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.086501:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677473.086502:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.086502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.086503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.086503:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.086503:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677473.086505:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.086507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.086517:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.086519:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.086520:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.086520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.086521:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.086522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.087390:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.087393:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.087394:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.087396:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.087398:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.087399:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.087400:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.087402:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.087403:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.087404:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.087407:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.087409:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.087412:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.087413:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.087414:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.087415:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.087416:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.087416:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.087417:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.087430:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.087433:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.087433:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.087434:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.087436:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.087439:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.087450:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.087452:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.087453:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.087453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.087454:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.087455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.087463:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.087466:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.087466:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.087468:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.087471:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.087472:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.087473:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.087475:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.087476:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.087477:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.087479:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.087480:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.087482:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.087482:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.087483:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.087484:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.087484:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.087486:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.087487:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.087487:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677473.087489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.087498:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.087499:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.087500:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.087502:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:10.0:1619677473.087502:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.087503:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.087503:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.087504:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.087504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.087505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.087641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.087648:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.087651:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.087652:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.087652:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.087654:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.087656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.088120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.088126:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.088128:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.088131:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.088131:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.088134:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.088135:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.088137:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.088138:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.088139:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.088140:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.088143:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.088145:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.088147:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.088148:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.088149:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.088150:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.088151:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.088151:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677473.088152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677473.088164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.088166:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.088167:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.088167:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.088169:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.088172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.088469:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.088472:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.088473:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.088474:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.088476:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.088478:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.088479:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.088480:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.088481:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.088482:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.088485:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.088486:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.088489:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.088490:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.088491:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.088492:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.088493:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.088494:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.088494:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.088507:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.088510:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.088510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677473.088511:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677473.088511:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.088513:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.088514:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.088515:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677473.088516:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:24.0:1619677473.088517:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.088519:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.088520:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.088522:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.088523:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.088524:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.088525:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.088527:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.088529:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.088530:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.088531:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.088531:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.088533:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.088533:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.088534:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.088535:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.088538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.088545:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.088547:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.088547:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.088548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.088550:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.088551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.088558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.088564:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.088567:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.088568:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.088568:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.088568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.088569:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.088646:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.088649:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.088650:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.088651:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.088654:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.088655:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.088656:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.088658:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.088659:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.088659:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.088662:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.088664:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.088666:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.088666:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.088667:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.088668:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.088669:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.088669:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.088670:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.088674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.088680:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.088682:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.088683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.088683:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.088685:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.088687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.088700:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.088703:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.088704:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.088705:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.088707:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.088709:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.088710:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.088711:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.088712:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.088713:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.088716:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.088717:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.088720:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.088721:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.088721:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.088723:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.088723:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.088724:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.088725:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.088729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.088737:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.088740:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.088741:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.088743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.088743:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.088746:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.089558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.089565:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.089567:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.089568:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.089568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.089570:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.089572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.090124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.090129:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.090132:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.090134:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.090134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.090136:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.090138:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.090139:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.090140:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.090141:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.090142:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.090146:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.090148:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.090150:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.090151:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.090152:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.090154:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.090154:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.090155:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.090155:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.090160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.090168:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.090171:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.090172:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.090172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.090174:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.090177:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.090185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.090192:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.090195:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.090195:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.090196:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.090197:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.090197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677473.090586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.090593:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.090595:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.090596:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.090599:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.090602:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.090603:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.090604:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.090606:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.090607:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.090608:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.090612:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.090614:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.090617:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.090618:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.090619:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.090621:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.090621:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.090622:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.090623:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677473.090819:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.090822:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.090825:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.090825:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.090827:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.090829:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.090831:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.090832:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.090833:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.090834:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.090838:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.090840:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.090842:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.090842:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677473.090843:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.090843:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.090845:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.090845:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.2:1619677473.090845:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.090846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:20.0:1619677473.090846:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.090847:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677473.090850:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677473.090852:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.090854:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.090855:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:8.0:1619677473.090856:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677473.090857:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:8.0:1619677473.090857:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96510947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.090858:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.090858:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677473.090858:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.090859:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.090859:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.090861:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.090861:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.090862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.2:1619677473.090862:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:6.0:1619677473.090862:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.090864:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.090865:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:8.0:1619677473.090865:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677473.090867:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677473.090867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677473.090869:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.090870:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000400:00000200:8.0:1619677473.090870:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.090870:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.090871:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.090872:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677473.090872:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.090875:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.090876:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.090876:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.090878:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.090879:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.090879:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677473.090880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.090884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.090892:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.090895:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.090896:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.090896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.090899:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.090902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.090904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.090911:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.090914:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.090915:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.090915:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.090916:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.090917:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677473.091162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.091168:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.091171:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.091172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.091173:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.091175:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.091177:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.091178:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.091180:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.091181:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.091182:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.091185:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.091187:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.091189:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.091190:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.091191:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.091192:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.091193:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.091194:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.091194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.091206:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.091209:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.091210:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.091210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.091212:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.091214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.091241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.091247:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.091250:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.091251:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.091251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.091253:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.091255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677473.092190:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.092193:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.092194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.092197:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.092199:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.092200:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.092202:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.092203:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.092204:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.092205:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.092208:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.092210:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677473.092212:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677473.092213:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.092213:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.092214:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677473.092215:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.092216:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677473.092216:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.092217:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.092217:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.092218:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:10.0:1619677473.092218:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.092221:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.092223:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.2:1619677473.092223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:10.0:1619677473.092224:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.092225:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.092226:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.092227:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.092231:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:10.0:1619677473.092231:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.092232:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.092233:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.092234:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.092235:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.092236:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.092236:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:10.0:1619677473.092236:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.092237:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.092239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.092239:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.092239:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.092240:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.092242:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.092242:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677473.092244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.092253:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677473.092253:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.092256:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:20.0:1619677473.092256:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677473.092256:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.092257:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.092257:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000400:00000200:6.0:1619677473.092257:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.092258:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:20.0:1619677473.092258:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:26.0:1619677473.092259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677473.092259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:6.0:1619677473.092259:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.092260:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.092261:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.092262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.092533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.092536:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.092537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.092538:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.092541:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.092543:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.092544:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.092546:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.092547:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.092548:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.092552:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.092553:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.092556:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.092557:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.092557:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.092559:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.092559:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.092561:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.092562:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.092562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677473.092565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.092573:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.092576:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677473.092576:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.092577:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.092577:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.092578:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.092579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677473.092579:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.092580:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677473.092583:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.093288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.093295:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.093299:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.093303:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.093304:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.093305:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.093308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.093311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.093317:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.093319:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.093320:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.093320:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.093321:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.093321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.093349:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.093351:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.093352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.093354:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.093357:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.093358:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.093359:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.093361:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.093362:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.093362:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.093365:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.093367:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.093371:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.093372:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.093372:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.093374:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.093375:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.093375:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.093376:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.093376:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:26.0:1619677473.093378:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.093378:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.093378:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.093379:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.093381:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.093382:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.093383:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.093385:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.093385:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.093387:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677473.093388:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:26.0:1619677473.093390:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.093391:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.093392:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:22.0:1619677473.093392:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.093392:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.093392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.093393:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677473.093394:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.093394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.093395:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.093395:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677473.093396:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.093397:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.093397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.093398:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.093399:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.093400:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.093401:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.093404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.093411:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.093413:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.093414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.093414:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.093416:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.093418:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.094094:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.094097:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.094098:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.094099:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.094102:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.094103:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.094104:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.094105:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.094106:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.094107:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.094110:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.094111:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.094113:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.094113:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.094114:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.094115:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.094115:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.094116:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.094117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.094124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.094130:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.094131:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.094132:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.094133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.094133:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.094134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677473.094434:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.094437:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.094439:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.094440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.094442:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.094443:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.094445:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.094446:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.094447:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.094448:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.094451:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.094453:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.094455:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.094456:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.094457:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.094458:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.094459:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.094460:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.094460:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.094464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.094472:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.094474:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.094476:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.094476:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.094478:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.094481:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.094486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.094493:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.094496:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.094497:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.094497:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.094498:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.094499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.094547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.094554:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.094556:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.094557:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.094557:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.094559:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.094560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.095231:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.095233:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.095234:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.095236:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.095238:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.095240:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.095241:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.095242:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.095243:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.095244:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.095247:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.095248:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.095251:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.095252:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.095253:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.095255:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.095255:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.095255:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.095256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.095261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.095269:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.095271:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.095272:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.095274:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.095275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.095277:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677473.095476:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.095479:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.095479:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.095481:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.095483:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.095485:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677473.095486:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.095488:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.095489:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.095490:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677473.095493:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.095495:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.095497:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.095497:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.095498:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.095499:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.095500:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.095501:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.095501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677473.095506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.095514:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.095516:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.095517:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.095517:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.095518:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.095520:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677473.095930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.095934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.095936:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.095938:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.095940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.095941:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.095942:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.095944:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.095945:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.095946:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.095947:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.095950:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.095951:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.095953:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.095954:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.095955:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.095956:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.095957:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.095958:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.095959:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.095960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.095962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677473.095965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677473.095970:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677473.095971:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.095973:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677473.095973:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677473.095974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677473.095974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.095975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677473.095975:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.095977:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.095979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677473.095986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.095991:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.095994:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.095994:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.095996:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.095998:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.095999:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.096000:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.096001:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.096002:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.096003:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.096006:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.096008:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.096010:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.096011:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.096012:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.096014:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.096014:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.096014:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.096015:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.096028:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.096031:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.096032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.096032:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.096034:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.096036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.096046:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.096049:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.096049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.096050:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.096051:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.096051:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.096286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.096293:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.096297:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.096298:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.096299:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.096301:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.096302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677473.096395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.096404:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.096406:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.096407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.096408:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.096410:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.096413:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677473.096434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.096443:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.096448:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.096449:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.096453:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.096456:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.096458:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.096460:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.096462:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.096464:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.096465:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.096471:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.096473:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.096477:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.096478:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.096479:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.096481:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.096482:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.096484:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.096484:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.096487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.096495:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.096498:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.096499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.096499:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.096502:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.096504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.097097:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.097100:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.097101:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.097103:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.097106:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.097107:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.097109:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.097110:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.097111:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.097112:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.097115:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.097117:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.097120:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.097121:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.097122:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.097124:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.097124:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.097125:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.097125:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.097138:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.097141:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.097142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.097142:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.097145:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.097148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.097155:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.097158:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.097158:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.097159:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.097160:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.097160:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.097204:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.097206:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677473.097207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677473.097209:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.097212:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.097213:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677473.097215:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.097216:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.097217:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.097218:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677473.097222:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.097224:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677473.097227:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.097228:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677473.097228:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677473.097230:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677473.097231:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.097232:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.097232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677473.097237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.097243:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.097246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.097246:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.097247:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.097248:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:10.0:1619677473.097249:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.097250:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.097252:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677473.097252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.0:1619677473.097253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.097255:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.097257:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:9.2:1619677473.097257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:21.0:1619677473.097258:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.097259:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.097260:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.097261:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.097263:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:21.0:1619677473.097265:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.097266:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.097266:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.097266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677473.097267:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.097268:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.097268:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.0:1619677473.097269:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.097270:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.097270:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.097272:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.097273:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.097273:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.097274:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.097285:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.097288:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.097288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.097289:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.097291:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.097298:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.097526:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.097529:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.097529:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.097532:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.097534:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.097536:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.097537:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.097538:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.097540:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.097540:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.097544:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.097546:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.097549:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.097549:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.097550:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.097552:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.097552:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.097553:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.097553:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.097559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.097567:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.097570:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.097570:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.097571:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.097573:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.097576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677473.097633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.097639:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.097642:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.097643:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.097643:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.097645:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.097646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.098202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.098208:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.098210:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.098211:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.098211:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.098213:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.098215:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.098234:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.098237:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.098238:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.098240:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.098243:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.098244:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.098246:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.098247:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.098248:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.098249:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.098252:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.098253:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.098256:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.098257:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.098258:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.098259:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.098260:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.098261:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.098261:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.098274:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.098276:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.098277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.098278:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.098280:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.098283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.098323:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.098326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.098326:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.098328:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.098331:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.098332:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.098334:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.098335:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.098336:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.098337:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.098340:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.098341:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.098344:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.098345:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.098345:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.098347:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.098347:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.098348:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.098348:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.098354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.098362:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.098365:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.098366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.098366:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.098368:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.098371:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.098374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.098381:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.098384:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.098385:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.098385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.098386:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.098387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677473.098641:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.098644:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.098645:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.098647:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.098649:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.098651:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.098652:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.098653:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.098654:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.098655:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.098658:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.098659:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.098661:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.098662:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.098663:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.098664:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.098665:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.098665:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.098666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.098670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.098676:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.098678:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.098679:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.098679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.098681:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.098683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.098854:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.098856:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.098857:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.098858:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.098860:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.098862:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.098863:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.098864:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.098865:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.098866:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.098868:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.098870:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.098871:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.098872:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.098872:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.098874:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.098874:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.098875:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.098875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.098890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.098896:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.098897:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.098898:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.098899:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.098899:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.098900:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.098917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.098923:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.098925:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.098926:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.098926:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.098927:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.098927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.098956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677473.098963:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.098966:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.098967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.098968:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.098971:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.098972:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677473.098974:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.098975:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.098976:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.098977:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677473.098981:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.098983:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.098985:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.098986:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677473.098987:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677473.098988:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677473.098989:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.098990:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.098990:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:22.0:1619677473.099002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.099005:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.099006:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.099006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.099008:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.099011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.099182:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.099184:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.099187:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.099187:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.099188:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.099190:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.099191:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.099193:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.099194:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.099195:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.099198:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.099200:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.099203:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.099204:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.099204:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.099206:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.099207:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.099208:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.099209:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.099210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677473.099212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.099218:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677473.099220:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677473.099221:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.099222:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.099222:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:8.0:1619677473.099222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.099223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:24.0:1619677473.099224:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677473.099224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677473.099227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.099266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.099272:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.099275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.099275:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.099276:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.099278:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.099280:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.099287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.099293:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.099299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.099301:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.099302:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.099302:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.099303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.099506:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.099509:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.099510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.099512:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.099514:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.099516:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.099517:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.099518:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.099519:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.099520:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.099523:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.099524:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.099527:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.099528:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.099528:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.099530:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.099530:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.099531:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.099532:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.099537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.099546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.099548:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.099549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.099550:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.099552:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.099555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677473.099672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.099681:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.099684:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.099684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.099685:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.099687:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.099690:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.100025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.100032:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.100034:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.100035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.100035:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.100037:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.100039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.100150:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.100152:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.100153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.100155:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.100158:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.100159:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.100160:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.100161:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.100162:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.100163:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.100166:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.100168:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.100170:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.100171:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.100172:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.100173:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.100174:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.100175:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.100176:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.100177:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677473.100179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.100188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677473.100189:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:21.0:1619677473.100190:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.100191:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.100192:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.100192:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677473.100193:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.100193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.100193:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677473.100196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.100484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.100490:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.100492:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.100493:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.100495:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.100497:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.100499:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.100500:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.100501:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.100502:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.100504:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.100507:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.100509:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.100510:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.100511:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.100512:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.100513:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.100514:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.100515:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.100515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.100520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.100528:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.100530:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.100530:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.100531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.100532:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.100533:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677473.100896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.100903:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677473.100904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677473.100905:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.100906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.100908:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.100910:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.100910:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677473.100911:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677473.100912:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:17.0:1619677473.100913:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677473.100914:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677473.100915:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.100916:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.100916:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677473.100917:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677473.100918:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.100920:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677473.100921:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677473.100921:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.100923:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677473.100924:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677473.100924:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.100925:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677473.100927:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.100928:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.100928:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:2.0:1619677473.100929:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677473.100930:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.100931:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.100931:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677473.100931:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677473.100932:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677473.100934:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.100935:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.100936:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.100938:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.100938:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.100939:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.100940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677473.100944:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.100947:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.100948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.100948:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.100950:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.100952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:12.0:1619677473.100953:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.100957:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.100958:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.100961:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.100963:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.100964:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.0:1619677473.101238:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.101240:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.101243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.101244:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.101247:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.101249:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.101251:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.101254:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.101255:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.101257:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.101262:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.101265:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.101267:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.101268:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.101269:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.101271:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.101271:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.101272:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.101272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.101278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.101285:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.101286:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677473.101288:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677473.101289:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677473.101289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:24.0:1619677473.101290:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.101290:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.101290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.101296:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.101298:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677473.101298:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.101300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.101684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.101691:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.101693:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.101694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.101695:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.101697:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.101699:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.101793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.101799:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.101801:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.101802:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.101803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.101804:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.101806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.101810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.101816:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.101819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.101819:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.101820:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.101820:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.101821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.101932:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.101935:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.101936:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.101938:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.101940:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.101942:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.101943:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.101944:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.101945:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.101946:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.101949:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.101951:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.101954:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.101955:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.101955:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.101957:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.101957:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.101958:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.101960:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.101960:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677473.101970:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.101973:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.101973:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.101974:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.101974:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.101976:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.101977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677473.101977:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.101978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.101980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.101991:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.101994:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.101994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.101995:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.101996:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.101996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.102190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.102195:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.102198:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.102199:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.102200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.102202:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.102203:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.102205:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.102206:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.102207:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.102208:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.102211:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.102213:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.102215:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.102216:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.102216:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.102218:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.102219:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.102219:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.102220:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.102225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.102233:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.102236:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.102237:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.102237:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.102239:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.102241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677473.102411:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.102414:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.102414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.102417:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.102419:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.102421:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.102422:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.102423:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.102424:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.102425:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.102428:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.102430:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.102432:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.102433:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.102434:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.102436:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.102436:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.102437:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.102437:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.102442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.102451:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.102453:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.102454:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.102456:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.102457:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.102459:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677473.102833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.102842:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.102845:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.102846:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.102846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.102848:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.102851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.103105:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.103108:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.103109:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.103111:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.103113:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.103115:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.103116:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.103118:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.103119:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.103120:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.103123:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.103125:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.103128:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.103129:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.103129:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.103131:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.103131:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.103132:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.103132:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.103145:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.103148:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.103149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.103149:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.103152:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.103154:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677473.103239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.103247:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.103250:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.103250:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.103251:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.103253:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.103256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.103262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.103269:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.103272:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.103272:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.103273:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.103274:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.103274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677473.103397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.103436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.103445:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.103447:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.103449:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.103449:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.103451:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.103454:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677473.103713:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.103716:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.103716:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677473.103719:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.103721:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.103722:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.103724:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.103725:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.103727:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.103727:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.103731:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.103733:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.103735:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.103736:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.103737:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.103738:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.103739:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.103740:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677473.103741:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.103742:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:22.0:1619677473.103752:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.103754:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677473.103754:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:22.0:1619677473.103755:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.103756:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:6.0:1619677473.103756:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.103757:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:22.0:1619677473.103758:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.103758:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677473.103760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.103956:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.103959:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.103960:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.103961:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.103964:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.103965:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.103967:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.103968:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.103969:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.103970:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.103974:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.103976:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.103978:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.103979:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.103980:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.103981:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.103982:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.103983:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.103983:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.103998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.104005:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:24.0:1619677473.104007:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:24.0:1619677473.104008:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.104009:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000400:00000200:24.0:1619677473.104010:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.104010:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:8.0:1619677473.104011:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.104012:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.104013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.104014:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.104015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.104182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.104185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.104188:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677473.104188:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.104189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.104189:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.104191:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.104192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.104192:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.104192:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.104194:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.104194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.104197:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.104201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.104207:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.104210:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.104210:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.104211:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.104212:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.104212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.104236:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.104239:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.104240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.104242:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.104245:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.104246:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.104247:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96510888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.104248:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.104249:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.104250:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.104253:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.104255:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.104256:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.104257:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.104257:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.104259:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.104259:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.104260:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.104261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.104263:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.104265:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:9.2:1619677473.104265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677473.104266:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.104267:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.104269:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.104271:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677473.104271:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.104273:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.104273:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.104274:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.104275:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.104277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.104861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.104869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.104871:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.104872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.104872:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.104874:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.104876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.105797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.105803:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.105806:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.105807:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.105807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.105808:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.105811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.106421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.106428:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.106430:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.106431:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.106432:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.106433:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.106435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.106757:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.106761:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.106763:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.106766:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.106769:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.106771:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.106773:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.106776:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.106777:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.106779:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.106784:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.106786:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.106790:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.106792:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.106793:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.106795:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.106796:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.106797:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.106798:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.106810:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.106813:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.106814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.106814:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.106816:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.106819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.106829:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.106832:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.106832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.106833:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.106834:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.106834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.107470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.107477:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.107480:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.107481:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.107481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.107483:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.107485:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.107610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.107618:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.107622:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.107622:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.107625:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.107628:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.107630:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.107631:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.107633:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.107634:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.107635:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.107639:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.107641:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.107644:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.107645:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.107645:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.107647:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.107648:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.107649:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.107649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677473.107654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.107664:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.107668:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.107670:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.107671:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.107674:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.107678:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.108003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.108010:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.108012:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.108013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.108013:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.108015:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.108017:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.108023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.108029:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.108031:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.108032:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.108032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.108033:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.108034:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.108085:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.108087:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677473.108088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677473.108089:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.108091:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.108093:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.108094:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.108095:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.108096:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.108097:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.108100:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.108101:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.108104:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.108105:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.108106:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.108107:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.108108:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.108108:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677473.108109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.108114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.108121:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.108124:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.108125:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.108125:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.108127:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.108130:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.108134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.108141:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.108144:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.108145:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.108145:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.108146:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.108147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.108350:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.108353:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.108354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.108356:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.108358:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.108360:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.108361:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.108362:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.108363:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.108364:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.108367:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.108369:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.108371:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.108372:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.108373:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.108374:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.108375:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.108375:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.108376:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.108381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.108389:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.108392:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.108393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.108393:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.108396:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.108399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677473.108790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.108796:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.108800:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.108800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.108803:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.108805:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.108806:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.108808:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.108809:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.108811:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.108812:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.108815:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.108817:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.108819:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.108820:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.108821:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.108822:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.108823:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.108824:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.108825:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.108825:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.108830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.108836:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.108836:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.108838:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677473.108839:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.108839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677473.108840:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.108840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.108841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.108842:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.108845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.108856:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.108858:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.108859:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.108859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.108860:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.108861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.109075:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.109077:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.109078:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.109080:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.109083:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.109085:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.109086:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.109087:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.109088:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.109089:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.109092:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.109093:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.109096:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.109097:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.109098:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.109099:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.109100:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.109101:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.109101:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.109112:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.109115:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.109116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.109116:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.109118:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.109121:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.109219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.109226:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.109228:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.109229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.109231:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.109234:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.109235:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.109236:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.109238:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.109239:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.109240:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.109243:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.109245:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.109247:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.109248:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.109249:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.109250:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.109251:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.109252:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.109253:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.109254:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.109257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.109259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.109264:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677473.109265:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677473.109266:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.109267:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677473.109268:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677473.109268:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:17.0:1619677473.109269:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.109269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.109271:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.109273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677473.109356:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.109359:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.109360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.109362:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.109364:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.109365:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.109366:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.109368:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.109369:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.109370:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.109373:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.109375:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.109377:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.109378:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.109379:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.109380:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.109381:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.109381:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.109382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.109387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.109396:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.109399:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.109399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.109401:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.109402:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.109404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.109480:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.109483:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.109484:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.109485:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.109488:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.109490:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.109491:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.109492:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.109493:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.109494:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.109497:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.109499:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.109501:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.109502:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.109502:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.109504:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.109504:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.109505:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.109505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.109510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.109517:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.109520:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.109521:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.109521:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.109523:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.109525:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.109740:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.109742:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.109743:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.109745:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.109747:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.109749:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.109750:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.109751:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.109752:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.109753:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.109755:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.109757:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.109758:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.109759:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.109759:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.109761:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.109761:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.109762:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.109763:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.109764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.109766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.109774:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.109775:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.109776:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.109777:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.109778:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.109779:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.109779:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.109780:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.109780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.109781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.109938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.109945:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.109947:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.109948:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.109948:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.109950:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.109952:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677473.110916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.110924:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.110927:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.110928:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.110929:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.110930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.110932:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.110935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.110942:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.110945:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.110946:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.110947:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.110947:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.110949:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.111485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.111492:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.111494:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.111495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.111495:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.111497:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.111499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.111551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.111558:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.111561:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.111562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.111564:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.111566:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.111568:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677473.111569:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.111570:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.111571:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.111572:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677473.111576:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.111577:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677473.111581:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.111582:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677473.111582:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677473.111585:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677473.111585:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.111586:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.111587:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677473.111591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.111598:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.111601:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.111602:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.111602:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.111605:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.111608:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677473.112096:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.112098:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.112099:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.112100:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.112102:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.112104:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.112105:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.112106:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.112107:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.112108:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.112111:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.112112:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.112115:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.112116:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.112117:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.112118:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.112119:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.112120:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.112121:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.112121:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.112132:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.112134:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.112135:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.112136:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677473.112136:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.112136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.112137:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.112138:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.112138:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.112141:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.112167:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.112169:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.112171:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.112172:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.112174:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.112176:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.112178:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.112179:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.112180:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.112181:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.112184:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.112186:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.112189:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.112190:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.112190:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.112192:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.112193:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.112198:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.112199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677473.112816:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b780e00] (21)++
00000800:00000200:18.0:1619677473.112820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b780e00] (22)--
00000800:00000200:2.0:1619677473.112820:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.52@o2ib3
00000400:00000200:2.0:1619677473.112823:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.52@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.112826:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.112828:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677473.112829:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.112830:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.112831:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.112832:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677473.112836:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.112838:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677473.112840:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.112841:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677473.112842:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677473.112844:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677473.112845:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907b780e00] (21)++
00000800:00000200:2.0:1619677473.112845:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907b780e00] (22)--
00000800:00000200:2.0:1619677473.112846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b780e00] (21)--
00000800:00000200:20.0:1619677473.112859:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.112861:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.112862:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.112863:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.112864:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.112866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.113083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.113089:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.113091:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.113092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.113094:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.113096:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.113098:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.113099:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.113101:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.113102:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.113103:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.113106:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.113108:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.113111:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.113112:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.113113:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.113114:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.113115:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.113116:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.113116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.113138:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.113141:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.113142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.113142:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.113144:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.113146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.113471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.113478:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.113480:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.113481:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.113481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.113483:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.113484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.113493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.113499:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.113502:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.113503:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.113503:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.113504:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.113504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.113673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.113680:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.113683:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.113684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.113686:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.113688:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.113690:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.113691:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.113692:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.113693:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.113694:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.113698:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.113699:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.113702:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.113702:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.113703:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.113705:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.113705:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.113706:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.113707:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.113711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.113719:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.113721:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.113722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.113722:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.113724:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.113727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.114026:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.114028:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.114030:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677473.114032:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000400:00000200:10.0:1619677473.114032:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.114034:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.114036:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.114037:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.114039:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.114040:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.114041:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.114042:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.114044:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.114046:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.114046:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.114047:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.114048:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.114048:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.114049:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.114050:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.114054:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.114054:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.114055:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.114055:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.114055:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.114058:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.114059:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.114059:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.114060:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.114060:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (23)++
00000800:00000200:26.0:1619677473.114061:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.114062:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (24)--
00000800:00000200:26.0:1619677473.114063:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.114063:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (22)++
00000800:00000200:10.0:1619677473.114063:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.114064:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:10.0:1619677473.114064:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.114065:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677473.114065:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:22.0:1619677473.114073:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.114075:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.114076:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.114077:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677473.114077:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.114077:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.114077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.114078:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.114079:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.114079:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.114080:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.114081:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.114082:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:21.0:1619677473.114083:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.114083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.114086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.114096:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.114099:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.114100:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.114100:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.114101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.114102:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.114246:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.114249:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.114250:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.114252:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.114255:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.114256:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.114257:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.114258:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.114259:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.114260:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.114263:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.114265:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.114267:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.114267:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.114268:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.114269:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.114270:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.114271:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.114271:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.114275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.114281:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.114283:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.114284:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.114284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.114286:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.114288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.114297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.114303:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.114306:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.114307:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.114307:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.114308:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.114308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.114587:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.114589:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.114590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.114592:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.114593:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.114595:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.114596:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.114597:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.114598:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.114599:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.114602:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.114603:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.114605:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.114606:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.114606:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.114608:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.114608:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.114608:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.114609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.114614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.114620:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.114621:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.114622:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.114623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.114624:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.114624:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.115089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.115095:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.115098:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.115099:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.115099:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.115100:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.115102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.115315:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.115318:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.115319:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.115321:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.112@o2ib7 : GET - for me
00000400:00000200:6.0:1619677473.115324:0:19937:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.112@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:6.0:1619677473.115326:0:19937:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.112@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:6.0:1619677473.115328:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.115329:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.115331:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677473.115335:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : REPLY try# 0
00000800:00000200:6.0:1619677473.115337:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.115338:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.115339:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.115340:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.115341:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.115342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.115365:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.115367:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.115368:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.115368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.115369:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: REPLY: OK
00000800:00000200:22.0:1619677473.115371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.115447:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.115450:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.115450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.115452:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.115454:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.115456:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.115457:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.115458:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.115459:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.115460:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.115463:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.115465:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.115467:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.115468:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.115469:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.115470:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.115471:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.115472:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677473.115473:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.115474:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.115477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.115484:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677473.115486:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677473.115486:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.115486:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.115487:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.0:1619677473.115488:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.115489:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.115490:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.115492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.115493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.115513:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.115516:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.115517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.115518:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.115521:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.115522:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.115524:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.115525:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.115526:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.115527:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.115530:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.115532:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.115535:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.115536:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.115537:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.115538:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.115539:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.115540:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.115540:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.115544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.115552:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.115554:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.115555:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.115557:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.115557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.115560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.115868:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.115871:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.115872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.115873:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.115877:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.115878:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.115879:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.115881:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.115882:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.115883:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.115886:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.115887:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.115890:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.115890:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.115891:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.115892:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.115893:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.115894:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.115895:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.115896:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:13.0:1619677473.115906:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677473.115908:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:13.0:1619677473.115909:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.115909:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.115910:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.115910:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.115911:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:13.0:1619677473.115911:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.115913:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.115914:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677473.116112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.116119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.116121:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.116122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.116124:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.116127:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.116128:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.116130:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.116131:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.116132:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.116133:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.116136:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.116138:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.116140:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.116141:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.116141:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.116143:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.116143:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.116144:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.116145:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.116149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.116156:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.116159:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.116159:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.116160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.116161:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.116162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.116892:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.116895:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.116895:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.116898:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.116900:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.116901:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.116902:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.116904:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.116905:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.116906:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.116909:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.116911:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.116914:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.116915:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.116916:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.116917:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.116918:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.116918:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.116919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.116924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.116932:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.116935:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.116936:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.116937:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.116939:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.116941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677473.116949:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.116952:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.116952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.116954:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.116957:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.116958:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.116959:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.116961:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.116962:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.116962:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.116965:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.116966:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.116969:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.116970:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.116970:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.116972:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.116973:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.116973:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.116973:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.116987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.116990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.116991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.116991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.116993:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.116997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677473.117055:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.117057:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677473.117058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677473.117060:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.117062:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.117063:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677473.117065:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.117066:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.117067:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.117068:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677473.117072:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.117073:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677473.117076:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.117077:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677473.117078:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677473.117080:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677473.117080:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.117082:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677473.117083:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677473.117083:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677473.117089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677473.117095:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:10.0:1619677473.117096:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:18.0:1619677473.117097:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677473.117097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677473.117098:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:10.0:1619677473.117098:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.117099:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.117100:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.117102:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.117105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.117115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677473.117121:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.117123:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677473.117124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.117125:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.117126:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.117127:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.117267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.117274:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.117276:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.117277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.117277:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.117279:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.117281:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.118069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.118075:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.118078:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.118078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.118081:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.118083:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.118085:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.118086:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.118088:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.118089:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.118090:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.118093:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.118095:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.118098:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.118098:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.118099:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.118101:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.118101:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.118101:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.118102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.118116:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.118118:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.118119:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.118119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.118121:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.118123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.118221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.118227:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.118230:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.118231:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.118231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.118233:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.118238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.118238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.118246:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.118248:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.118249:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.118250:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:9.2:1619677473.118251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:2.0:1619677473.118252:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.118255:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.118258:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.118260:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.118261:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.118261:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.118264:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.118266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677473.118322:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.118325:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.118326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.118326:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.118328:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.118331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:12.0:1619677473.118363:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.118366:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.118366:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.118368:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.118370:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.118371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677473.118383:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677473.118385:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677473.118386:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.118387:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677473.118387:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.118388:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.0:1619677473.118468:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.118470:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.118471:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.118473:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.118473:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.118476:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.118675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677473.118681:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.118684:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677473.118685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677473.118687:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.118690:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.118691:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677473.118692:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.118694:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.118695:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.118696:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677473.118699:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.118701:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677473.118704:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.118705:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677473.118706:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677473.118708:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:1.0:1619677473.118708:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.118709:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677473.118709:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:20.2:1619677473.118876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.118882:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.118885:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.118886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.118888:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.118890:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.118893:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.118894:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.118895:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.118896:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.118897:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.118902:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.118905:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.118908:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.118909:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.118910:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.118912:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.118913:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.118914:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.118914:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677473.118921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:12.0:1619677473.118927:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:8.0:1619677473.118928:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:12.0:1619677473.118930:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:8.0:1619677473.118931:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.118932:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:12.0:1619677473.118932:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.118932:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.118935:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.118936:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:8.0:1619677473.118936:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.118937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677473.118939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.118944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.118950:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.118952:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.118953:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.118953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.118954:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.118955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.119266:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.119269:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.119270:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.119272:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.119275:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.119276:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.119278:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.119279:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.119280:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.119281:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.119284:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.119286:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.119289:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.119290:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.119291:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.119292:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.119299:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.119300:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.119301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.119308:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.119311:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.119312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.119313:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.119315:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.119318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.119326:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.119329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.119329:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.119330:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.119331:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.119331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.119351:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.119354:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.119356:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.119357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.119359:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.119361:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.119362:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.119363:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.119364:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.119365:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.119369:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.119371:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.119373:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.119374:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.119374:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.119376:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.119377:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.119377:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.119378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.119390:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.119392:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.119393:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.119393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.119395:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.119397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.119516:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.119519:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.119520:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.119522:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.119525:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.119526:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.119528:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.119529:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.119530:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.119531:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.119533:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.119535:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.119537:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.119538:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.119539:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.119540:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.119541:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.119541:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.119542:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.119546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.119553:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.119556:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.119557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.119557:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.119559:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.119562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677473.119765:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.119767:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677473.119769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677473.119771:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.119774:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.119776:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677473.119777:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.119779:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.119779:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.119781:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677473.119784:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.119786:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677473.119789:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.119790:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677473.119791:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677473.119793:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677473.119793:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.119794:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677473.119795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677473.119798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.119805:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.119808:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677473.119808:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.119809:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.119811:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.119814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677473.119845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.119851:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.119853:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.119856:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.119856:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:24.0:1619677473.119858:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677473.119858:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.119860:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677473.119861:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:21.0:1619677473.119861:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.119862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:21.0:1619677473.119863:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.119865:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.119865:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.119866:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.119866:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677473.119868:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.119869:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96510830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.119870:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.119871:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.119872:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.119872:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:24.0:1619677473.119873:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.119875:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.119876:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.119876:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:24.0:1619677473.119877:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.119878:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677473.119879:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.119879:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.119879:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.119880:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:24.0:1619677473.119881:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.119882:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.119883:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.119884:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.2:1619677473.119884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677473.119885:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.119886:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.119887:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.119890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.119893:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.119895:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.119896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.119896:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:20.0:1619677473.119897:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.119898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677473.119898:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.119898:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.119899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.119899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:8.0:1619677473.119900:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.119902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.120152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.120160:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.120163:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.120164:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.120164:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.120166:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.120170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677473.120274:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.120277:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.120277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.120280:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.120283:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.120284:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.120285:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.120287:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.120288:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.120289:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.120292:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.120299:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.120301:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.120302:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.120303:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.120304:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.120305:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.120305:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.120306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.120311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.120320:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.120322:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.120323:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.120324:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.120326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.120327:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.120332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.120340:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.120343:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.120344:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.120345:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.120345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.120346:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677473.120942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.120948:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.120951:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.120952:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.120952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.120954:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.120956:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.121058:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.121061:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.121062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.121064:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.121066:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.121067:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.121068:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.121070:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.121071:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.121071:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.121075:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.121076:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.121079:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.121080:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.121080:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.121082:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.121083:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.121083:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.121084:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.121088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.121096:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.121099:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.121100:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.121101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.121102:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.121106:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.122556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.122563:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.122566:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.122567:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.122567:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.122569:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.122571:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.122835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.122841:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.122844:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.122845:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.122847:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.122849:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.122851:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.122852:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.122853:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.122854:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.122855:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.122858:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.122860:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.122862:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.122863:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.122864:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.122866:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.122866:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.122867:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.122867:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.122879:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.122882:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.122883:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.122883:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.122885:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.122888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.123089:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.123092:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.123093:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.123095:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.123098:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.123099:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.123100:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.123101:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.123102:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.123103:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.123106:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.123108:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.123111:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.123111:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.123112:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.123114:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.123114:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.123116:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.123117:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.123117:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.123127:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.123129:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.123130:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.123130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.123131:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.123132:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677473.123133:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.123133:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.123134:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.123136:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.123146:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.123148:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.123149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.123149:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.123150:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.123151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:13.0:1619677473.123505:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.123508:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.123510:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.123513:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.123514:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:12.0:1619677473.123514:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.123516:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.123517:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.123518:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.123519:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677473.123523:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.123524:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.123527:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.123527:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677473.123528:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.123529:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.123530:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.123531:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.123531:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677473.123543:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.123546:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.123546:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.123548:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.123549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.123551:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.123564:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.123566:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.123567:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.123568:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.123568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.123569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.123755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.123762:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.123765:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.123766:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.123766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.123768:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.123770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.124212:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.124215:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.124215:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.124218:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.124220:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.124222:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.124223:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.124224:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.124225:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.124226:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.124229:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.124230:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.124233:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.124234:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.124234:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.124236:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.124237:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.124237:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.124238:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.124250:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.124253:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.124254:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.124254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.124256:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.124259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.124343:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.124346:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677473.124347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677473.124348:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.124350:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.124352:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.124353:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.124354:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.124356:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.124357:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.124360:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.124361:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.124364:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.124365:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.124365:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.124367:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.124368:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.124368:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677473.124369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.124374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.124383:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.124385:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.124386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.124387:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.124389:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.124392:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.124394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.124401:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.124404:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.124405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.124405:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.124406:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.124407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.124459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.124466:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.124468:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.124469:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.124469:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.124471:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.124473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.124482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.124488:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.124491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.124491:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.124492:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.124493:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.124493:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.124800:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.124802:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.124803:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.124805:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.124808:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.124809:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.124810:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.124812:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.124813:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.124813:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.124816:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.124818:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.124820:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.124820:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.124821:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.124823:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.124823:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.124824:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.124824:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.124828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.124834:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.124836:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.124837:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.124838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.124839:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.124841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.124848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.124853:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.124856:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.124857:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.124857:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.124858:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.124858:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677473.124890:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.124892:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.124893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.124896:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.124898:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.124899:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.124900:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.124902:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.124903:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.124904:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.124907:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.124909:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.124911:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.124912:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.124913:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.124914:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.124915:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.124915:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.124916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.124920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.124929:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.124931:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.124932:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.124934:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.124935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.124936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677473.125243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.125246:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.125247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.125248:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.125251:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.125253:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.125254:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.125255:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.125256:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.125257:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.125259:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.125261:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.125263:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.125263:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.125264:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.125265:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.125266:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.125267:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.125267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.125271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.125277:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.125279:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.125279:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.125280:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.125281:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.125282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.125286:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.125289:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.125291:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.125292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.125301:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.125302:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.125307:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.125309:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.125310:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.125310:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:21.0:1619677473.125311:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.125313:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.125313:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:21.0:1619677473.125314:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.125316:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:6.0:1619677473.125317:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.125319:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677473.125319:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.125320:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.125320:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677473.125321:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:21.0:1619677473.125322:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.125322:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.125323:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.125323:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.125324:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677473.125324:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.125325:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.125326:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677473.125330:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.125332:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677473.125336:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.125337:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677473.125338:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.125338:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677473.125339:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.125340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.125340:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:22.0:1619677473.125341:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.125341:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.125342:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.125343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:22.0:1619677473.125344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.125344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:12.0:1619677473.125354:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.125357:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.125358:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.125359:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.125361:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.125362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.126073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.126080:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.126083:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.126084:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.126084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.126085:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.126088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.126623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.126629:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.126633:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.126634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.126636:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.126639:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.126641:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.126642:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.126644:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.126645:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.126646:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.126650:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.126652:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.126655:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.126656:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.126657:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.126658:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.126659:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.126660:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.126661:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.126665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.126674:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.126677:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.126678:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.126679:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.126681:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:8.2:1619677473.126682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.126685:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.126687:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.126690:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.126691:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.126691:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.126692:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.126693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677473.126822:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677473.126825:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.5@o2ib3
00000800:00000200:18.0:1619677473.126827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:2.0:1619677473.126829:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.5@o2ib3 : GET - for me
00000400:00000200:2.0:1619677473.126832:0:19932:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.5@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:2.0:1619677473.126835:0:19932:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.5@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:2.0:1619677473.126836:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677473.126837:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782400] (22)--
00000400:00000200:2.0:1619677473.126839:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.5@o2ib3 local destination
00000400:00000200:2.0:1619677473.126844:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.5@o2ib3(10.51.5.5@o2ib3:10.51.5.5@o2ib3) : REPLY try# 0
00000800:00000200:2.0:1619677473.126846:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.5@o2ib3
00000800:00000200:2.0:1619677473.126849:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c610f180] -> 10.51.5.5@o2ib3 (2) version: 12
00000800:00000200:2.0:1619677473.126849:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919065782400] (21)++
00000800:00000200:2.0:1619677473.126850:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782400] (22)++
00000800:00000200:2.0:1619677473.126851:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919065782400] (23)--
00000800:00000200:2.0:1619677473.126852:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000800:00000200:10.0:1619677473.126855:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.126857:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.126858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.126860:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.126862:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.126864:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677473.126865:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782400] (22)++
00000400:00000200:10.0:1619677473.126865:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677473.126867:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782400] (23)--
00000400:00000200:10.0:1619677473.126867:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.126867:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.126868:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.126868:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.126869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (22)--
00000400:00000200:18.0:1619677473.126871:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.5@o2ib3: REPLY: OK
00000400:00000200:10.0:1619677473.126872:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.126873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782400] (21)--
00000800:00000200:10.0:1619677473.126874:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.126876:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.126877:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.126878:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.126880:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.126880:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.126881:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.126882:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.126883:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:13.0:1619677473.126893:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.126895:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.126896:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.126897:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:13.0:1619677473.126897:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.126899:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.126900:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:13.0:1619677473.126900:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.126901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.126901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677473.127619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.127626:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.127631:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.127632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.127635:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.127639:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.127641:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.127643:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.127646:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.127647:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.127649:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.127655:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.127659:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.127663:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.127664:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.127665:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.127668:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.127668:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.127670:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.127672:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.127673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.127678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.127682:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.127685:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.127687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.127687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.127687:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677473.127689:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.127690:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677473.127692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677473.127692:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.127694:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677473.127888:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.127891:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.127892:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.127894:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.127896:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.127898:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.127899:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.127900:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.127901:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.127902:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.127906:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.127908:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.127911:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.127911:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.127912:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.127914:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.127914:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.127914:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.127915:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.127927:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.127930:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.127931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.127931:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.127933:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.127937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.128110:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.128113:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.128114:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.128116:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.128119:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.128121:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677473.128122:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.128123:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.128124:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.128125:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677473.128128:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.128130:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677473.128133:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.128134:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677473.128134:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677473.128136:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677473.128136:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.128137:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.128137:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.128143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.128151:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.128154:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.128155:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.128155:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.128157:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.128160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.128324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.128331:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.128334:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.128334:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.128335:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.128337:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.128339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.129189:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.129192:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.129193:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.129194:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.129197:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.129199:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.129200:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.129201:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.129202:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.129203:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.129206:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.129208:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.129210:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.129211:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.129211:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.129213:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.129213:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.129214:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.129215:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.129227:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.129230:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.129230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.129231:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.129233:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.129235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.129244:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.129247:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.129248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.129248:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.129249:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.129249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677473.129527:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.129530:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.129530:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.129532:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.129534:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.129536:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.129537:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.129538:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.129539:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.129540:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.129544:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.129546:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.129548:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.129549:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.129549:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.129551:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.129551:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.129552:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.129553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.129559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.129568:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.129570:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.129571:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.129573:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.129574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.129575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677473.129600:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.129602:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.129604:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.129605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.129606:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.129609:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.129610:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.129611:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.129612:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.129613:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.129617:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.129619:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.129621:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.129622:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.129623:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.129625:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.129625:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.129627:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.129628:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.129628:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677473.129631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.129637:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677473.129638:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.129639:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.129640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677473.129641:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677473.129641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:24.0:1619677473.129642:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.129642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.129644:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.129647:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.129654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.129661:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.129663:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.129664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.129664:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.129665:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.129666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.129696:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.129698:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677473.129699:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677473.129700:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.129702:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.129704:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677473.129705:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.129706:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.129707:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.129708:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677473.129711:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.129713:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677473.129715:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.129716:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:24.0:1619677473.129716:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:24.0:1619677473.129718:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:24.0:1619677473.129718:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.129719:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677473.129720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:13.0:1619677473.129731:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.129734:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.129734:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.129736:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.129738:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.129739:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:26.0:1619677473.130049:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.130052:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.130053:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.130055:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.130058:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.130059:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.130060:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.130061:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.130062:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.130063:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.130066:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.130067:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.130070:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.130070:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.130071:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.130073:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.130073:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.130074:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.130074:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.130078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.130084:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.130087:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.130088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.130088:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.130090:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.130092:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.130311:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.130314:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.130315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.130316:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.130318:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.130320:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.130321:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.130322:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.130323:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.130323:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.130326:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.130328:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.130329:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.130329:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.130330:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.130331:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.130332:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.130333:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.130334:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.130335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.130337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.130346:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.130347:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.130348:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.130349:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.130350:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.130350:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.130351:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.130351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.130352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.130352:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.130401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.130407:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.130410:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.130411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.130411:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.130412:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.130414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.131149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.131155:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.131158:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.131159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.131161:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.131163:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.131164:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.131166:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.131167:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.131168:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.131169:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.131173:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.131175:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.131178:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.131179:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.131179:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.131181:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.131182:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.131183:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.131184:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.131184:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.131190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.131196:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.131197:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.131198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.131199:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677473.131404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.131410:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.131413:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.131414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.131416:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.131419:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.131421:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.131422:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.131423:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.131424:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.131425:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.131428:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.131430:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.131432:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.131432:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.131433:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.131435:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.131436:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.131437:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.131437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.131441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.131448:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.131451:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.131451:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.131452:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.131453:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.131454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.131634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.131637:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.131639:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:21.0:1619677473.131642:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.131642:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.131644:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.131646:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677473.131647:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.131649:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.131650:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.131651:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677473.131654:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.131656:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677473.131660:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.131661:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677473.131661:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677473.131663:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677473.131664:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.131665:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677473.131666:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677473.131667:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677473.131671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.131677:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:13.0:1619677473.131677:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:21.0:1619677473.131679:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677473.131680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677473.131680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:13.0:1619677473.131680:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.131681:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.131683:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.131685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.131686:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:9.2:1619677473.131953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.131960:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.131963:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.131964:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.131964:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.131966:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.131968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.132234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.132240:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.132243:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.132244:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.132244:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.132245:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.132248:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.132369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.132374:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.132377:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.132378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.132380:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.132382:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.132384:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.132385:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.132387:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.132388:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.132389:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.132391:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.132393:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.132396:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.132396:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.132397:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.132399:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.132399:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.132400:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.132400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.132414:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.132416:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.132417:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.132417:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.132419:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.132421:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.133199:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.133201:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.133202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.133204:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.133206:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.133208:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.133209:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.133210:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.133211:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.133212:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.133215:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.133217:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.133220:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.133220:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.133221:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.133223:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.133223:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.133224:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.133225:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.133226:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677473.133236:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.133239:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.133239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.133239:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.133240:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.133242:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.133242:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677473.133242:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.133243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.133245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.134111:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.134114:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.134115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.134116:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.134119:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.134121:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.134122:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.134123:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.134124:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.134125:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.134128:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.134129:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.134132:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.134133:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.134133:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.134135:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.134135:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.134137:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.134138:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.134139:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.134149:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.134152:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677473.134152:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677473.134153:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.134153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.134154:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:9.2:1619677473.134154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677473.134155:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.134156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.134156:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.134158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.134161:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.134163:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.134165:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.134165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.134166:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.134168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.134172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.134179:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.134181:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.134182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.134182:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.134183:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.134183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.134183:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.134185:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.134186:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.134188:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.134191:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.134192:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.134194:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.134195:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.134196:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.134197:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.134201:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.134202:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.134205:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.134206:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.134206:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.134208:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.134208:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.134210:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.134211:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.134211:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.134214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.134222:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677473.134224:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677473.134225:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677473.134225:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.134226:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677473.134227:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.134228:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.134230:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.134231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.134232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.134367:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a0717e5c00] (21)++
00000800:00000200:26.0:1619677473.134371:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a0717e5c00] (22)--
00000800:00000200:10.0:1619677473.134371:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.70@o2ib3
00000400:00000200:10.0:1619677473.134374:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.2.70@o2ib3 : GET - for me
00000400:00000200:10.0:1619677473.134377:0:19940:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.2.70@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:10.0:1619677473.134379:0:19940:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.2.70@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:10.0:1619677473.134380:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a0717e5c00] (21)++
00000800:00000200:10.0:1619677473.134381:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a0717e5c00] (22)--
00000400:00000200:10.0:1619677473.134383:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.2.70@o2ib3 local destination
00000400:00000200:10.0:1619677473.134386:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.2.70@o2ib3(10.51.2.70@o2ib3:10.51.2.70@o2ib3) : REPLY try# 0
00000800:00000200:10.0:1619677473.134388:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.2.70@o2ib3
00000800:00000200:10.0:1619677473.134389:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3c4599600] -> 10.51.2.70@o2ib3 (2) version: 12
00000800:00000200:10.0:1619677473.134390:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a0717e5c00] (21)++
00000800:00000200:10.0:1619677473.134391:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a0717e5c00] (22)++
00000800:00000200:10.0:1619677473.134392:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a0717e5c00] (23)--
00000800:00000200:10.0:1619677473.134393:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a0717e5c00] (22)--
00000800:00000200:26.0:1619677473.134406:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a0717e5c00] (22)++
00000800:00000200:26.0:1619677473.134408:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a0717e5c00] (23)--
00000400:00000200:26.0:1619677473.134409:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.134409:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a0717e5c00] (22)--
00000400:00000200:26.0:1619677473.134410:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.2.70@o2ib3: REPLY: OK
00000800:00000200:26.0:1619677473.134412:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a0717e5c00] (21)--
00000800:00000200:10.0:1619677473.134661:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.134664:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.134665:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.134666:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.134668:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.134669:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.134671:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.134672:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.134673:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.134674:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.134677:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.134679:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.134682:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.134682:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.134683:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.134685:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.134685:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.134686:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.134686:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.134691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.134700:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.134703:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.134704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.134704:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.134706:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.134709:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.134710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.134718:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.134720:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.134721:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.134722:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.134723:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.134723:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677473.134983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.134992:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.134994:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.134995:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.134996:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.134998:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.135001:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.135300:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.135303:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.135304:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.135306:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.135309:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.135310:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.135312:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.135313:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.135314:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.135315:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.135318:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.135320:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.135322:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.135323:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.135323:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.135325:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.135325:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.135326:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.135326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.135331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.135338:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.135341:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.135341:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.135342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.135343:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.135346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.135363:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.135365:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.135367:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.135368:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.135370:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.135372:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677473.135373:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.135374:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.135375:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.135376:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.135379:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.135381:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.135384:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.135385:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677473.135385:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677473.135387:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677473.135388:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.135388:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.135389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677473.135727:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.135730:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.135731:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.135733:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.135735:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.135736:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.135737:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.135739:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.135740:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.135741:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.135744:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.135746:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.135747:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.135748:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.135749:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.135750:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.135751:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.135751:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.135752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.135755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.135761:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.135763:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.135764:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.135765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.135766:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.135768:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.135777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.135784:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.135786:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.135787:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.135787:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.135789:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.135791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.135838:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.135840:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.135841:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.135843:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.135846:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.135847:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677473.135849:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.135850:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.135851:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.135852:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677473.135855:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.135857:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.135859:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.135860:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677473.135861:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677473.135862:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677473.135863:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.135863:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.135864:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677473.136792:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.136795:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677473.136796:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677473.136798:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.136801:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.136803:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677473.136804:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.136805:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.136806:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.136807:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677473.136810:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.136812:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.136815:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.136815:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677473.136816:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677473.136818:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677473.136819:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.136820:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677473.136821:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677473.136822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677473.136825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677473.136832:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677473.136833:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:2.0:1619677473.136833:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:22.0:1619677473.136834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677473.136835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:2.0:1619677473.136835:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.136836:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.136837:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.136838:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.136841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677473.137097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.137102:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.137105:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.137106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.137108:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.137110:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.137112:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.137113:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.137114:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.137115:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.137116:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.137120:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.137122:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.137124:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.137125:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.137126:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.137127:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.137128:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.137129:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.137129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.137142:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.137144:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.137145:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.137146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.137148:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.137150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.137412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.137419:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.137422:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.137422:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.137424:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.137427:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.137428:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.137430:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.137431:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.137432:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.137433:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.137436:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.137438:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.137441:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.137442:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.137442:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.137444:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.137445:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.137446:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.137447:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.137460:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.137464:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.137465:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.137468:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.137469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.137472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.137494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.137501:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.137504:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.137505:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.137505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.137506:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.137508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.137513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.137519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.137522:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.137523:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.137523:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.137524:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.137524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.137664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.137670:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.137672:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.137673:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.137674:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.137675:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.137677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.138038:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.138041:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.138042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.138043:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.138045:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.138047:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.138048:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.138049:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.138050:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.138051:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.138054:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.138055:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.138058:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.138059:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.138059:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.138061:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.138061:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.138062:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.138062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.138075:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.138078:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.138079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.138080:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.138081:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.138084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.138096:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.138099:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.138100:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.138100:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.138101:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.138102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.138628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.138635:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.138637:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.138638:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.138638:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.138640:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.138642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.138963:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.138965:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.138966:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.138968:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.138971:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.138972:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.138974:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.138975:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.138976:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.138976:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.138979:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.138981:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.138983:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.138984:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.138985:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.138986:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.138987:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.138987:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.138988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.139000:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.139003:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.139004:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.139004:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.139006:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.139009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.139017:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.139020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.139020:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.139021:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.139022:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.139022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:13.0:1619677473.139068:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.139071:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.139073:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.139075:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.139076:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.139077:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677473.139078:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.139080:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.139081:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.139082:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677473.139085:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.139087:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677473.139089:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.139090:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677473.139090:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677473.139092:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677473.139092:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.139093:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.139094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677473.139241:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.139244:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.139245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.139247:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.139249:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.139251:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.139252:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.139254:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.139255:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.139256:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.139259:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.139261:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.139265:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.139265:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.139266:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.139268:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.139268:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.139269:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.139270:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.139274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:9.2:1619677473.139281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.139283:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.139285:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.139286:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.139288:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:21.0:1619677473.139288:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.139288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.139290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677473.139291:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.139292:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.139292:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.2:1619677473.139299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:24.0:1619677473.139301:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.139303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677473.139307:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.139310:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.139311:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.139312:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.139312:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.139314:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677473.139316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.139324:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.139327:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.139329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.139329:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.139331:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.139334:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.139563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.139570:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.139573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.139573:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.139573:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.139575:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.139575:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.139712:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.139715:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677473.139716:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677473.139717:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.139720:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.139721:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.139722:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.139724:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.139725:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.139726:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.139729:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.139731:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.139734:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.139735:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.139735:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.139737:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.139738:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.139739:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677473.139740:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677473.139741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677473.139743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.139751:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.139753:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677473.139754:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677473.139754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.139755:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.139756:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:18.0:1619677473.139757:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.139757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677473.139758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:18.0:1619677473.139760:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677473.140099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.140107:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.140109:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.140110:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.140112:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.140113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.140115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.140123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.140131:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.140134:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.140135:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.140135:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.140136:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.140137:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677473.140541:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.140544:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.140545:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.140547:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.140550:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.140551:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.140553:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.140554:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.140555:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.140556:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.140559:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.140560:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.140562:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.140563:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.140563:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.140565:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.140565:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.140566:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.140566:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.140571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.140578:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.140580:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.140581:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.140582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.140583:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.140586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.141129:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.141132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.141132:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.141134:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.141136:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.141138:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.141139:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.141141:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.141141:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.141142:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.141145:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.141147:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.141148:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.141149:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.141149:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.141150:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.141151:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.141152:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.141152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.141156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.141162:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.141164:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.141164:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.141165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.141166:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.141167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.141557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.141565:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.141567:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.141568:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.141569:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.141571:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.141573:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.141909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.141915:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.141918:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.141919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.141921:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.141923:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.141924:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.141926:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.141927:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.141928:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.141929:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.141931:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.141933:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.141936:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.141937:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.141937:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.141939:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.141939:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.141940:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.141940:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.141953:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.141955:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.141956:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.141956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.141958:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.141960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.142869:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.142872:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.142873:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.142874:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.142877:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.142879:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.142880:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.142881:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.142882:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.142883:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.142886:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.142888:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.142890:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.142891:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.142892:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.142893:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.142894:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.142895:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.142896:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.142897:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677473.142908:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.142910:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.142910:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.142911:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.142912:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677473.142912:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.142913:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.142914:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.142914:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.142917:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.142947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.142954:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.142956:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.142957:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.142957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.142959:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.142961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.143036:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.143039:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.143040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.143042:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.143045:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.143046:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.143048:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.143049:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.143050:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.143051:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.143054:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.143055:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.143058:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.143059:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.143059:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.143061:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.143062:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.143063:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.143063:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.143068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.143076:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.143078:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.143080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.143080:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.143082:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.143085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677473.143402:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.143404:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.143407:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.143407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.143409:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.143411:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.143412:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.143414:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.143415:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.143416:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.143419:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.143421:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.143423:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.143424:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.143425:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.143426:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.143427:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.143427:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.143428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.143442:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.143445:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.143445:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.143446:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.143448:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.143450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.143584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.143591:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.143594:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.143594:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.143596:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.143599:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.143601:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.143602:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.143603:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.143604:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.143606:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.143609:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.143611:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.143613:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.143614:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.143615:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.143616:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.143617:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.143618:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.143618:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.143623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.143632:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.143635:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.143635:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.143636:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.143638:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.143641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.143745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.143751:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.143753:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.143754:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.143754:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.143756:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.143758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677473.143824:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.143826:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.143827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.143829:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.143831:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.143833:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677473.143834:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.143835:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.143836:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.143837:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677473.143840:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.143842:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677473.143846:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.143846:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677473.143847:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677473.143849:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677473.143849:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.143850:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.143851:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.143855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.143862:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.143864:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677473.143865:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.143866:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.143868:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.143872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:1.0:1619677473.143937:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.143940:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.143941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.143943:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.143945:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.143947:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.143948:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.143949:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.143950:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.143951:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.143954:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.143955:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.143958:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.143958:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.143959:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.143960:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.143961:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.143962:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.143962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.143967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.143975:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.143977:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.143978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.143979:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.143981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.143982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.144525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.144530:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.144533:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.144535:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.144536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.144538:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.144539:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.144541:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.144542:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.144543:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.144544:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.144547:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.144549:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.144551:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.144552:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.144553:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.144554:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.144555:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.144555:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.144556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.144561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.144570:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.144572:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.144573:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.144573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.144576:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.144578:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677473.144634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.144641:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.144643:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.144644:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.144644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.144646:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.144648:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.144652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.144658:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.144661:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.144661:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.144662:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.144663:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.144663:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.145614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677473.145620:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.145623:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.145624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.145626:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.145628:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.145630:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.145631:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.145633:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.145633:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.145634:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.145638:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.145639:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.145642:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.145643:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.145643:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.145645:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.145646:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.145647:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677473.145648:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677473.145649:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677473.145654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.145659:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677473.145662:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677473.145662:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.145663:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677473.145732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.145739:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.145741:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.145742:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.145743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.145744:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.145746:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.145772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.145774:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.145777:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677473.145778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.0:1619677473.145779:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:1.0:1619677473.145780:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.145781:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.145782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.145782:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.145782:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.145784:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677473.145785:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.145786:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677473.145786:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.145787:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:1.0:1619677473.145788:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.145789:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.145789:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:1.0:1619677473.145789:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677473.145790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:1.0:1619677473.145791:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.145792:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.145795:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.145795:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.145796:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:10.0:1619677473.145796:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.145797:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.145797:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677473.145799:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:10.0:1619677473.145799:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.145799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677473.145799:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.145800:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000400:00000200:10.0:1619677473.145800:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.145800:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677473.145801:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.145801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:10.0:1619677473.145801:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677473.145801:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.145802:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.145802:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.145803:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:10.0:1619677473.145804:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.145804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:10.0:1619677473.145806:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.145808:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677473.145808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.145809:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.145810:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.145811:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.145812:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.145812:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.145813:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677473.145816:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:9.2:1619677473.145817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677473.145818:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.145819:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.145819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.145821:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.145822:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677473.145823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677473.145824:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.145824:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.145825:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.145827:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.145827:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.145828:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.145829:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677473.145831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.145832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.145833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:8.0:1619677473.145834:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.145836:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:1.0:1619677473.145836:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:8.0:1619677473.145837:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.145838:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.145839:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.145839:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.145840:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:8.0:1619677473.145840:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:1.0:1619677473.145840:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.145841:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.145842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:8.0:1619677473.145843:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.145845:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677473.145847:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.145848:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677473.145848:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677473.145850:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677473.145851:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.145851:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.145852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.146386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.146395:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.146397:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.146398:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.146400:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.146401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.146403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677473.146508:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.146511:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.146512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.146514:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.146517:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.146518:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677473.146519:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.146520:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.146521:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.146522:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.146525:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.146527:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.146529:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.146530:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677473.146530:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677473.146532:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677473.146533:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.146534:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.146535:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.146536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.146548:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.146550:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.146551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.146552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677473.147187:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.147190:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.147191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.147192:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.147195:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.147196:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.147197:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.147198:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.147199:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.147200:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.147203:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.147204:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.147207:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.147208:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.147208:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.147210:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.147210:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.147211:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.147212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.147216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.147224:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.147227:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.147228:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.147228:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.147230:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.147233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.147236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:21.0:1619677473.147240:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.147243:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:1.0:1619677473.147244:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000400:00000200:21.0:1619677473.147245:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.147245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:1.0:1619677473.147246:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.147247:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.147247:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000400:00000200:21.0:1619677473.147248:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.147248:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:21.0:1619677473.147249:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677473.147249:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:21.0:1619677473.147250:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.147252:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.147253:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.147254:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.147257:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.147259:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.147261:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.147262:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.147262:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.147264:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.147264:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.147265:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.147266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.147278:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.147280:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.147281:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.147281:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.147283:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.147286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.147332:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.147335:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.147336:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.147338:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.147341:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.147342:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.147344:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.147345:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.147346:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.147347:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.147352:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.147353:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.147353:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.147354:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.147357:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.147357:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.147358:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.147358:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:10.0:1619677473.147359:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.147360:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:10.0:1619677473.147360:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.147362:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.147363:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.147364:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.147364:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.147365:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.147366:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.147367:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677473.147368:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.147368:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.147370:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.147373:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.147374:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677473.147374:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.147375:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.147376:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:12.0:1619677473.147377:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.147377:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.147377:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.147378:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000400:00000200:12.0:1619677473.147379:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:10.0:1619677473.147379:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.147379:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:12.0:1619677473.147381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.147382:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677473.147383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:12.0:1619677473.147390:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.0:1619677473.147392:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:12.0:1619677473.147393:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.147394:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:12.0:1619677473.147394:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.147394:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:20.0:1619677473.147395:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:12.0:1619677473.147395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.147396:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:20.0:1619677473.147396:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.147396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.147397:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.147398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:20.0:1619677473.147398:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.147399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.147401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.147416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.147423:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.147426:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.147426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.147429:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.147432:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.147433:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.147434:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.147435:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.147436:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.147437:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.147440:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.147442:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.147444:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.147445:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.147445:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.147447:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.147448:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.147449:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677473.147450:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.147450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.147453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.147456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.147462:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.147463:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.147463:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.147464:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677473.147464:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.147465:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.147466:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.147466:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.147467:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677473.147467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.147476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.147483:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.147485:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.147486:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.147486:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.147487:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.147487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.147739:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.147742:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.147743:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.147745:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.147748:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.147749:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.147751:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.147752:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.147753:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.147754:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.147757:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.147758:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.147762:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.147763:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.147764:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.147765:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.147766:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.147767:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.147767:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.147780:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.147783:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.147784:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.147784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.147786:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.147789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.147798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677473.147800:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.147803:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.147804:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.147804:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:6.0:1619677473.147804:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.147805:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.147806:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:6.0:1619677473.147806:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.147807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.147808:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.147810:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.147812:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.148005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.148011:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.148013:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.148014:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.148014:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.148016:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.148018:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.148212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.148218:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.148221:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.148221:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.148224:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.148225:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.148226:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.148228:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.148229:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.148230:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.148231:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.148234:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.148236:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.148239:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.148239:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.148240:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.148242:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.148242:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.148244:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.148245:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.148245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.148251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.148255:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.148257:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.148257:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677473.148258:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677473.148259:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.148259:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.148259:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.148260:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677473.148261:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.148261:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677473.148262:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677473.148262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.148262:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677473.148264:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:8.0:1619677473.148264:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.148265:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.148266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677473.148267:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.148268:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.148268:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.148270:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.148271:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.148271:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.148273:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:8.0:1619677473.148273:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.148274:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.148274:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677473.148276:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.148277:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.148277:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.148278:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677473.148280:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677473.148282:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.148282:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.148282:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.148283:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677473.148284:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677473.148284:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.148285:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.148285:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.148286:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677473.148287:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.148287:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.148287:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:26.0:1619677473.148288:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.148290:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.148290:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677473.148290:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.148290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:26.0:1619677473.148291:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.148291:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:17.0:1619677473.148292:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677473.148300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:17.0:1619677473.148303:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.148304:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.148305:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96510729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.148307:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:17.0:1619677473.148307:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.148308:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.148308:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:24.0:1619677473.148309:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:17.0:1619677473.148309:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677473.148310:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.148311:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677473.148311:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.148312:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:17.0:1619677473.148313:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.148313:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.148314:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.148315:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:6.0:1619677473.148315:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.148316:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677473.148318:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.148318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.148319:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.148320:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.148322:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.148322:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.148323:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.148323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.148328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.148337:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.148339:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.148340:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.148342:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.148343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.148345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.148681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.148687:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.148690:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.148690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.148693:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.148695:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.148696:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.148698:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.148699:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.148700:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.148701:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.148704:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.148706:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.148709:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.148710:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.148711:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.148712:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.148713:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.148714:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.148715:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677473.149118:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.149120:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.149123:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.149123:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.149125:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.149127:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.149128:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.149129:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.149130:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.149131:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.149134:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.149136:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.149139:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.149140:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.149140:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.149142:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.149143:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.149144:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.149145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.149148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.149155:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.149157:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.149158:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.149159:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.149160:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.149163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677473.149172:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.149175:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.149177:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.149179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.149179:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.149181:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677473.149182:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.149183:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.149185:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.149186:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677473.149189:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.149191:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677473.149193:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.149194:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677473.149195:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677473.149197:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677473.149197:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.149199:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677473.149200:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677473.149201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677473.149202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.149209:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.149211:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.149212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.149212:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.149214:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.149215:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677473.149217:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677473.149217:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:10.0:1619677473.149217:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:13.0:1619677473.149218:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.149230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677473.149236:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.149239:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677473.149240:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.149240:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.149241:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.149241:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.149926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.149933:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.149935:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.149936:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.149937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.149938:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.149941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.149943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.149949:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.149952:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.149952:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.149953:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.149954:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.149954:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.149977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.149983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.149985:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.149986:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.149987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.149988:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.149990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.149996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.150002:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.150004:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.150005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.150005:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.150006:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.150006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.150661:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.150664:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.150665:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.150667:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.150669:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.150670:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.150671:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.150673:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.150674:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.150674:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.150677:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.150679:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.150681:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.150682:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.150682:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.150683:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.150684:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.150684:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.150685:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.150690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.150696:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.150699:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.150700:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.150700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.150701:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.150703:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.150808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.150814:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.150817:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.150818:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.150818:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.150819:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.150821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.150877:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.150879:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.150880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.150882:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.150883:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.150884:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.150885:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.150887:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.150888:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.150888:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.150891:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.150893:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.150896:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.150897:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.150897:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.150899:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.150899:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.150900:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.150900:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.150905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.150913:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.150916:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.150917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.150917:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.150919:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.150923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.151573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.151581:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.151583:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.151584:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.151584:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.151586:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.151589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.152639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.152646:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.152649:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.152650:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.152650:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.152651:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.152653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.152705:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.152707:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.152708:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.152710:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.152712:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.152714:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.152715:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.152716:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.152717:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.152718:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.152721:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.152722:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.152725:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.152726:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.152726:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.152728:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.152728:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.152729:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.152729:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.152745:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.152749:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.152751:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.152751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.152755:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.152759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.152772:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.152777:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.152778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.152779:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.152781:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.152782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.152836:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.152838:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.152841:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.152841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.152844:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.152846:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.152847:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.152849:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.152851:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.152852:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.152855:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.152858:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.152860:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.152861:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.152862:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.152864:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.152864:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.152866:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.152867:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.152868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677473.152879:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.152883:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.152885:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.152886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.152889:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:20.0:1619677473.152892:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677473.152893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.152894:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.152895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.152896:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:1.0:1619677473.152929:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.152934:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.152935:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.152939:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.152942:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.152944:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.152947:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.152949:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.152951:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.152952:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.152958:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.152961:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.152964:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.152965:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.152966:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.152968:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.152969:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.152971:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.152972:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.152973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.152975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.152982:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.152985:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677473.152985:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000400:00000200:20.0:1619677473.152986:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.152987:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.152988:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677473.152989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.152990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677473.152990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.152992:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:5.2:1619677473.153056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.153063:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677473.153064:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677473.153067:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677473.153068:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677473.153068:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677473.153068:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.153072:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.153073:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.153075:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.153076:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.153076:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.153078:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.153079:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.153080:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.153081:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.153081:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.153082:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.153084:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.153086:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.153087:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677473.153088:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.153091:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.153094:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677473.153094:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.153095:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.153096:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.153097:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.153098:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.153098:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.153100:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.153101:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:1.0:1619677473.153101:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.153102:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677473.153102:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.153103:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.153105:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.153106:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.153107:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.153108:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677473.153115:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.153117:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677473.153117:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.153118:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.153119:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.153120:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.153121:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:6.0:1619677473.153122:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.153123:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.153124:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.153125:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.153127:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.153130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.153131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.153132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.153133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.153138:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.153141:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.153142:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.153144:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.153147:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.153148:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.153150:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.153151:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.153152:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.153153:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.153156:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.153158:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.153161:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.153162:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.153163:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.153164:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.153165:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.153166:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.153168:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.153168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677473.153170:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.153180:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.153181:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.153183:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.153184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677473.153184:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677473.153185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677473.153186:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.153186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.153190:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.153194:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.153433:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.153435:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.153436:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.153438:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.153442:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.153443:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.153444:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.153446:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.153447:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.153447:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.153451:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.153453:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.153456:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.153457:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.153457:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.153459:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.153459:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.153461:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.153462:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.153462:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677473.153466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.153474:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677473.153476:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:21.0:1619677473.153476:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.153477:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.153478:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.153479:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677473.153479:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.153479:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.153480:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677473.153482:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.153543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.153550:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.153553:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.153554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.153554:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.153556:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.153558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.153730:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.153736:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677473.153737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677473.153741:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.153745:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.153748:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677473.153751:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.153754:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.153756:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.153757:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677473.153764:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.153767:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677473.153772:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.153773:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:6.0:1619677473.153775:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677473.153778:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677473.153779:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.153780:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.153781:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.2:1619677473.153784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.153794:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.153799:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.153801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.153801:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.153804:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.153808:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677473.153930:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.153933:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677473.153934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677473.153935:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.153938:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.153939:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.153941:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.153942:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.153943:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.153944:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.153947:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.153948:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.153951:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.153952:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.153953:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.153955:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.153955:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.153956:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677473.153956:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.153961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.153969:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.153972:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.153973:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.153973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.153976:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.153979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677473.154105:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.154108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.154108:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.154110:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.154113:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.154114:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.154115:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.154116:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.154117:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.154118:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.154120:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.154122:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.154123:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.154124:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.154124:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.154126:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.154126:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.154127:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.154127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.154131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.154137:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.154139:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.154140:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.154141:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.154141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.154143:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677473.154654:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.154656:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.154659:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.154660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.154661:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.154663:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.154664:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.154666:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.154667:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.154668:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.154671:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.154673:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.154676:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.154677:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.154677:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.154679:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.154680:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.154680:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.154681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.154685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.154692:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.154695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.154695:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.154696:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.154698:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.154701:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.154703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.154709:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.154711:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.154712:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.154712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.154713:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.154714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.155117:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.155120:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.155121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677473.155122:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677473.155124:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.155126:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.155127:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.155128:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.155129:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.155130:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.155135:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.155137:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.155143:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.155144:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.155144:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.155146:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.155147:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.155147:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.155148:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677473.155152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.155159:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.155161:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:22.0:1619677473.155161:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:24.0:1619677473.155162:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.155163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.155164:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.155164:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.155165:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.155165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.155167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:22.0:1619677473.155167:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.155170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.155453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.155459:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.155462:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.155463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.155465:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.155467:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.155469:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.155470:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.155471:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.155472:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.155474:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.155477:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.155479:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.155482:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.155483:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.155483:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.155485:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.155486:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.155487:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.155488:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.155488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.155494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.155500:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677473.155500:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677473.155503:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.155503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677473.155503:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.155504:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.155504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677473.155506:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.155507:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.155509:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.155895:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.155898:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.155899:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.155901:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.155904:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.155905:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.155906:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.155907:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.155908:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.155909:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.155912:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.155913:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.155915:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.155916:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.155917:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.155918:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.155919:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.155919:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.155920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.155924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.155931:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.155933:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.155934:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.155935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.155936:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.155939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.155942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.155948:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.155951:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.155951:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.155952:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.155952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.155953:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677473.156546:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.156549:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.156551:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.156551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.156554:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.156556:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.156557:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.156559:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.156560:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.156561:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.156564:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.156566:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.156569:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.156570:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.156571:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.156573:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.156574:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.156575:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677473.156577:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677473.156577:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.156586:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677473.156588:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677473.156589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.156590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677473.156900:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.156902:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.156904:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.156905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.156907:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.156909:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.156910:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.156911:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.156912:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.156913:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.156916:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.156917:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.156920:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.156920:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.156921:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.156923:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.156923:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.156924:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.156924:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.156937:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.156939:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.156940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.156940:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.156942:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.156945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.156953:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.156955:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.156956:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.156956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.156957:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.156958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.157110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.157118:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.157120:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.157121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.157121:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.157123:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.157125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.157268:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.157271:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.157272:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.157273:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.157275:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.157276:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.157277:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.157279:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.157279:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.157280:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.157283:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.157285:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.157288:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.157288:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.157289:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.157291:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.157291:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.157292:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.157293:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.157297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.157306:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.157309:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.157310:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.157310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.157312:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.157316:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677473.157462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.157471:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.157474:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.157475:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.157476:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.157477:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677473.157479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.157482:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.157484:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.157486:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.157487:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.157488:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.157488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.157489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.157646:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.157648:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.157649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.157651:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.157654:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.157655:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.157656:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.157658:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.157659:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.157660:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.157663:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.157665:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.157668:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.157668:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.157669:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.157671:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.157671:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.157672:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.157674:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.157674:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.157687:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.157689:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.157690:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.157691:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.157695:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.157697:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.157698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.157699:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.157701:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.157703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.157942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.157948:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.157951:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.157952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.157954:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.157956:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.157958:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.157959:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.157960:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.157961:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.157962:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.157966:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.157968:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.157970:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.157970:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677473.157971:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.157971:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.157973:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677473.157973:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.157973:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677473.157973:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677473.157974:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.157975:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677473.157975:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.157979:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.157980:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.157981:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.157983:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.157984:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.157985:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.157988:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.157990:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.157993:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.157994:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.157994:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.157996:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.157996:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.157997:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.157997:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.157998:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.157999:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.158000:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.158001:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.158002:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.158005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.158008:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.158009:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.158010:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.158010:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.158012:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.158014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.158927:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.158930:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.158931:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.158933:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.158936:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.158937:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:1.0:1619677473.158937:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:8.0:1619677473.158938:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.158940:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677473.158940:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.158941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:8.0:1619677473.158942:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.158943:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677473.158944:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.158946:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.158947:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677473.158948:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:8.0:1619677473.158949:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:1.0:1619677473.158950:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.158951:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.158951:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:1.0:1619677473.158951:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.158952:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:1.0:1619677473.158952:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.158953:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:1.0:1619677473.158953:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.0:1619677473.158954:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.158954:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.158955:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677473.158957:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.158958:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.2:1619677473.158961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.158961:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.158962:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.158962:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.158964:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.158964:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.158965:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.158966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:24.0:1619677473.158967:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.158969:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.158970:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677473.158971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677473.158972:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.158974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.158975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677473.158979:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.158981:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.158982:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.158984:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.158985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.158986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.158998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.159006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.159009:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.159010:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.159010:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677473.159011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.159012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.159353:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.159355:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.159356:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.159359:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.159361:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.159362:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.159364:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.159365:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.159366:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.159367:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.159370:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.159373:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.159375:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.159376:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.159376:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.159378:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.159378:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.159379:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.159380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.159397:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:9.2:1619677473.159397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:12.0:1619677473.159399:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.159400:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.159402:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.159403:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:13.0:1619677473.159404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.159405:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677473.159406:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.159407:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.159407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.159408:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.159410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.159423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.159429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.159431:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.159432:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.159432:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.159433:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.159434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.159990:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.159993:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.159995:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.159996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.159998:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.160000:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.160001:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.160002:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.160003:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.160004:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.160007:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.160009:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.160011:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.160012:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.160013:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.160014:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.160015:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.160016:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.160016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.160028:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.160031:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.160032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.160032:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.160034:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.160036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.160277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.160285:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.160288:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.160289:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.160289:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.160291:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.160301:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.160751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.160760:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.160762:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.160764:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.107@o2ib7 : GET - for me
00000800:00000200:21.0:1619677473.160766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.160766:0:19934:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.107@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:20.0:1619677473.160768:0:19934:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.107@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:20.0:1619677473.160770:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.160771:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.160773:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.160776:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : REPLY try# 0
00000800:00000200:20.0:1619677473.160778:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.160779:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.160780:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.160781:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.160782:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.160783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677473.160788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.160796:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.160799:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.160799:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.160801:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: REPLY: OK
00000800:00000200:20.0:1619677473.160802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.160803:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.161162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.161169:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.161171:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.161173:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.161174:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.161177:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.161178:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.161180:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.161181:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.161182:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.161183:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.161187:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.161188:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.161190:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.161191:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.161192:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.161194:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.161194:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.161195:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.161196:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.161200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.161208:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.161210:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.161211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.161211:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.161213:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.161216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.161220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.161227:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.161230:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.161231:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.161231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.161232:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.161233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.161279:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.161282:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.161283:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.161285:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.161288:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.161289:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.161290:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.161291:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.161292:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.161299:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.161302:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.161303:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.161306:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.161307:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.161308:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.161309:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.161309:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.161310:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.161310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.161315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.161325:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.161328:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.161329:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.161330:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:24.0:1619677473.161331:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.161332:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.161333:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.161334:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:8.0:1619677473.161334:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677473.161335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677473.161336:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.161337:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.161338:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.161340:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.161341:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.161341:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.161344:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.161346:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.161347:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.161347:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.161348:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.161349:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.161350:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.161351:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677473.161352:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677473.161353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.161365:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:9.2:1619677473.161366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.161367:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677473.161368:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.161369:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:24.0:1619677473.161370:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.161371:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.161372:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.161373:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.161373:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.161375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677473.161504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.161513:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.161515:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.161516:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.161517:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.161519:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.161522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677473.161788:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.161791:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.161793:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.161794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.161795:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.161797:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.161798:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.161799:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.161800:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.161801:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.161804:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.161806:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.161809:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.161810:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.161810:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.161812:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.161812:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.161813:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.161815:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.161815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677473.161823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.161826:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.161828:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.161829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.161829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677473.161830:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.161832:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.161833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.161834:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.161836:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.161838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.161870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.161876:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.161878:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.161880:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.161881:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.161883:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.161884:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677473.161886:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.161887:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.161888:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.161889:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677473.161892:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.161893:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.161895:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.161896:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.161896:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.161898:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.161899:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.161899:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.161900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.161910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.161918:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.161920:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.161921:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.161923:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.161924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.161925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677473.161975:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.161978:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.161979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.161981:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.161982:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.161984:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.161985:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.161987:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.161988:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.161989:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.161992:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.161994:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.161997:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.161997:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.161998:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.162000:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.162000:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.162001:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.162003:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.162003:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.162006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.162014:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.162016:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677473.162016:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.162017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.162018:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.162018:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.162019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677473.162020:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.162020:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677473.162023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.162026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.162034:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.162036:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.162037:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.162038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.162039:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.162039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677473.162063:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.162065:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.162066:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.162068:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.162071:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.162072:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677473.162074:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.162075:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.162076:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.162077:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677473.162080:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.162082:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677473.162085:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.162086:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677473.162087:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677473.162089:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.162089:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.162090:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.162090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.162095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.162102:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.162105:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.162105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.162106:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.162108:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.162111:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.162807:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.162809:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.162810:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.162812:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.162814:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.162816:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.162817:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.162818:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.162819:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.162820:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.162823:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.162825:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.162828:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.162829:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.162830:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.162831:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.162832:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.162832:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.162833:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677473.162838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.162844:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.162847:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.162847:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.162848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.162850:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.162851:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.162851:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:1.0:1619677473.162851:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.162853:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.162853:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.162855:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677473.162856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:1.0:1619677473.162857:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.162858:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.162859:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.162860:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.162863:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.162865:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.162867:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.162868:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.162869:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.162870:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.162871:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.162872:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.162872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.162882:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.162885:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:6.0:1619677473.162885:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.162886:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (24)++
00000800:00000200:22.0:1619677473.162887:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000800:00000200:6.0:1619677473.162887:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (25)--
00000400:00000200:22.0:1619677473.162888:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.162888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.162890:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.162890:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.162892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.162954:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.162956:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.162957:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.162959:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.162962:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.162964:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.162965:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.162966:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.162967:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.162968:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.162972:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.162973:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.162977:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.162977:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.162978:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.162980:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.162980:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.162981:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.162982:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.162983:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.162996:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.162997:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.162998:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.162999:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.163003:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:9.2:1619677473.163005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.163005:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.163006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.163007:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.163009:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.163012:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677473.163012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.163015:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.163016:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.163016:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.163018:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.163021:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.163062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.163069:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.163071:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.163072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.163072:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.163074:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.163076:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.163358:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.163361:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.163363:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.163364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.163365:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.163367:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.163368:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.163369:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.163370:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.163371:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.163374:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.163376:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.163378:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.163379:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.163380:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.163381:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.163382:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.163383:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.163383:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.163395:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.163398:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.163399:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.163399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.163401:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.163403:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.163412:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.163415:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.163415:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.163416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.163416:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.163417:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677473.163783:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.163786:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.163786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.163789:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.163791:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.163792:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.163794:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.163795:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.163796:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.163797:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.163801:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.163802:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.163805:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.163806:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.163806:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.163808:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.163809:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.163809:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.163810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.163815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.163823:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.163825:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.163826:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.163828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.163828:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.163830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677473.163870:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.163872:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677473.163873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677473.163875:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.163877:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.163878:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677473.163880:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.163881:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.163882:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.163883:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677473.163886:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.163888:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.163889:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.163890:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.163891:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.163892:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.163893:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.163893:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677473.163894:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677473.163898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.163904:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.163905:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.163906:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.163907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.163908:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.163910:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.163927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.163934:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.163936:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.163937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.163940:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.163942:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.163943:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.163944:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.163945:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.163947:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.163947:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.163951:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.163952:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.163954:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.163955:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.163955:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.163957:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.163958:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.163959:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.163959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.163963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.163971:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.163973:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.163973:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.163974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.163975:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.163977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677473.164024:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.164027:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.164027:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.164029:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.164032:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.164033:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.164034:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.164035:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.164036:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.164037:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.164040:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.164042:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.164043:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.164044:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.164045:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.164046:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.164046:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.164047:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.164047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.164051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.164058:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.164059:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.164060:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.164061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.164062:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.164063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.164074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.164080:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.164082:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.164083:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.164083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.164084:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.164084:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.164766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.164773:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.164776:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.164777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.164777:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.164778:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.164781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.166228:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.166230:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.166231:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.166233:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.166236:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.166238:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.166239:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.166240:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.166241:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.166242:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.166244:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.166246:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.166248:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.166248:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.166249:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.166251:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.166251:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.166252:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.166252:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.166256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.166263:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.166265:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.166266:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.166267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.166268:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.166270:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.166276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.166282:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.166284:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.166285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.166285:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.166286:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.166287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.166682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.166689:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.166691:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.166692:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.166692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.166694:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.166696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.166713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.166719:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.166722:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.166723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.166725:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.166727:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.166728:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.166729:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.166731:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.166732:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.166733:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.166736:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.166738:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.166741:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.166742:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.166743:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.166745:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.166745:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.166746:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.166747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677473.166959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.166966:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.166968:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.166969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.166969:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.166971:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.166973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.167016:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.167019:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.167021:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.167021:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.167023:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.167025:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.167026:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.167027:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.167028:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.167029:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.167032:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.167034:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.167036:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.167037:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.167038:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.167039:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.167040:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.167040:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.167041:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.167045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.167051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.167054:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.167055:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.167055:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.167057:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.167060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677473.167088:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.167091:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.167095:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.167095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.167098:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.167101:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677473.167103:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.167105:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.167107:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.167108:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677473.167114:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.167117:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677473.167121:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.167122:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677473.167123:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677473.167125:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677473.167126:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.167127:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.167128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.167131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.167139:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.167142:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.167143:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.167144:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.167146:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.167149:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.167444:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.167447:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.167448:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.167450:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.167452:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.167453:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.167454:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.167456:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.167456:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.167457:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.167460:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.167462:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.167464:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.167465:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.167465:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.167467:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.167467:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.167468:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.167468:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.167481:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.167483:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.167484:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.167485:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.167487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.167488:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677473.167822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.167828:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.167831:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.167832:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.167834:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.167837:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.167838:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.167839:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.167840:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.167841:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.167842:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.167846:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.167848:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.167850:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.167851:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.167852:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.167853:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.167853:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.167855:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.167856:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.167856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.167861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.167867:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.167867:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.167869:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.167869:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.167870:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.167870:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.167871:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.167871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.167873:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.167875:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.167937:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.167940:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.167941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.167943:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.167945:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.167947:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.167948:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.167950:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.167951:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.167952:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.167955:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.167958:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.167961:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.167962:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.167962:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.167963:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.167963:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677473.167964:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.167965:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.167966:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677473.167966:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.167967:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.167968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:10.0:1619677473.167968:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677473.167969:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.167969:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.167971:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.167973:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.167974:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.167977:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.167978:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.167979:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.167981:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.167982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.167982:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677473.167982:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.167983:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.167983:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677473.167984:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.167985:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.167986:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.167986:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.167987:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.167988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.167998:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.168001:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.168002:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.168002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677473.168002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.168004:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.168007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677473.168011:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.168014:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.168015:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.168015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.168017:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.168021:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677473.168264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.168273:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.168275:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.168276:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.168276:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.168279:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.168282:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677473.168463:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.168466:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.168467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.168468:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.168470:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.168472:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.168473:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.168474:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.168475:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.168476:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.168479:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.168481:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.168483:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.168484:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.168485:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.168487:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.168487:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.168488:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.168488:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.168493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.168502:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.168505:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.168506:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.168506:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.168508:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.168511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677473.168611:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.168613:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.168616:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.168617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.168618:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.168620:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.168621:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.168622:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.168623:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.168624:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.168628:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.168629:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.168632:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.168633:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.168634:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.168635:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.168636:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.168637:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.168637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.168641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.168647:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.168650:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.168651:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.168651:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.168654:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.168656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.168661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.168667:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.168670:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.168671:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.168671:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.168672:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.168672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.169015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.169021:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.169023:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.169024:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.169025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.169026:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.169029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.169348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.169355:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.169357:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.169358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.169358:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.169360:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.169363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.170038:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.170040:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.170041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.170042:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.170045:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.170046:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.170047:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.170048:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.170049:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.170050:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.170052:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.170054:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.170055:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.170056:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.170056:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.170057:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.170058:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.170059:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.170059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.170063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.170069:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.170071:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.170072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.170072:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.170073:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.170074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677473.170364:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.170367:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.170368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.170370:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.170372:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.170374:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.170375:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.170376:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.170377:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.170378:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.170381:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.170383:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.170386:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.170386:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.170387:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.170389:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.170389:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.170390:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.170390:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.170396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.170404:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.170407:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.170408:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.170409:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.170410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.170412:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677473.170582:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.170585:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677473.170586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677473.170588:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.170590:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.170591:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.170593:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.170594:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.170595:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.170596:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.170599:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.170601:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.170604:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.170604:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.170605:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.170607:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.170607:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.170608:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677473.170608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.170614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.170623:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.170626:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.170627:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.170628:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.170630:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.170633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.170637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.170639:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.170641:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.170641:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.170642:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.170643:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.170643:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.171109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.171115:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.171118:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.171119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.171119:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.171121:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.171123:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.171618:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.171620:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.171621:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.171623:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.171626:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.171627:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.171628:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.171630:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.171631:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.171631:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.171635:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.171636:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.171638:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.171639:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.171640:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.171641:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.171642:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.171642:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.171643:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.171659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.171666:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.171668:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.171669:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.171670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.171671:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.171673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.172153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.172160:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.172162:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.172163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.172165:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.172168:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.172169:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.172171:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.172173:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.172174:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.172175:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.172178:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.172179:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.172182:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.172183:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.172183:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.172185:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.172185:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.172186:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.172186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.172201:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.172205:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.172206:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.172207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.172209:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.172213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.172226:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.172231:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.172232:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.172232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.172234:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.172235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.172689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.172696:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.172699:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.172700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.172700:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.172702:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.172704:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.172721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.172727:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.172729:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.172730:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.172730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.172731:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.172731:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.172823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.172830:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.172832:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.172833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.172833:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.172834:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.172836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.172841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.172847:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.172847:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.172850:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677473.172850:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.172850:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:24.0:1619677473.172851:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.172851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.172852:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677473.172852:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.172853:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:10.0:1619677473.172855:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.172856:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.172857:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.172858:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.172859:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.172860:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.172863:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.172864:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.172868:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.172869:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.172870:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.172870:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.172871:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.172872:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677473.172873:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.172873:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.172874:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:26.0:1619677473.172875:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.172875:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.172876:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.172877:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677473.172877:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.172878:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.172880:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.172881:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.172881:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.172885:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.172886:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.172886:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.172889:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.172889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.172889:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.172889:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.172890:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:6.0:1619677473.172890:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.172891:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.172891:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.172891:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.172892:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.172892:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.172893:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.172895:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677473.172895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.172896:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.172896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.172906:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.172909:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.172910:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.172910:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.172912:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.172914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.173169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.173172:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.173174:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.173177:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.173178:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.173180:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.173181:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.173183:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.173184:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.173185:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.173186:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.173189:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.173191:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.173193:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.173194:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.173195:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.173196:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.173197:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.173197:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.173198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.173203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.173211:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.173214:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.173215:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.173215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.173217:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.173219:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677473.173391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.173432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.173441:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.173444:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.173445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.173445:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.173447:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.173450:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.173454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.173462:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.173464:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.173465:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.173465:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.173466:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.173467:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:10.0:1619677473.173496:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.173499:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.173500:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.173502:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.173505:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.173506:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.173507:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.173509:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.173510:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.173511:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.173514:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.173516:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.173518:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.173519:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.173520:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.173521:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.173522:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.173523:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.173523:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.173528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.173537:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.173539:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.2:1619677473.173539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:21.0:1619677473.173540:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.173542:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.173545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677473.173545:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677473.173546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:2.0:1619677473.173548:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.173549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.173551:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.173553:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.173555:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.173556:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.173557:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.173558:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.173559:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.173563:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.173565:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.173567:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.173568:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.173569:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.173571:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.173572:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.173572:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.173573:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.173585:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.173588:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.173589:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.173590:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.173593:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.173594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.173603:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.173605:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.173606:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.173607:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.173607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.173608:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.174676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.174683:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:1.0:1619677473.174683:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:8.0:1619677473.174686:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677473.174686:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677473.174687:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:8.0:1619677473.174687:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.174688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.174690:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.174690:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.174692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:1.0:1619677473.174693:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.174694:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677473.174696:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.174697:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.174698:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.174699:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677473.174702:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.174704:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677473.174706:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.174707:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677473.174708:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677473.174710:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677473.174710:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.174711:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.174712:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677473.174716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.174724:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.174726:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.174727:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.174727:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.174730:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.174732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677473.174755:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.174758:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.174759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.174761:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.174763:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.174765:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.174766:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.174767:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.174768:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.174769:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.174772:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.174774:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.174777:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.174778:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.174778:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.174780:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.174780:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.174781:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.174783:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.174783:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.174786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.174794:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.174796:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.174797:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.174798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.174798:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.174798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677473.174798:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.174799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677473.174800:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.174803:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677473.175860:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.175862:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.175863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.175865:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.175868:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.175869:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.175870:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.175872:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.175873:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.175874:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.175877:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.175878:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.175881:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.175882:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.175882:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.175884:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.175884:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.175885:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.175886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.175891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.175899:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.175902:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.175903:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.175903:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.175905:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.175908:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677473.175947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.175956:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.175958:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.175959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.175960:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.175962:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.175965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677473.176147:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.176150:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.176151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.176153:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.176154:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.176156:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677473.176157:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.176159:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.176159:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.176160:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677473.176163:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.176165:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677473.176168:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.176169:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677473.176169:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.176171:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677473.176172:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.176172:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.176172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.176177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.176184:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.176186:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.176187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.176187:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.176189:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.176193:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.176232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.176239:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.176241:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.176242:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.176242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.176244:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.176246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.176957:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.176960:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.176961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.176962:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.176965:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.176966:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.176968:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.176969:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677473.176969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677473.176970:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.176970:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.176973:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.176974:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677473.176975:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677473.176976:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.176977:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.176978:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677473.176978:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.176978:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677473.176979:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.176980:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677473.176980:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.176981:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.176981:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677473.176983:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.176985:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.176986:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.176986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677473.176987:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.176988:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.176989:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677473.176992:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677473.176993:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.176994:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677473.176995:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.176996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.176996:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.176997:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.176998:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.176998:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:8.0:1619677473.176998:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.177000:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.177000:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677473.177000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677473.177001:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.177002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.177025:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.177027:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.177028:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.177028:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.177030:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.177033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.177051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.177054:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.177056:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.177056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.177058:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.177060:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.177061:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.177063:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.177064:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.177065:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.177069:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.177070:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.177073:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.177074:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.177075:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.177077:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.177078:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.177079:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.177079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677473.177322:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.177325:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.177325:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.177327:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.177330:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.177331:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.177332:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.177334:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.177335:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.177336:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.177339:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.177341:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.177344:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.177345:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.177345:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.177347:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.177347:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.177349:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.177350:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.177351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.177362:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.177364:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.177364:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.177365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.177366:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.177366:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677473.177366:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.177367:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.177368:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.177370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677473.177388:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.177391:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.177392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.177394:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.177396:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.177398:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.177399:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.177400:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.177401:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.177402:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.177406:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.177408:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.177410:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.177411:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.177412:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.177413:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.177414:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.177415:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.177416:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.177417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.177420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.177427:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677473.177428:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677473.177429:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.177429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677473.177430:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677473.177430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677473.177431:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.177433:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.177434:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.177436:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.177441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.177449:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.177452:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.177453:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.177454:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677473.177454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.177456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.177864:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.177867:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.177868:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.177870:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.177873:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.177874:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.177876:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.177877:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.177878:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.177879:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.177883:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.177884:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.177887:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.177888:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.177889:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.177891:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.177891:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.177892:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.177892:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.177897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.177905:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.177907:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.177908:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.177910:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.177910:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.177912:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677473.177951:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.177954:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.177955:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.177956:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.177959:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.177960:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.177962:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.177963:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.177964:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.177964:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.177967:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.177969:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.177971:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.177972:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.177972:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.177974:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.177974:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.177975:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.177976:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.177988:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.177990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.177991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.177991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.177994:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.177996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.178409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.178416:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.178419:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.178420:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.178420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.178422:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.178424:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.178428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.178434:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.178436:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.178437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.178437:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.178438:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.178439:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.178758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.178765:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.178767:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.178769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.178770:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.178773:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.178775:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.178777:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.178779:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.178780:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.178781:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.178785:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.178787:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.178789:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.178790:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.178791:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.178792:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.178793:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.178794:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.178796:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.178796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.178798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.178802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677473.178806:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677473.178808:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677473.178809:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677473.178810:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:17.0:1619677473.178810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.178810:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.178811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.178812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677473.178813:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.178815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.178817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.178824:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.178827:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.178828:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.178828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.178829:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.178830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.178833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.178839:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.178842:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.178843:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.178843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.178845:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.178847:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.178924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.178930:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.178933:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.178934:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.178936:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.178938:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.178939:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.178941:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.178942:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.178943:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.178944:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.178947:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.178949:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.178951:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.178952:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.178953:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.178954:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.178955:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.178956:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.178956:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.178960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.178968:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.178970:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.178971:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.178971:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.178972:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.178974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.179121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.179124:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.179126:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.179128:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.179129:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.179131:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.179133:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.179134:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.179135:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.179136:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.179137:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.179140:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.179142:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.179145:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.179146:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.179146:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.179148:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.179148:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.179150:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.179151:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.179152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.179154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677473.179157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677473.179161:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677473.179162:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.179164:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677473.179164:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677473.179165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677473.179165:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.179165:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.179167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677473.179168:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.179170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677473.179610:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.179613:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.179613:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.179615:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.179617:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.179619:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677473.179620:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.179621:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.179622:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.179623:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.179627:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.179629:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.179631:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.179632:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.179633:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.179634:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.179635:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.179635:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677473.179636:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677473.179637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:22.0:1619677473.179649:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.179652:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.179653:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.179653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.179655:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.179657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.179672:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.179674:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.179675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.179676:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.179677:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.179677:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.179721:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677473.179723:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.179724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.179725:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.179957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.179964:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.179967:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.179968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.179968:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.179970:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.179972:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.180448:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.180450:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.180451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.180453:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.180455:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.180456:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.180457:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.180459:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.180460:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.180461:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.180465:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.180467:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.180469:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.180469:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.180470:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.180471:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.180472:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.180472:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.180473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.180478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.180484:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:8.0:1619677473.180486:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677473.180487:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000400:00000200:8.0:1619677473.180487:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.180488:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:24.0:1619677473.180489:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.180489:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.180490:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.180491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.180492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.180493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.181597:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.181600:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.181601:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.181602:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.181604:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.181606:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.181607:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.181608:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.181609:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.181610:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.181613:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.181615:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.181617:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.181618:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.181619:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.181620:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.181621:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.181621:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.181622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.181627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.181636:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.181638:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.181640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.181640:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.181642:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.181645:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.181647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.181654:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.181656:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.181657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.181658:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.181659:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.181659:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677473.181758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.181764:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.181767:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.181768:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.181770:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.181772:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.181774:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.181775:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.181776:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.181777:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.181778:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.181782:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.181783:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.181786:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.181786:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.181787:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.181789:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.181789:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.181790:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.181790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677473.181799:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.181802:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.181803:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.181804:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:10.0:1619677473.181805:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.181808:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:10.0:1619677473.181808:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.181809:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677473.181810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.181810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:10.0:1619677473.181811:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.181812:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.181814:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.181814:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.181815:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.181819:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.181821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.181821:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677473.181823:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.181823:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.181824:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.181824:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.181826:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.181826:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.181826:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.181827:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.181828:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.181828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.181828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.181830:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.181832:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.181832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.181838:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.181840:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.181841:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.181841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.181844:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.181846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.181917:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.181920:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.181921:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.181923:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.181925:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.181927:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.181928:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.181930:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.181931:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.181932:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.181935:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.181937:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.181940:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.181941:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.181941:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.181943:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.181943:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.181945:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.181946:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.181946:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.181949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.181957:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.181959:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677473.181960:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.181961:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.181961:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:2.0:1619677473.181961:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.181962:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.181963:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677473.181963:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.181967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677473.182077:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.182080:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.182081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.182083:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.182085:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.182087:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.182089:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.182091:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.182092:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.182093:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.182097:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.182100:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.182102:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.182103:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.182104:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.182105:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.182106:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.182107:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.182107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.182112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:9.2:1619677473.182118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677473.182122:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677473.182124:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677473.182125:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.182126:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.182127:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.182128:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677473.182128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:8.0:1619677473.182128:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.182129:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.182130:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.182133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677473.182133:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.182138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.182144:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.182147:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.182147:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.182148:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.182148:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.182149:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.182204:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.182206:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.182207:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.182209:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.182212:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.182214:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.182215:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.182216:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.182217:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.182218:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.182222:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.182223:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.182226:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.182227:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.182228:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.182229:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.182230:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.182230:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.182231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.182245:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.182249:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.182251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.182251:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.182254:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.182258:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.182560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.182568:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.182570:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.182571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.182571:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.182573:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.182575:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.182581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.182587:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.182589:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.182590:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.182590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.182591:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.182591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.183168:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.183170:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.183172:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.183173:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.183176:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.183178:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.183179:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.183180:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.183181:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.183182:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.183185:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.183186:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.183189:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.183190:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.183190:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.183192:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.183192:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.183193:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.183194:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.183195:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.183207:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.183209:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.183210:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.183211:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.183216:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.183219:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.183220:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.183220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.183222:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.183225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.183373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.183375:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677473.183376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677473.183378:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.183381:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.183382:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677473.183383:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.183385:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.183386:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.183387:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677473.183390:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.183392:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677473.183394:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.183395:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677473.183396:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677473.183398:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677473.183398:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.183399:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.183399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677473.183404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.183413:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.183415:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.183416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.183416:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.183419:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.183421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677473.183517:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.183520:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.183522:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.183523:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.183524:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.183526:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.183527:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.183528:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.183529:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.183530:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.183534:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.183535:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.183537:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.183538:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.183539:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.183541:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.183541:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.183542:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.183543:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.183544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677473.183554:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.183554:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677473.183557:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677473.183557:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:22.0:1619677473.183558:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.183558:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677473.183559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.183560:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.0:1619677473.183560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.183563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.184164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.184170:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.184173:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.184174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.184174:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.184176:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.184178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.184744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.184751:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.184754:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.184754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.184756:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.184758:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.184760:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.184762:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.184763:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.184764:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.184765:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.184768:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.184769:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.184772:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.184773:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.184774:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.184775:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.184776:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.184777:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.184777:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677473.185216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.185223:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.185226:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.185227:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.185227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.185228:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.185231:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.185489:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.185492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.185492:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677473.185494:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.185497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.185499:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677473.185500:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.185501:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.185502:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.185503:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677473.185507:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.185509:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677473.185511:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.185512:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677473.185513:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677473.185514:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677473.185515:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.185516:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.185516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677473.185521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.185529:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.185532:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.185533:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.185534:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.185535:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.185538:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677473.185783:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.185786:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677473.185787:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677473.185789:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.185791:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.185793:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.185794:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.185795:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.185796:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.185797:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.185801:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.185803:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.185806:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.185807:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.185807:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.185809:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.185810:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.185811:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677473.185811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.185815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.185823:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.185826:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.185827:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.185827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.185829:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.185832:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677473.185848:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.185851:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.185852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.185853:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.185856:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.185857:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.185858:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.185859:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.185860:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.185861:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.185863:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.185864:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.185866:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.185866:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.185867:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.185868:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.185868:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.185870:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.185871:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.185871:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.185880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.185886:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.185887:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.185887:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.185888:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.185888:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.185889:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.185889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:24.0:1619677473.185890:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.185890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.185891:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.185987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.185990:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.185992:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.185994:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.185996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.185997:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.185998:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.185999:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.186000:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.186001:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.186003:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.186006:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.186007:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.186010:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.186010:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.186011:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.186013:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.186013:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.186014:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.186015:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.186019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.186027:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.186030:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.186031:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.186031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.186033:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.186035:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.186042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.186050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:1.0:1619677473.186050:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.186053:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.186053:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.186054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677473.186054:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:1.0:1619677473.186054:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.186055:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.186056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:10.0:1619677473.186057:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.186060:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.186061:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.186063:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.186064:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.186065:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.186066:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.186070:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.186071:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.186075:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.186075:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.186076:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.186078:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.186078:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677473.186079:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677473.186079:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:21.0:1619677473.186091:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.186094:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.186095:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.186097:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.186097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.186099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.186208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.186214:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.186217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.186217:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.186218:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.186220:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.186222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677473.186395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.186405:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.186408:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.186409:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.186410:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.186411:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.186413:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.186413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.186414:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.186415:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.186415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.186416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677473.186542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.186548:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.186551:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.186552:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.186553:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.186556:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.186557:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.186558:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.186560:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.186561:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.186562:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.186565:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.186567:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.186569:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.186570:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.186571:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.186572:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.186573:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.186574:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.186575:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.186575:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.186580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.186586:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.186587:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.186588:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.186588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.186589:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.186589:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.186591:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.186591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.186592:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.186595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.186847:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.186850:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.186851:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.186852:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.186855:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.186856:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.186857:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.186858:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.186859:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.186860:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.186863:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.186864:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.186867:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.186867:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.186868:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.186869:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.186870:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.186870:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.186871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.186875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.186883:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.186885:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.186886:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.186888:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.186889:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.186890:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.187069:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.187072:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.187073:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.187075:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.187078:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.187079:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.187080:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.187082:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.187083:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.187084:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.187087:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.187089:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.187092:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.187093:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.187093:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.187093:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.187094:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.187095:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.187096:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677473.187096:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.187097:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.187098:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677473.187098:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.187099:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.187100:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.187101:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.187101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677473.187103:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.187103:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.187104:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.187107:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.187108:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677473.187110:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677473.187111:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.187112:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:10.0:1619677473.187112:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.187112:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:24.0:1619677473.187113:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.187114:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.187114:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677473.187115:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.187115:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.187115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677473.187115:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.187116:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.187117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.187118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:8.0:1619677473.187118:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.187120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.187120:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.187121:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.187122:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96510581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.187124:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.187124:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.187125:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:6.0:1619677473.187127:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:8.0:1619677473.187128:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.187130:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677473.187130:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.187131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.187131:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.187132:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.187132:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.187133:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677473.187133:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.187134:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.187135:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.187136:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.187136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.187136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.187140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.187145:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.187147:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.187148:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.187149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.187150:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.187151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.187156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.187162:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.187164:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.187165:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.187165:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.187165:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.187166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.187743:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.187745:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.187748:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.187748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.187750:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.187757:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.187758:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.187759:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.187760:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.187761:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.187766:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.187769:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.187771:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.187772:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.187772:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.187774:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.187775:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.187775:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.187776:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.187789:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.187792:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.187793:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.187793:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.2:1619677473.187794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:22.0:1619677473.187795:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.187798:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.187803:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.187806:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.187807:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.187807:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.187809:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.187812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.187832:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.187835:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.187836:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.187838:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.187841:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.187843:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677473.187844:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.187845:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.187846:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.187847:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677473.187851:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.187852:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677473.187854:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.187855:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677473.187855:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677473.187857:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677473.187857:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.187858:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.187859:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677473.188016:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.188018:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.188019:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.188021:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.188023:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.188025:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.188026:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.188027:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.188028:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.188028:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.188031:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.188032:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.188035:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.188035:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.188036:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.188037:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.188038:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.188039:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.188039:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.188051:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.188054:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.188054:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.188056:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.188058:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.188059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.188175:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.188178:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.188179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.188180:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.188182:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.188184:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.188185:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.188186:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.188187:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.188188:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.188190:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.188192:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.188194:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.188194:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.188195:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.188196:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.188197:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.188198:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.188198:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.188202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.188208:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.188211:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.188212:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.188212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.188213:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.188215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677473.188633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.188642:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.188644:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.188645:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.188647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.188647:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.188650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.188726:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.188729:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.188729:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.188731:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.188734:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.188735:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677473.188736:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.188737:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.188738:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.188739:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677473.188742:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.188744:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677473.188746:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.188746:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677473.188747:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677473.188748:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677473.188749:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.188749:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.188750:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.189203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.189212:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.189214:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.189215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.189216:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.189218:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.189221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.190207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.190214:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.190216:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.190217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.190217:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.190219:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.190221:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.190227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.190232:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.190235:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.190236:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.190236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.190237:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.190237:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.190413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.190420:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.190422:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.190423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.190423:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.190425:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.190428:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.190485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:21.0:1619677473.190494:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.190496:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:20.0:1619677473.190499:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.190500:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677473.190502:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.190504:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.190506:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.190508:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.190509:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.190511:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.190515:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.190518:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.190521:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.190521:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.190522:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.190524:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.190525:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.190526:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.190526:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.190530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.190537:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.190539:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:12.0:1619677473.190539:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:24.0:1619677473.190540:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.190540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:12.0:1619677473.190541:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:24.0:1619677473.190542:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:12.0:1619677473.190542:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.190544:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.190545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677473.190546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.190547:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677473.191053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.191059:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.191062:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.191063:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.191065:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.191068:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.191070:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.191072:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.191073:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.191074:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.191075:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.191079:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.191080:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.191084:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.191085:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.191086:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.191088:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.191088:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.191089:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.191090:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.191102:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.191105:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.191106:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.191109:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.191110:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.191112:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677473.191204:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.191207:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.191208:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.191210:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.191211:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.191213:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.191214:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.191215:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.191216:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.191217:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.191220:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.191222:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.191224:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.191224:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.191225:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.191227:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.191227:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.191228:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.191228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.2:1619677473.191230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677473.191234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.191236:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.191238:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.191239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677473.191240:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:1.0:1619677473.191241:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.191242:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.191243:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.191243:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.191244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:1.0:1619677473.191245:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677473.191246:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.191247:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.191248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:1.0:1619677473.191248:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.191249:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.191250:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.191253:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.191255:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.191257:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.191258:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.191259:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.191261:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.191261:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.191262:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.191262:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.191275:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.191277:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.191278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.191278:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.191280:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.191282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.191293:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.191299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.191300:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.191301:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.191302:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.191303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.191459:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.191462:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.191463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.191465:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.191467:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.191468:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.191469:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.191471:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.191472:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.191472:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.191475:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.191477:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.191479:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.191480:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.191481:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.191482:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.191483:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.191483:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.191484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.191489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.191497:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.191500:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.191501:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.191502:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.191503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.191505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.191978:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.191981:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.191982:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.191984:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.191986:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.191988:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.191989:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.191990:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.191991:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.191992:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.191996:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.191997:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.192000:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.192001:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.192002:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.192003:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.192004:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.192004:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.192005:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.192017:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.192020:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.192021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.192021:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.192023:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.192026:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.192344:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.192347:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.192348:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.192349:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.192352:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.192353:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.192355:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.192356:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.192357:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.192357:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.192360:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.192361:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.192363:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.192364:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.192364:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.192366:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.192366:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.192367:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.192367:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.192372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.192379:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.192382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.192382:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.192383:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.192384:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.192386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.192531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.192538:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.192540:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.192541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.192544:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.192547:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.192548:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.192549:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.192551:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.192552:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.192553:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.192556:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.192558:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.192560:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.192560:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.192561:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.192563:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.192564:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.192565:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.192565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.192569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.192576:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.192579:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.192579:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.192580:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.192582:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.192583:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.192878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.192884:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.192886:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.192887:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.192887:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.192889:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.192891:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.192932:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.192935:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.192935:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.192938:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.192940:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.192941:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677473.192943:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.192944:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.192945:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.192945:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677473.192948:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.192950:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.192953:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.192954:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677473.192954:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.192956:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.192956:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.192958:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.192959:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.192959:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677473.192962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.192971:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:10.0:1619677473.192972:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:1.0:1619677473.192973:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.192974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.192974:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.192975:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677473.192975:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:1.0:1619677473.192975:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.192977:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.192980:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677473.193227:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.193229:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.193230:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.193232:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.193235:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.193236:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.193238:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.193239:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.193240:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.193241:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.193244:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.193245:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.193248:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.193248:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.193249:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.193251:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.193251:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.193252:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.193254:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.193254:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.193264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.193266:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.193267:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.193268:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.193268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.193269:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.193270:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.193270:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.193270:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.193273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.193282:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.193285:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.193286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.193286:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.193287:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.193288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.193601:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.193604:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.193606:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.193606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.193609:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.193611:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.193612:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.193613:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.193614:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.193615:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.193618:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.193620:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.193622:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.193623:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.193624:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.193626:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.193626:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.193627:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.193628:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.193632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.193640:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.193642:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.193643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.193644:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.193646:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.193648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.193651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.193657:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.193659:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.193660:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.193660:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.193661:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.193662:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.193771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.193778:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.193780:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.193781:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.193781:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.193783:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.193785:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.193988:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.193990:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.193992:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.193993:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.193994:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.193996:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.193997:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.193998:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.193999:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.194000:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.194003:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.194005:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.194007:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.194008:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.194009:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.194010:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.194011:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.194012:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.194012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.194017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.194025:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.194028:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.194029:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.194029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.194031:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.194035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.194038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.194040:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.194042:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.194043:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.194043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.194044:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.194045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.194092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.194095:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.194098:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677473.194100:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.194101:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.194103:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.194105:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677473.194106:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.194107:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.194109:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.194110:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677473.194113:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.194115:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677473.194118:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.194119:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677473.194119:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677473.194121:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677473.194122:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.194123:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.194124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:13.0:1619677473.194136:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.194138:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.194139:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.194141:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.194143:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677473.194143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.194153:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677473.194156:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677473.194157:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.194158:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677473.194158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.194159:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:26.0:1619677473.194395:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.194398:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.194399:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.194401:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.194404:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.194406:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677473.194407:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.194408:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.194409:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.194410:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677473.194414:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.194415:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.194418:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.194419:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677473.194420:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.194421:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.194422:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.194422:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.194423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.194428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.194437:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.194440:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.194440:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.194441:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.194443:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.194446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677473.194805:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.194808:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.194809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.194811:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.194813:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.194814:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.194815:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.194817:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.194818:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.194818:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.194821:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.194823:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.194826:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.194827:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.194827:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.194829:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.194829:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.194830:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.194830:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.194835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.194843:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.194846:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.194847:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.194847:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.194849:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.194853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677473.194973:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.194976:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.194978:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.194979:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.194980:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.194981:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.194983:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.194984:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.194985:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.194986:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.194989:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.194991:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.194994:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.194994:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.194995:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.194997:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.194997:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.194999:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.195000:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.195001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677473.195003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.195009:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677473.195009:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.195012:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.195012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677473.195012:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.195013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.195013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677473.195013:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.195015:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.195018:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677473.195123:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.195126:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.195128:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.195130:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.195130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.195132:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677473.195133:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.195135:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.195135:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.195136:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677473.195140:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.195141:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677473.195144:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.195145:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677473.195145:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677473.195147:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677473.195147:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.195148:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.195149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.195152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.195159:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.195161:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.195162:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.195162:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.195164:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.195167:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.195449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.195456:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.195459:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.195460:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.195460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.195462:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.195464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.195587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.195593:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.195595:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.195596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.195596:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.195598:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.195600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.195603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.195609:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.195611:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.195612:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.195612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.195613:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.195614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.195998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.196004:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.196007:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.196007:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.196009:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.196011:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.196013:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.196014:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.196015:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.196016:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.196017:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.196020:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.196022:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.196024:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.196025:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.196026:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.196027:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.196028:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.196029:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.196029:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.196041:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.196044:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.196045:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.196045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.196047:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.196049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.196217:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677473.196220:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677473.196221:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:10.0:1619677473.196222:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.196225:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.196227:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.196228:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.196229:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.196230:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.196231:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.196234:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.196236:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.196239:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.196240:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.196240:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.196242:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.196242:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677473.196243:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:10.0:1619677473.196245:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677473.196245:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:8.2:1619677473.196248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.196256:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677473.196258:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:17.0:1619677473.196259:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.196260:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.196260:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.196261:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:26.0:1619677473.196261:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677473.196262:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000400:00000200:17.0:1619677473.196262:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.196266:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677473.196411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.196419:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677473.196419:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.196422:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.196423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677473.196423:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.196423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.196427:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.196427:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.196430:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.196430:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.196431:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677473.196432:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677473.196433:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.196434:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.196435:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.196436:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.196436:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.196437:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.196438:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677473.196439:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677473.196443:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677473.196445:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.196446:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.196448:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.196448:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677473.196449:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.196450:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.196451:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677473.196451:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.196451:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:18.0:1619677473.196452:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677473.196452:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:18.0:1619677473.196453:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.196454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677473.196454:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.196454:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.196455:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.196457:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.196457:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677473.196458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.196467:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677473.196468:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.196469:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:26.0:1619677473.196470:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677473.196470:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.196471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677473.196471:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.196472:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677473.196472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.196472:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.196473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677473.196473:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:26.0:1619677473.196474:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.196475:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.196476:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.196478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.196555:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.196558:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.196560:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.196561:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.196563:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.196564:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.196566:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.196567:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.196569:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.196570:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.196573:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.196575:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.196578:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.196579:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.196579:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.196581:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.196581:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.196582:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.196583:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.196587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.196593:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.196596:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.196597:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.196597:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.196599:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.196601:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677473.197186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:9.2:1619677473.197189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677473.197194:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.197195:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:17.0:1619677473.197196:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677473.197197:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677473.197197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:24.0:1619677473.197198:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.197198:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.197199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:17.0:1619677473.197200:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.197202:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.197203:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677473.197205:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677473.197205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:9.2:1619677473.197211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677473.197213:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.197216:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.197217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.197217:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.197217:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.197218:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.197219:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677473.197219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677473.197220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.197220:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.197221:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.197222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.197489:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.197492:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.197493:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.197495:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.197498:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.197499:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.197500:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.197502:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.197503:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.197504:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.197507:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.197508:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.197511:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.197512:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.197513:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.197514:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.197515:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.197515:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.197516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.197540:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.197543:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.197544:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.197544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.197546:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.197549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.198162:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.198164:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.198167:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.198167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.198170:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.198172:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.198173:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.198174:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.198175:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.198176:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.198180:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.198181:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.198184:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.198185:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.198186:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.198188:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.198188:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.198189:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.198190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677473.198263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.198266:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.198269:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:21.0:1619677473.198271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.198271:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.198274:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.198276:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.198277:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.198279:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.198280:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.198282:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.198285:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.198287:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677473.198287:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.198289:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.198290:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677473.198290:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.198291:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.198291:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.198298:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.198298:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.198299:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.198299:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.198300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:26.0:1619677473.198301:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.198302:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.198304:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.198305:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.198306:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.198307:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.198307:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.198309:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:26.0:1619677473.198310:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.198310:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.198311:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.198311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.198312:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.198315:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.198315:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.198316:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.198316:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.198318:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.198319:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.198320:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.198320:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.198332:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.198335:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.198336:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.198336:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.198338:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.198341:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.198443:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.198446:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.198447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.198449:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.198452:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.198453:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.198454:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.198455:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.198457:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.198458:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.198461:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.198464:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.198465:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.198466:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.198466:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.198468:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.198468:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.198469:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.198470:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.198471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.198473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677473.198481:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677473.198485:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.198486:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.198486:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:24.0:1619677473.198487:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.198487:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.198488:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.198489:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.198489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.198490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677473.198491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.198491:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.198493:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.198494:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.198496:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.198497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.198499:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.199099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.199106:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.199109:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.199110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.199110:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.199112:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.199114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.199734:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.199737:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.199739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.199740:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.199741:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.199743:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.199744:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.199745:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.199746:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.199747:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.199749:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.199751:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.199752:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.199753:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.199754:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.199755:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.199755:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.199756:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.199756:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.199761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.199767:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.199769:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.199770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.199770:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.199771:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.199772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.200180:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.200182:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.200183:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.200185:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.200188:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.200189:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.200190:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.200191:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.200192:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.200193:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.200196:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.200197:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.200199:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.200200:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.200201:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.200202:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.200203:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.200203:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.200204:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.200216:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.200218:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.200218:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.200220:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.200222:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.200223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677473.200224:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.200227:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.200228:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.200230:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.200233:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.200235:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.200236:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.200237:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.200238:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.200239:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.200243:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.200244:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.200247:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.200248:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.200248:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.200250:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.200251:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.200252:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677473.200253:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.200254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.200256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.200264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677473.200264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.200266:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.200266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677473.200267:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677473.200267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677473.200268:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.200269:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.200271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.200272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.200285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.200293:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.200300:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.200301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.200302:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.200302:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.200303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.200487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.200496:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.200498:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.200499:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.200501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.200501:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.200504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.200666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.200673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.200675:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.200676:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.200676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.200678:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.200680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.200686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.200691:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.200694:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.200695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.200695:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.200696:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.200696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.200826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.200833:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.200836:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.200836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.200838:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.200841:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.200842:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.200844:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.200845:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.200846:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.200847:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.200850:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.200851:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.200854:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.200855:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.200856:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.200857:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.200858:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.200858:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.200859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.200871:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.200874:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.200876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.200876:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.200877:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.200880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.200889:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.200892:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.200893:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.200893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.200894:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.200895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.201256:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f392800] (21)++
00000800:00000200:26.0:1619677473.201259:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.45@o2ib3
00000800:00000200:10.0:1619677473.201259:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f392800] (22)--
00000800:00000200:10.0:1619677473.201260:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677473.201262:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.45@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.201262:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.201264:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.201266:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.201267:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.201269:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677473.201270:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.201272:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.201273:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.201274:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677473.201276:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.201277:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.201278:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.201279:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.201280:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1832 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:10.0:1619677473.201280:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.201281:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.201285:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.201285:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.201286:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677473.201287:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677473.201287:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.201289:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677473.201290:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f392800] (21)++
00000800:00000200:10.0:1619677473.201290:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.201291:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f392800] (22)--
00000800:00000200:26.0:1619677473.201291:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f392800] (21)--
00000800:00000200:10.0:1619677473.201291:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.201292:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.201292:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.2:1619677473.201297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677473.201298:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.201299:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.201300:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.201300:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677473.201307:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677473.201309:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.201310:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.201311:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.201312:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677473.201312:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.201313:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.201313:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.201313:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.201315:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.201317:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.201319:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.201344:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f392800] (21)++
00000800:00000200:26.0:1619677473.201347:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.45@o2ib3
00000800:00000200:10.0:1619677473.201348:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f392800] (22)--
00000400:00000200:26.0:1619677473.201350:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.45@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.201353:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.201355:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.201356:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.201358:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.201359:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.201359:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.201363:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.201364:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 2776 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.201368:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.201368:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.201369:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.201371:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.201371:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f392800] (21)++
00000800:00000200:26.0:1619677473.201372:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f392800] (22)++
00000800:00000200:26.0:1619677473.201374:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f392800] (23)--
00000800:00000200:26.0:1619677473.201375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f392800] (22)--
00000800:00000200:8.2:1619677473.201378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.201386:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677473.201387:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f392800] (22)++
00000800:00000200:21.0:1619677473.201389:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.201390:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.201390:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f392800] (23)--
00000800:00000200:10.0:1619677473.201390:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f392800] (22)--
00000400:00000200:21.0:1619677473.201391:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.201391:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f392800] (21)--
00000800:00000200:20.0:1619677473.201392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.201395:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.201399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.201406:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.201409:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.201410:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.201411:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.201412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.201413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.201849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677473.201852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677473.201854:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677473.201856:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.201857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677473.201859:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.201860:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.201861:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.201863:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.201863:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.201864:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.201868:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.201869:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.201871:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.201872:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.201873:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.201875:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.201875:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677473.201876:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677473.201876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.201880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.201888:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.201890:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.201891:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.201892:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.201893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.201894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.201951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.201957:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.201960:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.201962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.201963:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.201966:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.201968:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.201969:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.201970:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.201971:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.201972:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.201975:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.201977:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.201979:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.201980:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.201981:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.201982:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.201983:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.201984:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.201985:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.201986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.201989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.201992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.201998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.201999:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.201999:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677473.202000:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.202000:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.202000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.202001:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.202001:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.202002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677473.202003:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677473.202278:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (21)++
00000800:00000200:20.0:1619677473.202281:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.3.1@o2ib3
00000400:00000200:20.0:1619677473.202283:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.202284:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000400:00000200:20.0:1619677473.202285:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.202286:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000400:00000200:20.0:1619677473.202288:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677473.202289:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.202290:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.202291:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.202292:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:21.0:1619677473.202297:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.202300:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.202300:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677473.202302:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677473.202302:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:21.0:1619677473.202303:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.202304:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.202305:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.202306:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.2:1619677473.202308:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:21.0:1619677473.202310:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.202311:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677473.202312:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.202313:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:20.0:1619677473.202313:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.202314:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677473.202315:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.202315:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:21.0:1619677473.202316:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677473.202316:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677473.202316:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:2.0:1619677473.202316:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:20.0:1619677473.202317:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677473.202317:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:21.0:1619677473.202318:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677473.202318:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)--
00000800:00000200:18.0:1619677473.202318:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677473.202319:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (22)++
00000400:00000200:2.0:1619677473.202319:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.202320:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (22)--
00000800:00000200:21.0:1619677473.202321:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (21)--
00000400:00000200:2.0:1619677473.202321:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.202322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:2.0:1619677473.202323:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.202324:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96510518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.202326:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.202327:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.202328:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.202330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (21)++
00000800:00000200:18.0:1619677473.202331:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:20.0:1619677473.202333:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:18.0:1619677473.202333:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:12.0:1619677473.202333:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:2.0:1619677473.202333:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:18.0:1619677473.202334:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.202336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)--
00000400:00000200:18.0:1619677473.202336:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.202336:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:2.0:1619677473.202336:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.202337:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677473.202337:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.202339:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.202340:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.202340:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.202341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677473.202341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:2.0:1619677473.202341:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:20.0:1619677473.202342:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677473.202342:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:20.0:1619677473.202343:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677473.202343:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000400:00000200:20.0:1619677473.202344:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.202345:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677473.202345:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.202345:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.202345:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.202345:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677473.202346:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677473.202347:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.202347:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.202348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.202348:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.202348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:20.0:1619677473.202349:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.202349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677473.202349:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.0:1619677473.202351:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.2:1619677473.202353:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677473.202357:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.202358:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.202358:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677473.202359:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:20.0:1619677473.202360:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.202361:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (21)++
00000800:00000200:20.0:1619677473.202361:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (22)--
00000800:00000200:2.0:1619677473.202361:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:20.0:1619677473.202362:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (21)--
00000800:00000200:2.0:1619677473.202362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.202363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.0:1619677473.202370:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (21)++
00000800:00000200:21.0:1619677473.202373:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:22.0:1619677473.202374:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677473.202375:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.202376:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)--
00000800:00000200:22.0:1619677473.202377:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.202378:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.202378:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.202379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:21.0:1619677473.202380:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677473.202381:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:21.0:1619677473.202382:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.202383:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.202384:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.202385:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677473.202386:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:22.0:1619677473.202387:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.202388:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000400:00000200:21.0:1619677473.202389:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677473.202391:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.202392:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677473.202393:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.202395:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677473.202395:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:21.0:1619677473.202396:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:20.0:1619677473.202396:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.202397:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677473.202397:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677473.202398:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.202398:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.202399:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677473.202400:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.202400:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000400:00000200:20.0:1619677473.202400:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677473.202400:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.202401:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.202402:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b84000] (24)++
00000800:00000200:6.0:1619677473.202402:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.202403:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000400:00000200:20.0:1619677473.202403:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.202403:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.202404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000800:00000200:21.0:1619677473.202405:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000800:00000200:20.0:1619677473.202405:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677473.202406:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:20.0:1619677473.202408:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.202409:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.202409:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677473.202409:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:21.0:1619677473.202411:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.202411:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677473.202412:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)++
00000800:00000200:20.0:1619677473.202412:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000400:00000200:21.0:1619677473.202413:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677473.202413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000400:00000200:21.0:1619677473.202414:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96510514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.202414:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000400:00000200:21.0:1619677473.202415:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.202415:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677473.202416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000400:00000200:21.0:1619677473.202417:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.202417:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:9.2:1619677473.202417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677473.202418:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000400:00000200:21.0:1619677473.202419:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677473.202422:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.202423:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677473.202424:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.202425:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.202425:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:20.0:1619677473.202426:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677473.202427:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96510513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677473.202427:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:20.0:1619677473.202428:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.202429:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677473.202429:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.202429:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.202430:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.202431:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.202433:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.202434:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (21)++
00000400:00000200:20.0:1619677473.202434:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.202435:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.202435:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677473.202437:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677473.202437:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:20.0:1619677473.202437:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.0:1619677473.202438:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:21.0:1619677473.202439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:21.0:1619677473.202439:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:20.0:1619677473.202440:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.202441:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:20.0:1619677473.202441:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:20.0:1619677473.202442:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:9.2:1619677473.202442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:21.0:1619677473.202443:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.202443:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:20.0:1619677473.202444:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:20.0:1619677473.202445:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677473.202445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000400:00000200:21.0:1619677473.202446:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.202446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)--
00000400:00000200:21.0:1619677473.202447:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:21.0:1619677473.202448:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96510512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.202449:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.202450:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677473.202450:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:21.0:1619677473.202451:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677473.202452:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:18.0:1619677473.202452:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.202453:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.202453:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677473.202454:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000400:00000200:21.0:1619677473.202456:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677473.202456:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.202456:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:20.0:1619677473.202457:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677473.202458:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677473.202459:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677473.202459:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:20.0:1619677473.202460:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.202460:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.202462:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677473.202462:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:12.0:1619677473.202462:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677473.202463:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:20.0:1619677473.202463:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96510511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.202463:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.202464:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677473.202465:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:20.0:1619677473.202465:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.202466:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.202467:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677473.202469:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:13.0:1619677473.202469:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677473.202470:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b84000] (24)++
00000400:00000200:20.0:1619677473.202470:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.202470:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.0:1619677473.202471:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000800:00000200:21.0:1619677473.202472:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677473.202472:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677473.202472:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:21.0:1619677473.202473:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000800:00000200:13.0:1619677473.202473:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.202473:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677473.202474:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:20.0:1619677473.202474:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.202474:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.202474:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.0:1619677473.202475:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (22)++
00000400:00000200:21.0:1619677473.202476:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.202476:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (23)++
00000800:00000200:20.0:1619677473.202477:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (24)--
00000800:00000200:22.0:1619677473.202478:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000400:00000200:21.0:1619677473.202479:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.202479:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)++
00000800:00000200:20.0:1619677473.202480:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000800:00000200:22.0:1619677473.202481:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000400:00000200:21.0:1619677473.202481:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677473.202481:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677473.202481:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000400:00000200:22.0:1619677473.202482:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.202482:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96510510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.202482:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:20.0:1619677473.202483:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:6.0:1619677473.202483:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000400:00000200:6.0:1619677473.202483:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.202484:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:21.0:1619677473.202484:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.202485:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.202486:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677473.202486:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:21.0:1619677473.202490:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.202492:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677473.202493:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:20.0:1619677473.202493:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.202493:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.202494:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.202494:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.202495:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:20.0:1619677473.202495:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.202496:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.202497:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.202497:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677473.202498:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.202499:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.202500:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.202501:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.2:1619677473.202504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:20.0:1619677473.202505:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.202506:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)++
00000800:00000200:20.0:1619677473.202506:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.202507:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000800:00000200:21.0:1619677473.202508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677473.202508:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.202509:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000800:00000200:21.0:1619677473.202509:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677473.202509:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.202510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:20.0:1619677473.202510:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (23)++
00000800:00000200:21.0:1619677473.202511:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (24)++
00000800:00000200:20.0:1619677473.202512:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (25)--
00000800:00000200:21.0:1619677473.202513:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (24)--
00000800:00000200:20.0:1619677473.202513:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (22)++
00000400:00000200:21.0:1619677473.202514:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.202514:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)--
00000800:00000200:20.0:1619677473.202514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)--
00000400:00000200:21.0:1619677473.202515:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.202515:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (21)++
00000800:00000200:20.0:1619677473.202516:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:21.0:1619677473.202517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.202518:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.202519:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.202519:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.202520:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.202521:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.202522:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.202522:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.202523:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.202525:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.202525:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.202526:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.202527:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.202528:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:21.0:1619677473.202529:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000400:00000200:20.0:1619677473.202529:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.202531:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.202532:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.202533:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.202533:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.202533:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.202534:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677473.202535:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677473.202535:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:21.0:1619677473.202536:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.202537:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.202538:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.202539:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.202540:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:20.0:1619677473.202541:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b84000] (24)++
00000800:00000200:20.0:1619677473.202543:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000800:00000200:20.0:1619677473.202543:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677473.202545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677473.202545:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000400:00000200:21.0:1619677473.202546:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.202547:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:21.0:1619677473.202548:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.202549:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.202549:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.202550:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.202550:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:21.0:1619677473.202551:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (23)++
00000400:00000200:20.0:1619677473.202551:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.202551:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677473.202552:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (24)++
00000400:00000200:6.0:1619677473.202552:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.202553:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (24)--
00000800:00000200:21.0:1619677473.202553:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000400:00000200:20.0:1619677473.202553:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:17.0:1619677473.202554:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:6.0:1619677473.202554:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.202554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:20.0:1619677473.202555:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96510506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.202557:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.202557:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.202558:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.202559:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.202560:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)++
00000400:00000200:17.0:1619677473.202560:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.202561:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000800:00000200:21.0:1619677473.202562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000800:00000200:6.0:1619677473.202562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677473.202563:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000400:00000200:20.0:1619677473.202563:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.202564:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b84000] (24)--
00000800:00000200:21.0:1619677473.202564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:6.0:1619677473.202564:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.202565:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000800:00000200:20.0:1619677473.202565:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:17.0:1619677473.202565:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677473.202566:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.202567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.202567:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:20.0:1619677473.202567:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677473.202567:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96510505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.202567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.202568:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.202568:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:17.0:1619677473.202569:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.202570:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.202570:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:6.0:1619677473.202570:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.202571:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000400:00000200:17.0:1619677473.202571:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.202572:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677473.202572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000400:00000200:17.0:1619677473.202572:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677473.202572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:21.0:1619677473.202573:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.202573:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)++
00000400:00000200:21.0:1619677473.202574:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677473.202575:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000400:00000200:21.0:1619677473.202576:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96510504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677473.202576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:21.0:1619677473.202577:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.202577:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677473.202578:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.202579:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677473.202580:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677473.202582:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.202584:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677473.202584:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.202585:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.202585:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677473.202586:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:20.0:1619677473.202586:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677473.202586:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677473.202586:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:20.0:1619677473.202587:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96510503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.202587:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:1.0:1619677473.202587:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.202588:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.202590:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677473.202590:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677473.202590:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:1.0:1619677473.202590:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.202591:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677473.202592:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677473.202593:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677473.202593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.202594:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.202594:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:17.0:1619677473.202594:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.202594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677473.202595:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:20.0:1619677473.202595:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.202597:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677473.202597:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.2:1619677473.202597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.202600:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.202601:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:20.0:1619677473.202601:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.0:1619677473.202602:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (23)++
00000800:00000200:12.0:1619677473.202602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:21.0:1619677473.202603:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)--
00000800:00000200:20.0:1619677473.202603:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:9.2:1619677473.202603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677473.202604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:20.0:1619677473.202604:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)++
00000800:00000200:12.0:1619677473.202604:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:21.0:1619677473.202605:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000800:00000200:17.0:1619677473.202605:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:12.0:1619677473.202605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.202606:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b84000] (24)++
00000800:00000200:1.0:1619677473.202606:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:21.0:1619677473.202607:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:20.0:1619677473.202608:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000800:00000200:17.0:1619677473.202608:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677473.202608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677473.202608:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.202608:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677473.202608:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:21.0:1619677473.202609:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.202609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000800:00000200:17.0:1619677473.202609:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677473.202610:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000800:00000200:17.0:1619677473.202610:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:2.0:1619677473.202610:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:1.0:1619677473.202610:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.202611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:12.0:1619677473.202611:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:1.0:1619677473.202611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:20.0:1619677473.202612:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.1@o2ib3
00000800:00000200:12.0:1619677473.202613:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677473.202613:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:26.0:1619677473.202614:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677473.202614:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.1@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.202614:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.202615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677473.202615:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:20.0:1619677473.202616:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.202616:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677473.202616:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.202617:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.202617:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.202618:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:12.0:1619677473.202618:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.202619:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677473.202619:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677473.202620:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.202622:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.202623:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.202624:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.202625:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:9.2:1619677473.202625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:26.0:1619677473.202626:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.202628:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677473.202629:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677473.202629:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:26.0:1619677473.202630:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.202630:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677473.202630:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.202630:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:26.0:1619677473.202631:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.202632:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.202633:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.202633:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.202633:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677473.202634:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677473.202634:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.202634:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000400:00000200:26.0:1619677473.202635:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.202635:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000400:00000200:21.0:1619677473.202636:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677473.202636:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:21.0:1619677473.202637:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677473.202640:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.202641:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.202641:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:20.0:1619677473.202641:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:21.0:1619677473.202643:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.202643:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677473.202644:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.202644:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (24)++
00000800:00000200:20.0:1619677473.202644:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (25)--
00000800:00000200:20.0:1619677473.202645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (24)--
00000800:00000200:21.0:1619677473.202646:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677473.202646:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)++
00000800:00000200:20.0:1619677473.202646:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b84000] (24)--
00000800:00000200:26.0:1619677473.202647:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.202647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (23)--
00000800:00000200:26.0:1619677473.202648:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.202648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (22)--
00000800:00000200:26.0:1619677473.202649:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.202649:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.202649:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:8.2:1619677473.202649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677473.202650:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.202651:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.202651:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.202652:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.202652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.0:1619677473.202652:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677473.202653:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b84000] (21)++
00000800:00000200:21.0:1619677473.202654:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b84000] (22)--
00000800:00000200:21.0:1619677473.202654:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b84000] (21)--
00000800:00000200:17.0:1619677473.202657:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:9.2:1619677473.202658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677473.202660:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.202661:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.202661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:9.2:1619677473.202662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:17.0:1619677473.202663:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.202665:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.202665:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:17.0:1619677473.202666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.202668:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.202668:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.202669:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:24.0:1619677473.202669:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:6.0:1619677473.202669:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.202670:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.202670:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.202673:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:22.0:1619677473.202673:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.202673:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.202674:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.202675:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.202677:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.202677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.202677:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677473.202679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.203415:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.203418:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.203419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.203420:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.203423:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.203425:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.203426:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.203427:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.203428:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.203429:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.203431:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.203433:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.203436:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.203436:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.203437:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.203439:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.203439:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.203441:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.203442:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.203442:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677473.203453:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.203456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.203456:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.203456:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.203457:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.203458:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.203458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677473.203459:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.203460:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.203462:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.203478:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.203481:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.203482:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.203482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.203483:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.203484:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.204094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:9.2:1619677473.204096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677473.204097:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.204100:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.204102:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.204103:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677473.204103:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:20.0:1619677473.204105:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.204106:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677473.204106:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.204107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:20.0:1619677473.204107:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.204107:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.204109:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.204110:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.204111:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.204112:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677473.204112:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:20.0:1619677473.204115:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.204117:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.204119:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.204120:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.204121:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.204122:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.204123:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.204124:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.204125:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.204129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.204137:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.204139:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.204140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.204140:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.204142:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.2:1619677473.204143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:1.0:1619677473.204144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677473.204150:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677473.204153:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:2.0:1619677473.204154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:18.0:1619677473.204156:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.204158:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.204160:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677473.204161:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.204162:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.204163:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.204164:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677473.204167:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.204169:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.204172:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.204173:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677473.204173:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677473.204175:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677473.204176:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677473.204177:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:18.0:1619677473.204177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:21.0:1619677473.204189:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.204191:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.204192:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.204194:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.204194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.204196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.204327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.204334:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.204336:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.204337:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.204337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.204340:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.204342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.204534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.204540:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.204543:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.204543:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.204544:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.204545:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.204547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.204941:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.204943:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677473.204944:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677473.204946:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.204949:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.204950:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677473.204951:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.204953:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.204953:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.204954:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677473.204957:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.204959:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.204962:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.204963:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677473.204963:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.204965:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.204965:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.204966:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677473.204966:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677473.204971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.204979:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.204982:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.204982:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.204983:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.204985:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.204988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677473.205070:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.205072:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.205074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.205075:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.205077:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.205079:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.205080:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.205081:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.205082:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.205083:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.205086:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.205088:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.205089:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.205090:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.205090:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.205092:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.205092:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.205093:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.205093:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.205097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.205103:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.205105:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.205106:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.205107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.205108:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.205109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677473.205495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.205503:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.205506:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.205507:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.205509:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.205510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.205512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677473.205595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.205601:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.205604:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.205605:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.205607:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.205609:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.205610:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.205612:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.205613:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.205614:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.205615:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.205618:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.205620:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.205622:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.205623:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.205624:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.205625:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.205626:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.205627:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.205628:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.205629:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.205633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.205638:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.205640:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677473.205640:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.205641:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.205642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.205643:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.205644:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.205644:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.205646:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.205648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.205657:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.205660:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.205660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.205661:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.205661:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.205662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.206213:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.206215:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.206216:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.206219:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.206221:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.206223:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.206224:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.206226:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.206227:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.206228:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.206231:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.206233:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.206236:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.206236:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.206237:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.206239:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.206239:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.206240:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.206240:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.206252:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.206255:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.206256:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.206257:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.206259:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.206262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.206283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.206290:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.206292:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.206293:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.206298:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.206300:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.206302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.206305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.206306:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.206308:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.206309:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.206309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.206310:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.206311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.206591:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.206593:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.206594:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.206596:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.206600:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.206601:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.206602:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.206603:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.206604:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.206605:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.206608:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.206609:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.206612:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.206613:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.206613:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.206615:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.206616:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.206616:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.206617:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677473.206629:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.206632:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.206633:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.206633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.206635:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.206638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.207224:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.207227:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.207228:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.207230:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.207232:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.207234:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.207235:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.207237:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.207238:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.207239:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.207242:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.207243:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.207246:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.207247:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.207247:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.207249:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.207249:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.207250:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.207250:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.207255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.207264:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.207266:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.207267:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.207269:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.207269:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.207271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.207539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677473.207547:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.207552:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.207554:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.207557:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.207560:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.207563:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677473.207565:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.207567:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.207569:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.207571:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677473.207576:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.207579:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677473.207583:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.207585:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677473.207586:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677473.207588:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677473.207589:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.207590:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.207591:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677473.207595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.207602:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.207605:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.207606:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.207607:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.207609:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.207612:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677473.207827:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.207830:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.207831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.207832:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.207835:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.207836:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.207837:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.207838:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.207839:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.207840:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.207843:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.207845:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.207848:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.207849:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.207849:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.207851:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.207851:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.207852:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.207853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.207866:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.207869:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.207871:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.207871:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.207873:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.207876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.207894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:9.2:1619677473.207895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677473.207897:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.207897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.207899:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.207902:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.207902:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:22.0:1619677473.207904:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677473.207905:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.207905:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.207906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.207906:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.207907:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.207908:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.207908:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.207909:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677473.207914:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.207915:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.207915:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.207919:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.207920:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677473.207920:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.207922:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.207923:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.207924:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677473.207925:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677473.207926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677473.207928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.207934:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.207936:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677473.207936:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000400:00000200:24.0:1619677473.207937:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.207938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.207938:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000400:00000200:24.0:1619677473.207939:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.207939:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.207940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:24.0:1619677473.207942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.208438:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.208441:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.208441:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.208444:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.208446:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.208448:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.208449:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.208450:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.208451:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.208452:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.208456:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.208457:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.208460:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.208461:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.208462:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.208464:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.208464:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.208465:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.208466:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.208467:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.208470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.208478:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677473.208480:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:18.0:1619677473.208481:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:26.0:1619677473.208482:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:18.0:1619677473.208482:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.208482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.208483:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.208484:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677473.208485:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.208488:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:10.0:1619677473.208615:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.208617:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.208618:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.208620:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.208623:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.208625:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.208626:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.208627:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.208628:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.208629:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.208632:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.208634:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.208637:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.208638:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.208638:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.208640:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.208640:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.208641:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.208641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.208654:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.208657:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.208658:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.208658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.208660:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.208663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.208781:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677473.208784:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.18@o2ib3
00000800:00000200:22.0:1619677473.208785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (22)--
00000400:00000200:6.0:1619677473.208787:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.18@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.208790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.208791:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.208792:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.208794:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.208795:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.208796:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.208799:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.208801:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 968 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.208803:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.208804:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.208805:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.208807:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.208807:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f7d3000] (21)++
00000800:00000200:6.0:1619677473.208808:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f7d3000] (22)--
00000800:00000200:6.0:1619677473.208808:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f7d3000] (21)--
00000800:00000200:20.0:1619677473.208820:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.208823:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.208823:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.208825:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.208826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.208827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.208854:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.208857:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.208858:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.208858:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.208859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.208860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.209336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.209342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.209345:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.209346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.209348:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.209350:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.209352:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.209353:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.209355:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.209355:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.209357:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.209360:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.209362:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.209364:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.209365:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.209366:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.209367:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.209368:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.209369:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.209370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.209374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.209383:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.209386:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.209387:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.209387:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.209389:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.209391:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677473.209809:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.209812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.209812:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677473.209814:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.209816:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.209817:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677473.209818:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.209820:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.209821:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.209822:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677473.209824:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.209826:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677473.209829:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.209830:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677473.209830:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677473.209832:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677473.209832:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.209833:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.209833:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677473.209839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.209846:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.209849:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.209850:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.209850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.209852:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.209854:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.209882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.209889:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.209891:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.209892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.209892:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.209894:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.209896:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.210214:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.210217:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.210217:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.210219:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.210222:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.210223:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677473.210224:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.210226:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.210227:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.210228:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.2:1619677473.210230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677473.210231:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.210233:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677473.210236:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.210237:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677473.210237:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677473.210238:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:22.0:1619677473.210239:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.210239:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.210240:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:21.0:1619677473.210240:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:22.0:1619677473.210241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:21.0:1619677473.210241:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.210243:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.210244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:9.2:1619677473.210245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677473.210246:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.210249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677473.210253:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.210256:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.210257:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.210257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677473.210258:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000400:00000200:24.0:1619677473.210259:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.210260:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677473.210261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.210261:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.210262:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.210263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677473.210263:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:9.2:1619677473.210263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.210269:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.210272:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.210273:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.210273:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.210274:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.210275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.210455:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.210457:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.210458:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.210460:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.210462:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.210463:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.210464:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.210466:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.210467:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.210468:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.210471:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.210473:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.210474:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.210475:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.210476:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.210477:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.210478:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.210478:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.210479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677473.210484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:9.2:1619677473.210484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.210487:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.210490:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677473.210490:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:20.0:1619677473.210492:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.210492:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.210492:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.210493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.210493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.210494:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.210495:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.210496:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677473.210496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:20.0:1619677473.210497:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.210498:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.210499:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.210500:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.210504:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.210506:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.210508:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.210509:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.210509:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.210511:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.210512:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.210512:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.210513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.210517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.210525:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.210528:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.210529:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.210529:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.210531:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.210533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677473.210824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.210830:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.210832:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.210833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.210835:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.210837:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.210838:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.210839:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.210841:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.210842:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.210843:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.210845:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.210847:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.210849:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.210850:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.210851:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.210852:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.210853:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.210853:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.210854:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.210866:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.210869:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.210870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.210870:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.210872:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.210874:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.211323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.211330:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.211333:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.211333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.211334:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.211336:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.211338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.211468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.211474:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.211477:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.211478:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.211481:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.211482:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677473.211483:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.211484:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677473.211485:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.211485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677473.211486:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.211487:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.211489:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.211489:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.211490:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677473.211491:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.211493:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677473.211494:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.211495:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96510464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.211496:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.211497:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677473.211497:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677473.211498:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677473.211499:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.211500:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.211500:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:26.0:1619677473.211502:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.211502:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:9.2:1619677473.211503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677473.211503:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677473.211504:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677473.211504:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.211505:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677473.211507:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.211508:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.211508:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:9.2:1619677473.211508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.211509:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:26.0:1619677473.211510:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.211510:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.211511:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.211512:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677473.211512:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.211513:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.211513:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.211515:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.211516:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.211517:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677473.211518:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.211519:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.211519:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.211521:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.211522:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.211523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:22.0:1619677473.211524:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.211527:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.211528:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.211528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677473.211529:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:22.0:1619677473.211530:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.211532:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677473.211532:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.211533:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.211533:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.211534:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.211535:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.212704:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.212706:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.212707:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.212709:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.212712:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.212713:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.212714:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.212715:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.212716:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.212717:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.212720:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.212721:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.212724:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.212725:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.212725:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.212727:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.212728:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.212729:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.212729:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.212742:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.212745:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.212746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.212746:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.212748:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.212751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.212761:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.212764:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.212764:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.212765:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.212766:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.212766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.213461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.213469:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.213471:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.213472:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.213472:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.213475:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.213476:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.213517:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.213520:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.213520:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.213522:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.213526:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.213527:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.213528:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.213529:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.213530:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.213531:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.213534:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.213535:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.213537:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.213538:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.213539:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.213540:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.213541:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.213542:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.213543:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.213544:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.213546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.213552:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.213554:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.213555:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.213555:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.213556:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.213557:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677473.213559:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.213559:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.213560:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.213561:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677473.214465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.214471:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.214474:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.214475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.214477:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.214479:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.214480:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.214482:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.214483:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.214484:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.214485:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.214488:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.214490:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.214492:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.214493:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.214494:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.214495:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.214496:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.214497:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.214497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.214513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.214521:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.214524:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.214525:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.214525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.214527:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.214529:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.214536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.214544:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.214546:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.214547:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.214547:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.214548:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.214549:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677473.214733:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.214736:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.214738:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.214739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.214740:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.214742:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.214743:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.214744:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.214745:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.214746:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.214749:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.214751:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.214753:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.214754:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.214755:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.214756:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.214757:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.214758:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.214758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.214772:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.214773:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.214774:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.214775:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.214775:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.214776:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.214777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677473.214778:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:8.0:1619677473.214780:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.214780:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:8.0:1619677473.214781:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.214783:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.214785:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.214786:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.214787:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.214788:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.214792:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.214794:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.214797:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.214798:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.214798:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.214800:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.214801:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.214802:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.214803:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.214804:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.214806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.214814:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.214816:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.214816:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.214817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.214818:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677473.214818:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.214819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.214820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677473.214820:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.214823:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677473.215131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.215137:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.215140:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.215141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.215143:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.215144:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.215146:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.215147:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.215149:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.215150:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.215151:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.215154:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.215156:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.215158:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.215159:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.215160:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.215162:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.215162:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.215163:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.215163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.215175:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.215178:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.215178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.215179:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.215181:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.215183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.215195:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.215198:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.215199:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.215199:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.215200:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.215201:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.215549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.215556:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.215559:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.215559:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.215560:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.215562:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.215564:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.215867:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.215870:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.215871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.215873:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.215876:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.215877:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.215878:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.215880:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.215881:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.215882:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.215885:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.215887:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.215890:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.215890:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.215891:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.215893:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.215893:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.215894:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.215894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.215899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677473.215906:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677473.215907:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:26.0:1619677473.215909:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677473.215910:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:10.0:1619677473.215910:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677473.215911:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.215913:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.215913:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.215913:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.215915:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.215915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:26.0:1619677473.215917:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.215918:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.215919:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.215920:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.215921:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.215924:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.215926:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.215929:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.215930:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.215931:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.215932:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.215933:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.215934:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.215935:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.215936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.215947:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.215948:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.215949:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677473.215950:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677473.215950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.215951:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.215951:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.215952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.215953:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.215956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.216303:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.216306:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.216306:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.216308:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.216312:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.216313:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.216315:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.216316:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.216317:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.216318:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.216321:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.216322:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.216325:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.216325:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.216326:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.216327:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.216328:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.216328:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.216329:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.216342:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.216344:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.216345:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.216347:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.216349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.216349:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.216502:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677473.216504:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677473.216505:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:10.0:1619677473.216506:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.216509:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.216511:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.216512:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.216513:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.216514:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.216515:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.216518:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.216520:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.216522:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.216523:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.216524:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.216526:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.216526:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677473.216527:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677473.216527:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:9.2:1619677473.216532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.216540:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.216543:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.216544:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.216545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.216547:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.216551:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677473.216593:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677473.216596:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677473.216597:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677473.216598:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.216601:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.216603:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.216604:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.216605:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.216606:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.216607:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.216610:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.216611:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 600 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.216614:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.216615:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677473.216616:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.216618:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.216618:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677473.216618:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677473.216619:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:21.0:1619677473.216631:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.216633:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.216634:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.216636:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.216636:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.216639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.217517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.217524:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.217527:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.217528:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.217528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.217529:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.217531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.217657:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.217660:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.217661:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.217662:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.217665:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.217667:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.217668:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.217669:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.217670:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.217671:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.217674:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.217675:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.217678:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.217679:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.217679:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.217681:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.217682:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.217682:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.217683:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.217695:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.217698:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.217699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.217699:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.217702:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.217705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.218110:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.218113:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677473.218114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677473.218116:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.218119:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.218120:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.218121:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.218122:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.218124:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.218124:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.218127:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.218129:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.218132:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.218133:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.218133:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.218135:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.218135:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.218137:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677473.218138:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677473.218138:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677473.218141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.218149:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.218151:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677473.218152:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.218153:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000400:00000200:18.0:1619677473.218153:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.218153:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.218154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677473.218155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677473.218155:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.218159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.218336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677473.218342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.218344:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.218345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.218347:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.218349:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.218350:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.218351:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.218352:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.218354:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.218355:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.218358:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.218360:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.218362:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.218363:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.218364:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.218366:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.218366:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.218367:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.218367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677473.218372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.218381:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.218384:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.218385:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.218385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.218387:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.218390:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.218579:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.218582:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.218583:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.218585:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.218588:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.218589:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.218590:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.218592:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.218593:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.218593:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.218597:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.218598:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.218600:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.218601:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.218602:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.218603:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.218604:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.218605:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.218605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.218610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.218616:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.218619:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.218619:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.218620:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.218621:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.218624:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.218690:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.218693:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.218695:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.218696:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.218698:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.218699:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677473.218701:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.218702:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.218703:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.218704:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.218707:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.218709:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.218712:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.218713:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677473.218714:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677473.218716:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677473.218716:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.218717:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.218718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:21.0:1619677473.218916:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.218918:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.218920:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.218921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.218923:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.218924:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.218926:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.218927:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.218928:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.218929:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.218932:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.218933:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.218935:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.218936:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.218937:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.218939:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.218939:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.218940:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.218941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.218953:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.218955:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.218956:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.218956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.218958:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.218960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.219224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.219231:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.219233:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.219234:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.219234:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.219236:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.219238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.219257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.219263:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.219265:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.219266:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.219266:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.219267:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.219268:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.219287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677473.219293:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677473.219298:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677473.219302:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000400:00000200:1.0:1619677473.219304:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.219306:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.219308:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677473.219309:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.219310:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.219311:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.219312:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677473.219315:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.219317:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677473.219320:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.219321:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677473.219322:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677473.219323:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677473.219324:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.219324:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677473.219325:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677473.219330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.219337:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.219340:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.219341:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.219341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.219343:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.219346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.219351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.2:1619677473.219354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677473.219357:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.219360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677473.219360:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677473.219360:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.219361:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.219362:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.219362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677473.219363:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.219364:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.219366:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.219369:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.219370:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.219372:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.219373:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.219374:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.219375:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.219379:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.219381:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.219384:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.219385:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.219386:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.219387:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.219388:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.219389:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.219389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677473.219426:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.219429:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.219429:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.219432:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.219434:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.219435:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677473.219436:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.219438:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.219439:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.219440:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677473.219443:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.219445:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.219446:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.219447:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.219448:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.219449:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.219449:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.219450:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.219451:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.219455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:24.0:1619677473.219457:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.219459:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.219460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677473.219461:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:24.0:1619677473.219462:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.219463:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.219463:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.219464:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.219465:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.219465:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:10.0:1619677473.219465:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:24.0:1619677473.219466:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.219467:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:24.0:1619677473.219468:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.219469:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.219469:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.219472:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.219473:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.219475:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.219476:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.219476:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.219478:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.219478:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.219479:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.219479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.219483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.219489:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.219491:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.219492:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.219492:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.219493:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.219495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.219876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.219882:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.219885:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.219886:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.219888:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.219891:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.219892:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.219894:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.219895:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.219896:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.219897:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.219900:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.219902:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.219905:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.219906:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.219906:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.219908:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.219908:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.219909:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.219910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.219922:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.219924:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.219926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.219926:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.219928:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.219930:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677473.220075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.220084:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.220086:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.220087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.220088:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.220090:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.220092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.220301:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.220304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.220304:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.220306:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.220308:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.220309:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677473.220310:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.220311:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.220312:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.220313:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677473.220316:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.220317:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.220319:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.220319:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.220320:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.220321:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.220321:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.220323:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.220324:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.220324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677473.220327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.220336:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.220337:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.220337:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.220339:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.220340:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.220340:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.220340:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.220341:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.220341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.220342:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.220345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.220351:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.220353:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.220354:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.220354:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.220355:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.220355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.220915:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.220917:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.220918:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.220919:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.220921:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.220923:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.220924:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.220925:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.220926:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.220926:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.220929:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.220930:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.220933:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.220934:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.220935:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.220936:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.220937:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.220937:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.220938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.220950:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.220953:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.220954:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.220954:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.220956:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.220959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.221135:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.221138:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.221139:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.221140:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.221143:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.221144:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.221146:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.221147:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.221148:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.221148:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.221151:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.221153:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.221155:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.221156:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.221157:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.221158:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.221159:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.221159:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.221160:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.221165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.221173:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.221175:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.221176:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.221177:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.221179:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.221183:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.221184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.221191:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.221194:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.221195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.221196:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.221197:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.221197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677473.221262:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.221265:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677473.221266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677473.221268:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.221271:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.221273:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677473.221274:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.221276:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.221277:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.221278:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677473.221281:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.221282:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677473.221285:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.221286:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677473.221287:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677473.221288:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677473.221289:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.221290:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.221290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677473.221296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.221303:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.221305:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.221306:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.221307:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.221309:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.221312:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677473.221620:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.221622:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.221624:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.221625:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.221626:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.221627:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.221629:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.221630:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.221631:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.221632:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.221635:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.221636:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.221637:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.221638:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.221639:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.221640:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.221640:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.221641:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.221641:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.221656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.221662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.221663:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.221664:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.221665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.221666:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.221667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677473.222341:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.222344:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677473.222344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677473.222347:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.222349:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.222350:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677473.222351:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.222352:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.222353:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.222354:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677473.222357:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.222358:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.222361:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.222361:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.222362:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.222364:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.222364:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.222365:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677473.222365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677473.222371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.222379:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.222380:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.222381:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.222382:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.222383:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.222384:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.222390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.222397:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.222400:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.222401:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.222401:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.222401:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.222402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677473.222409:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.222412:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.222413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.222414:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.222416:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.222417:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.222418:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.222419:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.222420:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.222421:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.222424:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.222425:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.222427:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.222428:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.222428:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.222429:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.222430:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.222430:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.222431:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.222436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.222442:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.222443:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.222444:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.222445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.222446:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.222447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.222607:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.222609:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.222610:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.222611:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.222615:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.222616:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.222617:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.222619:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.222619:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.222620:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.222623:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.222625:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.222628:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.222628:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.222629:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.222630:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.222631:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.222631:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.222632:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.222645:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.222648:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.222648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.222649:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.222651:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.222654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.222667:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.222670:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.222671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.222671:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.222672:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.222673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.222760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.222767:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.222769:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.222770:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.222770:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.222772:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.222774:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.222782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.222788:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.222790:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.222791:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.222791:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.222792:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.222793:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.223724:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.223727:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.223728:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.223729:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.223733:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.223734:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.223735:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.223736:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.223737:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.223738:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.223741:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.223742:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.223744:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.223745:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.223746:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.223747:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.223747:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.223748:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.223750:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.223750:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.223753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.223760:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.223762:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.223763:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.223764:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677473.223764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.223765:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.223766:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.223766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677473.223767:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.223767:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.224320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.224327:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.224330:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.224331:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.224331:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.224332:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.224335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.224655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.224662:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.224665:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.224665:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.224667:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.224669:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.224671:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.224672:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.224673:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.224674:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.224675:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.224678:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.224679:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.224682:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.224683:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.224683:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.224685:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.224686:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.224687:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.224688:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.224689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.224694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.224699:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.224699:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.224701:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677473.224702:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.224702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677473.224703:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.224703:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.224704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.224705:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.224708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.224720:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.224722:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.224723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.224723:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.224725:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.224725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.224813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.224819:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.224822:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.224823:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.224823:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.224825:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.224827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.225618:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.225621:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.225622:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.225624:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.225627:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.225628:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.225629:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.225631:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.225632:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.225632:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.225636:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.225638:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.225641:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.225642:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.225642:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.225644:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.225644:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.225645:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.225646:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.225651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.225659:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.225662:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.225663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.225663:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.225666:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.225669:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677473.225724:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.225727:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.225727:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.225730:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.225732:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.225733:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.225735:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.225736:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.225737:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.225738:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.225741:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.225742:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.225745:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.225746:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.225746:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.225748:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.225749:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.225749:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.225750:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.225755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.225764:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.225766:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.225767:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.225769:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.225770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.225772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.225775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.225783:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.225785:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677473.225785:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677473.225786:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.225787:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677473.225788:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.225788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.225788:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.225789:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.225791:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.225793:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.225795:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.225796:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.225798:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.225799:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.225800:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.225803:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.225805:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.225808:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.225808:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.225809:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.225811:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.225811:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.225812:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.225813:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.225824:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:20.2:1619677473.225824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:22.0:1619677473.225827:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.225828:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.225828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677473.225829:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000400:00000200:22.0:1619677473.225830:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.225832:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:22.0:1619677473.225833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:21.0:1619677473.225834:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.225834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.225837:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.225838:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677473.225839:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.225841:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.225842:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.225843:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677473.225846:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.225848:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.225850:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.225851:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.225851:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.225853:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.225854:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.225856:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:21.0:1619677473.225857:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677473.225858:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677473.225859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677473.225864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.225870:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.225872:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.225872:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.225873:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:21.0:1619677473.225874:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.225874:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677473.225875:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.225877:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.225878:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677473.225879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:9.2:1619677473.226086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.226093:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.226095:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.226096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.226096:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.226098:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.226100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.226428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.226437:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.226441:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.226442:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.226446:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.226449:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.226451:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.226453:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.226456:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.226457:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.226459:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.226464:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.226467:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.226470:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.226471:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.226472:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.226474:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.226476:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.226477:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.226478:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.226481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.226490:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.226493:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.226495:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.226496:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.226498:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.226500:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.226504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.226513:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.226516:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.226518:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.226518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.226519:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.226520:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.226834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.226841:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.226843:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.226844:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.226844:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.226846:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.226848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.227021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.227027:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.227030:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.227031:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.227033:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.227035:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.227036:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.227038:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.227039:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.227040:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.227041:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.227044:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.227046:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.227048:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.227049:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.227050:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.227051:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.227052:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.227053:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.227055:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.227055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.227058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.227060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677473.227066:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677473.227067:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677473.227068:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677473.227069:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.227069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677473.227069:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.227069:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.227071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:17.0:1619677473.227071:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.227074:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677473.227078:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.227081:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.227082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.227083:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.227086:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.227087:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.227089:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.227090:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.227091:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.227092:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.227094:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.227096:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.227097:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.227098:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.227099:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.227100:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.227101:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.227102:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.227104:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.227104:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.227105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.227115:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.227116:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.227116:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.227118:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.227119:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.227119:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.227120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.227120:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.227121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.227121:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677473.227533:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.227535:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.227536:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.227538:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.227540:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.227542:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.227543:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.227544:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.227545:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.227546:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.227548:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.227550:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.227553:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.227553:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.227554:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.227556:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.227556:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.227557:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.227557:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.227582:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.227585:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.227586:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.227586:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.227588:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.227591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.227744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.227753:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.227756:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.227756:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.227757:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.227759:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.227760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.227763:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.227764:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.227766:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.227767:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.227768:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.227768:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.227769:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677473.228018:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.228020:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.228021:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.228023:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.228024:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.228026:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.228027:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.228028:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.228029:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.228030:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.228033:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.228035:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.228036:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.228037:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.228038:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.228039:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.228039:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.228040:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.228040:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.228044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.228050:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.228052:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.228053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.228053:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.228055:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.228056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.228063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.228069:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.228072:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.228073:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.228073:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.228073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.228074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.228099:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.228101:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.228102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.228104:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.228107:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.228108:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677473.228110:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.228111:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.228112:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.228113:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677473.228116:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.228118:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677473.228121:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.228122:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677473.228122:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677473.228124:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677473.228125:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.228125:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.228126:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:20.2:1619677473.228611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.228649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677473.228652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:10.0:1619677473.228656:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.228658:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.228659:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.228659:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.228661:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.228664:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.228848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.228856:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.228858:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.228859:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.228859:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.228861:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.228863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.228872:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.228875:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.228875:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.228877:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.228880:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.228881:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.228882:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.228883:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.228884:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.228885:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.228888:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.228889:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.228891:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.228892:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.228893:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.228894:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.228895:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.228895:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.228896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.228900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.228906:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.228908:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.228909:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.228909:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.228911:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.228913:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.228924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.228930:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.228932:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.228933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.228933:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.228934:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.228935:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.229476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.229482:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.229484:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.229485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.229487:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.229489:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.229491:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.229492:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.229493:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.229494:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.229495:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.229498:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.229499:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.229502:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.229503:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.229503:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.229505:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.229506:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.229506:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.229507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.229521:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.229524:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.229525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.229525:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.229527:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.229530:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.229640:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.229643:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677473.229644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677473.229646:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.229648:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.229649:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.229650:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.229652:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.229653:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.229654:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.229658:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.229659:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.229662:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.229663:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.229664:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.229666:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.229667:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.229667:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677473.229668:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677473.229672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.229680:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.229683:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.229684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.229684:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.229686:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.229689:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.230146:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.230149:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.230150:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.230152:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.230154:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.230156:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.230157:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.230159:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.230159:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.230160:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.230164:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.230166:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.230168:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.230169:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.230170:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.230171:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.230172:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.230172:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.230173:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.230185:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.230188:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.230189:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.230189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.230192:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.230195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.230212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.230219:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.230221:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.230222:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.230222:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.230224:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.230227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.230397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.230404:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.230406:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.230407:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.230408:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.230409:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.230411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.230972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.230979:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.230981:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.230982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.230982:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.230983:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.230986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677473.231507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.231517:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.231519:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.231520:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.231522:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.231522:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.231525:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.231672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.231674:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.231677:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677473.231679:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.231680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.231681:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.231683:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677473.231684:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.231685:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.231686:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.231687:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677473.231690:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.231692:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677473.231695:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.231696:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:20.0:1619677473.231697:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677473.231698:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677473.231699:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.231700:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.231700:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:9.2:1619677473.231705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.231712:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.231715:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.231716:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.231716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.231719:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.231722:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677473.231974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.231981:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.231984:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.231985:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.231987:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.231989:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.231991:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.231992:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.231993:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.231994:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.231995:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.231998:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.231999:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.232002:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.232003:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.232004:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.232006:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.232007:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.232007:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.232008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.232020:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.232022:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.232023:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.232025:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.232027:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.232028:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677473.232039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.232047:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.232049:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.232050:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.232052:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.232052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.232055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677473.232192:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.232196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.232196:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.232198:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.232201:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.232202:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.232203:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.232205:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.232206:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.232207:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.232210:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.232211:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.232214:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.232215:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.232215:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.232217:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.232217:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.232219:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.232220:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.232220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.232223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.232231:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677473.232231:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677473.232233:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677473.232233:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.232234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677473.232235:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.232235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677473.232236:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.232237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.232239:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677473.232325:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.232328:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.232330:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.232330:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.232332:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.232334:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.232335:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.232337:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.232338:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.232339:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.232342:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.232344:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.232346:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.232347:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.232348:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.232349:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.232350:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.232351:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.232352:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.232353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.232362:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.232364:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:6.0:1619677473.232364:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.232365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.232365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.232366:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.232367:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.232368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.232369:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.232371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.232542:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.232545:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.232546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.232547:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.232550:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.232551:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.232552:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.232554:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.232555:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.232556:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.232559:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.232560:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.232562:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.232562:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.232563:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.232564:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.232565:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.232565:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.232566:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.232570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.232576:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.232578:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.232578:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.232579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677473.232579:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677473.232580:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.232581:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677473.232581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.232582:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.232584:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.232587:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.232588:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.232589:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.232590:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.232591:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.232592:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.232595:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.232597:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.232599:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.232600:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.232601:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.232603:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.232603:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.232604:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.232604:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.232616:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.232619:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.232619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.232620:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.232622:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.232625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.232818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.232826:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.232828:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.232829:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.232831:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.232832:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.232834:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.232834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.232836:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.232836:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.232837:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.232837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677473.233761:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.233763:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.233764:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.233766:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.233769:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.233771:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.233772:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.233773:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.233774:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.233775:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.233779:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.233780:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.233783:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.233784:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.233784:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.233786:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.233786:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.233787:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.233789:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.233789:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677473.233801:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.233803:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.233803:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.233804:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.233805:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677473.233805:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.233806:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.233807:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.233807:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.233809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.233821:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.233823:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.233824:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.233824:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.233825:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.233826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.234241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.234248:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.234251:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.234252:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.234252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.234253:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.234256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.234262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.234268:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.234270:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.234271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.234271:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.234272:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.234273:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.234292:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:5.2:1619677473.234296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677473.234297:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.234299:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677473.234301:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677473.234302:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677473.234304:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.234305:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677473.234305:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677473.234306:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.234306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677473.234308:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.234309:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.234309:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.234310:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677473.234311:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.234312:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.234314:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.234315:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677473.234316:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.234318:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677473.234318:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.234319:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.234321:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.234322:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:1.0:1619677473.234322:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.234323:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.234324:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.234325:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.234325:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:1.0:1619677473.234325:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.234327:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.234327:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:1.0:1619677473.234327:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.234328:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.234329:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.234330:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:9.2:1619677473.234331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677473.234331:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.234331:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.234332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677473.234339:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.234340:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677473.234341:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.234342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.234343:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.234343:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677473.234343:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.234344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.234345:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677473.234345:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.234347:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.234348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.234348:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.234348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:6.0:1619677473.234350:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.234352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.234361:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.234364:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.234365:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.234365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.234366:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.234367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.235121:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.235124:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.235125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.235127:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.235130:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.235132:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.235133:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.235134:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.235135:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.235136:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.235140:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.235142:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.235144:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.235145:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.235146:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.235148:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.235148:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.235149:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.235150:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.235151:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677473.235161:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.235163:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.235163:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.235164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.235165:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677473.235165:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.235166:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677473.235166:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677473.235167:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.235170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677473.235348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.235357:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.235360:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.235360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.235361:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.235363:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.235366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677473.235934:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.235936:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677473.235938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677473.235939:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.235941:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.235942:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.235943:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.235944:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.235945:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.235946:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.235949:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.235951:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.235954:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.235955:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.235955:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.235957:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.235958:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.235958:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677473.235959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.235964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.235973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.235976:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.235977:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.235977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.235980:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.235982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.236181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.236188:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.236191:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.236192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.236192:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.236194:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.236196:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.236896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.236902:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.236905:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.236906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.236908:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.236911:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.236912:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.236914:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.236915:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.236916:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.236917:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.236921:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.236922:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.236925:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.236926:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.236927:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.236929:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.236929:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.236930:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.236931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677473.237335:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.237338:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.237339:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.237341:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.237344:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.237345:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677473.237346:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.237348:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.237349:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.237350:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677473.237352:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.237354:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.237356:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.237357:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.237357:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.237359:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.237359:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.237360:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.237360:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.237365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.237371:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.237373:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.237373:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.237374:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.237375:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.237376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.237386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.237392:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.237394:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.237395:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.237395:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.237395:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.237396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677473.237504:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.237506:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.237507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.237508:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.237510:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.237511:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.237512:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.237513:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.237514:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.237515:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.237518:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.237519:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.237521:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.237522:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.237523:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.237524:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.237525:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.237525:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.237526:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.237538:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.237541:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.237542:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.237542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.237544:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.237547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.238304:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.238306:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.238309:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.238310:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.238311:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.238312:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.238313:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.238315:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.238316:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.238317:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.238320:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.238322:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.238324:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.238325:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.238326:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.238327:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.238328:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.238329:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.238329:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.238342:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.238345:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.238346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.238346:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.238348:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.238350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.238655:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.238658:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.238660:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.238662:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.238662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.238664:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.238665:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.238667:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677473.238667:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.238668:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.238669:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.238670:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.238671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:21.0:1619677473.238673:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.238674:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.238676:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.238677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.238678:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:21.0:1619677473.238679:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677473.238680:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96510375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.238680:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:24.0:1619677473.238681:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.238681:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:24.0:1619677473.238682:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.238683:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.238683:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.238683:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.238685:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677473.238686:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000400:00000200:24.0:1619677473.238687:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.238687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:24.0:1619677473.238688:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.238690:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.238690:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.238691:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.238692:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.238693:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.238693:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.238694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677473.238695:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677473.238698:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677473.238698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.238699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677473.238725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.238731:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.238733:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.238734:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.238735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.238736:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.238737:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.238909:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.238912:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.238912:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.238914:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.238917:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.238919:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.238920:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.238921:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.238922:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.238923:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.238926:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.238927:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.238929:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.238930:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.238930:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.238932:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.238932:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.238933:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.238933:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677473.238933:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.238936:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.238937:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:9.2:1619677473.238937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:1.0:1619677473.238939:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.238941:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.238943:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:1.0:1619677473.238943:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.238944:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.238945:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.238946:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:1.0:1619677473.238946:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.238947:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.238947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:1.0:1619677473.238947:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677473.238950:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.238951:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.238952:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677473.238953:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.238956:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.238957:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.238957:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.238959:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.238959:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.238960:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.238961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.238965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.238973:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.238975:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.238976:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.238977:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.238979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.238980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677473.239032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.239038:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.239041:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.239042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.239043:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.239045:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.239047:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.239048:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.239050:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.239051:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.239052:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.239054:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:17.0:1619677473.239056:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.239056:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677473.239058:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.239058:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.239060:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.239061:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677473.239061:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:17.0:1619677473.239062:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.239062:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:1.0:1619677473.239062:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.239064:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:1.0:1619677473.239064:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.239065:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.239066:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677473.239069:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.239070:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.239070:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.239071:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.239071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677473.239072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.239072:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.239073:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.239073:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.239075:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.239076:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.239076:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.239077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:22.0:1619677473.239078:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.239081:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:8.2:1619677473.239081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:22.0:1619677473.239082:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.239082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.239084:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.239086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.239089:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.239091:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.239092:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.239093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.239094:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.239096:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677473.239505:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.239508:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.239508:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.239510:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.239513:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.239514:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.239515:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.239517:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.239518:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.239519:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.239522:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.239524:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.239527:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.239528:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.239528:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.239530:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.239531:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.239531:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.239531:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.239545:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.239548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.239548:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.239549:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.239551:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.239553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.239586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:18.0:1619677473.239592:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677473.239595:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:2.0:1619677473.239596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:18.0:1619677473.239598:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.239601:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.239603:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677473.239604:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.239605:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.239606:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.239607:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677473.239611:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.239613:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.239616:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.239616:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677473.239617:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677473.239619:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677473.239619:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677473.239620:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:18.0:1619677473.239621:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:21.0:1619677473.239633:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.239635:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.239636:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.239638:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.239638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.239640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.239672:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.239675:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.239676:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.239678:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.239680:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.239682:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677473.239683:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.239685:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.239686:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.239687:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677473.239689:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.239691:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677473.239692:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.239693:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677473.239694:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.239695:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.239696:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.239696:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.239697:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.239701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.239707:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.239708:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.239709:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.239710:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.239711:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.239712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.240414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.240421:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.240423:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.240424:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.240424:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.240426:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.240428:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677473.240537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.240545:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.240548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.240548:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.240549:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.240551:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.240554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.240556:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.240559:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.240559:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.240561:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.240563:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.240565:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.240566:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.240567:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.240568:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.240569:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.240571:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.240573:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.240575:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.240576:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.240576:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.240578:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.240578:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.240579:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.240580:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.240593:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.240596:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.240597:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.240599:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.240601:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.240603:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.240610:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.240613:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.240615:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.240615:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.240616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.240617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677473.242116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.242124:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.242127:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.242128:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.242128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.242131:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.242133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.242296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.242302:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.242305:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.242306:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.242306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.242308:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.242310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.242315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.242321:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.242324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.242324:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.242325:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.242326:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.242326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.242399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.242405:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.242408:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.242409:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.242409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.242410:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.242413:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.242528:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.242530:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.242531:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.242533:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.242536:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.242537:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.242539:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.242540:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.242541:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.242542:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.242544:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.242546:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.242549:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.242550:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.242550:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.242552:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.242552:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.242553:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.242553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.242569:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.242573:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.242576:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.242576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.242579:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.242583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.242706:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.242710:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.242714:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.242714:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.242717:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.242720:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.242722:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.242724:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.242726:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.242727:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.242733:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.242736:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.242739:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.242740:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.242741:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.242744:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.242745:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.242746:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.242747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.242760:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.242764:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.242765:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.242766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.242769:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.242773:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.242775:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.242778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.242778:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.242780:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.242782:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.242782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.243199:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.243202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.243202:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.243205:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.243207:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.243209:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.243210:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.243211:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.243212:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.243213:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.243216:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.243217:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.243220:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.243221:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.243222:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.243223:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.243224:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.243224:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.243225:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.243238:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.243241:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.243241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.243242:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.243244:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.243244:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:22.0:1619677473.243247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.243247:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.243248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677473.243250:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.243252:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.243253:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.243255:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.243256:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.243257:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.243257:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:8.0:1619677473.243258:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.243260:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.243260:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.243261:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000400:00000200:8.0:1619677473.243263:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.243263:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.243263:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.243264:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.243267:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.243268:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.243269:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.243271:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.243271:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.243272:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.243273:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677473.243284:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.243286:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.243287:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.243287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.243289:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.243292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.243800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.243807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.243810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.243810:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.243812:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.243814:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.243815:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.243817:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.243818:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.243819:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.243820:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.243824:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.243825:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.243828:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.243829:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.243830:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.243832:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.243832:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.243834:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.243835:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.243835:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.243840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.243845:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.243847:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.243847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677473.243847:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.243848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.243850:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.243851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.243851:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.243853:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.243855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.244078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.244085:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.244087:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.244088:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.244088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.244090:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.244092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.244159:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.244162:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.244163:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.244164:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.244167:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.244168:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.244169:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.244171:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.244171:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.244172:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.244175:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.244176:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.244178:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.244179:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.244180:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.244181:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.244182:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.244183:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.244184:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.244185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.244187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.244193:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.244195:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.244196:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.244197:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.244197:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.244198:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.244199:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.244200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.244200:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.244201:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.244209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.244215:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.244217:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.244218:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.244218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.244219:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.244219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.244264:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.244266:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.244267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.244269:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.244271:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.244272:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.244273:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.244274:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.244275:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.244276:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.244278:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.244280:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.244281:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.244282:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.244282:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.244284:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.244284:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.244285:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.244286:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.244287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.244300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.244304:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.244305:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.244305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.244306:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.244307:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.244307:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.244308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:24.0:1619677473.244309:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.244309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.244310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.244580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.244586:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.244589:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.244590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677473.244592:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:2.0:1619677473.244592:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.244595:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:2.0:1619677473.244595:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.244597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:2.0:1619677473.244597:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:21.0:1619677473.244598:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.244598:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.244600:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.244600:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.244601:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.244602:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677473.244602:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:21.0:1619677473.244604:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.244605:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:21.0:1619677473.244606:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.244607:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.244608:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677473.244608:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.244610:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.244611:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.244611:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.244612:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677473.244613:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677473.244614:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.244614:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.244615:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677473.244616:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.244616:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677473.244616:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.0:1619677473.244617:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.244619:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.244619:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.244620:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.2:1619677473.244620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677473.244621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.244624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.244628:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.244630:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.244631:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.244631:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:1.0:1619677473.244631:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.244633:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:1.0:1619677473.244633:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.244634:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.244635:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.244636:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.244637:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000400:00000200:8.0:1619677473.244641:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:9.2:1619677473.244643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677473.244645:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.244646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677473.244648:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.244648:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.244648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.244649:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.244650:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677473.244650:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.244650:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677473.244651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677473.244651:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.244652:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.244653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.244749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.244755:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.244758:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.244759:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.244761:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.244763:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.244764:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.244766:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.244767:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.244768:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.244769:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.244772:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.244774:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.244776:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.244777:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.244777:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.244779:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.244780:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.244781:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.244781:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.244785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.244792:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.244794:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.244795:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.244796:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.244796:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.244798:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.245550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.245556:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.245559:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.245560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.245561:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.245563:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.245564:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.245565:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.245566:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.245567:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.245568:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.245571:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.245572:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.245575:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.245576:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.245577:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.245578:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.245579:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.245580:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.245580:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:12.0:1619677473.245595:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.245598:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.245599:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.245602:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.245603:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.245605:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677473.245943:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.245946:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.245946:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.245948:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.245950:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.245952:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.245953:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.245955:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.245956:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.245957:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.245960:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.245962:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.245965:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.245966:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.245966:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.245968:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.245968:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.245969:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.245970:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.245975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.245984:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.245986:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.245987:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.245989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.245989:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.245992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677473.246111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.246117:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.246120:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.246121:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.246121:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.246123:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.246125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.246867:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.246870:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.246871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.246872:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.246874:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.246876:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.246877:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.246878:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.246879:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.246880:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.246883:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.246884:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.246887:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.246888:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.246888:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.246890:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.246891:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.246891:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.246892:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.246905:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.246908:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.246909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.246910:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.246912:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.246914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.247015:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.247018:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.247019:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.247020:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.247023:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.247025:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.247026:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.247027:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.247028:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.247029:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.247032:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.247033:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.247035:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.247036:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.247037:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.247038:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.247039:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.247039:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.247040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.247044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.247052:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.247055:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.247056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.247056:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.247058:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.247062:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677473.247130:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.247132:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.247134:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.247135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.247136:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.247138:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.247139:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.247141:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.247142:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.247143:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.247146:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.247148:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.247150:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.247150:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.247151:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.247152:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.247153:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.247154:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.247155:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.247156:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.247165:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.247167:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.247168:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.247169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.247176:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.247179:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.247180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.247180:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.247182:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.247184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.247716:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.247719:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677473.247720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677473.247722:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.247727:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.247729:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677473.247730:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96510342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.247731:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.247732:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.247733:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677473.247736:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677473.247738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.247738:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677473.247740:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.247741:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677473.247742:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677473.247744:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677473.247744:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.247745:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677473.247746:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.247746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:24.0:1619677473.247747:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.247747:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.247748:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.247749:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.247749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677473.247751:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.247756:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.247758:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.247759:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.247760:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.247761:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.247763:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.247770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677473.247776:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.247779:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677473.247779:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.247780:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.247780:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.247781:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677473.248013:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.248015:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.248016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.248018:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.248020:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.248022:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.248023:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.248024:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.248025:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.248026:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.248029:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.248030:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.248033:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.248034:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.248035:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.248036:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.248037:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.248037:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.248038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.248061:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.248064:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.248065:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.248065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.248068:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.248071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.248386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.248395:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.248398:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.248398:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.248399:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.248401:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.248404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.248540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.248547:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.248550:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.248551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.248551:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.248553:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.248557:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677473.248930:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.248932:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.248935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.248935:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.248937:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.248939:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.248940:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.248941:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.248942:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.248943:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.248947:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.248949:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.248952:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.248953:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.248953:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.248955:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.248956:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.248957:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.248957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.248962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.248968:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.248971:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.248972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.248972:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.248974:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.248977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.249042:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.249044:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.249045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.249047:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.249049:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.249050:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.249051:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.249052:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.249053:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.249054:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.249057:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.249058:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.249060:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.249061:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.249061:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.249063:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.249063:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.249064:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.249064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.249068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.249074:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.249076:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.249076:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.249077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.249078:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.249079:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.249090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.249096:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.249098:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.249099:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.249100:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.249100:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.249100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:5.2:1619677473.249106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.249111:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.249114:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.249115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.249117:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.249119:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.249121:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.249122:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.249123:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.249124:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.249125:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.249128:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.249130:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.249133:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.249133:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.249134:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.249136:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.249136:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.249137:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.249137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.249149:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.249152:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.249153:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.249153:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.249155:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.249157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.249496:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.249498:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.249499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.249501:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.249504:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.249505:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.2:1619677473.249505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:26.0:1619677473.249506:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.249508:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.249509:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.249510:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677473.249512:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677473.249513:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.249514:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:26.0:1619677473.249515:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.249515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.249517:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677473.249518:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.249519:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.249519:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:18.0:1619677473.249519:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.249521:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:18.0:1619677473.249521:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677473.249522:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.249522:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677473.249522:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.249523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677473.249523:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.249524:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.249525:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.249528:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.249530:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.249533:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.249534:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.249535:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677473.249536:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677473.249537:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.249537:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677473.249538:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:18.0:1619677473.249539:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:6.0:1619677473.249539:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.249540:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.249540:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.249540:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677473.249542:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.249544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.249545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.249551:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677473.249552:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677473.249553:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.249554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677473.249555:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677473.249555:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.249556:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.249557:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.249557:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.249558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.249559:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:13.0:1619677473.249560:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.249560:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.249560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:12.0:1619677473.249562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.249564:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.249890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677473.249895:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.249897:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.249898:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677473.249899:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:10.0:1619677473.249900:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.249901:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.249901:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.249902:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677473.249902:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.249903:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.249904:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.249905:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.249905:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.249906:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.249906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.249907:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.249907:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.249907:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:10.0:1619677473.249908:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.249908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677473.249909:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.249909:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677473.249910:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.249912:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.249913:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.249914:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.249915:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.249916:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677473.249918:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.249919:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.249919:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.249919:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.249920:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.249922:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.249922:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677473.249923:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.249923:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:24.0:1619677473.249924:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.249924:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.249924:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.249925:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677473.249926:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.249927:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.249927:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.249928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677473.249935:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.249937:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.249938:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.249939:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677473.249939:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.249940:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677473.249940:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.249941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.249942:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.249947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.249949:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.249950:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.249951:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.249951:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677473.249953:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.249953:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.249954:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.249955:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.249956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.249956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.249957:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.249957:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.250188:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.250191:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.250192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.250194:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.250197:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.250198:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.250199:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.250201:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.250202:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.250202:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.250205:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.250206:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.250208:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.250208:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.250209:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.250210:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.250211:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.250211:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.250212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.250215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.250221:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.250223:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.250224:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.250225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.250226:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.250227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.251066:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.251068:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.251071:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.251071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.251074:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.251076:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.251077:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.251079:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.251079:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.251080:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.251083:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.251085:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.251088:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.251088:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.251089:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.251091:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.251092:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.251093:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.251093:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.251097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.251104:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.251107:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.251108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.251108:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.251110:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.251113:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.251159:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.251161:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677473.251162:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677473.251164:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.251165:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.251167:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.251167:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.251169:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.251170:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.251171:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.251174:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.251175:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.251178:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.251179:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.251180:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.251181:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.251182:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.251182:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677473.251183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.251187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.251196:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.251199:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.251200:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.251200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.251202:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.251205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.251209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.251216:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.251219:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.251220:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.251220:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.251221:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.251221:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.251600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.251607:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.251610:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.251611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.251611:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.251613:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.251615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.251619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.251625:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.251628:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.251629:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.251629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.251630:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.251630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.252566:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.252568:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.252569:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.252571:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.252574:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.252576:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.252577:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.252578:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.252579:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.252580:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.252583:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.252584:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.252587:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.252587:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.252588:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.252590:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.252590:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.252591:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.252591:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.252604:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.252606:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.252607:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.252609:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.252611:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.252612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.253285:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.253289:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.253290:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.253300:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.253303:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.253306:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.253308:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.253310:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.253311:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.253313:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.253318:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.253321:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.253324:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.253326:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.253326:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.253328:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.253329:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.253330:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.253331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.253344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.253346:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.253347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.253348:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.253350:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.253353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.253879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.253888:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.253893:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.253895:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.253899:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.253902:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.253905:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.253907:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.253910:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.253912:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.253914:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.253920:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.253924:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.253927:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.253929:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.253930:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.253932:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.253933:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.253934:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.253935:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.253946:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.253949:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.253951:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.253951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.253953:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.253955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.254134:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.254137:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.254138:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.254140:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.254143:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.254144:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.254145:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.254147:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.254148:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.254149:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.254152:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.254154:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.254157:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.254158:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.254158:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.254160:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.254161:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.254161:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.254162:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.254173:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.254176:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.254177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.254177:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.254179:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.254182:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.254190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:18.0:1619677473.254197:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677473.254200:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:2.0:1619677473.254200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:18.0:1619677473.254203:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.254205:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.254207:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677473.254208:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.254209:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.254210:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.254211:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677473.254215:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.254217:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.254219:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.254220:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677473.254221:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677473.254222:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677473.254223:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677473.254224:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907cbe3400] (22)++
00000800:00000200:18.0:1619677473.254225:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (23)--
00000800:00000200:18.0:1619677473.254226:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:20.2:1619677473.254231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677473.254236:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)++
00000800:00000200:20.0:1619677473.254238:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677473.254238:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907cbe3400] (23)--
00000800:00000200:2.0:1619677473.254239:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:20.0:1619677473.254240:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:2.0:1619677473.254240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000400:00000200:20.0:1619677473.254241:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.254243:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.254244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.254246:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.254585:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.254588:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.254590:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.254591:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.254592:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.254594:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.254595:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.254596:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.254598:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.254599:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.254602:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.254604:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.254607:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.254608:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.254608:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.254610:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.254611:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.254612:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.254612:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.254628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.254636:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.254638:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.254639:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.254640:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.254642:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.254644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.254672:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.254675:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.254676:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.254678:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.254681:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.254682:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:24.0:1619677473.254682:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:26.0:1619677473.254683:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.254685:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.254685:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677473.254685:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:26.0:1619677473.254686:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.254687:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677473.254688:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.254691:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.254692:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:24.0:1619677473.254693:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.254694:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:26.0:1619677473.254695:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.254696:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:24.0:1619677473.254696:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96510318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.254697:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:24.0:1619677473.254697:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.254698:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.254698:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.254698:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677473.254699:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.254700:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.254701:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:24.0:1619677473.254701:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.254702:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677473.254703:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:9.2:1619677473.254704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.254705:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.254705:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.254706:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.254708:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.254709:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.254710:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.254711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677473.254711:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.254713:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.254714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.254714:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.254715:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.254716:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.254717:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.254717:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.254718:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.254718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.254723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.254729:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.254731:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.254732:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.254732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.254734:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.254735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677473.254742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.254751:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.254753:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.254754:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.254754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.254757:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.254759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677473.254948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.254955:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.254957:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.254958:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.254960:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.254963:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.254965:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.254966:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.254967:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.254968:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.254969:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.254972:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.254974:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.254976:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.254977:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.254978:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.254980:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.254981:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.254982:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.254982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677473.255160:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.255162:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.255163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.255165:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.255167:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.255168:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.255169:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.255171:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.255172:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.255173:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.255176:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.255177:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.255179:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.255179:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.255180:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.255181:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.255182:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.255182:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.255183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.255197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.255203:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.255205:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.255206:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.255206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.255207:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.255208:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677473.255315:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.255318:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.255321:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.255321:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.255324:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.255326:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.255327:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.255329:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.255331:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.255332:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.255336:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.255338:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.255341:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.255342:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.255342:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.255344:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677473.255345:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:9.2:1619677473.255345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.255346:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.255347:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.255348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677473.255352:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:8.0:1619677473.255354:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677473.255355:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)++
00000800:00000200:24.0:1619677473.255355:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:24.0:1619677473.255356:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.255356:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.255358:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677473.255358:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:8.0:1619677473.255358:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.255360:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.255361:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677473.255361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.255362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.255363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677473.255363:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.255365:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.255366:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.255366:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.255367:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.255368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.256214:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.256217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.256217:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677473.256219:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.256222:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.256223:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677473.256225:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.256226:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.256227:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.256228:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677473.256231:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.256233:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677473.256237:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.256237:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677473.256238:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677473.256240:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677473.256240:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.256241:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.256241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.256246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.256253:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.256256:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.256257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.256257:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.256259:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.256262:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.256915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.256922:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.256925:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.256926:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.256926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.256928:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.256930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677473.257300:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (21)++
00000800:00000200:17.0:1619677473.257304:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.4@o2ib3
00000800:00000200:1.0:1619677473.257304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (22)--
00000400:00000200:17.0:1619677473.257307:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.4@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.257310:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.257311:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677473.257313:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.257314:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.257315:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.257316:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677473.257320:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.257322:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677473.257325:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.257326:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:17.0:1619677473.257327:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677473.257328:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:17.0:1619677473.257329:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf5e00] (21)++
00000800:00000200:17.0:1619677473.257330:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf5e00] (22)--
00000800:00000200:17.0:1619677473.257330:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (21)--
00000800:00000200:1.0:1619677473.257345:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (21)++
00000800:00000200:1.0:1619677473.257348:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.4@o2ib3
00000800:00000200:17.0:1619677473.257349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (22)--
00000400:00000200:1.0:1619677473.257351:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.4@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.257354:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.257355:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677473.257357:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.257358:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.257359:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.257360:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677473.257363:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.257364:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677473.257367:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.257368:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677473.257368:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677473.257370:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677473.257371:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf5e00] (21)++
00000800:00000200:1.0:1619677473.257372:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce0bf5e00] (22)++
00000800:00000200:1.0:1619677473.257374:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf5e00] (23)--
00000800:00000200:1.0:1619677473.257374:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (22)--
00000800:00000200:9.2:1619677473.257376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.257383:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.257385:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.257386:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677473.257386:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (22)++
00000400:00000200:8.0:1619677473.257387:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.257388:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce0bf5e00] (23)--
00000800:00000200:17.0:1619677473.257388:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (22)--
00000800:00000200:17.0:1619677473.257389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (21)--
00000400:00000200:8.0:1619677473.257389:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.257391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677473.257451:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.257454:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.257455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.257457:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.257460:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.257461:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677473.257462:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.257464:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.257465:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.257465:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677473.257468:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.257470:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677473.257472:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.257473:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677473.257473:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677473.257475:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677473.257476:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.257476:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.257477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.257481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.257488:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.257491:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.257492:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.257492:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.257494:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.257497:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:17.0:1619677473.257577:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (21)++
00000800:00000200:17.0:1619677473.257580:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.4@o2ib3
00000800:00000200:1.0:1619677473.257580:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (22)--
00000400:00000200:17.0:1619677473.257582:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.4@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.257585:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.257587:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677473.257588:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.257590:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.257591:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.257591:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677473.257595:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.257596:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677473.257599:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.257599:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677473.257600:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677473.257602:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677473.257602:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf5e00] (21)++
00000800:00000200:17.0:1619677473.257603:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf5e00] (22)--
00000800:00000200:17.0:1619677473.257603:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (21)--
00000800:00000200:9.2:1619677473.257607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.257614:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.257616:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.257617:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.257617:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.257620:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.257622:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.257656:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (21)++
00000800:00000200:1.0:1619677473.257659:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.4@o2ib3
00000800:00000200:17.0:1619677473.257660:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (22)--
00000400:00000200:1.0:1619677473.257662:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.4@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.257664:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.257666:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677473.257667:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.257668:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.257669:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.257670:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677473.257673:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.257674:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677473.257677:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.257677:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677473.257678:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677473.257680:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677473.257681:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf5e00] (21)++
00000800:00000200:1.0:1619677473.257681:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf5e00] (22)--
00000800:00000200:1.0:1619677473.257682:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf5e00] (21)--
00000800:00000200:12.0:1619677473.257696:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.257699:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.257700:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.257703:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.257706:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.257708:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:10.0:1619677473.258212:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.258214:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.258215:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.258217:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.258219:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.258220:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.258222:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.258223:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.258224:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.258225:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.258227:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.258229:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.258232:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.258233:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.258233:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.258235:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.258235:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.258236:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.258236:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.258249:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.258252:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.258252:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.258253:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.258255:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.258258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.258620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.258627:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.258629:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.258630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.258630:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.258632:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.258635:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.258655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.258661:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.258664:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.258665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.258667:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.258670:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.258671:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.258673:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.258674:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.258675:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.258676:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.258679:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.258681:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.258684:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.258685:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.258685:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.258687:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.258688:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.258689:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.258689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.258702:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.258704:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.258705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.258705:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.258708:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.258710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.258817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.258820:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677473.258821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677473.258823:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.258824:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.258826:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677473.258827:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.258829:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.258830:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.258831:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.258834:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.258836:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.258838:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.258839:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677473.258840:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677473.258841:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677473.258842:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.258843:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677473.258845:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677473.258845:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.258858:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677473.258860:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677473.258860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677473.258861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677473.259023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.259030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.259032:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.259033:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.259033:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.259035:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.259037:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.259044:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677473.259044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677473.259047:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.259048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.259050:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.259050:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.259052:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677473.259053:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.259053:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.259053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:26.0:1619677473.259054:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677473.259054:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.259054:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677473.259055:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.259057:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.259057:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.259058:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.259061:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.259063:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.259066:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.259066:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.259067:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.259069:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.259069:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.259070:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.259071:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.259082:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.259085:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.259086:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.259086:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.259088:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.259091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.259104:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.259107:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.259108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.259108:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.259109:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.259110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.259121:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.259123:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.259126:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.259126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.259129:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.259130:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.259131:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.259133:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.259134:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.259135:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.259138:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.259140:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.259143:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.259144:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.259144:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.259146:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.259147:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.259147:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.259148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.259152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.259158:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.259161:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.259161:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.259162:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.259164:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.259166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.259369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.259375:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.259378:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.259379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.259381:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.259384:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.259385:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.259387:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.259388:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.259389:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.259390:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.259394:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.259395:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.259397:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.259398:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.259398:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.259400:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.259401:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.259402:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.259403:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.259403:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.259407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.259410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.259416:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.259417:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.259417:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677473.259418:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.259418:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.259419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.259420:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.259420:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.259421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677473.259422:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.259521:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.259524:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.259525:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.259527:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.259530:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.259531:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.259532:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.259533:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.259534:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.259535:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.259538:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.259540:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.259542:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.259543:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.259544:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.259545:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.259546:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.259546:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.259547:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.259551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.259557:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.259559:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.259560:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.259561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.259562:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.259565:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.259571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.259577:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.259579:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.259580:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.259580:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.259581:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.259581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.260054:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.260057:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.260059:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.260060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.260061:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.260063:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.260064:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.260065:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.260066:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.260067:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.260070:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.260072:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.260074:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.260075:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.260075:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.260077:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.260078:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.260078:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.260079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:26.0:1619677473.260090:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:22.0:1619677473.260091:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.260093:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:22.0:1619677473.260093:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.260094:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.260094:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:6.0:1619677473.260095:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677473.260096:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:22.0:1619677473.260097:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:26.0:1619677473.260099:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.260099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677473.260101:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.260102:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.260104:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.260104:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.260106:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.260108:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677473.260109:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.260111:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677473.260111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.260111:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.260112:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.260113:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.260113:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.260114:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.260115:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.260115:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.260117:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.260118:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.260119:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.260120:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.260121:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.260124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.260131:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.260133:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677473.260134:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.260135:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.260135:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:2.0:1619677473.260135:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.260136:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.260137:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677473.260138:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.260141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.260143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.260151:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.260153:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.260154:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.260154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.260156:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.260156:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677473.260340:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.260342:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.260344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.260345:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.260347:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.260348:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.260349:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.260351:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.260352:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.260352:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.260355:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.260357:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.260358:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.260359:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.260359:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.260361:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.260361:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.260362:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.260363:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.260364:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.260367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.260374:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.260375:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.260376:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.260378:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.260378:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677473.260378:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.260379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.260379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.260380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.260380:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.260844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.260850:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.260853:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.260854:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.260854:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.260856:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.260857:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.261699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.261705:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.261707:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.261709:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.261710:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.261713:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.261715:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.261716:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.261717:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.261718:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.261719:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.261722:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.261724:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.261727:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.261727:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.261728:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.261730:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.261731:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.261732:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.261732:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.261737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.261745:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.261748:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.261749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.261749:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.261751:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.261754:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.262555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.262564:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.262567:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.262568:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.262568:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.262570:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.262574:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677473.262603:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.262605:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.262606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.262607:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.262609:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.262611:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.262612:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.262613:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.262614:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.262615:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.262618:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.262620:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.262622:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.262623:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.262624:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.262625:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.262626:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.262626:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.262627:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.262631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.262640:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.262642:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.262643:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.262643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.262646:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.262649:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.262652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.262659:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.262662:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.262663:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.262663:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.262664:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.262664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.262777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.262783:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.262786:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.262787:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.262787:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.262789:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.262791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.262796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.262802:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.262804:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.262805:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.262805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.262806:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.262807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.263091:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.263095:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.263097:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.263100:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.263103:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.263106:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.263108:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.263110:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.263111:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.263113:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.263118:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.263120:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.263124:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.263126:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.263127:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.263129:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.263130:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.263131:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.263131:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.263144:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.263147:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.263148:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.263148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.263151:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.263154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.263417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2400] (20)++
00000800:00000200:5.2:1619677473.263417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.263424:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.263426:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.263427:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.263430:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.263432:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.263434:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.263436:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.263438:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.263439:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.263440:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.263445:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.263448:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.263451:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.263452:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.263453:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.263454:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.263455:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.263456:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.263457:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.263457:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.2:1619677473.263462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:5.2:1619677473.263462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677473.263466:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:20.2:1619677473.263468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2400] (21)++
00000800:00000200:2.0:1619677473.263468:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:24.0:1619677473.263469:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:6.0:1619677473.263469:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.263469:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:24.0:1619677473.263470:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.263470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:1.0:1619677473.263471:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:24.0:1619677473.263472:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.263472:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:2.0:1619677473.263472:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:1.0:1619677473.263472:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.263472:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.263473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:18.0:1619677473.263473:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:6.0:1619677473.263473:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.263475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:6.0:1619677473.263475:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.263475:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.263478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:2.0:1619677473.263478:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.263480:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677473.263482:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.263483:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.263484:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.263485:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.2:1619677473.263486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000400:00000200:2.0:1619677473.263488:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.263490:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:2.0:1619677473.263490:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.263493:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.263493:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:2.0:1619677473.263493:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.263494:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.263495:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.263495:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:2.0:1619677473.263495:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.263496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:2.0:1619677473.263497:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677473.263497:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.263498:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677473.263499:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:22.0:1619677473.263510:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.263513:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.263514:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.263514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.263516:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.263518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.263527:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.263531:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.263533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.263536:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.263540:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.263542:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.263545:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.263547:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.263549:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.263550:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.263556:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.263559:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.263563:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.263564:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.263565:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.263567:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.263568:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.263569:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.263570:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.263580:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.263583:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.263584:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.263584:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.263587:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.263589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.264401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.264408:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.264411:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.264411:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.264412:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.264413:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.264415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.264786:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.264789:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.264790:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.264791:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.264794:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.264795:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.264796:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.264798:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.264798:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.264799:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.264802:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.264803:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.264806:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.264806:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.264807:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.264808:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.264809:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.264810:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.264811:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.264811:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.264814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.264820:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.264822:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.264823:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.264824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.264824:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.264825:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.264826:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.264827:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.264827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.264828:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677473.265742:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.265744:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.265745:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.265747:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.265749:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.265750:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.265752:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.265753:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.265754:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.265755:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.265758:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.265759:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.265761:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.265761:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.265762:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.265763:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.265764:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.265764:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.265765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.265769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.265775:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.265777:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.265778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.265779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.265780:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.265781:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.266031:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.266033:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.266034:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.266036:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.266038:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.266040:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677473.266041:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.266042:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.266043:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.266044:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677473.266047:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.266048:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.266050:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.266051:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677473.266052:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677473.266053:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677473.266054:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.266054:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.266055:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:18.0:1619677473.266352:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.266356:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677473.266358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677473.266362:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.266365:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.266368:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.266370:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.266372:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677473.266372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:18.0:1619677473.266374:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.266376:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.266379:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:18.0:1619677473.266381:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.266382:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.266383:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.266383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677473.266384:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.266385:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.266386:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.266387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677473.266388:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.266389:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.266391:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.266392:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.266394:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677473.266395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677473.266397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.266406:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.266410:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.266411:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.266412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.266414:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.266417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.266569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.266575:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.266578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.266578:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.266578:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.266580:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.266582:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.266678:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.266680:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.266682:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:24.0:1619677473.266683:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.266684:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.266684:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677473.266686:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.266686:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.266687:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.266689:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.266690:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.266691:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.266692:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.266694:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.266694:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.266695:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96510272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.266696:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.266697:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.266697:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.266698:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.266701:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.266702:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.266703:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.266703:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.266704:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.266706:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.266706:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.266706:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.266707:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.266707:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.266708:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.266708:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.266708:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677473.266709:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677473.266709:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677473.266710:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.266710:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677473.266711:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.266712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.266721:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000800:00000200:18.0:1619677473.266725:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000400:00000200:18.0:1619677473.266727:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.266727:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000800:00000200:2.0:1619677473.266728:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000400:00000200:18.0:1619677473.266730:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.266730:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.266733:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.266735:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.266736:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.266737:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.266741:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.266742:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.266742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.266743:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.266744:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677473.266802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.266811:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.266814:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.266815:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.266815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.266817:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.266821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677473.267379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.267385:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.267388:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.267389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.267391:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.267393:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.267394:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.267396:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.267397:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.267398:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.267399:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.267402:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.267404:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.267407:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.267408:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.267409:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.267411:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.267412:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.267413:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.267413:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.267426:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.267429:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.267429:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.267431:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.267433:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.267434:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.267444:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.267447:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.267448:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.267449:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.267449:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.267451:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.267814:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.267817:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.267818:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.267820:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.267822:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.267824:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.267825:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.267826:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.267827:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.267828:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.267831:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.267833:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.267835:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.267836:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.267837:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.267839:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.267839:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.267840:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.267840:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.267846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.267855:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.267857:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.267858:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.267860:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.267860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.267863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.267885:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.267887:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.267889:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.267890:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.267893:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.267894:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.267895:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.267897:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.267898:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.267899:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.267902:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.267903:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.267906:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.267907:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.267907:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.267909:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.267909:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.267910:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.267912:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.267912:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.267922:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.267925:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.267925:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.267926:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677473.267926:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.267926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.267927:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.267928:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.267928:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.267930:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.267966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.267972:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.267975:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.267976:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.267976:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.267978:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.267980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.267985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.267991:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.267994:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.267994:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.267995:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.267995:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.267996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.268008:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.268010:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.268012:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.268013:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.268016:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.268017:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.268018:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.268019:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.268020:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.268021:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.268024:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.268026:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.268029:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.268029:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.268030:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.268032:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.268032:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.268033:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.268033:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.268045:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.268048:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.268049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.268049:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.268052:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.268054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.268166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.268173:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.268176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.268176:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.268178:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.268180:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.268182:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.268183:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.268184:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.268185:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.268186:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.268190:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.268191:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.268194:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.268195:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.268195:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.268197:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.268198:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.268198:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.268199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.268210:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.268213:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.268214:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.268214:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.268216:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.268219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.268233:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.268236:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.268237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.268237:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.268238:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.268238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.268379:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.268381:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.268382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.268384:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.268386:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.268388:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.268389:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.268391:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.268392:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.268393:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.268396:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.268398:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.268400:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.268401:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.268402:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.268403:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.268404:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.268405:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.268406:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.268407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677473.268409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.268418:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.268420:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677473.268420:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677473.268422:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677473.268422:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.268422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.268423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677473.268424:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.268424:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677473.268427:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.268957:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.268960:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.268961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.268963:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.268966:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.268967:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677473.268968:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.268970:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.268971:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.268972:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677473.268975:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.268977:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677473.268980:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.268980:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:26.0:1619677473.268981:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677473.268982:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677473.268983:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.268984:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.268985:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.268985:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:13.0:1619677473.268996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:10.0:1619677473.268998:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:13.0:1619677473.268999:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.268999:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.269000:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.269000:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:13.0:1619677473.269001:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.269001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:13.0:1619677473.269003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.269004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677473.269017:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677473.269020:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677473.269020:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.269021:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677473.269022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.269023:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677473.269658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.269661:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.269664:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677473.269666:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.269667:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.269669:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.269670:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677473.269672:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.269673:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.269674:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.269675:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677473.269678:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.269679:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.269682:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.269682:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677473.269683:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.269685:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.269685:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.269686:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677473.269688:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677473.269688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677473.269694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.269697:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.269698:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.269699:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000400:00000200:20.0:1619677473.269699:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.269700:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000400:00000200:20.0:1619677473.269700:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.269701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.269702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.269703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.269704:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:21.0:1619677473.269721:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.269724:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.269725:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.269726:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.269726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.269727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.269930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.269937:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.269940:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.269941:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.269941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.269943:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.269945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.270041:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.270044:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.270044:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.270047:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.270048:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.270050:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.270051:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.270052:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.270053:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.270054:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.270057:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.270059:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.270061:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.270062:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.270063:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.270064:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.270065:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.270065:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.270066:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.270071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677473.270077:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677473.270079:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:26.0:1619677473.270080:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.270080:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677473.270081:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:26.0:1619677473.270082:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.270082:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.270083:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.270084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:26.0:1619677473.270085:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.270086:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:21.0:1619677473.270086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:26.0:1619677473.270087:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.270089:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.270090:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.270091:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.2:1619677473.270092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:26.0:1619677473.270094:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.270096:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.270098:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.270099:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.270100:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677473.270100:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677473.270101:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.270101:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.270102:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.270103:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677473.270103:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.270104:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.270104:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.270105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.270106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677473.270106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.270113:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.270115:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.270116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.270116:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.270118:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.270120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677473.270358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.270367:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.270369:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.270370:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.270372:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.270372:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.270375:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.270908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.270915:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.270918:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.270918:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.270921:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.270924:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.270925:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677473.270927:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.270928:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.270929:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.270930:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677473.270933:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.270935:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677473.270938:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.270939:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677473.270940:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677473.270942:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677473.270942:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.270943:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677473.270944:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677473.270945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677473.270950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.270956:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:9.2:1619677473.270958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677473.270959:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677473.270959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.270960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:10.0:1619677473.270965:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.270968:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.270969:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.270969:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.270971:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.270975:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677473.271122:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.271125:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.271126:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.271128:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.271130:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.271131:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.271132:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.271133:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.271134:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.271135:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.271138:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.271140:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.271141:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.271142:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.271143:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.271144:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.271145:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.271145:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.271146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.271150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.271155:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.271157:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.271158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.271158:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.271160:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.271161:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.271632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.271639:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.271642:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.271643:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.271643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.271644:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.271646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.272520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.272527:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.272530:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.272530:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.272533:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.272535:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.272537:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.272538:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.272539:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.272540:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.272541:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.272545:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.272547:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.272550:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.272551:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.272551:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.272553:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.272554:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.272555:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.272556:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.272557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.272563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.272568:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.272570:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.272571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.272572:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677473.272830:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.272834:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.272835:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.272836:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.272837:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.272839:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.272840:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.272842:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.272843:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.272844:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.272847:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.272848:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.272851:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.272852:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.272852:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.272854:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.272854:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.272855:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.272856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.272860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.272869:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.272871:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.272872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.272873:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.272875:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.272878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.272909:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.272912:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.272913:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.272914:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.272917:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.272918:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.272919:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.272921:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.272922:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.272923:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.272926:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.272927:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.272930:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.272931:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.272931:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.272933:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.272933:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.272934:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.272934:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.272947:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.272949:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.272950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.272951:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.272953:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.272956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.272968:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.272971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.272971:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.272972:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.272973:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.272973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.272998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.273004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.273006:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.273008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.273009:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.273012:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.273014:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.273015:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.273016:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.273017:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.273018:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.273022:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.273023:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.273025:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.273026:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.273027:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.273029:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.273029:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.273030:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.273031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.273035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.273042:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.273044:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.273045:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.273045:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.273047:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.273048:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.273223:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.273226:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.273227:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.273229:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.273232:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.273234:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.273235:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.273236:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.273237:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.273238:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.273241:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.273242:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.273245:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.273246:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.273247:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.273248:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.273249:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.273250:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.273250:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.273263:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.273267:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.273268:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.273270:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.273273:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.273275:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.273697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.273704:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.273707:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.273708:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.273708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.273710:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.273712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.273716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.273722:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.273724:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.273725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.273725:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.273726:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.273726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.274581:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.274583:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.274586:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.274586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.274588:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.274590:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.274591:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.274592:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.274593:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.274594:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.274598:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.274599:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.274602:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.274603:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.274604:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.274606:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.274606:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.274607:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.274608:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677473.274736:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.274739:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.274739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.274741:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.274743:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.274744:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.274746:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.274747:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.274748:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.274749:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.274752:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.274754:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.274757:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.274757:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.274758:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.274760:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.274760:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.274761:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.274761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.274766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.274774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.274777:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.274778:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.274778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.274781:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.274784:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.275345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.275352:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:26.0:1619677473.275353:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677473.275354:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:26.0:1619677473.275355:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:24.0:1619677473.275355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.275355:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.275356:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677473.275358:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.275359:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.275360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677473.275361:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.275362:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.275364:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.275365:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.275366:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.275366:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.275369:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.275371:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.275373:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.275373:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.275374:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.275376:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.275376:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.275377:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.275378:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:12.0:1619677473.275378:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677473.275379:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677473.275381:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:9.2:1619677473.275381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:12.0:1619677473.275384:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.275386:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.275386:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.275387:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:12.0:1619677473.275387:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677473.275388:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.275389:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.275390:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.275390:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.275390:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.275391:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.275391:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677473.275391:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.275392:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.275393:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677473.275394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:12.0:1619677473.275394:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.275394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.275395:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:12.0:1619677473.275396:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677473.275399:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.275400:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:12.0:1619677473.275400:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:12.0:1619677473.275402:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:12.0:1619677473.275403:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.275403:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677473.275403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:12.0:1619677473.275404:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677473.275409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.275410:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.275412:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.275413:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677473.275414:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:26.0:1619677473.275415:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.275416:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.275417:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.275417:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:26.0:1619677473.275418:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.275418:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677473.275418:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:26.0:1619677473.275419:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677473.275419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:26.0:1619677473.275420:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.275422:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.275422:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.275422:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:26.0:1619677473.275423:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.275424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.275424:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.275427:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677473.275427:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.275429:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.275431:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677473.275432:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.275433:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.275434:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.275436:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.275436:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.275437:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.275437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.275450:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.275452:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.275453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.275454:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.275456:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.275459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.275798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.275806:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.275810:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.275811:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.275815:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.275818:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.275820:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.275823:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.275825:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.275826:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.275828:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.275834:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.275837:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.275840:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.275842:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.275843:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.275845:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.275845:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.275846:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.275847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.275859:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.275861:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.275862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.275862:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.275864:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.275867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.275915:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.275918:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.275921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.275921:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.275923:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.275924:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.275926:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.275927:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.275928:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.275929:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.275933:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.275935:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.275937:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.275938:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.275938:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.275940:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.275941:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.275942:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.275942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.275954:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.275957:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.275958:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.275958:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.275960:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.275963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.276069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677473.276076:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.276079:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.276081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.276082:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.276085:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.276087:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677473.276089:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.276090:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.276091:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.276093:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677473.276097:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.276099:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677473.276103:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.276104:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677473.276105:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677473.276107:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677473.276107:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.276108:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.276109:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677473.276113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.276119:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.276122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.276122:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.276123:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.276125:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.276128:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:1.0:1619677473.276225:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.276228:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.276229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.276231:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.276233:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.276235:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.276236:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.276237:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.276238:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.276239:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.276242:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.276244:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.276246:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.276247:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.276248:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.276249:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.276250:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.276251:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.276252:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.276252:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.276255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.276263:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677473.276265:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677473.276266:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677473.276266:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.276267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677473.276268:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.276268:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.276270:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.276271:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.276273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677473.276518:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.276521:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.276522:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.276523:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.276526:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.276527:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.276529:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.276530:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.276531:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.276532:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.276535:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.276537:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.276538:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.276539:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.276540:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.276541:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.276541:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.276543:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.276544:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.276544:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.276548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.276557:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.276557:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.276558:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.276560:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677473.276560:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.276561:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.276562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.276562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.276563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.276563:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.276804:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.276806:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.276807:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.276809:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.276811:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.276813:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677473.276814:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.276815:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.276816:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.276817:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677473.276820:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.276822:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.276825:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.276825:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677473.276826:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.276828:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.276828:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.276829:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.276829:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.276834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.276843:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.276845:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.276846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.276847:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.276849:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.276852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.276853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.276861:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.276863:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.276864:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.276864:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.276865:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.276866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677473.277381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.277388:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.277391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.277391:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.277392:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.277394:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.277395:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.277442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677473.277447:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.277450:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677473.277451:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677473.277453:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.277456:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.277457:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677473.277458:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.277460:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.277461:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.277462:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677473.277466:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.277468:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677473.277470:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.277471:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:1.0:1619677473.277472:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677473.277474:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677473.277475:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.277476:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677473.277477:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677473.277478:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:9.2:1619677473.277480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:5.2:1619677473.277484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:18.0:1619677473.277488:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:17.0:1619677473.277489:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:18.0:1619677473.277491:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:17.0:1619677473.277491:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000400:00000200:18.0:1619677473.277492:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.277492:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:2.0:1619677473.277492:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677473.277493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000400:00000200:18.0:1619677473.277495:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.277497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.277501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.277508:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.277511:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.277512:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.277512:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.277513:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.277514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.277637:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.277640:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.277641:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.277643:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.277646:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.277647:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.277648:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.277650:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.277651:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.277652:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.277655:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.277657:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.277659:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.277660:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.277661:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.277663:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.277663:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.277664:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.277664:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.277669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.277677:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.277679:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.277680:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.277681:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.277683:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.277686:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677473.277759:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.277762:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.277763:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.277764:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.277767:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.277769:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.277770:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.277772:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.277773:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.277774:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.277776:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.277778:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.277781:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.277781:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.277782:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.277783:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.277784:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.277785:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.277785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.277797:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.277800:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.277801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.277802:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.277804:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.277807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.278830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.278836:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.278839:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.278840:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.278841:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.278844:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.278846:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.278847:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.278849:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.278850:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.278851:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.278854:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.278856:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.278858:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.278859:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.278859:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.278861:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.278862:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.278863:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.278864:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.278868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.278877:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.278880:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.278881:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.278881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.278883:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:8.2:1619677473.278885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.278888:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.278889:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.278891:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.278892:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.278893:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.278893:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.278894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677473.279016:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.279019:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.279020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.279022:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.279024:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.279025:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.279026:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.279027:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.279028:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.279029:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.279032:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.279034:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.279036:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.279037:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.279038:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.279040:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.279040:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.279041:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.279041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.279045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.279054:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.279056:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.279057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.279058:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.279060:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.279062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.279144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.279150:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.279153:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.279154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.279154:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.279156:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.279158:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.279180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.279186:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.279188:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.279189:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.279189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.279190:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.279191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.279431:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677473.279434:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:1.0:1619677473.279434:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000400:00000200:1.0:1619677473.279436:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : GET - for me
00000400:00000200:1.0:1619677473.279439:0:19931:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.4.65@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1619677473.279441:0:19931:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.4.65@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:1.0:1619677473.279443:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677473.279444:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:1.0:1619677473.279446:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.4.65@o2ib3 local destination
00000400:00000200:1.0:1619677473.279450:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.4.65@o2ib3(10.51.4.65@o2ib3:10.51.4.65@o2ib3) : REPLY try# 0
00000800:00000200:1.0:1619677473.279452:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.4.65@o2ib3
00000800:00000200:1.0:1619677473.279454:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918ce37f3300] -> 10.51.4.65@o2ib3 (2) version: 12
00000800:00000200:1.0:1619677473.279455:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677473.279455:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb3800] (22)++
00000800:00000200:1.0:1619677473.279457:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918cfdbb3800] (23)--
00000800:00000200:1.0:1619677473.279457:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:17.0:1619677473.279467:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)++
00000800:00000200:17.0:1619677473.279469:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb3800] (23)--
00000400:00000200:17.0:1619677473.279470:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.279470:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:17.0:1619677473.279472:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.4.65@o2ib3: REPLY: OK
00000800:00000200:17.0:1619677473.279473:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:20.0:1619677473.279710:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.279713:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.279716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.279716:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.279717:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.279719:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.279720:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.279722:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.279723:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.279724:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.279727:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.279729:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.279731:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.279732:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.279733:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.279734:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.279735:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.279736:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.279737:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.279738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.279748:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677473.279749:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.279750:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.279751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677473.279752:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677473.279752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:22.0:1619677473.279753:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.279753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.279755:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.279757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.279998:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.280000:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.280001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.280004:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.280006:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.280008:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.280009:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.280010:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.280011:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.280012:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.280015:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.280017:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.280020:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.280021:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.280021:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.280023:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.280023:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.280024:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.280025:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.280036:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.280039:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.280040:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.280040:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.280042:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.280045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.280143:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.280146:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.280147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.280149:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.280152:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.280153:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.280154:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.280155:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.280157:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.280157:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.280161:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.280162:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.280164:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.280164:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.280165:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.280166:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.280167:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.280167:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.280168:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.280172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.280179:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.280181:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.280182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.280182:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.280184:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.280186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.280519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.280522:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.280522:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.280524:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.280526:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.280527:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.280528:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.280529:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.280530:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.280531:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.280534:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.280536:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.280538:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.280539:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.280540:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.280541:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.280542:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.280542:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.280543:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.280547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.280556:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.280559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.280559:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.280560:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.280562:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.280565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677473.280722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.280727:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.280730:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.280730:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.280732:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.280734:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.280736:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.280737:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.280738:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.280739:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.280740:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.280743:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.280745:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.280748:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.280749:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.280749:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.280751:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.280751:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.280752:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.280752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.280766:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.280768:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.280769:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.280769:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.280772:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.280774:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.280782:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.280785:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.280786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.280786:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.280787:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.280787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.281356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.281363:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.281366:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.281367:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.281367:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.281369:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.281371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.281558:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.281560:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.281561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.281563:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.281565:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.281566:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.281567:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.281569:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.281569:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.281571:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.281574:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.281575:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.281577:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.281578:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.281578:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.281579:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.281580:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.281580:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.281581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.281585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.281590:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.281592:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.281593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.281593:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.281594:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.281596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.282155:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.282158:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677473.282159:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677473.282160:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.282162:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.282163:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.282164:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.282166:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.282166:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.282167:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.282171:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.282172:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.282175:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.282176:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.282176:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.282178:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.282178:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.282180:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677473.282181:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677473.282181:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677473.282184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.282192:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.282194:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677473.282195:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.282196:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.282196:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:2.0:1619677473.282196:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.282197:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.0:1619677473.282197:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:18.0:1619677473.282198:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.282202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.283123:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.283125:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.283126:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.283128:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.283130:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.283131:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.283133:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.283134:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.283135:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.283136:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.283139:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.283140:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.283143:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.283144:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.283144:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.283146:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.283146:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.283147:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.283148:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.283161:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.283163:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.283164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.283165:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.283167:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.283174:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.283176:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.283178:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.283178:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.283179:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.283179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.283180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.283230:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.283233:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.283235:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.283235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.283238:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.283239:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.283241:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.283242:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.283243:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.283244:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.283247:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.283249:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.283252:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.283252:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.283253:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.283255:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.283255:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.283256:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.283257:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.283261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.283268:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.283270:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.283271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.283271:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.283273:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.283276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.283394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.283401:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.283403:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.283404:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.283404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.283406:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.283408:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.283414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.283420:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.283423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.283423:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.283423:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.283424:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.283425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.283434:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.283436:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.283437:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.283439:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.283442:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.283443:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677473.283444:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.283445:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.283446:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.283447:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677473.283450:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.283451:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.283454:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.283455:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677473.283456:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.283457:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.283458:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.283458:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.283459:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.283464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.283472:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.283475:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.283476:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.283476:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.283478:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.283481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677473.283587:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.283589:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.283590:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.283592:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.283594:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.283596:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.283596:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.283598:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.283599:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.283600:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.283603:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.283604:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.283607:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.283607:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.283608:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.283610:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.283610:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.283610:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.283611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.283616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.283624:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.283626:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.283627:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.283629:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.283630:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.283631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.283653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.283658:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.283661:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.283663:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.283663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.283666:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.283667:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.283669:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.283670:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.283671:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.283672:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.283676:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.283677:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.283680:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.283681:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.283681:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.283683:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.283683:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.283684:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677473.283685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677473.283697:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.283699:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.283700:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.283701:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.283702:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.283705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.284707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.284713:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.284716:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.284717:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.284719:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.284721:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.284723:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.284724:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.284725:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.284726:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.284728:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.284731:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.284733:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.284736:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.284737:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.284738:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.284740:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.284740:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.284741:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.284742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.284756:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.284760:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.284761:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.284763:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.284765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.284767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.284965:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.284967:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.284968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.284970:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.284974:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.284975:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.284976:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.284978:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.284979:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.284980:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.284983:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.284984:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.284987:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.284988:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.284989:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.284990:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.284991:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.284992:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.284992:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.285004:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.285007:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.285008:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.285008:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.285010:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.285013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.285105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.285112:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.285114:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.285115:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.285115:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.285117:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.285119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.285122:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.285124:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.285125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.285127:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.285130:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.285131:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.285132:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.285134:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.285134:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.285135:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.285138:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.285140:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.285142:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.285143:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.285143:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.285145:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.285145:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.285147:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.285148:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.285148:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677473.285157:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.285160:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.285160:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.285161:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:12.0:1619677473.285162:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.285163:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:13.0:1619677473.285164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.285164:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677473.285165:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.285165:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677473.285413:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.285416:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.285416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.285418:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.285420:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.285422:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.285423:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.285424:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.285425:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.285425:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.285428:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.285430:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.285432:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.285432:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.285433:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.285434:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.285435:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.285436:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.285437:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.285437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.285441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.285447:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.285450:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.285451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.285451:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.285451:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.285452:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.285453:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.285454:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.285455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.285455:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.285941:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.285944:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.285946:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.285947:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.285948:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.285950:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.285951:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.285952:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.285953:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.285954:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.285957:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.285958:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.285960:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.285961:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.285962:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.285964:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.285964:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.285965:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.285966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.285977:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.285980:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.285981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.285981:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.285983:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.285985:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.286073:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.286076:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.286077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.286078:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.286081:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677473.286081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:24.0:1619677473.286083:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.286084:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.286085:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.286086:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.286087:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677473.286087:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.286090:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:24.0:1619677473.286091:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.286091:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677473.286093:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:1.0:1619677473.286094:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.286095:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.286096:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.286096:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:1.0:1619677473.286096:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.286097:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677473.286098:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.286098:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677473.286099:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.286100:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:1.0:1619677473.286100:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.286101:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677473.286101:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.286102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677473.286102:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677473.286103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:1.0:1619677473.286105:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.286107:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.286109:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.286110:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:8.0:1619677473.286111:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:1.0:1619677473.286111:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.286112:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:8.0:1619677473.286113:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677473.286113:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677473.286114:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.286114:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677473.286115:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677473.286115:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:24.0:1619677473.286116:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:8.0:1619677473.286116:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.286116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677473.286117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.286118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.286118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.286119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:5.2:1619677473.286120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.286125:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.286126:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.286128:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.286128:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677473.286129:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.286129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677473.286129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.286131:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.286131:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.286133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.287032:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.287035:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.287036:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.287038:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.287039:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.287041:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.287042:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.287043:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.287044:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.287046:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.287049:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.287051:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.287054:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.287054:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.287055:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.287057:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.287057:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.287058:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.287058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.287062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.287071:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.287074:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.287075:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.287075:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.287078:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.287081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.287086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.287092:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.287095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.287095:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.287096:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.287098:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.287100:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.287925:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.287928:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.287929:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.287931:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.287933:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.287935:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.287936:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.287937:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.287938:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.287939:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.287942:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.287944:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.287947:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.287947:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.287948:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.287950:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.287950:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.287951:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.287951:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.287964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.287967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.287968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.287968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.287970:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.287973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.288833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.288841:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.288843:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.288844:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.288844:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.288846:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.288848:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.288853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.288859:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.288862:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.288863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.288863:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.288864:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.288864:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677473.288867:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.288871:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.288874:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.288876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.288877:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.288880:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677473.288882:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.288884:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.288886:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.288887:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677473.288892:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.288895:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677473.288898:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.288899:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:13.0:1619677473.288900:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:13.0:1619677473.288902:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677473.288903:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.288904:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677473.288906:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677473.288907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677473.288908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.288917:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.288919:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:12.0:1619677473.288920:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:2.0:1619677473.288920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.288921:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.288922:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:18.0:1619677473.288923:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.288923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.288924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677473.288927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677473.289346:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.289351:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.289353:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.289355:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.289358:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.289360:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.289362:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.289364:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.289365:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.289366:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.289370:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.289373:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.289376:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.289377:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.289378:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.289380:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.289380:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.289381:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.289382:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.289386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677473.289393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.289394:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.289396:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.289397:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.289399:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.289400:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.289402:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.289403:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677473.289404:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.289405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.289406:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.289408:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.289410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.289413:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.289414:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.289416:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.289417:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.289418:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.289418:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.289419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677473.289607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.289615:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.289619:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.289621:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.289624:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.289627:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.289630:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.289632:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.289634:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.289636:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.289638:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.289644:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.289647:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.289650:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.289652:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.289653:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.289655:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.289656:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.289658:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.289658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677473.289788:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.289790:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.289791:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.289793:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.289796:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.289797:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.289799:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.289800:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.289801:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.289802:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.289805:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.289807:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.289810:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.289811:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.289811:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.289813:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.289813:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.289815:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.289816:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.289816:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.289827:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.289829:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.289830:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.289831:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677473.289831:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.289831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.289832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.289833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.289833:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.289836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.289888:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.289891:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.289892:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.289893:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.289896:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.289898:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.289899:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.289900:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.289901:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.289902:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.289905:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.289906:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.289909:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.289910:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.289910:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.289912:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.289913:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.289913:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.289914:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.289918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.289926:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.289928:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.289929:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.289931:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.289932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.289934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.290076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.290113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.290119:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.290121:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.290122:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.290122:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.290124:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.290127:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.290697:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.290700:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.290701:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.290702:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.290705:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.290706:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.290707:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.290708:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.290709:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.290710:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.290713:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.290714:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.290716:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.290717:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.290717:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.290719:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.290719:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.290720:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.290720:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.290725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.290733:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.290735:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.290736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.290736:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.290737:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.290740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.291029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.291035:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.291037:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.291038:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.291038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.291040:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.291041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.291084:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.291087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.291087:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.291089:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.291092:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.291093:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.291094:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.291095:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.291096:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.291097:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.291100:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.291102:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.291103:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.291104:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.291104:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.291106:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.291106:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.291107:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.291107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.291111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.291117:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.291119:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.291119:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.291120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.291121:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.291122:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.291307:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.291310:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.291310:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.291312:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.291315:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.291316:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.291317:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.291319:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.291320:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.291321:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.291324:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.291326:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.291328:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.291329:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.291330:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.291331:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.291332:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.291332:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.291333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.291338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.291346:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.291349:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.291350:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677473.291350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:21.0:1619677473.291352:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.291352:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.291355:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677473.291356:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.291359:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.291360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.291361:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.291364:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.291366:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.291367:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.291368:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.291369:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.291370:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.291374:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.291375:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.291378:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.291378:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.291379:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.291380:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.291381:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.291381:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.291382:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.291395:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.291397:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.291398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.291398:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.291400:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.291402:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.291404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3a3000] (20)++
00000800:00000200:18.0:1619677473.291410:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3a3000] (21)++
00000800:00000200:22.0:1619677473.291413:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:18.0:1619677473.291413:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.12.22@o2ib3
00000800:00000200:2.0:1619677473.291414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)--
00000800:00000200:22.0:1619677473.291416:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.291417:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000400:00000200:18.0:1619677473.291417:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.22@o2ib3 : REPLY - routed
00000800:00000200:6.0:1619677473.291417:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.291418:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.291419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677473.291420:0:19933:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:18.0:1619677473.291425:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907a3a3000] (21)++
00000800:00000200:18.0:1619677473.291426:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3a3000] (22)++
00000800:00000200:18.0:1619677473.291427:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3a3000] (23)--
00000800:00000200:18.0:1619677473.291428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)--
00000800:00000200:20.2:1619677473.291434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3a3000] (21)++
00000800:00000200:2.0:1619677473.291439:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)++
00000800:00000200:2.0:1619677473.291441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (23)--
00000800:00000200:2.0:1619677473.291442:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)--
00000800:00000200:20.2:1619677473.291559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3a3000] (21)++
00000800:00000200:18.0:1619677473.291565:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)++
00000800:00000200:18.0:1619677473.291568:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.12.22@o2ib3
00000800:00000200:2.0:1619677473.291568:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (23)--
00000800:00000200:18.0:1619677473.291570:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907a3a3000] (22)--
00000400:00000200:18.0:1619677473.291571:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.291573:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677473.291574:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.291575:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.291576:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.291577:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677473.291581:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:18.0:1619677473.291582:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:18.0:1619677473.291584:0:19933:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:18.0:1619677473.291590:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.291591:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:18.0:1619677473.291592:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:18.0:1619677473.291594:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:18.0:1619677473.291594:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3a3000] (21)++
00000800:00000200:18.0:1619677473.291594:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3a3000] (22)--
00000800:00000200:18.0:1619677473.291595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (21)--
00000800:00000200:12.0:1619677473.291607:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.291610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677473.291610:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.291634:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.291637:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677473.291641:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677473.291642:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.291656:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677473.291656:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.291793:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.291798:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.291799:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.291799:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:13.0:1619677473.291800:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:13.0:1619677473.291803:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:6.0:1619677473.291836:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (21)++
00000800:00000200:22.0:1619677473.291840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000800:00000200:6.0:1619677473.291840:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000400:00000200:6.0:1619677473.291843:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.291846:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.291847:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677473.291849:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.291850:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.291851:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.291852:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677473.291856:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.291857:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.291859:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.291860:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677473.291861:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.291862:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.291863:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (21)++
00000800:00000200:6.0:1619677473.291864:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (22)--
00000800:00000200:6.0:1619677473.291864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (21)--
00000800:00000200:22.0:1619677473.291875:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.291877:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.291878:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.291878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.291879:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.291881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.291994:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (21)++
00000800:00000200:6.0:1619677473.291997:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000800:00000200:22.0:1619677473.291998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000400:00000200:6.0:1619677473.291999:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.292000:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.292002:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677473.292003:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.292004:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.292005:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.292005:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677473.292008:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.292010:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677473.292013:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.292014:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:6.0:1619677473.292014:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677473.292016:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:6.0:1619677473.292016:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (21)++
00000800:00000200:6.0:1619677473.292017:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (22)--
00000800:00000200:6.0:1619677473.292017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (21)--
00000800:00000200:9.2:1619677473.292022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.292029:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.292031:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677473.292032:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.292033:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.292035:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.292038:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:22.0:1619677473.292041:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (21)++
00000800:00000200:22.0:1619677473.292044:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000800:00000200:6.0:1619677473.292044:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000400:00000200:22.0:1619677473.292047:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.292050:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.292051:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677473.292052:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.292054:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.292055:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.292055:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677473.292059:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.292060:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677473.292062:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.292063:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677473.292063:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677473.292065:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677473.292065:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (21)++
00000800:00000200:22.0:1619677473.292066:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (22)--
00000800:00000200:22.0:1619677473.292067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (21)--
00000800:00000200:8.2:1619677473.292071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.292078:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.292081:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.292082:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.292082:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.292084:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.292087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.292107:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (21)++
00000800:00000200:22.0:1619677473.292110:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000800:00000200:6.0:1619677473.292110:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000400:00000200:6.0:1619677473.292112:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.292114:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677473.292115:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.292116:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:8.0:1619677473.292117:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.292118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677473.292118:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.292120:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.292120:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.292121:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.292122:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.292122:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677473.292124:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:6.0:1619677473.292125:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.292126:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96510181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.292127:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.292128:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:8.0:1619677473.292129:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.292130:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:6.0:1619677473.292132:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.292133:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:6.0:1619677473.292133:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:8.0:1619677473.292134:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.292135:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:8.0:1619677473.292136:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677473.292136:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (21)++
00000800:00000200:6.0:1619677473.292137:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919874bc4600] (22)++
00000800:00000200:8.0:1619677473.292138:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.292139:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.292139:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.292140:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677473.292140:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (24)--
00000800:00000200:6.0:1619677473.292140:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (23)--
00000800:00000200:8.0:1619677473.292141:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.292141:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (22)++
00000800:00000200:8.0:1619677473.292142:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.292142:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.292143:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000800:00000200:22.0:1619677473.292144:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (23)++
00000800:00000200:22.0:1619677473.292145:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919874bc4600] (24)--
00000800:00000200:22.0:1619677473.292146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (23)--
00000800:00000200:9.2:1619677473.292146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677473.292146:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.292147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000800:00000200:12.0:1619677473.292148:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:6.0:1619677473.292148:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.292150:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:6.0:1619677473.292150:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677473.292151:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.292151:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96510180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.292152:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:6.0:1619677473.292153:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.292154:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.292154:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.292154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677473.292154:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.292154:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.292155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:6.0:1619677473.292155:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.292156:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.292157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:24.0:1619677473.292158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677473.292159:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.292161:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 344 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677473.292163:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.292164:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:9.2:1619677473.292165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677473.292165:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677473.292167:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677473.292167:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (21)++
00000800:00000200:6.0:1619677473.292168:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (22)--
00000800:00000200:6.0:1619677473.292169:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (21)--
00000800:00000200:8.0:1619677473.292172:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:9.2:1619677473.292173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.292174:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.292175:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.292175:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.292176:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.292177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677473.292180:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.292183:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.292184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.292184:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.292186:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.292189:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.292196:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (21)++
00000800:00000200:22.0:1619677473.292199:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000800:00000200:6.0:1619677473.292199:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000400:00000200:22.0:1619677473.292201:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.292204:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.292206:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:22.0:1619677473.292207:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.292208:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.292209:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.292210:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:22.0:1619677473.292213:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.292214:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677473.292216:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.292217:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:22.0:1619677473.292217:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677473.292219:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:22.0:1619677473.292219:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (21)++
00000800:00000200:22.0:1619677473.292220:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (22)--
00000800:00000200:22.0:1619677473.292221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (21)--
00000800:00000200:8.2:1619677473.292225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.292233:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.292235:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.292236:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.292236:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (21)++
00000400:00000200:20.0:1619677473.292238:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.292239:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (22)++
00000800:00000200:21.0:1619677473.292239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:6.0:1619677473.292239:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000800:00000200:20.0:1619677473.292240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677473.292241:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000400:00000200:6.0:1619677473.292242:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.292243:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677473.292243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:22.0:1619677473.292245:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.292245:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.292246:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:22.0:1619677473.292248:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.292249:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.292250:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.292251:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.292251:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677473.292252:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.0:1619677473.292254:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677473.292254:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.292255:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.292255:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.292256:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000400:00000200:6.0:1619677473.292256:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.292257:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.292257:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:6.0:1619677473.292257:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.292258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:6.0:1619677473.292258:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677473.292260:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.292261:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:22.0:1619677473.292262:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677473.292262:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.292264:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:6.0:1619677473.292264:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677473.292265:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (23)++
00000800:00000200:22.0:1619677473.292266:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (24)--
00000800:00000200:22.0:1619677473.292266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (23)--
00000800:00000200:6.0:1619677473.292266:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.292267:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:6.0:1619677473.292267:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677473.292268:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (22)++
00000800:00000200:22.0:1619677473.292269:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000800:00000200:8.2:1619677473.292269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677473.292269:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677473.292270:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (23)++
00000400:00000200:22.0:1619677473.292271:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.292271:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919874bc4600] (24)++
00000800:00000200:6.0:1619677473.292272:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (25)--
00000400:00000200:22.0:1619677473.292273:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.292273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (24)--
00000400:00000200:22.0:1619677473.292274:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:6.0:1619677473.292274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (23)--
00000400:00000200:22.0:1619677473.292275:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96510176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677473.292275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677473.292277:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.292277:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.292278:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677473.292282:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.292282:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:22.0:1619677473.292283:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 376 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.0:1619677473.292283:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.292284:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.292284:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:6.0:1619677473.292284:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (23)++
00000400:00000200:21.0:1619677473.292285:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.292285:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:6.0:1619677473.292285:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919874bc4600] (24)--
00000400:00000200:20.0:1619677473.292286:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.292286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (23)--
00000800:00000200:22.0:1619677473.292287:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.292287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000800:00000200:22.0:1619677473.292288:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000400:00000200:20.0:1619677473.292288:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.292289:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:22.0:1619677473.292292:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677473.292292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.292293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677473.292296:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677473.292299:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (22)++
00000800:00000200:22.0:1619677473.292300:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (23)++
00000800:00000200:21.0:1619677473.292300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677473.292301:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (24)--
00000800:00000200:6.0:1619677473.292301:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000800:00000200:22.0:1619677473.292302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (23)--
00000800:00000200:22.0:1619677473.292303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000800:00000200:21.0:1619677473.292303:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677473.292303:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.292304:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.292305:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.292305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.292306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:13.0:1619677473.292306:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:6.0:1619677473.292307:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.292309:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:13.0:1619677473.292310:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:6.0:1619677473.292310:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.292311:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.292312:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.292313:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.292314:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.292314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:6.0:1619677473.292314:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677473.292317:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:6.0:1619677473.292318:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.292320:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677473.292322:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.292323:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677473.292323:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677473.292325:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677473.292326:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (21)++
00000800:00000200:6.0:1619677473.292327:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (22)--
00000800:00000200:6.0:1619677473.292329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000800:00000200:6.0:1619677473.292330:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc4600] (21)++
00000800:00000200:8.2:1619677473.292332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677473.292332:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.64@o2ib3
00000800:00000200:22.0:1619677473.292333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (22)--
00000400:00000200:6.0:1619677473.292335:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.292338:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.292339:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677473.292340:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:6.0:1619677473.292340:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.292341:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.292342:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677473.292343:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:6.0:1619677473.292343:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.292344:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.292344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.292347:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.292347:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.292349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.292349:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.292350:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.292351:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677473.292351:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.292353:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.292353:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc4600] (21)++
00000800:00000200:6.0:1619677473.292354:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc4600] (22)--
00000800:00000200:6.0:1619677473.292355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc4600] (21)--
00000800:00000200:22.0:1619677473.292366:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.292368:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.292369:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.292369:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.292371:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.292372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.292384:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.292387:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.292387:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.292388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.292388:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.292389:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.292606:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.292609:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.292610:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.292613:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.292615:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.292616:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.292617:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.292619:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.292620:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.292620:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.292623:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.292625:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.292628:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.292628:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.292629:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.292631:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.292631:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.292632:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.292633:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677473.292643:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.292645:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.292646:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.292647:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.292649:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.292650:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.292665:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.292668:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.292668:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.292669:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.292670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.292670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.292761:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.292763:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.292764:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.292766:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.292768:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.292770:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.292771:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.292772:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.292773:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.292774:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.292776:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.292778:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.292781:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.292782:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.292782:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.292784:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.292784:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.292785:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.292786:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.292797:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.292800:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.292801:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.292801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.292803:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.292806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.293015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.293023:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.293026:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.293026:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.293029:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.293032:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.293033:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.293034:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.293036:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.293037:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.293038:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.293041:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.293042:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.293045:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.293045:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.293046:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.293048:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.293048:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.293049:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.293050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.293054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.293061:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.293063:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.293064:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.293065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.293066:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.293067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.293115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.293122:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.293124:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.293125:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.293125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.293127:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.293129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.293758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.293761:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677473.293762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677473.293764:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.293765:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.293767:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677473.293768:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.293769:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.293770:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.293771:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677473.293775:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.293776:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677473.293780:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.293780:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677473.293781:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677473.293782:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677473.293783:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.293783:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.293784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677473.293788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.293797:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.293799:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.293800:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.293800:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.293802:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.293805:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.293812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.293819:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.293822:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.293822:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.293822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.293823:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.293824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677473.293875:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.293878:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.293879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.293881:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.293884:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.293886:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.293887:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.293888:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.293889:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.293890:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.293893:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.293895:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.293897:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.293898:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.293898:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.293900:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.293901:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.293902:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.293903:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.293904:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.293907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.293916:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.293916:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.293919:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677473.293919:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.293919:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.293920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677473.293920:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.293921:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.293923:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.293923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.293927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.293929:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.293931:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.293931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.293932:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.293933:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.293933:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.294257:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.294260:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.294261:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.294263:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.294266:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.294267:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.294269:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.294270:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.294271:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.294272:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.294275:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.294276:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.294279:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.294280:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.294280:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.294282:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.294282:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.294284:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.294285:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.294285:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677473.294288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.294296:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677473.294299:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:1.0:1619677473.294299:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.294300:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.294300:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.294302:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.294302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.294303:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:1.0:1619677473.294303:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.294306:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677473.294609:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.294612:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.294614:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.294616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.294617:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.294618:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677473.294620:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.294621:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.294622:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.294623:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.294627:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.294629:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.294632:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.294632:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677473.294633:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677473.294635:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677473.294635:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.294636:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.294637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677473.294754:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.294756:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.294757:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.294759:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.294762:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.294763:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.294764:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.294766:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.294767:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.294768:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.294771:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.294773:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.294776:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.294777:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.294778:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.294779:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.294779:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.294780:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.294780:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.294793:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.294796:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.294797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.294797:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.294799:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.294803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.295047:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.295050:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.295051:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.295053:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.295056:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.295057:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.295058:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.295060:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.295061:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.295062:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.295065:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.295066:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.295069:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.295069:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.295070:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.295072:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.295072:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.295073:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.295073:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.295078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.295086:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.295089:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.295090:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.295090:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.295092:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.295095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677473.295329:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.295331:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.295333:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.295334:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.295336:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.295338:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.295339:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.295340:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.295341:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.295342:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.295346:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.295348:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.295350:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.295351:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.295352:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.295354:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.295354:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.295356:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.295357:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.295357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.295366:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.295368:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.295369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.295370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.295370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.295377:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.295379:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.295380:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.295380:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.295383:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.295385:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.295511:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.295513:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.295514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.295516:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.295519:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.295521:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.295522:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.295523:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.295524:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.295525:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.295528:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.295530:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.295532:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.295533:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.295534:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.295535:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.295535:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.295536:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.295537:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.295538:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.295541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.295547:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.295549:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.295550:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.295550:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.295551:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677473.295551:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.295553:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.295553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677473.295554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.295555:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.295563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.2:1619677473.295568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3a3000] (20)++
00000800:00000200:8.0:1619677473.295569:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.295572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.295572:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.295572:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.295573:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.295574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677473.295575:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3a3000] (21)++
00000800:00000200:18.0:1619677473.295578:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.12.22@o2ib3
00000800:00000200:2.0:1619677473.295578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)--
00000400:00000200:18.0:1619677473.295580:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.22@o2ib3 : REPLY - routed
00000800:00000200:18.0:1619677473.295583:0:19933:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:18.0:1619677473.295588:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907a3a3000] (21)++
00000800:00000200:18.0:1619677473.295589:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3a3000] (22)++
00000800:00000200:18.0:1619677473.295589:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3a3000] (23)--
00000800:00000200:18.0:1619677473.295590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)--
00000800:00000200:20.2:1619677473.295595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3a3000] (21)++
00000800:00000200:2.0:1619677473.295600:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)++
00000800:00000200:2.0:1619677473.295602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (23)--
00000800:00000200:2.0:1619677473.295603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)--
00000800:00000200:20.2:1619677473.295657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.295693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.295700:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.295702:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.295703:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.295703:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.295705:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.295707:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.295717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.295722:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.295725:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.295725:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.295726:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.295726:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.295727:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677473.295903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.295909:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.295912:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.295912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.295914:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.295917:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.295918:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.295920:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.295921:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.295922:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.295923:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.295926:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.295927:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.295929:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.295930:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.295931:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.295932:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.295933:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.2:1619677473.295934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3a3000] (21)++
00000800:00000200:18.0:1619677473.295934:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.295935:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.295936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.295937:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)++
00000800:00000200:18.0:1619677473.295939:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.12.22@o2ib3
00000800:00000200:8.2:1619677473.295939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.295940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (23)--
00000800:00000200:20.2:1619677473.295941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.295941:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907a3a3000] (22)--
00000400:00000200:18.0:1619677473.295942:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.295944:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677473.295945:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.295946:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.295947:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677473.295947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677473.295947:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:18.0:1619677473.295949:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.295949:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.295949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677473.295950:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677473.295950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:17.0:1619677473.295951:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.295951:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:18.0:1619677473.295953:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : REPLY try# 0
00000400:00000200:17.0:1619677473.295953:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.295955:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.295956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677473.295957:0:19933:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:18.0:1619677473.295966:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.295967:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677473.295967:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677473.295969:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677473.295970:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3a3000] (21)++
00000800:00000200:18.0:1619677473.295970:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3a3000] (22)--
00000800:00000200:18.0:1619677473.295971:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (21)--
00000800:00000200:22.0:1619677473.295984:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.295986:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.295987:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.296019:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.296022:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.296023:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.296027:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.296041:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.296041:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:5.2:1619677473.296205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.296211:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.296213:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.296214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.296216:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.296218:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.296219:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.296221:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.296222:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.296223:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.296224:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.296227:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.296229:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.296231:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.296232:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.296233:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.296235:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.296235:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.296236:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.296236:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.296293:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.296299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.296300:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.296301:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.296303:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.296305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677473.296438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.296447:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.296449:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.296451:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.296452:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.296453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.296456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.296460:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.296464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.296464:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.296465:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.296467:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: REPLY: OK
00000800:00000200:6.0:1619677473.296469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.296657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.296664:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.296666:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.296667:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.296667:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.296669:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.296671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.297026:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.297029:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.297030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.297031:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.297034:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.297035:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.297036:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.297038:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.297038:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.297039:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.297042:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.297044:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.297045:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.297046:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.297046:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.297048:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.297048:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.297049:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.297049:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.297064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.297070:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.297071:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.297072:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.297073:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.297074:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.297075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.297084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.297090:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.297092:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.297093:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.297093:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.297094:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.297094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677473.297488:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.297491:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.297492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.297494:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.297496:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.297497:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.297498:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.297500:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.297500:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.297501:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.297505:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.297506:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.297509:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.297510:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.297510:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.297512:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.297512:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.297513:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.297514:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.297519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.297528:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.297530:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.297531:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.297532:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.297535:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.297536:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.297538:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.297541:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.297542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.297544:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.297546:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.297547:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.297549:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.297550:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.297551:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.297552:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.297555:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.297556:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.297559:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.297560:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.297561:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.297563:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.297563:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.297564:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.297564:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.297578:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.297580:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.297581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.297582:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.297584:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.297587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.297954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.297962:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.297965:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.297966:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.297966:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.297968:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.297971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677473.298201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3a3000] (20)++
00000800:00000200:24.0:1619677473.298205:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.298208:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677473.298208:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3a3000] (21)++
00000800:00000200:8.0:1619677473.298209:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.298211:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.298211:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.12.22@o2ib3
00000800:00000200:18.0:1619677473.298212:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)--
00000400:00000200:24.0:1619677473.298213:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.298214:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.22@o2ib3 : REPLY - routed
00000400:00000200:24.0:1619677473.298215:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.298216:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.298217:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677473.298217:0:19932:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000400:00000200:24.0:1619677473.298218:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.298219:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.298222:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.298222:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907a3a3000] (21)++
00000800:00000200:24.0:1619677473.298223:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.298223:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3a3000] (22)++
00000800:00000200:2.0:1619677473.298224:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3a3000] (23)--
00000800:00000200:2.0:1619677473.298224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)--
00000800:00000200:24.0:1619677473.298226:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.298227:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.298228:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.298229:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.298230:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.298230:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.298231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677473.298232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3a3000] (21)++
00000800:00000200:8.2:1619677473.298235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.298238:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)++
00000800:00000200:18.0:1619677473.298240:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (23)--
00000800:00000200:18.0:1619677473.298241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)--
00000800:00000200:17.0:1619677473.298243:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.298246:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.298247:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.298247:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.298249:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.298252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.298535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.298542:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.298545:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.298546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.298546:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.298548:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.298550:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.298568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3a3000] (21)++
00000800:00000200:2.0:1619677473.298574:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3a3000] (22)++
00000800:00000200:2.0:1619677473.298577:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.12.22@o2ib3
00000800:00000200:18.0:1619677473.298578:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (23)--
00000800:00000200:2.0:1619677473.298579:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907a3a3000] (22)--
00000400:00000200:2.0:1619677473.298580:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.298582:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677473.298584:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.298585:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.298586:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.298587:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677473.298591:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : REPLY try# 0
00000800:00000200:2.0:1619677473.298593:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677473.298595:0:19932:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:2.0:1619677473.298608:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.298609:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677473.298610:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677473.298612:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677473.298612:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3a3000] (21)++
00000800:00000200:2.0:1619677473.298613:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3a3000] (22)--
00000800:00000200:2.0:1619677473.298614:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3a3000] (21)--
00000800:00000200:26.0:1619677473.299794:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.299797:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.299797:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.299799:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.299802:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.299804:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677473.299805:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.299806:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.299807:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.299808:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677473.299811:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.299813:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677473.299816:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.299817:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677473.299817:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677473.299819:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677473.299819:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.299820:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.299821:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.299826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677473.299833:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677473.299835:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:10.0:1619677473.299836:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.299837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.299838:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.299838:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.299839:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.299839:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.299840:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.299842:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.299842:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.299843:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677473.299844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:10.0:1619677473.299845:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.299845:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.299846:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.299848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:10.0:1619677473.299849:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.299850:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677473.299851:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.299852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.299852:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.299853:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677473.299854:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.299854:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.299854:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.299855:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.299856:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.299857:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.299858:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.299859:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.299860:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.299873:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.299876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.299876:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.299877:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.299879:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.299882:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.299895:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.299898:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.299899:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.299899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.299900:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.299901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.300342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.300348:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.300351:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.300352:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.300354:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.300357:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.300358:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677473.300360:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.300361:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.300362:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.300363:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677473.300366:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.300368:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677473.300372:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.300373:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677473.300373:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677473.300375:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677473.300376:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.300377:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.300377:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677473.300381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.300388:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.300391:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.300392:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.300392:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.300394:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.300398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.300400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677473.300406:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.300408:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677473.300409:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.300409:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.300410:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.300410:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677473.300495:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.300497:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.300498:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.300500:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.300502:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.300504:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.300504:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.300506:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.300507:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.300508:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.300510:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.300512:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.300514:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.300515:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.300516:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.300517:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.300518:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.300519:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.300519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.300523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.300532:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.300534:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.300535:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.300537:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.300537:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.300539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.300539:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677473.300540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.300541:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677473.300541:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.300544:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.300545:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.300546:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.300548:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.300548:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.300550:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.300553:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.300555:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.300558:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.300558:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.300559:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.300561:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.300561:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.300562:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677473.300562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.300567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.300574:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.300577:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.300578:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.300578:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.300580:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.300583:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.300714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.300751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.300757:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.300759:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.300760:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.300760:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.300762:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.300764:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.300779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.300785:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.300788:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.300789:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.300789:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.300790:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.300792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.301138:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.301140:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.301141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.301143:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.301145:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.301146:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.301147:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.301149:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.301150:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.301150:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.301153:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.301155:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.301157:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.301158:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.301159:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.301160:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.301161:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.301161:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.301162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.301167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.301175:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.301177:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.301178:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.301179:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.301181:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.301184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.301323:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.301326:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.301327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.301328:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.301331:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.301332:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.301333:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.301334:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.301335:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.301336:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.301339:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.301340:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.301343:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.301343:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.301344:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.301345:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.301346:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.301347:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.301347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.301360:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.301363:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.301363:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.301365:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.301367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.301368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677473.301585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.301591:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.301594:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.301595:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.301596:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.301599:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.301601:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.301602:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.301603:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.301604:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.301605:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.301609:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.301610:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.301613:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.301614:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.301614:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.301616:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.301617:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.301617:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.301618:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.301630:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.301633:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.301634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.301634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.301636:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.301638:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677473.301688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.301697:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.301699:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.301700:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.301702:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.301703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.301705:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677473.302275:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.302278:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.302279:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.302281:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.302284:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.302285:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.302286:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.302288:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.302289:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.302290:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.302299:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.302301:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.302304:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.302305:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.302305:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.302307:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.302307:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.302308:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.302310:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.302310:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.302314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.302322:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.302324:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.302324:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.302326:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.302326:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:2.0:1619677473.302326:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.302327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.302327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677473.302328:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.302331:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.302413:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.302416:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.302416:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.302418:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.302421:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.302422:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.302423:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.302425:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.302426:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.302427:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.302430:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.302431:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.302434:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.302435:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.302435:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.302437:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.302437:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.302438:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.302438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.302451:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.302454:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.302454:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.302455:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.302457:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.302460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.302590:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.302592:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.302593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.302596:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.302598:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.302600:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677473.302601:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.302602:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.302604:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.302605:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677473.302609:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.302611:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677473.302614:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.302615:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677473.302616:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677473.302618:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677473.302618:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.302619:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.302620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.302623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.302630:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.302632:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.302633:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:9.2:1619677473.302633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:26.0:1619677473.302634:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.302636:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.302639:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:8.0:1619677473.302639:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.302641:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.302642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.302642:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.302644:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.302647:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.302669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.302675:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.302678:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.302678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.302680:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.302683:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.302684:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.302686:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.302687:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.302688:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.302689:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.302693:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.302695:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.302698:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.302698:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.302699:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.302701:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.302702:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.302703:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.302703:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.302714:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.302717:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.302718:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.302720:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.302722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.302723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.302904:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.302907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.302907:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.302909:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.302912:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.302913:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.302914:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.302915:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.302916:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.302917:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.302920:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.302922:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.302925:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.302926:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.302927:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.302928:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.302929:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.302930:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.302930:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.302935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.302943:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.302945:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.302946:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.302947:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.302949:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.302951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677473.303104:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.303107:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.303108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.303110:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.303112:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.303114:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.303115:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.303116:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.303117:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.303118:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.303121:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.303123:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.303126:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.303126:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.303127:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.303128:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.303129:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.303130:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.303131:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.303132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.303135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.303143:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677473.303145:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.303145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677473.303146:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677473.303146:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677473.303148:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.303149:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.303150:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.303151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.303153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.303265:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.303268:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.303268:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.303269:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:26.0:1619677473.303271:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.303272:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.303275:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.303275:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.303277:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.303278:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.303280:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.303281:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.303282:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.303284:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:10.0:1619677473.303285:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.303286:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.303286:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.303287:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.303288:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.303288:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677473.303292:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.303292:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.303293:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.303302:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.303302:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677473.303304:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.303304:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:10.0:1619677473.303304:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.303305:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677473.303306:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (24)--
00000800:00000200:10.0:1619677473.303306:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677473.303307:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.303308:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.303308:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677473.303308:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:26.0:1619677473.303309:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.303310:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (24)++
00000400:00000200:26.0:1619677473.303311:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677473.303311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.303312:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (25)--
00000800:00000200:10.0:1619677473.303312:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (24)--
00000400:00000200:26.0:1619677473.303313:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.303313:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)++
00000400:00000200:26.0:1619677473.303314:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677473.303314:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:26.0:1619677473.303315:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96510128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.303315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:26.0:1619677473.303317:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.303317:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.303318:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.303318:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:10.0:1619677473.303320:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.303322:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.303322:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:24.0:1619677473.303323:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:10.0:1619677473.303323:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.303323:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.303324:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677473.303324:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:10.0:1619677473.303325:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:24.0:1619677473.303326:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:2.0:1619677473.303326:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.303327:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.303327:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96510127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.303329:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.303329:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677473.303329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:8.0:1619677473.303329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:10.0:1619677473.303330:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.303331:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677473.303334:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.303335:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.303335:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677473.303335:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677473.303335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677473.303335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.303336:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.303337:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677473.303338:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.303338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677473.303338:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.303338:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.303339:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (24)++
00000800:00000200:2.0:1619677473.303339:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.303340:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (25)--
00000800:00000200:24.0:1619677473.303340:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677473.303341:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (24)--
00000800:00000200:10.0:1619677473.303341:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.303341:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.303342:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:24.0:1619677473.303342:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.303342:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:10.0:1619677473.303342:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:10.0:1619677473.303342:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677473.303342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677473.303342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677473.303343:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (24)--
00000800:00000200:24.0:1619677473.303343:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677473.303344:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.303344:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:24.0:1619677473.303344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.303345:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:10.0:1619677473.303345:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:26.0:1619677473.303346:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.303346:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (24)--
00000800:00000200:10.0:1619677473.303347:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)--
00000400:00000200:26.0:1619677473.303348:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.303348:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.303349:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:26.0:1619677473.303350:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677473.303350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:26.0:1619677473.303351:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677473.303351:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.303352:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96510126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.303353:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:26.0:1619677473.303354:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.303354:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.303355:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677473.303356:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.303357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677473.303357:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.303357:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.303358:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.303359:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:10.0:1619677473.303359:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677473.303360:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96510125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.303360:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.303361:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.303361:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.303362:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.303362:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.303363:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.303365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.303365:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.303365:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677473.303368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677473.303370:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.303370:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.303371:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.303371:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.303372:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:22.0:1619677473.303372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.2:1619677473.303372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:6.0:1619677473.303372:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.303373:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677473.303373:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677473.303373:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.303374:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.303375:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:10.0:1619677473.303375:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.303375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.303376:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:10.0:1619677473.303376:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677473.303377:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (24)--
00000800:00000200:10.0:1619677473.303377:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677473.303378:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.303379:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.303379:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677473.303379:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:26.0:1619677473.303380:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:21.0:1619677473.303381:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:10.0:1619677473.303381:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (24)++
00000800:00000200:10.0:1619677473.303382:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (25)--
00000800:00000200:10.0:1619677473.303382:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (24)--
00000400:00000200:26.0:1619677473.303383:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.303383:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677473.303383:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:21.0:1619677473.303384:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)++
00000800:00000200:21.0:1619677473.303385:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (24)--
00000800:00000200:10.0:1619677473.303385:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:26.0:1619677473.303386:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.303386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000400:00000200:26.0:1619677473.303387:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677473.303387:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:26.0:1619677473.303388:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96510124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.303388:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.303389:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:26.0:1619677473.303390:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.303390:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.303391:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.303392:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677473.303392:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.303393:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:10.0:1619677473.303394:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.303395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:26.0:1619677473.303396:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.303396:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677473.303398:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.303398:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96510123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.303400:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.303401:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.303402:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677473.303404:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.303405:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.303406:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677473.303406:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.303407:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677473.303408:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.303409:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (24)++
00000800:00000200:26.0:1619677473.303410:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (25)--
00000800:00000200:26.0:1619677473.303410:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (24)--
00000800:00000200:10.0:1619677473.303410:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.303411:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:10.0:1619677473.303411:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677473.303412:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.303413:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:9.2:1619677473.303413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.303414:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:26.0:1619677473.303415:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.303415:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (24)++
00000800:00000200:10.0:1619677473.303415:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (25)--
00000800:00000200:10.0:1619677473.303416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (24)--
00000800:00000200:10.0:1619677473.303417:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:10.0:1619677473.303417:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (24)--
00000400:00000200:26.0:1619677473.303418:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.303418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)--
00000400:00000200:26.0:1619677473.303419:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677473.303419:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.303419:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:26.0:1619677473.303420:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96510122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.303420:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:9.2:1619677473.303420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:26.0:1619677473.303422:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.303422:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.303422:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:26.0:1619677473.303423:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.303423:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.303424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677473.303425:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.303426:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.303427:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.303428:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:10.0:1619677473.303428:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.303428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.303429:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677473.303429:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677473.303430:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96510121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677473.303431:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:10.0:1619677473.303431:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.303432:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677473.303432:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.303433:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.303433:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.303435:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.303436:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.303437:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)++
00000400:00000200:10.0:1619677473.303437:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677473.303437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.303438:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (23)++
00000800:00000200:10.0:1619677473.303439:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677473.303440:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (24)--
00000800:00000200:26.0:1619677473.303440:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:26.0:1619677473.303441:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (24)--
00000800:00000200:10.0:1619677473.303441:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.303442:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.303442:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (23)++
00000800:00000200:26.0:1619677473.303443:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:18.0:1619677473.303443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (24)--
00000800:00000200:10.0:1619677473.303443:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (24)++
00000800:00000200:26.0:1619677473.303444:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:18.0:1619677473.303445:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (24)++
00000800:00000200:10.0:1619677473.303445:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (24)--
00000400:00000200:26.0:1619677473.303446:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.303446:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:10.0:1619677473.303447:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (24)++
00000800:00000200:10.0:1619677473.303448:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (25)--
00000400:00000200:26.0:1619677473.303449:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.303449:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677473.303449:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (24)--
00000800:00000200:18.0:1619677473.303450:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (24)++
00000800:00000200:10.0:1619677473.303450:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:2.0:1619677473.303450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (24)++
00000400:00000200:26.0:1619677473.303451:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:18.0:1619677473.303451:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (26)--
00000800:00000200:2.0:1619677473.303451:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (26)--
00000800:00000200:18.0:1619677473.303452:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (24)--
00000800:00000200:10.0:1619677473.303452:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:2.0:1619677473.303452:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.303453:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96510120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677473.303453:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000400:00000200:26.0:1619677473.303454:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677473.303454:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000400:00000200:18.0:1619677473.303455:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.303455:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.303456:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.303457:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.303457:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.303459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.303460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:10.0:1619677473.303460:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.303460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:26.0:1619677473.303461:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.303463:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.303463:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.303465:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677473.303466:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.303466:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:10.0:1619677473.303466:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.303467:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:10.0:1619677473.303467:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.303469:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:10.0:1619677473.303469:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.303469:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.303470:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (24)++
00000800:00000200:26.0:1619677473.303471:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (25)--
00000800:00000200:26.0:1619677473.303472:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (24)--
00000400:00000200:10.0:1619677473.303472:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.303473:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:10.0:1619677473.303474:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.303475:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:26.0:1619677473.303476:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.303477:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.303477:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:26.0:1619677473.303478:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.303478:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (23)++
00000800:00000200:10.0:1619677473.303479:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (24)--
00000400:00000200:26.0:1619677473.303480:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.303481:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96510118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.303482:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.303483:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.303483:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:12.0:1619677473.303483:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)++
00000400:00000200:26.0:1619677473.303486:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.303486:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (24)--
00000400:00000200:12.0:1619677473.303487:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.303487:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (24)++
00000800:00000200:10.0:1619677473.303487:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (25)--
00000800:00000200:26.0:1619677473.303488:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:12.0:1619677473.303488:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.303488:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (24)--
00000800:00000200:10.0:1619677473.303489:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:26.0:1619677473.303490:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.303490:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.303491:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:13.0:1619677473.303491:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)++
00000800:00000200:12.0:1619677473.303491:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (24)--
00000800:00000200:26.0:1619677473.303492:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:13.0:1619677473.303492:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (24)--
00000800:00000200:26.0:1619677473.303493:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:13.0:1619677473.303493:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.303493:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:10.0:1619677473.303493:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.303494:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (24)++
00000800:00000200:26.0:1619677473.303494:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (25)--
00000400:00000200:13.0:1619677473.303494:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.303495:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (24)--
00000400:00000200:10.0:1619677473.303495:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.303496:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:13.0:1619677473.303496:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.303497:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (24)--
00000400:00000200:10.0:1619677473.303497:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:26.0:1619677473.303498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.303499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.303499:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.303500:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677473.303500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:10.0:1619677473.303502:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.303503:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677473.303506:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.303507:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.303507:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677473.303509:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.303509:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:12.0:1619677473.303509:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.303510:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.303510:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677473.303510:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.303511:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.303511:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.303511:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.303511:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677473.303512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.303512:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677473.303514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.303514:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677473.303514:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.303515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.303515:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.303516:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.303517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.303529:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.303531:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.303532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.303534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.0:1619677473.303583:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.303585:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.303586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.303588:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.303591:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.303592:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.303593:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.303595:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.303596:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.303597:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.303600:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.303601:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.303604:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.303605:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.303605:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.303607:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.303607:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.303608:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.303608:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.303613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.303621:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.303624:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.303625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.303625:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.303627:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.303631:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.303718:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.303721:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.303722:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.303724:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.303726:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.303727:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.303729:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.303730:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.303731:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.303732:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.303735:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.303737:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.303740:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.303741:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.303741:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.303743:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.303744:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.303744:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.303745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.303757:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.303759:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.303760:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.303762:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.303762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.303765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.303802:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.303804:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.303805:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.303807:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.303809:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.303810:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.303811:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.303812:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.303813:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.303814:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.303817:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.303818:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.303819:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.303820:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.303820:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.303821:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.303821:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.303822:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.303823:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.303826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.303832:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.303834:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.303835:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.303836:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.303837:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.303838:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.303844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.303850:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.303852:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.303853:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.303853:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.303854:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.303854:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.303883:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.303885:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.303886:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.303887:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.303889:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.303890:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677473.303891:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.303892:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.303893:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.303894:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677473.303896:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.303897:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677473.303898:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.303899:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677473.303899:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.303901:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.303901:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.303902:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.303902:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.303906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.303913:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.303914:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.303915:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.303916:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.303916:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.303918:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677473.303933:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.303935:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.303936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.303937:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.303938:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.303940:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.303941:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.303942:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.303943:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.303943:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.303946:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.303947:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.303950:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.303951:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.303952:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.303954:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.303954:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.303955:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.303956:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.303957:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:8.2:1619677473.303960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.303968:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677473.303969:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.303971:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:20.0:1619677473.303971:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677473.303972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:20.0:1619677473.303972:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.303973:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000400:00000200:20.0:1619677473.303973:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.303974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.303976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.304341:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.304344:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.304345:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.304346:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.304349:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.304350:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.304351:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.304353:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.304354:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.304354:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.304357:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.304359:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.304361:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.304362:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.304362:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.304364:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.304364:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.304365:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.304366:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.304379:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.304381:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.304382:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.304384:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:20.0:1619677473.304384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.304387:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.304710:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.304713:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.304714:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.304715:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.304718:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.304719:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.304721:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.304722:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.304723:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.304724:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.304726:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.304728:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.304730:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.304731:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.304732:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.304733:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.304734:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.304734:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.304735:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.304747:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.304750:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.304751:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.304751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.304753:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.304756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.304764:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:9.2:1619677473.304766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677473.304767:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.304767:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.304769:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.304772:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.304773:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:10.0:1619677473.304774:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.304775:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.304776:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.304776:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.304776:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.304777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:10.0:1619677473.304777:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.304779:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:8.0:1619677473.304779:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.304782:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.304783:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.304784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.304785:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.304786:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.304787:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.304788:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.304789:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.304789:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.304790:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.304794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.304797:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.304799:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.304800:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.304800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.304801:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.304803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.304898:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.304900:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.304901:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.304903:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.304906:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.304907:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677473.304908:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.304909:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.304910:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.304911:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677473.304914:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.304915:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677473.304917:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.304918:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677473.304918:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677473.304920:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677473.304920:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.304921:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.304921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677473.305511:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.305514:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.305515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.305516:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.305519:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.305520:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.305521:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.305523:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.305524:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.305524:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.305527:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.305528:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.305531:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.305532:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.305532:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.305534:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.305534:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.305535:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.305536:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:8.2:1619677473.305541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.305550:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.305552:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.305553:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.305553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.305555:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.305558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677473.305810:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.305812:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.305814:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.305815:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.305817:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.305818:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.305820:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.305821:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.305822:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.305823:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.305825:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.305827:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.305828:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.305829:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.305830:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.305831:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.305831:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.305833:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.305834:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.305835:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.305837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.305843:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.305846:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.305847:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.305847:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.305847:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.305848:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.305849:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.305850:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.305851:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.305851:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.305957:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.305959:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.305960:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.305962:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.305963:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.305965:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.305966:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.305967:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.305968:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.305969:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.305971:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.305973:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.305976:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.305976:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677473.305977:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.305979:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.305979:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.305980:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.305981:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.305982:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:20.0:1619677473.305993:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.305995:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677473.305995:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:20.0:1619677473.305997:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.305997:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.305997:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:20.0:1619677473.305998:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.305998:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.306000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.306001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.306373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:5.2:1619677473.306373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677473.306380:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:1.0:1619677473.306380:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677473.306383:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:17.0:1619677473.306383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.306383:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:24.0:1619677473.306384:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.306384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:1.0:1619677473.306386:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.306387:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.306388:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.306389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:1.0:1619677473.306390:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.306391:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.306392:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.306393:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.306394:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.306398:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.306400:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.306402:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.306403:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.306404:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.306405:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.306406:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.306407:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.306408:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.306409:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.306414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.306419:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.306420:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.306422:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677473.306422:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.306423:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.306423:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.306424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.306424:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.306425:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.306428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.306766:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.306768:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.306769:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.306771:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.306774:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.306775:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.306776:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.306778:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.306778:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.306779:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.306782:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.306784:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.306786:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.306787:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.306788:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.306790:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.306790:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.306791:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.306791:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.306795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.306798:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:21.0:1619677473.306800:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677473.306801:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.306802:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.306802:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:8.0:1619677473.306802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.306804:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:21.0:1619677473.306805:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.306805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677473.306806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:21.0:1619677473.306807:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.306809:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.306810:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.306812:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.306812:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.306814:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.306817:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.306819:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.306821:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.306822:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.306822:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.306824:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.306825:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.306825:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.306826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.306838:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.306840:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.306841:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.306841:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.306843:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.306845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.307056:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.307059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.307059:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.307061:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.307065:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.307066:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.307067:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.307069:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.307070:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.307071:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.307074:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.307076:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.307077:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.307078:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.307079:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.307080:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.307081:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.307082:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.307082:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.307086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.307092:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.307094:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.307094:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.307095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.307096:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.307098:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.307128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.307135:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.307138:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.307139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.307141:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.307144:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.307145:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.307147:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.307148:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.307149:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.307150:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.307154:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.307156:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.307159:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.307160:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.307160:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.307162:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.307163:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.307164:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.307164:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677473.307166:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.307168:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.307169:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.307171:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.307174:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.307175:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677473.307176:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.307178:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.307178:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.307180:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677473.307182:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.307184:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.307185:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.307186:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:12.0:1619677473.307187:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677473.307187:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.307188:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:12.0:1619677473.307189:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677473.307189:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.307189:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:12.0:1619677473.307190:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.307190:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000400:00000200:12.0:1619677473.307192:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.307195:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.307196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.307226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.307232:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.307234:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.307235:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.307236:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.307237:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.307239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.307319:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.307322:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.307323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.307324:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.307326:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.307328:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.307329:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.307330:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.307331:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.307332:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.307335:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.307336:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.307339:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.307340:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.307340:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.307342:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.307342:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.307343:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.307344:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.307358:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.307360:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.307361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.307362:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.307364:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.307366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.307514:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.307517:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.307518:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.307519:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.307522:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.307523:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.307524:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.307525:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.307526:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.307527:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.307530:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.307531:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.307534:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.307535:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677473.307535:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.307537:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.307537:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.307538:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.307538:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:20.0:1619677473.307550:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.307552:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.307553:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.307555:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.307556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.307558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.307563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:21.0:1619677473.307568:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677473.307568:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:21.0:1619677473.307571:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.307572:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677473.307572:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:21.0:1619677473.307573:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:18.0:1619677473.307573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.0:1619677473.307574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.307575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:2.0:1619677473.307575:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.307578:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.307580:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.307582:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.307583:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.307584:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.307585:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.307588:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.307590:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.307592:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.307593:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.307594:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.307595:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.307596:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.307596:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.307597:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.307611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.307618:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.307620:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.307621:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.307622:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.307623:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.307624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.307653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.307660:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.307663:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.307664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.307664:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.307666:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.307668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.307672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.307677:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.307680:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.307681:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.307681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.307682:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.307682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.307900:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.307902:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.307903:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.307905:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.307907:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.307909:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.307910:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.307911:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.307912:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.307913:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.307916:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.307917:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.307921:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.307921:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.307922:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.307923:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.307924:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.307925:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.307927:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.307927:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:21.0:1619677473.307937:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677473.307939:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:21.0:1619677473.307939:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.307940:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.307941:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.307942:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:21.0:1619677473.307942:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.307943:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:8.0:1619677473.307943:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677473.307944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.307945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.307946:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.307947:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.307949:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.307950:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.307952:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.307953:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.307954:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.307955:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.307956:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.307959:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.307960:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.307962:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.307962:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.307963:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.307964:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.307964:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.307965:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.307966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.307970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.307976:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.307977:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.307978:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.307979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.307980:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.307981:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.307991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.307997:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.308000:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.308000:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.308001:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.308001:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.308001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.308048:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.308051:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.308052:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.308054:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.308056:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.308058:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.308059:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.308060:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.308061:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.308061:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.308064:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.308066:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.308069:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.308070:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.308070:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.308072:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.308072:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.308074:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.308075:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.308075:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677473.308078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.308087:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677473.308089:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677473.308089:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677473.308091:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.308091:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677473.308091:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.308092:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.308092:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677473.308093:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.308095:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.308098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.308106:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.308108:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.308109:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.308110:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.308111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.308111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677473.308249:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.308252:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.308253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.308255:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.308258:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.308259:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.308260:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.308262:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.308262:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.308263:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.308267:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.308268:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.308271:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.308272:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677473.308272:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.308274:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.308274:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.308275:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.308276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.308287:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.308289:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.308290:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.308291:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.308293:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.308299:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.308391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.308398:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.308401:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.308402:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.308402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.308404:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.308405:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.308761:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.308764:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.308765:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.308767:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.308770:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.308771:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.308772:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.308774:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.308775:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.308776:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.308779:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.308781:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.308783:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.308783:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.308784:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.308785:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.308786:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.308786:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.308787:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.308791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.308794:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.308796:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.308797:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.308797:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.308799:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.308801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.309178:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.309180:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.309181:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.309183:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.309185:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.309186:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.309188:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.309189:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.309190:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.309191:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.309194:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.309195:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.309198:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.309199:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.309199:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.309201:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.309201:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.309203:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.309204:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.309204:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.309215:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.309217:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.309218:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.309219:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.309219:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677473.309219:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.309220:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.309221:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.309221:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.309224:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.309233:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.309236:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.309237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.309237:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.309238:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.309239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.309272:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.309275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.309275:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.309277:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.309278:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.309280:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.309281:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.309283:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.309283:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.309284:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.309288:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.309289:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.309292:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.309293:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.309298:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.309300:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.309301:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.309302:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.309303:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.309304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677473.309306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.309315:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.309317:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677473.309317:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.309318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.309319:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677473.309319:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.309320:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.309321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677473.309321:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.309324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.309943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.309949:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.309952:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.309953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.309953:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.309955:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.309957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.310072:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.310075:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.310076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.310078:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.310080:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.310082:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.310083:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.310084:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.310085:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.310086:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.310089:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.310090:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.310093:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.310094:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.310094:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.310096:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.310097:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.310097:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.310098:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:13.0:1619677473.310102:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.310106:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:20.0:1619677473.310110:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:13.0:1619677473.310110:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.310110:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:20.0:1619677473.310112:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.310113:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.310114:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.310115:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.310116:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:13.0:1619677473.310117:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677473.310118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:13.0:1619677473.310119:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.310121:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.310123:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.310125:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677473.310130:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.310133:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677473.310137:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.310138:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677473.310139:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677473.310141:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677473.310142:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.310144:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.310145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677473.310147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.310156:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.310158:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.310159:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.310160:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.310162:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.310165:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677473.310554:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.310557:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.310557:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.310559:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.310561:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.310562:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.310563:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.310565:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.310566:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.310567:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.310570:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.310572:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.310574:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.310575:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.310575:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.310577:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.310578:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.310578:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.310579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.310584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.310592:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.310595:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.310595:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.310597:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.310598:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.310599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.310650:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.310652:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.310653:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.310655:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.310658:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.310660:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.310661:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.310662:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.310663:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.310664:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.310668:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.310670:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.310672:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.310673:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.310673:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.310675:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.310675:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.310676:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.310677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.310681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.310688:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.310690:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.310691:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.310691:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.310693:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.310695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.310745:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.310748:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.310749:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.310750:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.310753:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.310754:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.310755:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.310756:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.310757:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.310757:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.310760:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.310762:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.310764:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.310764:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.310765:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.310766:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.310766:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.310768:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.310769:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.310769:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.310772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.310775:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.310777:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.310778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.310778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.310779:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.310781:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.310782:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.310784:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.310785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.310785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.310790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.310792:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.310795:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.310795:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.310795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.310796:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.310797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.0:1619677473.310921:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.310924:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.310926:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.310927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.310928:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.310929:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.310931:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.310932:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.310933:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.310934:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.310938:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.310939:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.310941:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.310942:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.310943:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.310945:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.310945:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.310946:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.310947:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.310959:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.310961:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.310962:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.310962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.310965:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.310966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.311333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.311342:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.311345:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.311346:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.311347:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.311347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:1.0:1619677473.311348:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.311349:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:8.2:1619677473.311349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:17.0:1619677473.311352:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.311352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.311353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.311354:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.311355:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000400:00000200:17.0:1619677473.311356:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677473.311356:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677473.311357:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.311357:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000400:00000200:17.0:1619677473.311358:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677473.311358:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.311359:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.311360:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.311360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:17.0:1619677473.311361:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.311364:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.311366:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.311368:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.311369:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.311370:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.311371:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.311372:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.311373:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677473.311374:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677473.311375:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:6.0:1619677473.311386:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.311389:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.311390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.311390:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.311392:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.311394:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.311403:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.311405:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:22.0:1619677473.311406:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.311407:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.311407:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.311407:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:22.0:1619677473.311408:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.311408:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:22.0:1619677473.311409:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.311410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677473.311496:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.311498:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677473.311499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677473.311501:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.311503:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.311505:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677473.311506:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.311507:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.311508:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.311509:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677473.311513:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.311514:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677473.311518:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.311518:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:6.0:1619677473.311519:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:6.0:1619677473.311521:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677473.311522:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.311523:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677473.311524:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677473.311525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.311534:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:12.0:1619677473.311535:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:22.0:1619677473.311537:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677473.311537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:12.0:1619677473.311538:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:6.0:1619677473.311538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:12.0:1619677473.311539:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.311541:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.311542:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.311544:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.311628:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.311631:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.311632:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.311634:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.311636:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.311637:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.311638:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.311640:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.311641:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.311642:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.311645:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.311647:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.311649:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.311649:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.311650:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.311652:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.311652:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.311653:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.311654:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.311657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.311660:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.311663:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.311664:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.311664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.311665:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.311667:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:5.2:1619677473.311695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.311700:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.311703:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.311704:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.311706:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.311708:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.311710:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.311711:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.311712:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.311713:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.311714:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.311718:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.311719:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.311722:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.311722:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.311723:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.311725:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.311725:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.311726:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.311726:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.311738:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.311741:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.311742:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.311742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.311744:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.311747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.311913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.311919:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.311921:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.311922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.311922:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.311925:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.311926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.312183:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.312185:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.312186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.312188:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.312190:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.312191:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.312192:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.312194:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.312195:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.312195:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.312199:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.312200:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.312203:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.312204:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.312204:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.312206:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.312206:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.312207:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.312208:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677473.312218:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:6.0:1619677473.312219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.312220:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.312221:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:6.0:1619677473.312222:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:26.0:1619677473.312223:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.312223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.312223:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.312225:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.312226:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.312226:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.312228:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.312229:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.312230:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.312231:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.312233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677473.312235:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.312237:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.312240:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.312241:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677473.312241:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.312243:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.312243:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.312244:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.312245:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.312257:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.312259:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.312260:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.312262:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.312262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.312265:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.312275:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.312277:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.312278:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.312279:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.312280:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.312280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677473.312348:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.312350:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.312351:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.312353:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.312355:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.312357:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.312359:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.312360:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.312361:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.312362:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.312366:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.312367:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.312370:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.312371:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.312371:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.312373:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.312373:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.312374:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.312375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.312380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.312388:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:10.0:1619677473.312390:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:2.0:1619677473.312390:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.312392:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677473.312392:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:2.0:1619677473.312392:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.312393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:2.0:1619677473.312394:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.312396:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.312397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:10.0:1619677473.312399:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.312400:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.312401:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.312401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000400:00000200:10.0:1619677473.312403:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.312404:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.312405:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677473.312408:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677473.312408:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.312409:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677473.312411:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.312411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.312412:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.312413:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.312413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.312413:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.312413:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.312414:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.312416:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.312416:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.312417:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.312419:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.312419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677473.312422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.312430:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677473.312432:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:1.0:1619677473.312433:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:26.0:1619677473.312434:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:17.0:1619677473.312434:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.312434:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.312435:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.312435:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:1.0:1619677473.312437:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.312440:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.312442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.312450:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.312452:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.312453:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.312453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.312455:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.312455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677473.312595:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.312598:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.312598:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.312600:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.312603:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.312605:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.312606:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.312607:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.312608:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.312609:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.312612:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.312613:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.312615:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.312616:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.312616:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.312618:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.312618:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.312619:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.312620:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.312623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.312626:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.312628:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.312629:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.312629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.312631:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.312634:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.312698:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.312700:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.312701:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.312702:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.312705:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.312706:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.312707:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.312708:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.312709:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.312710:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.312713:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.312714:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.312717:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.312718:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.312718:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.312720:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.312720:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.312721:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.312721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.312726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.312734:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.312736:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.312737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.312738:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.312740:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.312743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677473.312985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.312990:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.312993:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.312994:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.312996:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.312998:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.312999:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.313000:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.313001:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.313002:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.313003:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.313006:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.313008:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.313011:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.313011:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.313012:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.313014:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.313014:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.313015:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.313016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.313020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.313028:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.313031:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.313032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.313032:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.313034:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.313036:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677473.313080:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.313082:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677473.313084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677473.313085:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.313087:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.313089:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677473.313090:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.313091:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.313092:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.313093:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677473.313096:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.313097:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677473.313100:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.313101:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677473.313102:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677473.313103:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677473.313104:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.313105:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677473.313106:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677473.313107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677473.313109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.313116:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:18.0:1619677473.313118:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:26.0:1619677473.313119:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677473.313119:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.313120:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.313120:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677473.313121:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:26.0:1619677473.313122:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.313122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:26.0:1619677473.313125:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677473.313146:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.313149:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.313149:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.313151:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.313154:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.313155:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.313156:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.313157:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.313158:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.313159:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.313163:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.313165:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.313167:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.313168:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.313168:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.313170:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.313170:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.313171:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.313173:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.313173:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.313175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.313178:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.313180:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.313181:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.313181:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.313183:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.313185:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:24.0:1619677473.313185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.313187:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.313188:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.313189:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:20.0:1619677473.314054:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.314056:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.314058:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.314059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.314060:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.314062:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.314064:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.314065:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.314066:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.314067:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.314070:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.314072:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.314075:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.314076:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.314076:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.314078:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.314079:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.314079:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.314080:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.314084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.314091:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.314094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.314094:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.314095:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.314097:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677473.314097:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:26.0:1619677473.314100:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.0:1619677473.314100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.314101:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.314102:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677473.314103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:26.0:1619677473.314104:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.314106:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.314107:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.314108:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.314109:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.314109:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:26.0:1619677473.314110:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677473.314111:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.314112:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.314112:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:26.0:1619677473.314113:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.314114:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.314114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.314115:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.314118:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.314118:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.314119:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.314121:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.314121:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.314122:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.314122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.314135:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.314138:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.314139:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.314139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.314141:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.314144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.314261:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.314263:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.314264:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.314266:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.314269:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.314270:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.314271:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.314272:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.314273:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.314274:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.314276:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.314278:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 968 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.314280:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.314281:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.314281:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.314283:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.314283:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.314284:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.314284:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.314288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.314291:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.314296:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.314298:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.314299:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.314300:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.314303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.314307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.314311:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.314311:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.314312:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.314313:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000400:00000200:24.0:1619677473.314314:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.314314:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.314314:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:8.0:1619677473.314315:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.314316:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.314317:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.314317:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:24.0:1619677473.314319:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.314320:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.314322:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.314322:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.314323:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.314326:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.314328:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.314330:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.314331:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.314332:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.314333:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.314334:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.314335:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.314335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.314339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.314347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.314350:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.314351:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.314351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.314353:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.314356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.314390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.314397:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.314399:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.314400:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.314400:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.314402:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.314404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.314615:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.314618:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677473.314619:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677473.314620:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.314623:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.314624:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.314625:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.314627:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.314628:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.314629:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.314632:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.314633:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.314636:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.314637:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.314637:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.314639:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.314640:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.314640:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677473.314641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.314645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.314653:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.314656:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.314657:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.314657:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.314659:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.314662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.314665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.314672:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.314674:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.314675:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.314675:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.314677:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.314679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.314774:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.314777:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677473.314778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677473.314780:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.314782:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.314784:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677473.314785:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.314786:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.314787:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.314788:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677473.314791:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.314793:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677473.314795:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.314796:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677473.314797:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677473.314799:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677473.314799:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.314800:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.314800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677473.314806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.314815:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.314817:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.314818:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.314818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.314820:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.314823:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.314964:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.314967:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.314968:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.314970:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.314972:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.314973:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.314974:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.314976:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.314977:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.314978:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.314981:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.314983:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.314985:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.314986:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.314987:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.314988:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.314989:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.314989:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.314990:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.314994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.314998:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.315000:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.315001:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.315001:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.315003:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.315005:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.315574:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.315577:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.315578:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.315579:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.315581:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.315583:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.315584:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.315585:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.315586:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.315587:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.315590:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.315591:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 968 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:20.2:1619677473.315592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:10.0:1619677473.315593:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.315594:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.315595:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.315596:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.315596:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.315597:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.315597:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:2.0:1619677473.315598:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.315601:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:24.2:1619677473.315602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:18.0:1619677473.315602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.315604:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.315605:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.315607:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:2.0:1619677473.315607:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.315608:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.315608:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:8.0:1619677473.315609:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:2.0:1619677473.315610:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.315611:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.315612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:2.0:1619677473.315612:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.315613:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.315614:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677473.315617:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.315619:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677473.315622:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.315623:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677473.315623:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677473.315625:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677473.315626:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.315627:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.315627:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677473.315632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.315641:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.315643:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.315644:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.315646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677473.315649:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.315652:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677473.315652:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.315654:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677473.315656:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677473.315657:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.315660:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.315661:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677473.315662:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.315664:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.315665:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.315666:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677473.315669:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.315671:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677473.315674:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.315674:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677473.315675:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677473.315677:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677473.315677:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.315679:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677473.315680:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677473.315680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677473.315682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.315690:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:17.0:1619677473.315690:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677473.315692:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:20.0:1619677473.315693:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:17.0:1619677473.315693:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:20.0:1619677473.315694:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.315694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:20.0:1619677473.315696:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.315697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.315698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677473.315896:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.315898:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.315899:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.315901:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.315903:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.315905:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.315906:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.315908:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.315909:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.315910:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.315913:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.315915:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.315918:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.315918:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.315919:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.315921:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.315921:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.315923:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.315924:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.315924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.315927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.315933:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.315935:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.315936:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.315937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.315937:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.315938:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.315939:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.315940:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.315940:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.315941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677473.316501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.316507:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.316510:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.316511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.316513:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.316515:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.316517:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.316518:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.316520:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.316521:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.316522:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.316525:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.316526:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.316529:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.316530:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.316530:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.316532:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.316532:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.316533:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.316534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.316547:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.316550:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.316551:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.316551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.316553:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.316555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.316634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.316641:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.316644:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.316645:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.316645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.316647:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.316649:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.316671:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.316674:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.316675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.316676:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.316679:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.316680:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.316681:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.316682:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.316683:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.316684:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.316687:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.316689:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.316691:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.316692:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677473.316693:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.316695:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.316695:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.316696:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.316697:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.316698:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:21.0:1619677473.316708:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677473.316710:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:21.0:1619677473.316711:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.316711:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.316713:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.316713:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.316713:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:20.0:1619677473.316714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.316715:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.316716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.317090:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.317092:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.317093:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.317095:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.317098:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.317099:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.317100:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.317102:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.317103:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.317103:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.317106:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.317107:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.317109:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.317110:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.317111:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.317112:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.317112:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.317113:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.317114:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.317118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.317124:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.317126:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.317127:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.317127:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.317129:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.317131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.317137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.317143:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.317146:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.317146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.317147:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.317147:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.317148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.317349:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677473.317351:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677473.317352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677473.317354:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677473.317356:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.317357:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.317358:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.317360:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.317361:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.317362:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.317365:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.317366:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.317369:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.317370:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.317370:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.317372:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.317372:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677473.317373:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677473.317374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:10.0:1619677473.317375:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.317377:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.317378:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:8.2:1619677473.317378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:10.0:1619677473.317380:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.317382:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.317383:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.317384:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.317385:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677473.317386:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:10.0:1619677473.317386:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.317387:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:17.0:1619677473.317389:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:10.0:1619677473.317389:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677473.317390:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.317390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677473.317391:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:17.0:1619677473.317392:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:10.0:1619677473.317393:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.317394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.317394:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.317395:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.317396:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.317397:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.317397:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.317398:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.317402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.317404:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.317407:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.317408:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.317408:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.317410:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.317412:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.317688:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.317690:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.317691:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.317692:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.317695:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.317696:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677473.317697:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.317699:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.317699:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.317701:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677473.317705:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.317705:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.317706:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.317707:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677473.317708:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.317710:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.317710:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.317711:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677473.317712:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677473.317712:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.317713:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.317714:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:10.0:1619677473.317715:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.317715:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.317716:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.317717:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.317718:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.317718:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677473.317719:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.317719:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.317721:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.2:1619677473.317721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677473.317723:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.317724:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.317725:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.317726:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.317727:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.317728:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.317728:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677473.317729:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.317732:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.317733:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.317733:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.317735:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.317739:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677473.317742:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.317745:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.317746:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.317749:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.317750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.317752:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677473.317826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.317863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677473.317867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:26.0:1619677473.317869:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.317871:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.317872:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.317872:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.317874:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.317876:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:8.0:1619677473.317953:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.317956:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.317957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.317958:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.317961:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.317963:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.317964:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.317965:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.317966:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.317967:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.317970:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.317971:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.317974:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.317975:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.317976:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.317977:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.317978:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.317979:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.317979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.317983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.317991:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.317993:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.317994:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.317995:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.317997:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.318000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.318005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.318012:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.318015:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.318015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.318016:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.318017:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.318017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.318076:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.318079:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.318080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.318081:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.318083:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.318085:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.318085:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.318087:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.318088:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.318088:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.318092:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.318093:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.318095:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.318096:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.318096:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.318098:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.318098:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.318099:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.318099:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.318103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.318106:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.318108:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.318109:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.318109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.318111:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.318112:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.318124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.318126:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.318129:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.318130:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.318130:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.318130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.318131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:9.2:1619677473.318160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.318167:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.318169:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.318170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.318170:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.318172:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.318174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.318781:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.318783:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.318784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.318786:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.318788:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.318789:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.318790:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.318791:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.318792:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.318793:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.318796:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.318797:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.318799:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.318800:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.318801:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.318802:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.318802:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.318803:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.318804:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.318809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.318814:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.318815:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.318816:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.318818:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.318818:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.318818:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.318820:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.318820:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.318822:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.318823:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96510033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.318824:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.318825:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.318826:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.318829:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.318830:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.318831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.318832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.318832:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.318833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.0:1619677473.318833:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.318834:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.318835:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.318836:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.318838:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.318839:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.318839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.318841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.318850:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.318851:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.318851:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.318853:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.318853:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.318855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.318855:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.318855:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.318856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.318856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.318858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.318864:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.318866:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.318867:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.318867:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.318868:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.318868:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.318949:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.318952:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.318952:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.318954:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.318956:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.318958:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.318959:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.318960:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.318961:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.318962:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.318965:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.318966:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.318969:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.318970:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.318971:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.318972:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.318973:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.318973:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.318974:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.318987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.318990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.318991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.318991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.318993:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.318996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.319004:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.319007:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.319007:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.319008:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.319009:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.319010:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.319747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.319754:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.319757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.319757:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.319758:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.319759:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.319761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.319898:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.319900:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.319901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.319902:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.319904:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.319905:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.319907:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.319908:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.319909:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.319910:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.319913:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.319914:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.319917:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.319918:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.319918:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.319920:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.319920:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.319921:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.319921:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.319926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.319934:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.319937:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.319938:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.319938:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.319940:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.319943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.319944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.319952:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.319954:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.319955:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.319955:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.319956:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.319957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677473.320004:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.320007:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677473.320008:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677473.320009:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.320011:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.320013:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677473.320014:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.320015:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.320016:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.320017:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677473.320020:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.320021:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.320023:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.320024:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.320025:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.320026:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.320026:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.320027:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.320027:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677473.320032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.320039:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.320040:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.320041:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.320042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.320043:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.320044:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.320139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.320145:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.320148:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.320149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.320149:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.320151:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.320153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.320247:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.320250:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.320251:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.320253:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.320255:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.320256:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.320257:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.320259:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.320260:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.320261:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.320264:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.320266:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.320269:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.320269:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.320270:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.320272:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.320272:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.320273:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.320273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.320278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.320286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.320288:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.320289:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.320291:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.320292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.320297:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.320362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.320368:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.320371:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.320372:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.320373:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.320376:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.320377:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:26.0:1619677473.320379:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:2.0:1619677473.320379:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.320380:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.320381:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.320383:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.320383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:2.0:1619677473.320383:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677473.320387:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.320388:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677473.320389:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.320390:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:26.0:1619677473.320391:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.320392:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677473.320393:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.320394:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677473.320394:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:26.0:1619677473.320395:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677473.320395:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:26.0:1619677473.320396:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:2.0:1619677473.320396:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.320397:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.320398:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:26.0:1619677473.320399:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.320399:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.320400:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677473.320401:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.320403:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.320404:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.320405:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:20.2:1619677473.320405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677473.320406:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.320407:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.320408:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.320409:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.320410:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:18.0:1619677473.320411:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:24.2:1619677473.320412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:18.0:1619677473.320413:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.320414:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.320414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677473.320415:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.320418:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.320419:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.320419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.320421:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.320422:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:24.0:1619677473.320423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.320425:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.320425:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.320426:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:10.0:1619677473.320743:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.320745:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.320746:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.320748:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.320750:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.320751:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.320752:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.320754:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.320755:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.320756:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.320759:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.320760:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.320762:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.320763:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.320763:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.320765:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.320765:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.320766:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.320767:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.320770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.320777:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.320779:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.320780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.320780:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.320782:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.320784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.321266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.321272:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.321275:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.321276:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.321278:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.321280:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.321282:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.321283:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.321285:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.321286:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.321287:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.321290:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.321292:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.321297:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:17.0:1619677473.321301:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.321302:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.321302:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.321302:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.321304:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:17.0:1619677473.321304:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.321305:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.321305:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677473.321306:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.321306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677473.321308:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.321310:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.321311:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.321312:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.321313:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.321314:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.321317:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.321319:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:6.0:1619677473.321319:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.321322:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.321322:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:6.0:1619677473.321322:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.321323:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:22.0:1619677473.321323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.321323:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.321325:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.321325:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:6.0:1619677473.321325:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.321326:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.321327:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:6.0:1619677473.321327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.2:1619677473.321331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.321334:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.321337:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.321338:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.321338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.321340:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.321342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:21.0:1619677473.321572:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.321575:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.321577:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.321577:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.321580:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.321581:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.321583:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.321584:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.321585:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.321586:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.321590:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.321592:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.321595:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.321596:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.321597:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.321599:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.321599:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.321600:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.321601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677473.321644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.321650:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.321652:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.321653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.321653:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.321655:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.321657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.321806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.321812:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.321814:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.321815:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.321816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.321817:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.321819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.321871:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.321873:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.321875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.321876:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.321879:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.321880:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.321881:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.321882:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.321883:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.321884:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.321887:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.321889:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.321891:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.321892:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.321893:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.321894:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.321895:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.321895:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.321896:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.321908:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.321911:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.321912:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.321912:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.321914:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.321917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.322212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.322221:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.322224:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.322225:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.322225:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.322227:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.322231:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.322358:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.322361:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.322362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.322363:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.322366:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.322367:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.322369:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.322370:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.322371:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.322372:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.322374:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.322376:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.322378:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.322378:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.322379:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.322381:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.322381:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.322382:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.322382:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.322387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.322390:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.322393:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.322393:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.322394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.322395:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.322397:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.322403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.322406:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.322408:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.322409:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.322409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.322410:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.322410:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.2:1619677473.322665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.322674:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.322677:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.322678:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.322678:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.322680:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.322683:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.322763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.322769:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.322772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.322772:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.322773:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.322775:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.322777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.323122:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.323124:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.323125:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.323127:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.323129:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.323130:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.323131:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.323133:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.323133:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.323134:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.323137:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.323139:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.323141:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.323141:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.323142:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.323143:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.323144:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.323144:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.323145:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.323149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.323151:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.323153:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.323154:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.323155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.323156:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.323158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.323675:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.323678:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.323679:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.323681:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.323683:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.323684:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.323685:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.323687:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.323688:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.323688:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.323691:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.323693:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:20.2:1619677473.323693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677473.323695:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.323695:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.323696:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.323698:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.323698:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.323699:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.323700:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:2.0:1619677473.323700:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677473.323701:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:2.0:1619677473.323702:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:24.2:1619677473.323704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:18.0:1619677473.323704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.323705:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.323707:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000400:00000200:2.0:1619677473.323708:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.323709:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:2.0:1619677473.323709:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.323710:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.323710:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:2.0:1619677473.323711:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96510010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.323713:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.323713:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:2.0:1619677473.323713:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.323714:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.323715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.323715:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000400:00000200:2.0:1619677473.323715:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677473.323716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.323717:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000400:00000200:2.0:1619677473.323718:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.323719:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.323721:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.323722:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.323723:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.323724:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.323725:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.323726:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.323727:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.323728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.323731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.323734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.323740:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.323741:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.323741:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677473.323742:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.323742:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.323742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.323743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.323744:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.323744:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677473.323745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.323838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.323845:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.323847:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.323848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.323848:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.323850:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.323852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.323857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.323863:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677473.323864:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.323867:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677473.323867:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677473.323867:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.0:1619677473.323867:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.323868:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.323869:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.323869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:10.0:1619677473.323869:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.323872:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.323873:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.323874:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.323876:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.323876:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.323877:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.323881:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.323882:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.323885:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.323886:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.323886:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.323888:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.323888:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.323889:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.323890:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.323902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.323905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.323905:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.323906:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.323908:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.323911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.324567:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.324570:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.324571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.324573:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.324575:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.324577:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.324578:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96510006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.324579:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.324580:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.324581:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.324584:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.324586:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.324588:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.324589:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.324590:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.324591:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.324592:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.324593:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.324593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.324597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.324606:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.324609:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.324609:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.324610:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.324612:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.324615:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677473.324893:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.324896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.324896:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.324898:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.324900:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.324902:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.324903:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.324904:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.324905:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.324906:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.324909:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.324911:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.324913:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.324914:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.324915:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.324916:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.324917:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.324917:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.324918:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.324923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.324931:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.324933:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.324934:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.324937:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.324937:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.324940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.324945:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.324948:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.324948:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:8.2:1619677473.324949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:26.0:1619677473.324951:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.324953:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.324954:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.324956:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.324957:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.324957:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:26.0:1619677473.324958:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.324959:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:21.0:1619677473.324960:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.324961:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.324962:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677473.324962:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:26.0:1619677473.324963:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.324963:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.324965:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.324967:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.324968:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.324969:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.324970:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.324971:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.324972:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.324972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.324976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.324979:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.324981:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.324982:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.324982:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.324984:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.324986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.324998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.325004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.325007:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.325008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.325010:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.325012:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.325014:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.325015:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96510003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.325017:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.325018:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.325019:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.325022:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.325024:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.325027:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.325028:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.325028:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.325030:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.325031:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.325031:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.325032:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.325046:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.325050:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.325051:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.325054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677473.325054:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.325057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.325065:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.325068:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.325069:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.325070:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.325070:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.325071:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.325506:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.325508:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.325509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.325511:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.325513:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.325515:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.325516:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.325517:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.325518:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.325519:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.325523:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.325524:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.325526:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.325527:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.325528:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.325529:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.325530:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.325531:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.325532:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.325533:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.325536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.325543:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.325545:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677473.325546:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677473.325546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.325546:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.325548:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677473.325548:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.325549:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.325550:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.325551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.325557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.325563:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.325565:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.325566:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.325566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.325567:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.325567:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.325621:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.325622:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.325624:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.325625:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.325625:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.325627:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.325628:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.325629:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.325630:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96510001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.325632:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.325632:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.325633:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.325635:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.325636:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677473.325637:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96510000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.325637:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677473.325639:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.325639:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.325640:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.325641:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677473.325643:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.325645:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677473.325646:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.325647:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677473.325647:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.325648:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.325648:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.325648:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.325649:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.325650:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.325650:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.325651:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.325651:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.325652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.325652:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.325652:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.325653:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677473.325653:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.325655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:9.2:1619677473.325659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.325661:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.325663:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.325663:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.325664:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.325665:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.325666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:18.0:1619677473.325666:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.325669:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.325670:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.325670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.325673:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.325676:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.325700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.325707:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.325710:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.325711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.325711:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.325713:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.325717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.325799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.325805:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.325807:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.325808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.325808:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.325810:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.325812:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.326044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.326050:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.326053:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.326053:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.326056:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.326058:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.326059:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.326061:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.326062:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.326063:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.326064:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.326068:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.326069:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.326072:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.326072:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.326073:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.326075:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.326075:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.326077:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.326078:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.326078:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.326083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.326089:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.326089:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.326091:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.326091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.326092:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.326092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.326093:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.326094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.326095:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.326097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.326313:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.326316:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.326317:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.326319:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.326322:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.326323:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.326325:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.326325:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:24.0:1619677473.326326:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.326327:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.326328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:24.0:1619677473.326328:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677473.326328:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.326331:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.326332:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.326334:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.326335:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:10.0:1619677473.326335:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.326336:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.326337:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.326338:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.326338:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.326339:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.326339:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:10.0:1619677473.326339:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.326341:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:10.0:1619677473.326342:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.326343:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.326344:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677473.326344:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.326345:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677473.326346:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.326346:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.326347:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.2:1619677473.326347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677473.326348:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.326349:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.326349:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.326350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.326354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677473.326356:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.326357:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:17.0:1619677473.326358:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677473.326359:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:1.0:1619677473.326359:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:24.0:1619677473.326360:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.326360:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.326360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:17.0:1619677473.326362:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.326363:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.326366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677473.326367:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.2:1619677473.326369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.2:1619677473.326372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.326375:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:1.0:1619677473.326377:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.326378:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.326379:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.326379:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.326379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.326380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:1.0:1619677473.326380:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.326381:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.326381:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.326382:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.326382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.326652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.326658:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.326660:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677473.326661:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677473.326661:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.326661:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.326663:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.326664:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677473.326664:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.326666:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.326666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:26.0:1619677473.326668:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.326669:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.326670:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.326672:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.326672:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.326673:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.326676:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.326678:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.326681:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.326681:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.326682:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.326684:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.326684:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.326685:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.326685:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.326698:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.326700:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.326701:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.326702:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.326704:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.326707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.326906:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.326908:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.326910:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.326911:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.326913:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.326914:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.326916:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.326917:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.326918:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.326918:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.326921:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.326922:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.326924:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.326925:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.326925:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.326927:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.326927:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.326928:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.326930:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.326930:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.326932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.326935:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.326937:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.326938:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.326938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.326940:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.326941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.326943:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.326945:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.326945:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.326946:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.327404:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.327407:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.327409:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.327410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.327412:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.327413:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.327414:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.327416:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.327417:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.327418:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.327421:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.327422:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.327425:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.327425:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.327426:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.327428:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.327428:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.327430:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.327431:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.327431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.327440:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677473.327442:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.327442:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.327443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.327444:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.327445:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.327446:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.327446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.327447:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.327450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.327464:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.327467:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.327468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.327468:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.327469:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.327470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.327752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.327759:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.327762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.327762:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.327763:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.327765:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.327767:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.328050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.328056:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.328058:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.328059:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.328059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.328061:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.328063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.328593:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.328596:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.328597:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.328598:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.328601:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.328602:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.328603:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.328604:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.328605:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.328606:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.328608:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.328610:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.328612:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.328613:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.328613:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.328615:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.328616:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.328616:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.328617:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.328621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.328624:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.328626:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.328627:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.328627:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.328629:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.328631:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.328761:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.328764:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.328764:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.328766:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.328768:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.328770:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.328771:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.328772:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.328773:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.328774:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.328777:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.328778:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.328781:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.328782:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.328782:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.328784:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.328784:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.328786:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.328787:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.328787:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.328799:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.328800:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.328801:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677473.328802:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.328803:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677473.328803:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.328803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677473.328804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.328805:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.328811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.328812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.328814:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.328816:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.328816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.328817:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.328817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.329017:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.329020:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.329021:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.329023:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.329025:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.329027:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677473.329029:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.329030:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.329031:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.329032:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677473.329035:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.329037:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677473.329040:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.329041:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677473.329042:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677473.329044:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677473.329044:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.329045:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.329045:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.329050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.329056:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.329059:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.329060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.329060:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.329063:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.329066:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.329256:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.329261:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.329262:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:9.2:1619677473.329262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:1.0:1619677473.329266:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.329269:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.329270:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:1.0:1619677473.329272:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:18.0:1619677473.329273:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.329274:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.329274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:1.0:1619677473.329274:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.329277:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.329277:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.329278:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677473.329280:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:1.0:1619677473.329280:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.329286:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.329289:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.329292:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.329294:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.329301:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.329304:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.329305:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.329306:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.329308:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.329309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.329310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.329318:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.329321:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677473.329321:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000400:00000200:20.0:1619677473.329322:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.329323:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.329324:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677473.329325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.329326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677473.329326:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.329328:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:26.0:1619677473.329565:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.329568:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.329569:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.329571:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.329574:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.329575:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.329576:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.329578:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.329579:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.329580:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.329583:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.329585:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.329588:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.329588:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.329589:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.329591:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.329591:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.329592:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.329592:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.329596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.329599:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.329601:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.329602:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.329602:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.329604:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.329606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.330048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.330055:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.330058:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.330059:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.330061:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.330065:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.330066:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.330068:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.330069:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.330070:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.330072:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.330076:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.330078:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.330080:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.330081:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.330082:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.330084:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.330085:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.330085:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.330086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.330090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.330100:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.330104:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.330106:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.330107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.330110:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.330114:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677473.330168:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.330173:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.330174:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.330178:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.330182:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.330185:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677473.330186:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.330189:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.330190:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.330192:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677473.330196:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.330199:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677473.330202:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.330203:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677473.330203:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677473.330205:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677473.330206:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.330207:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.330207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.330212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.330218:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.330221:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.330222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.330222:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.330224:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.330227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.330230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.330236:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.330238:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677473.330238:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677473.330239:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.330239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.330240:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.330241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.330241:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.330242:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.330243:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.330245:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.330247:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.330248:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.330249:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.330250:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.330251:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.330255:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.330257:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.330259:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.330260:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.330261:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.330262:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.330263:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.330263:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.330264:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.330268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.330274:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.330276:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.330277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.330277:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.330279:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.330281:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.330592:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.330594:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.330595:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.330596:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.330599:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.330600:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.330601:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.330603:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.330603:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.330604:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.330607:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.330609:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.330612:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.330613:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.330613:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.330615:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.330616:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.330617:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.330618:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.330619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677473.330622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.330630:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.330632:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.330633:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.330633:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:1.0:1619677473.330634:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.330635:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.330636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677473.330636:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.330637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677473.330639:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.330688:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.330691:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.330692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.330692:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.330695:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.330698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.330742:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.330745:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.330746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.330748:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.330750:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.330751:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.330752:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.330753:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.330755:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.330755:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.330759:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.330760:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.330763:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.330763:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.330764:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.330765:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.330766:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.330766:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.330767:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.330771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.330774:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.330776:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.330777:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.330777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.330779:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.330781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.330790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.330793:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.330795:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.330796:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.330796:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.330796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.330797:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:5.2:1619677473.330829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.330835:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.330838:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.330838:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.330840:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.330843:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.330844:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.330846:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.330847:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.330848:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.330849:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.330852:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.330854:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.330856:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.330857:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.330858:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.330860:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.330860:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.330860:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.330861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.330873:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.330875:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.330876:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.330876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.330878:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.330880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.331095:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.331098:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.331100:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.331100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.331102:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.331103:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.331105:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.331106:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.331107:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.331108:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.331111:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.331113:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.331115:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.331116:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.331116:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.331118:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.331118:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.331119:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.331120:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.331131:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.331133:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.331134:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.331135:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.331136:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.331139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.331216:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.331218:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.331219:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.331221:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.331223:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.331225:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.331226:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.331227:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.331228:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.331229:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.331232:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.331234:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.331236:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.331237:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.331237:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.331239:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.331239:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.331240:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.331242:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.331242:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.331244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.331247:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.331249:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.331250:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.331250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.331252:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.331254:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:24.0:1619677473.331254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.331256:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.331257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.331258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:26.0:1619677473.331529:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.331531:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.331532:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.331533:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.331535:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.331537:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.331538:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.331540:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.331540:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.331541:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.331544:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.331545:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.331548:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.331549:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.331550:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.331551:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.331552:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.331552:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.331553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.331566:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.331569:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.331569:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.331570:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.331572:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.331575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.331861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.331870:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.331873:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.331873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.331874:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.331876:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.331880:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.331882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.331889:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.331892:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.331893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.331893:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.331894:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.331894:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677473.332620:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.332623:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.332624:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.332625:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.332628:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.332630:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.332631:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.332632:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.332633:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.332634:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.332637:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.332639:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.332641:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.332642:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.332643:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.332644:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.332645:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.332646:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.332646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.332650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.332659:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.332662:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.332663:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.332663:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.332665:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.332669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.333035:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.333038:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.333039:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.333041:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.333044:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.333045:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.333046:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.333048:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.333049:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.333050:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.333053:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.333054:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.333056:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.333057:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.333057:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.333059:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.333059:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.333060:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.333060:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.333064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.333067:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.333069:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.333070:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.333071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.333072:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.333074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.0:1619677473.333270:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.333272:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677473.333273:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677473.333274:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.333277:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.333278:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.333279:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.333280:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.333281:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.333282:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.333285:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.333286:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.333289:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.333290:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.333290:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.333292:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.333292:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.333293:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677473.333298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.333298:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:2.0:1619677473.333306:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.333309:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.333310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.333310:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.333313:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.333316:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.333435:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.333438:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.333439:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.333439:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.333441:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.333444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.333454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.333456:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.333457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.333458:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.333459:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.333459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.333672:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.333675:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.333676:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.333678:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.333680:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.333682:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.333683:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.333684:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.333685:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.333686:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.333689:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.333691:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.333694:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.333694:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.333695:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.333696:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.333697:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.333698:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.333698:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.333710:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.333712:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.333713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.333714:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.333716:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.333719:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677473.333871:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.333874:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.333875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.333877:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.333880:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.333881:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.333883:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.333884:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.333885:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.333886:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.333890:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.333892:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.333894:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.333895:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.333895:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.333897:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.333898:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.333898:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.333899:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677473.333910:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.333913:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.333914:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.333914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.333916:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.333918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.333932:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.333935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.333935:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.333937:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.333940:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.333941:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.333942:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.333944:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.333945:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.333946:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.333949:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.333951:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.333953:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.333954:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.333955:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.333956:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.333957:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.333957:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.333958:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.333962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.333965:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.333967:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.333968:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.333968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.333970:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.333972:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.334082:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.334085:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.334086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.334087:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.334089:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.334091:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.334092:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.334093:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.334094:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.334094:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.334097:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.334099:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.334101:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.334101:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.334102:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.334103:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.334104:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.334105:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.334106:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.334107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:13.0:1619677473.334117:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.334119:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677473.334119:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:13.0:1619677473.334120:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.334121:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.334121:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.334122:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:13.0:1619677473.334123:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.334123:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.334124:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677473.334442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.334482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.334491:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.334494:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.334495:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.334495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.334497:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.334500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.334503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.334505:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.334507:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.334508:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.334508:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.334509:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.334510:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.334563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.334570:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.334572:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.334573:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.334573:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.334575:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.334577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.334580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.334586:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.334588:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.334589:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.334589:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.334590:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.334591:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.334795:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.334797:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.334798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.334800:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.334802:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.334804:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.334805:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.334806:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.334807:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.334808:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.334810:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.334812:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.334813:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.334814:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.334814:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.334816:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.334816:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.334817:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.334818:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.334822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.334828:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.334829:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.334830:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.334831:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.334831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.334833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677473.334934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.334937:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.334939:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.334940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.334942:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.334944:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.334945:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.334946:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.334947:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.334948:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.334952:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.334954:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.334956:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.334957:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.334957:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.334959:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.334960:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.334960:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.334961:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.334973:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.334975:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.334976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.334976:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.334978:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.334980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:13.0:1619677473.335082:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.335085:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.335087:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.335089:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.335089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.335091:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677473.335092:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.335094:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.335095:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.335097:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677473.335100:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.335102:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677473.335104:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.335105:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677473.335106:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677473.335107:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677473.335108:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.335108:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.335109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:26.0:1619677473.335116:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.335119:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.335120:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.335121:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.335123:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.335125:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.335126:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.335127:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.335128:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.335129:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.335132:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.335133:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.335135:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.335136:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.335137:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.335138:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.335139:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.335139:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.335140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.335144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.335147:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.335149:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.335150:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.335150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.335151:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.335153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.335156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.335162:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:24.2:1619677473.335164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:21.0:1619677473.335165:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:24.0:1619677473.335167:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:20.0:1619677473.335167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.335168:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.335169:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.335170:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.335170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.335171:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.335172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:21.0:1619677473.335172:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.335174:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.335175:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.335177:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.335179:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.335180:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.335185:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.335187:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.335190:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.335191:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.335191:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.335193:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.335193:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.335194:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.335195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.335199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677473.335204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677473.335207:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.335209:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.335210:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.335212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.335212:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.335212:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:20.0:1619677473.335214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677473.335215:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.335216:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.335216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.335218:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.335222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.335402:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.335405:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.335406:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.335408:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.335411:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.335412:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.335413:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.335415:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.335416:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.335416:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.335420:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.335422:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.335424:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.335425:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.335425:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.335427:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.335427:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.335428:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.335429:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.335430:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.335433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.335440:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.335443:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677473.335444:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677473.335444:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.335444:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.335446:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.335446:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677473.335446:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.335447:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.335448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.335595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.335601:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.335604:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.335604:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.335606:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.335609:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.335610:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.335611:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.335612:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677473.335613:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.335614:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:1.0:1619677473.335614:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.335615:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:1.0:1619677473.335615:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:22.0:1619677473.335616:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.335618:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.335619:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.335622:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.335623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.335624:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.335625:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.335626:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.335628:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.335628:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.335629:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.335630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.335641:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.335644:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.335644:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.335645:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.335647:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.335648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.335810:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.335813:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.335813:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.335816:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.335818:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.335819:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.335820:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.335821:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.335823:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.335824:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.335827:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.335829:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.335831:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.335832:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.335832:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.335834:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.335834:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.335835:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.335837:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.335837:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.335839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.335842:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.335844:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.335845:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.335845:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.335847:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.335849:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:24.0:1619677473.335849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.335851:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.335852:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.335853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.336065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.336071:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.336073:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.336074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.336074:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.336076:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.336078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.336496:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.336498:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.336499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.336501:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.336503:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.336504:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.336505:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.336506:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.336507:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.336508:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.336510:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.336512:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.336514:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.336515:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.336515:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.336517:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677473.336517:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.336517:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.336518:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.336518:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677473.336520:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.336521:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.336522:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677473.336523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:24.0:1619677473.336524:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.336527:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.336528:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.336529:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.336529:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:24.0:1619677473.336530:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.336530:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.336531:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.336531:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.336533:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.336535:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.336535:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.336536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.336537:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.336540:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.336541:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.336541:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.336543:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.336544:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.336544:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.336545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.336549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.336558:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.336560:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.336561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.336562:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.336564:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.336567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.336569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.336576:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.336579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.336579:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.336580:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.336581:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.336581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.336677:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.336680:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.336681:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.336682:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.336685:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.336686:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.336687:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.336688:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.336689:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.336690:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.336693:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.336694:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.336696:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.336697:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.336698:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.336699:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.336699:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.336700:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.336701:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.336704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.336708:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.336710:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.336711:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.336711:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.336712:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.336714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:21.0:1619677473.336756:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.336759:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.336761:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.336762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.336764:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.336766:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.336767:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.336768:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.336769:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.336770:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.336774:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.336775:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.336778:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.336779:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.336780:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.336782:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.336782:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.336784:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677473.336785:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677473.336785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.336794:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677473.336797:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677473.336797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.336798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.336921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.336929:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.336932:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.336933:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.336933:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.336936:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.336939:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677473.337121:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.337124:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.337125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677473.337127:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.337130:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.337131:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.337133:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.337134:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.337135:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.337136:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.337139:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.337141:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.337143:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.337144:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.337145:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.337147:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.337147:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.337149:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677473.337150:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677473.337150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:22.0:1619677473.337161:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.337163:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:22.0:1619677473.337164:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677473.337165:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000400:00000200:22.0:1619677473.337165:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.337165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.337166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.337167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:22.0:1619677473.337167:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.337170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.337184:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.337187:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.337187:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.337188:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.337189:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.337189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.337825:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.337827:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.337828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.337830:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.337832:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.337834:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677473.337835:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.337836:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.337837:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.337838:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677473.337842:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.337844:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677473.337847:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.337848:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677473.337848:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677473.337850:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677473.337850:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.337852:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677473.337853:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.337853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677473.337857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.2:1619677473.337860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677473.337864:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:17.0:1619677473.337865:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677473.337867:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:10.0:1619677473.337867:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:2.0:1619677473.337867:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677473.337868:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:17.0:1619677473.337868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:10.0:1619677473.337869:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.337869:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:1.0:1619677473.337869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677473.337871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677473.337872:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.337873:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677473.337875:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:2.0:1619677473.337875:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.337877:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.337878:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.337879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:2.0:1619677473.337880:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.337881:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.337882:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677473.337885:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:2.0:1619677473.337886:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.337888:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677473.337888:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677473.337888:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677473.337890:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.337890:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677473.337891:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.337892:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.337892:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677473.337894:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:2.0:1619677473.337894:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677473.337895:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:2.0:1619677473.337895:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.337896:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677473.337897:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:2.0:1619677473.337897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677473.337899:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:26.0:1619677473.337900:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.337900:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:26.0:1619677473.337902:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.337902:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.337904:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.337905:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.337906:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.337907:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.337908:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.337910:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.337911:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.337913:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.337913:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96509939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.337915:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.337915:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.337916:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.337917:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.337919:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.337920:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000400:00000200:10.0:1619677473.337920:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.337921:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.337921:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.337923:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.337923:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.337924:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)++
00000800:00000200:26.0:1619677473.337924:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (24)--
00000800:00000200:10.0:1619677473.337924:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.337925:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.337925:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:26.0:1619677473.337926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.337926:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:10.0:1619677473.337926:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.337927:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.337928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.337929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.337932:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (23)++
00000800:00000200:24.0:1619677473.337934:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (24)--
00000400:00000200:24.0:1619677473.337935:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.337935:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (23)++
00000800:00000200:8.0:1619677473.337936:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (24)--
00000400:00000200:24.0:1619677473.337937:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.337937:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.337939:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.337940:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:8.0:1619677473.337940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.337941:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.337953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.337957:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.337959:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.337960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.337960:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.337960:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.337961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.338568:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.338570:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.338571:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.338573:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.338575:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.338577:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.338578:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.338579:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.338580:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.338580:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.338584:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.338585:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.338588:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.338589:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.338589:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.338591:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.338591:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.338592:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.338592:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.338606:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.338608:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.338609:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.338610:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.338612:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.338615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.338623:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.338626:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.338627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.338627:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.338628:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.338629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.338686:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.338688:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.338689:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.338689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.338692:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.338695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.338759:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.338761:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.338762:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.338764:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.338767:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.338768:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.338770:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.338771:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.338772:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.338773:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.338775:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.338777:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.338779:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.338780:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.338780:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.338782:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.338782:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.338783:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.338784:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.338785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.338788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.338791:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.338793:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.338794:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.338794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.338795:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.338797:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:24.0:1619677473.338797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.338799:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.338800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.338801:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:20.0:1619677473.339438:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.339441:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.339444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.339444:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.339446:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.339448:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.339449:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.339450:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.339451:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.339452:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.339455:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.339457:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.339459:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.339460:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.339461:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.339462:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.339463:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677473.339463:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677473.339464:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.339464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.339466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.339466:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.339469:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.339471:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.339473:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.339474:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.339476:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.339476:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.339477:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.339480:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.339482:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.339486:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677473.339487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:6.0:1619677473.339487:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.339488:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.339488:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.339489:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.339490:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.339490:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.339491:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:22.0:1619677473.339492:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.339492:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.339492:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677473.339494:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677473.339494:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677473.339494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677473.339497:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.339497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.339500:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.339503:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:18.0:1619677473.339503:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.339504:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.339505:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.339506:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.339506:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.339507:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.339508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:18.0:1619677473.339508:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.339509:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:18.0:1619677473.339509:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677473.339511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:18.0:1619677473.339513:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.339515:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.339517:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.339518:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.339518:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.339520:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.339521:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.339522:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.339522:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.339527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.339533:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.339536:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.339536:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.339537:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.339538:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.339539:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.339747:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.339750:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.339751:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.339753:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.339755:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.339756:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.339758:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.339759:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.339760:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.339761:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.339764:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.339765:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.339768:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.339768:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.339769:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.339771:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.339771:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.339772:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.339772:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.339777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.339781:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.339783:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.339784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.339784:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.339786:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.339788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.0:1619677473.339850:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.339853:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.339854:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.339855:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.339858:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.339859:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.339860:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.339862:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.339863:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.339864:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.339866:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.339868:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.339871:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.339871:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.339872:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.339874:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.339874:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.339875:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.339876:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.339877:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.339880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.339887:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.339889:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.339890:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.339891:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.339891:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677473.339891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.339892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.339893:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677473.339894:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.339897:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.339898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.339905:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.339908:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.339909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.339909:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.339910:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.339911:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.340149:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.340152:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.340153:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.340156:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.340158:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.340159:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.340160:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.340162:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.340163:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.340164:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.340168:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.340169:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.340171:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.340172:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.340173:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.340174:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.340174:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.340175:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.340176:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.340179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.340186:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.340188:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.340189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.340189:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.340191:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.340193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.340200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.340206:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.340209:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.340210:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.340210:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.340211:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.340211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.340363:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.340366:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.340367:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.340369:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.340371:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.340373:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.340374:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.340375:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.340376:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.340377:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.340380:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.340381:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.340384:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.340385:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.340386:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.340388:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.340388:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.340389:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.340389:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.340396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.340404:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.340407:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.340408:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.340409:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:20.0:1619677473.340410:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.340411:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677473.340411:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.340412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.340413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:6.0:1619677473.340414:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.340417:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.340419:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.340420:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.340421:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.340422:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.340423:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.340426:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.340428:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.340430:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.340431:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.340431:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.340433:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:5.2:1619677473.340433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677473.340434:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.340435:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.340436:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.340437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:17.0:1619677473.340439:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.340442:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.340443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.340445:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.340448:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:17.0:1619677473.340448:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.340449:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:17.0:1619677473.340449:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677473.340450:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.340450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.340451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.340451:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:17.0:1619677473.340451:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.340452:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.340452:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.340453:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.340454:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.340454:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:17.0:1619677473.340454:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.340458:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.340460:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677473.340460:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.340462:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.340463:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.340463:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.340465:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.340466:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.340466:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.340467:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.340479:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.340481:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.340482:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.340483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.340485:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.340488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.340519:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.340522:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.340523:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.340523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.340525:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.340528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.341082:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.341085:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.341086:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.341088:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.341091:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.341092:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.341093:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.341095:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.341096:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.341097:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.341100:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.341102:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.341104:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.341104:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.341105:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.341106:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.341106:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.341107:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.341108:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.341112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.341116:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.341118:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.341119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.341119:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.341120:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.341122:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.341291:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.341297:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.341299:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677473.341302:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.341304:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.341305:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.341306:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.341308:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.341308:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.341309:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.341312:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.341314:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.341316:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.341317:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.341317:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.341319:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.341319:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.341320:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.341320:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.341325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.341332:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.341334:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.341335:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.341335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.341336:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.341339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.341515:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.341517:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.341518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.341520:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.341522:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.341523:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.341525:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.341526:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.341527:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.341528:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.341532:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.341533:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.341536:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.341537:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.341537:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.341539:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.341539:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.341540:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.341540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.341553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.341555:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.341556:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.341557:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.341559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.341559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.341568:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.341571:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.341572:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.341573:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.341573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.341574:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.341823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.341830:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.341832:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.341833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.341833:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.341835:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.341838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.342402:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.342405:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.342406:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.342407:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.342410:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.342411:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.342412:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.342413:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.342414:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.342415:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.342418:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.342419:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.342421:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.342422:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.342422:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.342424:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.342424:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:22.0:1619677473.342425:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:10.0:1619677473.342425:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.342425:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:22.0:1619677473.342428:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.342428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.2:1619677473.342430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:22.0:1619677473.342430:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.342433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000400:00000200:22.0:1619677473.342433:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.342434:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:24.0:1619677473.342435:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.342436:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.342436:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.342437:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.342437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:22.0:1619677473.342438:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.342439:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.342439:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.342441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:22.0:1619677473.342443:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.342445:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.2:1619677473.342447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:22.0:1619677473.342447:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.342448:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.342449:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.342450:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677473.342450:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:22.0:1619677473.342451:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.342452:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.342452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677473.342453:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.342454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.342454:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.342455:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.342455:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:21.0:1619677473.342465:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.342467:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.342468:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.342470:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.342471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.342472:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.342493:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.342496:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.342496:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.342498:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.342501:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.342502:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.342503:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.342505:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.342506:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.342507:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.342510:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.342512:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.342514:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.342515:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.342516:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.342517:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.342518:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.342519:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.342519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.342543:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.342546:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.342547:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.342549:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.342551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.342552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.342573:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.342576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.342576:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.342577:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.342579:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.342582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.342590:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.342593:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.342594:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.342594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.342595:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.342596:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.342722:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.342725:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.342726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.342728:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.342730:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.342732:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.342733:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.342734:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.342735:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.342736:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.342739:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.342740:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.342743:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.342744:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.342744:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.342746:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.342747:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.342747:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.342748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.342752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.342761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.342764:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.342765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.342765:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.342768:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.342771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.342805:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.342808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.342808:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677473.342810:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.342812:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.342814:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677473.342815:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.342816:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.342817:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.342818:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677473.342822:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.342823:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.342826:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.342827:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677473.342827:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677473.342829:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677473.342829:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.342830:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677473.342831:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677473.342832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677473.342835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677473.342841:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677473.342843:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:18.0:1619677473.342843:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:22.0:1619677473.342844:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677473.342845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:18.0:1619677473.342846:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.342847:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.342847:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.342849:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.342851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677473.342864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.342870:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.342872:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.342873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.342875:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.342877:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.342879:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.342880:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.342881:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.342882:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.342883:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.342887:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.342888:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.342892:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.342892:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.342893:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.342895:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.342895:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.342896:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.342897:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.342898:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.342903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.342909:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.342911:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.342912:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.342912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677473.343001:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.343003:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.343004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677473.343006:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.343009:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.343010:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.343011:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.343013:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.343014:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.343015:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.343017:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.343019:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.343020:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.343021:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.343022:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.343023:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.343023:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.343024:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.343024:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677473.343028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.343034:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.343036:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.343037:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.343038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.343039:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.343040:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.343293:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.343297:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.343301:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.343304:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.343306:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.343307:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.343308:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.343310:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.343311:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.343312:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.343315:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.343317:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.343319:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.343320:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.343321:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.343322:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.343323:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.343323:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.343324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.343336:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.343339:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.343339:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.343339:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:20.0:1619677473.343342:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.343342:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.343342:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:21.0:1619677473.343343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.343344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:17.0:1619677473.343345:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.343347:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.343348:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677473.343350:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.343351:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.343352:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.343353:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677473.343357:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.343358:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677473.343361:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.343362:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677473.343363:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677473.343365:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677473.343365:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.343366:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.343367:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:10.0:1619677473.343367:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.343370:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.343371:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677473.343371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:10.0:1619677473.343373:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.343377:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.343378:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:10.0:1619677473.343378:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677473.343380:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.343380:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.343381:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.343381:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.343383:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.343383:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.343384:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.343387:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.343387:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.343388:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677473.343389:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.343392:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.343393:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.343394:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.343396:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.343396:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.343397:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.343397:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.343410:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.343412:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.343413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.343414:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.343416:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.343419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.343484:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.343486:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.343489:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.343489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.343492:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.343493:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.343495:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.343496:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.343497:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.343498:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.343501:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.343503:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.343505:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.343506:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.343507:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.343509:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.343509:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.343510:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.343511:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.343512:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677473.343515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.343521:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677473.343521:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.343523:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.343524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677473.343524:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.343525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.343525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677473.343526:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.343527:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.343531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.343800:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.343802:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.343803:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.343805:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.343808:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.343809:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.343810:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.343811:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.343812:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.343813:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.343816:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.343818:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.343820:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.343821:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.343822:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.343823:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.343824:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.343825:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.343826:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.343827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.343829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.343832:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.343834:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.343835:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.343835:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.343836:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.343838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.343841:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.343843:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.343843:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.343844:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:8.0:1619677473.344322:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.344324:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.344325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.344327:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.344329:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.344330:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.344331:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.344332:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.344333:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.344334:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.344337:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.344338:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.344340:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.344341:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.344341:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.344343:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.344343:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.344344:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.344344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.344349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.344355:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.344357:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.344357:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.344358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.344359:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.344360:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.344367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.344370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.344370:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:9.2:1619677473.344371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:6.0:1619677473.344373:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.344375:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.344376:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.0:1619677473.344377:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677473.344377:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.344378:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.344380:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677473.344380:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.344380:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:24.0:1619677473.344381:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.344381:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.344381:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:6.0:1619677473.344381:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677473.344382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.344382:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.344383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:6.0:1619677473.344386:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677473.344386:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.344388:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:1.0:1619677473.344389:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.344390:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.344391:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:1.0:1619677473.344391:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:6.0:1619677473.344392:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:1.0:1619677473.344392:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.344393:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.344394:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:1.0:1619677473.344394:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.344395:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:1.0:1619677473.344395:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.344396:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:1.0:1619677473.344396:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677473.344397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:1.0:1619677473.344399:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.344401:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677473.344404:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.344405:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:20.0:1619677473.344406:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:1.0:1619677473.344406:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677473.344407:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:1.0:1619677473.344408:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677473.344408:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:22.0:1619677473.344409:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677473.344409:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:1.0:1619677473.344409:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.344409:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677473.344410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.344410:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.344410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.344412:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.344412:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.344414:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:13.0:1619677473.344421:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.344424:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.344425:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.344427:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.344429:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.344430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.0:1619677473.344676:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.344678:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.344680:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.344681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.344682:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.344685:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.344686:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.344687:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.344688:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.344689:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.344692:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.344694:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.344696:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.344697:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.344698:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.344699:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.344700:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.344701:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.344701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.344713:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.344715:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.344716:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.344717:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.344718:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.344721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.345013:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.345017:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.345017:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.345019:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.345021:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.345023:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.345024:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.345026:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.345027:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.345028:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.345031:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.345034:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.345036:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.345037:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.345038:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.345038:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.345039:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.345041:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.345041:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677473.345041:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.345043:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:10.0:1619677473.345043:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.345044:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:10.0:1619677473.345044:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:26.0:1619677473.345045:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.345045:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.345046:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:9.2:1619677473.345046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677473.345047:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.345048:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.345049:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.345051:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.345053:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677473.345053:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.345055:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.345056:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.345056:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.345056:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.345057:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.345057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.345058:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677473.345058:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.345059:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.345059:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.345059:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.345060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.345060:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.345061:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677473.345061:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.345062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.345065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.345068:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.345071:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.345071:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.345072:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.345073:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.345075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:12.0:1619677473.345164:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.345166:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.345168:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.345170:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.345170:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.345172:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677473.345173:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.345174:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.345175:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.345176:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677473.345179:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.345181:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677473.345184:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.345185:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677473.345185:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677473.345187:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677473.345188:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.345189:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.345189:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.345193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.345202:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.345205:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.345205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.345206:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.345208:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.345211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677473.345249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.345255:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.345255:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677473.345259:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677473.345259:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:1.0:1619677473.345259:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.345261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.345263:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.345264:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:17.0:1619677473.345264:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.345264:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.345266:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.345267:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.345268:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677473.345268:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.345270:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677473.345270:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677473.345273:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.345273:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.345275:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.345275:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.345277:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.345278:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.345280:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677473.345280:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677473.345287:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.345287:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.345289:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.345289:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.345290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677473.345290:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.345292:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.345293:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.345293:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677473.345296:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677473.345299:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677473.345300:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.345301:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.345301:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677473.345302:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.345302:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.345303:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.345303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:17.0:1619677473.345304:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.345305:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.345306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.345311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677473.345316:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:1.0:1619677473.345317:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677473.345318:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.345319:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.345319:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.345319:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.345320:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:20.0:1619677473.345320:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.345320:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.345322:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:21.0:1619677473.345322:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:22.0:1619677473.345323:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.345323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.345323:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.345325:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.345328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.345336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.345338:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.345339:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.345339:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.345340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.345341:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.345444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.345484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.345491:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.345494:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.345495:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.345495:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.345497:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.345499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.345766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.345774:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.345777:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.345778:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.345778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.345779:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.345782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.345785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.345791:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.345793:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.345794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.345794:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.345795:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.345796:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.346074:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.346077:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.346078:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.346079:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.346081:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.346082:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.346083:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.346085:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.346085:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.346086:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.346089:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.346091:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.346093:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.346093:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.346094:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.346095:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.346096:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.346097:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.346097:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.346101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.346104:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.346107:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.346107:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.346108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.346109:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.346111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.346122:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.346125:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.346126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.346127:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.346129:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.346130:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.346131:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.346133:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.346134:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.346134:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.346137:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.346138:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.346140:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.346141:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.346141:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.346143:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.346143:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.346144:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.346144:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.346148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.346154:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677473.346156:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.0:1619677473.346157:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.346157:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.346158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.346159:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.346160:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:8.0:1619677473.346160:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.346162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:6.0:1619677473.346162:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.346164:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.346166:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.346167:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.346169:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.346170:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.346171:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.346174:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.346176:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.346178:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.346179:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.346179:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.346181:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.346181:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.346182:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.346183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.346194:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.346197:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.346197:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.346199:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.346200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.346201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.346835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.346841:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.346844:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.346845:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.346847:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.101@o2ib7 : PUT - routed
00000800:00000200:24.0:1619677473.346848:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.347208:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.347210:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.347211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.347213:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.347216:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.347217:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.347218:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.347219:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.347220:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.347221:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.347224:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.347225:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.347227:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.347228:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.347229:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.347230:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.347231:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.347231:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.347232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.347245:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.347247:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.347248:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.347250:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.347253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.347255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.347259:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.347262:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.347263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.347265:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.347267:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.347268:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.347269:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.347271:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.347272:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.347273:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.347276:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.347278:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.347280:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.347281:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.347281:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.347283:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.347283:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.347284:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.347284:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.347288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:20.2:1619677473.347290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677473.347292:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.347296:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:2.0:1619677473.347296:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677473.347298:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.347299:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.347300:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.347301:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:8.0:1619677473.347301:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.347303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:2.0:1619677473.347303:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.347306:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.347307:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.347308:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.2:1619677473.347310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000400:00000200:2.0:1619677473.347310:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.347311:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.347312:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:2.0:1619677473.347312:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.347315:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.347315:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.347316:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000400:00000200:2.0:1619677473.347316:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.347317:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.347317:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:2.0:1619677473.347318:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.347321:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.347322:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.347323:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.347325:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.347325:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.347326:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.347327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677473.347496:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.347498:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.347499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677473.347501:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677473.347503:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.347505:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.347506:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.347507:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.347508:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.347509:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.347512:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.347513:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.347515:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.347516:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.347516:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.347517:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.347518:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677473.347518:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.347518:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.347519:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:20.0:1619677473.347521:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.347524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.347524:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677473.347526:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:20.0:1619677473.347526:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.347528:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:20.0:1619677473.347528:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.347529:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.347529:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:20.0:1619677473.347531:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.347532:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.347532:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.347533:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677473.347534:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.347535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:26.0:1619677473.347536:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.347537:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96509885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.347537:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677473.347539:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.347539:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677473.347540:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.347541:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677473.347541:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.347542:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.347543:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677473.347543:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)++
00000800:00000200:20.0:1619677473.347543:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (24)++
00000400:00000200:26.0:1619677473.347544:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.347544:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.347545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)--
00000800:00000200:26.0:1619677473.347546:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:24.0:1619677473.347546:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:20.0:1619677473.347546:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (24)--
00000800:00000200:20.0:1619677473.347547:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.347547:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677473.347548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.347548:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:26.0:1619677473.347549:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.347550:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.347551:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.347552:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.347553:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.347553:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.347554:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.347563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:13.0:1619677473.347567:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:8.0:1619677473.347569:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:13.0:1619677473.347571:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:8.0:1619677473.347571:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.347572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:13.0:1619677473.347572:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.347572:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.347575:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.347575:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:8.0:1619677473.347575:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.347577:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.347578:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.347578:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)++
00000800:00000200:8.0:1619677473.347578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677473.347579:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (24)--
00000800:00000200:12.0:1619677473.347580:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.347581:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.347645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.347648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.347648:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.347649:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.347651:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.347655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.347696:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.347698:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.347699:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.347701:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.347703:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.347705:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.347706:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.347707:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.347708:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.347709:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.347712:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.347714:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.347717:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.347718:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677473.347718:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.347718:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.347720:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.347720:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677473.347721:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677473.347721:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.0:1619677473.347721:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677473.347722:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:24.0:1619677473.347723:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.347723:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677473.347725:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.347726:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.347727:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.347728:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.347729:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.347730:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.347733:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.347733:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:24.0:1619677473.347734:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677473.347735:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.347736:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.347736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677473.347737:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677473.347737:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.347738:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.347738:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677473.347738:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.347738:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677473.347739:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.347739:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.347740:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.2:1619677473.347740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:24.0:1619677473.347741:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677473.347741:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.347742:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.347742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677473.347746:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677473.347746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.347749:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.347750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.347752:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.347752:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:18.0:1619677473.347755:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.347755:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677473.347755:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.347756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:18.0:1619677473.347756:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:6.0:1619677473.347756:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.347757:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000400:00000200:18.0:1619677473.347758:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.347758:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:6.0:1619677473.347758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:18.0:1619677473.347759:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.347759:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.347759:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:18.0:1619677473.347761:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.347762:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677473.347762:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.347765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:18.0:1619677473.347766:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.347768:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.347770:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.347772:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.347772:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.347774:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.347775:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.347776:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.347777:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.347787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:9.2:1619677473.347792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.347793:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.347795:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.347796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.347797:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.347799:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.347799:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.347801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677473.347801:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.347802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.347802:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.347804:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.347806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.348012:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.348015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.348015:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.348017:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.348020:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.348021:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.348022:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.348024:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.348025:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.348025:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.348028:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.348030:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.348032:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.348032:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.348033:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.348034:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.348035:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.348036:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.348037:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.348038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.348040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.348043:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.348045:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.348046:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.348046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.348048:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.348050:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:24.0:1619677473.348050:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.348052:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.348053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.348054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:22.0:1619677473.348072:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.348075:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.348075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.348077:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.348080:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.348081:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.348083:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.348084:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.348085:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.348086:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.348090:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.348091:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.348094:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.348095:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.348095:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.348097:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.348097:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.348099:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.348100:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.348100:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.348111:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.348113:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.348113:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.348113:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.348114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.348115:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.348116:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.348117:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.348118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.348120:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.348594:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.348597:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.348598:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.348600:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.348602:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.348603:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.348604:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.348606:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.348607:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.348608:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.348611:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.348613:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.348615:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.348616:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.348616:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.348618:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.348618:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.348619:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.348619:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.348624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.348628:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.348630:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.348631:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.348631:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.348633:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.348634:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.0:1619677473.348747:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.348749:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.348751:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.348752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.348754:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.348755:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.348756:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.348757:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.348758:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.348759:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.348762:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.348764:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.348767:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.348767:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.348768:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.348770:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.348770:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.348771:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.348772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.348776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.348783:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.348785:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.348786:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.348786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.348788:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.348791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.348881:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a0717e5c00] (21)++
00000800:00000200:26.0:1619677473.348884:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.2.70@o2ib3
00000800:00000200:10.0:1619677473.348885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a0717e5c00] (22)--
00000400:00000200:26.0:1619677473.348887:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.70@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.348889:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.348890:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.348892:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.348893:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.348894:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.348895:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.348898:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.348899:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.348902:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.348902:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.348903:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.348905:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.348905:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a0717e5c00] (21)++
00000800:00000200:26.0:1619677473.348906:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a0717e5c00] (22)--
00000800:00000200:26.0:1619677473.348906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a0717e5c00] (21)--
00000800:00000200:24.2:1619677473.348910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.348914:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.348916:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.348917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.348917:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.348919:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.348921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:9.2:1619677473.348948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.348955:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.348957:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.348958:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.348958:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.348960:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.348961:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.348994:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.348996:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.348997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.348999:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.349002:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.349003:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.349004:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.349005:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.349007:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.349008:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.349011:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.349012:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.349015:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.349016:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.349016:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.349018:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.349018:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.349019:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.349019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.349032:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.349034:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.349035:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.349036:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.349037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.349039:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.349051:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.349054:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.349055:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.349056:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.349056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.349057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.349474:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.349477:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.349478:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.349478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.349480:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.349483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.349492:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.349494:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.349495:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.349495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.349496:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.349497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.349601:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.349604:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.349605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.349607:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.349609:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.349610:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.349612:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.349613:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.349614:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.349615:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.349618:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.349619:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.349622:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.349623:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.349623:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.349625:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.349625:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.349626:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.349627:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:6.0:1619677473.349638:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.349641:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.349642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.349642:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.349645:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.349647:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.349783:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.349786:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.349787:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.349789:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.349791:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.349793:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.349794:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.349795:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.349796:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.349797:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.349800:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.349801:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.349803:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.349804:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.349804:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.349806:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.349806:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.349807:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.349807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.349811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.349818:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.349821:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.349822:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.349822:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.349824:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.349825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.349832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.349838:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.349841:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.349841:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.349841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.349842:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.349843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.349920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.349926:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.349928:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.349929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.349929:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.349931:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.349933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.350017:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.350020:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.350021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.350023:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.350025:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.350027:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.350028:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.350029:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677473.350029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677473.350030:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.350031:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.350035:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.350035:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.350037:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677473.350038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.350038:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677473.350039:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.350040:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.350041:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:1.0:1619677473.350041:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.350043:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.350043:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:1.0:1619677473.350043:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.350044:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.350044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:1.0:1619677473.350045:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.350046:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.350047:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.350048:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.350050:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.350053:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.350055:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677473.350056:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.350058:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:1.0:1619677473.350058:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677473.350059:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.350059:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.350059:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:20.0:1619677473.350061:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.350061:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.350061:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.0:1619677473.350062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:1.0:1619677473.350062:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677473.350063:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.350063:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.350074:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.350077:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.350078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.350078:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.350080:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.350082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.350339:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.350342:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.350343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.350344:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.350347:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.350349:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.350350:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.350352:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.350353:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.350354:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.350357:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.350359:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.350362:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.350363:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.350363:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.350365:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.350366:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.350367:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.350369:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.350369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.350372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.350380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.350382:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.350383:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677473.350384:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677473.350384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.350385:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.350385:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.350386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677473.350387:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.350390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.350572:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.350575:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.350576:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.350578:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.350581:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.350582:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:10.0:1619677473.350582:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.350583:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.350585:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:10.0:1619677473.350586:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.350587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:10.0:1619677473.350587:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.350588:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.350588:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:21.0:1619677473.350591:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.350592:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:21.0:1619677473.350593:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:10.0:1619677473.350593:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:21.0:1619677473.350594:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.350595:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.350596:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.350597:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.350597:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:21.0:1619677473.350598:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677473.350598:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.350599:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.350600:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.350600:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677473.350601:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.350601:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677473.350603:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.350606:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.350607:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.350607:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.350609:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.350610:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.350611:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.350612:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:12.0:1619677473.350612:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:21.0:1619677473.350613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:12.0:1619677473.350615:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:9.2:1619677473.350615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:12.0:1619677473.350616:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.350618:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.350620:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:20.0:1619677473.350621:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:12.0:1619677473.350621:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677473.350621:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.350624:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:8.0:1619677473.350624:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.350625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677473.350625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.350626:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677473.350626:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.350628:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.350631:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.350635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.350641:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.350644:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.350644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.350645:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.350646:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.350646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.350762:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.350765:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.350766:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.350768:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.350770:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.350771:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.350773:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.350774:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.350775:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.350776:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.350779:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.350781:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.350783:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.350784:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.350785:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.350786:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.350787:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.350788:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.350788:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.350792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.350796:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.350798:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.350799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.350799:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.350801:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.350803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.350814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.350816:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.350819:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.350819:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.350820:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.350820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.350821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.350870:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.350872:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.350873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.350875:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.350877:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.350879:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.350880:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.350882:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.350882:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.350883:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.350886:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.350886:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.350888:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.350888:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.350889:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.350890:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.350891:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.350892:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:10.0:1619677473.350892:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.350894:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.350894:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677473.350894:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.350895:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.350895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:10.0:1619677473.350896:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.350897:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.350898:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.350899:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.350900:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.350903:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.350905:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677473.350907:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677473.350907:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.350908:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.0:1619677473.350909:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677473.350909:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:20.0:1619677473.350910:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.350910:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.350910:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.350911:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677473.350912:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.350912:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.0:1619677473.350913:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.350914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.350915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.350922:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.350924:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.350925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.350925:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.350927:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.350929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.351095:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.351098:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.351099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.351099:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.351101:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.351104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.351196:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.351199:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.351200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.351201:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.351204:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.351205:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677473.351206:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.351208:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.351209:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.351209:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677473.351212:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.351214:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.351217:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.351218:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677473.351218:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.351220:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.351220:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.351222:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.351223:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.351223:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677473.351226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.351235:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:10.0:1619677473.351236:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:17.0:1619677473.351238:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:10.0:1619677473.351238:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.351239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:1.0:1619677473.351239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:26.0:1619677473.351240:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:17.0:1619677473.351240:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.351242:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.351245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677473.351727:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.351730:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.351730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.351732:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.351735:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.351736:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.351737:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.351739:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.351740:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.351741:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.351744:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.351746:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.351748:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.351749:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.351750:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.351751:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.351752:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.351753:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.351754:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.351754:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.351765:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.351767:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.351767:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.351768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.351769:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.351769:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.351770:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.351772:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.351773:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.351774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.352437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.352444:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.352446:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.352447:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.352448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.352449:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.352451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.352692:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.352695:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.352696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.352697:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.352700:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.352701:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.352703:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.352704:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.352705:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.352706:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.352709:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.352710:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.352713:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.352714:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.352715:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.352717:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.352717:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.352718:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.352718:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.352731:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.352733:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.352734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.352735:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.352737:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.352738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677473.352740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677473.352746:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.352749:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.352750:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.352750:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.352752:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.352755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677473.352773:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.352776:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.352778:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677473.352778:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677473.352779:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.352780:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.352781:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.352782:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.352782:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.352783:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.352784:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.352785:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.352786:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.352786:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.352787:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677473.352788:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.352789:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.352790:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.352791:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.352792:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677473.352798:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:6.0:1619677473.352798:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.352801:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.352801:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.352804:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.352804:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.352805:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677473.352805:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677473.352806:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.352806:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.352807:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.352808:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677473.352809:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.352809:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677473.352809:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.352809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.352810:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.352811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:21.0:1619677473.352820:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.352823:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.352823:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.352825:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677473.352825:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.352826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.352827:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677473.352827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:22.0:1619677473.352828:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.352828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.352830:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.352833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.352840:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.352842:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.352843:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.352843:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.352844:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.352845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.352904:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.352907:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.352908:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.352910:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.352913:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.352914:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.352915:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.352916:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.352918:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.352918:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.352922:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.352924:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.352926:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.352927:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.352928:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.352929:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.352930:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.352931:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.352932:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.352933:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.352935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.352939:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.352941:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.352941:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.352942:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.352944:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.352945:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.352946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.352947:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.352948:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.352949:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.0:1619677473.353085:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.353087:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.353088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.353090:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.353092:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.353094:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.353095:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.353096:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.353097:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.353098:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.353101:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.353102:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.353105:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.353105:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.353106:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.353108:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.353108:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.353109:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.353109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.353114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.353122:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.353125:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.353125:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.353126:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.353128:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.353131:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.353646:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.353649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.353649:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.353651:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.353654:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.353655:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.353656:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.353657:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.353659:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.353659:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.353663:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.353665:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.353667:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.353668:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.353669:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.353671:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.353671:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.353672:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.353672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.353686:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.353688:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.353689:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.353690:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.353691:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.353693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.353766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.353774:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.353777:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.353778:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.353780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.353780:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.353783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.353787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.353795:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.353798:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.353799:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.353799:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.353800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.353801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677473.354291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:13.0:1619677473.354299:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:1.0:1619677473.354299:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.354303:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677473.354303:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:1.0:1619677473.354303:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:13.0:1619677473.354307:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.354307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:1.0:1619677473.354307:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.354309:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.354311:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.354311:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.354312:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.354313:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677473.354314:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.354314:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.354315:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.354316:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:13.0:1619677473.354317:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.354319:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.354320:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677473.354321:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677473.354324:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.354324:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.354326:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.354327:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677473.354327:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.354328:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:13.0:1619677473.354329:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.354329:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:13.0:1619677473.354330:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677473.354330:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:13.0:1619677473.354331:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677473.354331:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:13.0:1619677473.354332:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677473.354332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:13.0:1619677473.354333:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.354335:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677473.354336:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677473.354337:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677473.354339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:22.0:1619677473.354344:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.354346:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:22.0:1619677473.354347:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.354348:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.354348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.354349:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.354350:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.354350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:22.0:1619677473.354351:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.354351:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:26.0:1619677473.354352:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.354353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:12.0:1619677473.354353:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677473.354354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:26.0:1619677473.354355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:12.0:1619677473.354355:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:6.0:1619677473.354464:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.354467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.354467:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.354468:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.354470:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.354473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.354484:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.354487:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.354487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.354488:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.354489:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.354489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.354498:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.354500:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.354501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.354502:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.354504:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.354505:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.354507:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.354508:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.354509:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.354510:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.354513:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.354514:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.354517:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.354518:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.354518:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.354520:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.354520:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.354521:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.354521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.354534:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.354536:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.354537:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.354539:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.354539:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.354541:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.354579:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.354582:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.354582:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.354585:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.354587:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.354588:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.354589:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.354591:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.354592:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.354593:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.354596:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.354598:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.354600:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.354601:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.354602:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.354603:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.354604:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.354605:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.354606:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.354606:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.354619:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:9.2:1619677473.354619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.354621:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.354622:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.354623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.354626:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.354629:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.354630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.354630:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.354632:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.354634:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.354641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.354647:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.354649:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.354650:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.354650:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.354651:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.354651:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677473.355139:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.355142:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.355143:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.355144:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.355147:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.355148:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677473.355149:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.355151:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.355152:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.355153:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677473.355155:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.355157:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.355159:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.355159:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.355160:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.355162:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.355162:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.355163:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.355163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677473.355168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.355175:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.355177:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.355178:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.355179:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.355179:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.355181:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.355187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.355194:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.355196:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.355197:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.355197:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.355197:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.355198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.355263:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.355266:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.355267:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.355269:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.355271:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.355272:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.355274:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.355275:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.355276:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.355277:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.355280:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.355282:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.355285:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.355286:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.355286:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.355288:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.355288:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.355289:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.355289:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.355293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.355299:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.355301:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.355302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.355302:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.355304:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.355306:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.355335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.355341:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.355344:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.355345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.355347:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.355349:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.355350:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.355352:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.355353:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.355355:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.355355:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.355359:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.355361:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.355362:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.355363:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.355364:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.355366:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.355366:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.355367:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.355368:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.355372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.355380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.355382:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.355382:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.355383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.355384:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.355385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.355394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.355401:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.355403:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.355404:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.355405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.355405:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.355405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.355553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.355556:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.355557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.355559:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.355562:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.355563:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.355564:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.355566:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.355566:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.355568:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.355570:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.355572:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.355574:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.355575:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.355576:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.355577:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.355578:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.355579:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.355580:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.355580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.355590:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.355591:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.355593:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.355593:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.355593:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.355594:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.355594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.355596:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.355597:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.355599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.355635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.355641:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.355644:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.355645:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.355647:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.355649:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.355650:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.355651:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.355653:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.355654:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.355655:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.355659:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.355660:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.355664:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.355664:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.355665:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.355667:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.355667:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.355669:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.355669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677473.355847:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.355850:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.355851:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.355852:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.355854:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.355856:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.355857:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.355858:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.355859:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.355860:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.355864:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.355865:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.355868:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.355869:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.355870:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.355872:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.355872:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.355873:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.355873:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.355886:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.355888:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.355889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.355890:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.355892:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.355895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.355895:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.355897:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.355898:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.355900:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.355903:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.355904:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677473.355906:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.355907:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.355908:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.355909:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677473.355912:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.355913:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.355914:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.355915:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.355915:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.355916:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.355917:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.355918:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.355918:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.355922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.355929:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.355930:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.355931:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.355932:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.355933:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.355934:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.355942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.355947:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.355950:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.355950:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.355951:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.355951:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.355952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677473.356174:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.356177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.356177:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.356178:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.356180:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.356183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.356334:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.356337:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677473.356338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677473.356339:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.356341:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.356343:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677473.356344:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.356345:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.356346:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.356347:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677473.356350:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.356352:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.356355:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.356356:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677473.356356:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.356358:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.356358:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.356359:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677473.356359:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677473.356365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.356373:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.356376:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.356377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.356377:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.356379:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.356382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.356386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.356393:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.356395:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.356396:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.356396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.356397:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.356398:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677473.356447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.356450:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.356451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.356453:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.356455:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.356456:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.356458:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.356459:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.356460:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.356461:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.356464:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.356466:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.356468:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.356469:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.356470:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.356471:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.356472:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.356473:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.356473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.356487:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.356489:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.356490:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.356491:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.356493:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.356494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.356494:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.356495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.356495:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.356496:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.356496:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.356570:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.356572:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.356573:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.356575:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.356578:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.356579:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.356581:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.356582:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.356583:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.356584:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.356587:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.356589:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.356591:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.356592:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.356593:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.356594:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.356594:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.356595:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.356595:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.356599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.356602:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.356605:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.356605:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.356606:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.356608:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.356610:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:1.0:1619677473.356839:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.356842:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.356842:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677473.356844:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.356846:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.356848:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677473.356849:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.356850:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.356851:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.356852:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677473.356856:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.356858:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677473.356861:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.356861:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677473.356862:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677473.356864:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677473.356864:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.356866:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677473.356867:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.356867:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677473.356870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.356876:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:17.0:1619677473.356878:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:26.0:1619677473.356879:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.356880:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.356880:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:10.0:1619677473.356880:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:17.0:1619677473.356881:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:26.0:1619677473.356882:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.356882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677473.356883:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677473.356885:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.0:1619677473.356886:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.356887:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.356888:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.356891:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.356893:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.356894:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.356895:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.356896:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.356897:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.356901:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.356903:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.356905:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.356906:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.356907:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.356908:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.356909:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.356910:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.356910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.356915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.356922:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.356925:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.356926:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.356926:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.356929:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.356932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.357338:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.357341:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.357341:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.357343:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.357345:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.357347:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.357348:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.357349:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.357350:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.357351:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.357355:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.357356:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.357358:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.357359:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.357360:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.357361:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.357362:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.357363:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.357363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.357376:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.357378:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.357379:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.357381:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.357382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.357383:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.357465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.357472:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.357475:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.357476:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.357476:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.357478:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.357480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.357597:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.357599:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.357600:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.357601:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.357604:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.357606:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.357607:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.357608:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.357609:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.357610:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.357613:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.357614:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.357617:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.357618:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.357618:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.357620:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.357620:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.357621:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.357622:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.357634:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.357636:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.357637:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.357637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.357640:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.357642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.357654:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.357657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.357657:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.357658:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.357659:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.357659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.358253:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.358255:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.358256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.358257:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.358259:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.358261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.358342:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.358345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.358345:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.358347:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.358349:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.358350:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.358351:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.358353:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.358354:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.358354:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.358357:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.358359:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.358361:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.358362:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.358363:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.358364:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.358365:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.358365:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.358366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.358379:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.358381:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.358382:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.358384:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.358385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.358386:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.358758:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.358761:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.358762:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.358764:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.358767:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.358768:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.358769:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.358771:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.358772:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.358773:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.358776:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.358778:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.358780:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.358781:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.358781:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.358783:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.358783:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.358784:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.358784:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.358797:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.358800:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.358802:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.358804:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.358806:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.358807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677473.358844:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.358846:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.358847:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.358849:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.358852:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.358853:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.358854:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.358856:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.358857:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.358857:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.358861:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.358862:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.358865:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.358865:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.358866:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.358868:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.358868:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.358869:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.358869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.358874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.358882:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.358885:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.358886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.358886:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.358888:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.358891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.359075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:5.2:1619677473.359081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.359086:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.359088:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.359088:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:17.0:1619677473.359089:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.359090:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677473.359091:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.359093:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.359094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.359095:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:1.0:1619677473.359095:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.359096:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.359097:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.359099:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.359100:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.359101:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.359102:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.359106:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.359107:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.359110:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.359111:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.359112:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.359113:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.359114:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.359114:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.359114:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.359126:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.359129:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.359130:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.359130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.359132:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.359135:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.359208:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.359211:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.359211:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.359214:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.359215:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.359217:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.359218:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.359219:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.359220:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.359221:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.359224:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.359226:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.359228:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.359229:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.359230:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.359231:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.359232:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.359233:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.359234:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.359234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.359245:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.359246:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.359247:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.359247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.359248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.359249:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.359250:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.359251:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.359253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.359254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.359269:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.359272:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.359272:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.359275:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.359277:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.359279:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.359280:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.359281:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.359282:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.359283:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.359287:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.359289:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.359291:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.359291:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.359292:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.2:1619677473.359300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:10.0:1619677473.359301:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:10.0:1619677473.359301:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.359302:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.359302:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.0:1619677473.359303:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.359305:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.359306:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.359306:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.359308:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.359310:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.359318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.359321:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.359324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.359324:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.359325:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.359325:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.359326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.359559:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.359561:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.359562:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.359564:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.359567:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.359568:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.359569:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.359571:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.359571:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.359572:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.2:1619677473.359573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:26.0:1619677473.359575:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.359577:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.359579:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.359580:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.359580:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.359582:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.359582:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677473.359582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677473.359583:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.359583:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677473.359584:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.359585:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.359587:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.359588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:9.2:1619677473.359588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677473.359590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677473.359594:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.359597:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.359598:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.359598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.359600:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.359602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.359657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.359666:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.359669:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.359670:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.359670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.359673:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.359676:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.359849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.359855:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.359858:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.359860:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.359860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.359862:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.359864:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:21.0:1619677473.359866:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.359867:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.359868:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.359869:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677473.359873:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.359875:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.0:1619677473.359878:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.359879:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677473.359880:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677473.359882:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677473.359882:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.359883:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677473.359884:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:13.0:1619677473.359897:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.359900:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.359901:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.359904:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.359904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.359907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677473.359916:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677473.359919:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677473.359920:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.359921:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677473.359921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.359922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:6.0:1619677473.360090:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.360093:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.360094:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.360096:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.360098:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.360099:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.360101:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.360102:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.360103:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.360104:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.360108:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.360109:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.360112:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.360112:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.360113:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.360115:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.360115:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.360115:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.360116:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.360128:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.360130:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.360131:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.360133:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.360134:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.360135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.360149:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.360152:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.360153:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.360153:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.360154:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.360155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.360232:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.360235:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.360236:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.360236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.360239:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.360242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.360251:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.360253:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.360254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.360254:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.360256:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.360256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.360266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.360274:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:17.0:1619677473.360276:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677473.360277:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.16@o2ib3
00000800:00000200:24.0:1619677473.360278:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:17.0:1619677473.360278:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.360279:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.360279:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.16@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.360279:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.360280:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:17.0:1619677473.360281:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.360282:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.360284:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677473.360284:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.360285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:8.0:1619677473.360285:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.360286:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677473.360287:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:8.0:1619677473.360287:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.360287:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.360288:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.360289:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677473.360289:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.360291:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.360292:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.360298:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677473.360300:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.360301:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 400 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:1.0:1619677473.360301:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.0:1619677473.360302:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.360303:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.360304:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:1.0:1619677473.360305:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.360306:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.360306:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:8.0:1619677473.360307:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:8.0:1619677473.360307:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)--
00000800:00000200:1.0:1619677473.360307:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.360309:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.360310:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.360311:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:9.2:1619677473.360312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.360313:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.360313:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.360314:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.360315:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:24.0:1619677473.360319:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.2:1619677473.360319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677473.360321:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.360322:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.360324:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.360325:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.360326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.360329:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.360331:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.360332:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.360333:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.360335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.360336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677473.360409:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:8.0:1619677473.360412:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.16@o2ib3
00000800:00000200:24.0:1619677473.360413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000400:00000200:8.0:1619677473.360415:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.16@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.360417:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.360418:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677473.360420:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.360421:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.360422:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.360423:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677473.360426:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.360428:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 472 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677473.360431:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.360432:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677473.360432:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677473.360434:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677473.360435:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:8.0:1619677473.360435:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:8.0:1619677473.360436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)--
00000800:00000200:8.2:1619677473.360440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.360448:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.360451:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.360452:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.360452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.360454:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.360457:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677473.360487:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:24.0:1619677473.360489:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.16@o2ib3
00000800:00000200:8.0:1619677473.360490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000400:00000200:24.0:1619677473.360492:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.16@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.360495:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.360496:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.360498:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.360499:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.360500:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.360501:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.360504:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.360506:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 448 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.360508:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.360509:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.360509:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.360511:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.360511:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:24.0:1619677473.360513:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c78ba2e00] (22)++
00000800:00000200:24.0:1619677473.360514:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba2e00] (23)--
00000800:00000200:24.0:1619677473.360515:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:8.2:1619677473.360517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.360525:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677473.360527:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)++
00000800:00000200:17.0:1619677473.360528:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.360528:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:8.0:1619677473.360529:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c78ba2e00] (23)--
00000400:00000200:1.0:1619677473.360529:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.360530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:8.0:1619677473.360531:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)++
00000400:00000200:1.0:1619677473.360531:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.360533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677473.360535:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:8.2:1619677473.360535:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.360539:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.16@o2ib3
00000800:00000200:17.0:1619677473.360542:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000400:00000200:8.0:1619677473.360542:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.16@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.360545:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.360545:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.360546:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000400:00000200:8.0:1619677473.360546:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:1.0:1619677473.360546:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.360547:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:8.0:1619677473.360547:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.360548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:8.0:1619677473.360549:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.360550:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.360550:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.2:1619677473.360553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:8.0:1619677473.360553:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.360554:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 368 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.360555:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.360556:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.360556:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.360558:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.360558:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:8.0:1619677473.360559:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:2.0:1619677473.360559:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677473.360560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)--
00000800:00000200:2.0:1619677473.360562:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.360563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677473.360563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:2.0:1619677473.360564:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.360567:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.360568:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:24.0:1619677473.360569:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:2.0:1619677473.360570:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.360571:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:2.0:1619677473.360571:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.360572:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.360572:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.360573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:2.0:1619677473.360574:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.360575:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.360576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:2.0:1619677473.360578:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.360580:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.360582:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.360583:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.360584:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.360586:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.360586:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.360587:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.360588:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677473.360615:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:8.0:1619677473.360618:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.16@o2ib3
00000800:00000200:24.0:1619677473.360619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000400:00000200:8.0:1619677473.360620:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.16@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.360622:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.360623:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677473.360624:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.360626:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.360627:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.360628:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.360630:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677473.360631:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.360633:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677473.360633:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 496 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.360634:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.360636:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.360636:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.360637:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:10.0:1619677473.360638:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.360638:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:10.0:1619677473.360639:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.0:1619677473.360639:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:10.0:1619677473.360640:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.360640:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:8.0:1619677473.360640:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba2e00] (22)--
00000400:00000200:10.0:1619677473.360641:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.360641:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)--
00000400:00000200:10.0:1619677473.360642:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.360643:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.2:1619677473.360645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:10.0:1619677473.360646:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.360647:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.360649:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.360650:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.360651:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.360652:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.360652:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.360653:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.360653:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677473.360653:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.360656:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.360657:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:9.2:1619677473.360657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:1.0:1619677473.360658:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.360660:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.360662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677473.360666:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.360669:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.360670:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.360671:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)++
00000400:00000200:24.0:1619677473.360672:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.360672:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.16@o2ib3
00000400:00000200:8.0:1619677473.360674:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.16@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.360675:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.360676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.360677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000400:00000200:8.0:1619677473.360677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.360679:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:9.2:1619677473.360680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:8.0:1619677473.360680:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.360682:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.360683:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.360685:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677473.360686:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.360688:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.360689:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:8.0:1619677473.360689:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.360690:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.360690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.360691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.360691:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 416 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677473.360694:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.360695:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677473.360696:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677473.360697:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677473.360698:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:8.0:1619677473.360698:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:8.0:1619677473.360699:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)--
00000800:00000200:8.2:1619677473.360703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.360711:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.360713:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.360715:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.360716:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.360717:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.360719:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.360725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.360733:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.360735:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.360736:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.360737:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677473.360737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.360738:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677473.360756:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:24.0:1619677473.360759:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.16@o2ib3
00000800:00000200:8.0:1619677473.360760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000400:00000200:24.0:1619677473.360762:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.16@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.360765:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.360766:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677473.360768:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.360769:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.360770:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.360771:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.360774:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.360775:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 400 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.360777:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.360778:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677473.360779:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677473.360780:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677473.360781:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:24.0:1619677473.360782:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c78ba2e00] (22)++
00000800:00000200:24.0:1619677473.360784:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba2e00] (23)--
00000800:00000200:24.0:1619677473.360784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:8.0:1619677473.360797:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)++
00000800:00000200:8.0:1619677473.360798:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c78ba2e00] (23)--
00000800:00000200:8.0:1619677473.360799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:8.0:1619677473.360800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)--
00000800:00000200:22.0:1619677473.361164:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.361166:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.361167:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.361169:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.361172:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.361173:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.361174:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.361176:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.361176:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.361177:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)++
00000400:00000200:22.0:1619677473.361178:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.361180:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.16@o2ib3
00000400:00000200:22.0:1619677473.361181:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.361181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000400:00000200:24.0:1619677473.361183:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.16@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.361183:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.361185:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677473.361186:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.361186:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:24.0:1619677473.361187:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:22.0:1619677473.361187:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:24.0:1619677473.361188:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.361188:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.361189:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677473.361190:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.361190:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.361190:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:24.0:1619677473.361191:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.361191:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677473.361195:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.361196:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 400 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677473.361199:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.361200:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677473.361201:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.361201:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:24.0:1619677473.361202:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677473.361203:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:24.0:1619677473.361203:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:20.0:1619677473.361203:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:24.0:1619677473.361204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)--
00000400:00000200:20.0:1619677473.361204:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.361206:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.361207:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.361208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.361208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.361217:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.361219:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.361220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.361221:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.361223:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.361225:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677473.361248:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:8.0:1619677473.361251:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.16@o2ib3
00000800:00000200:24.0:1619677473.361252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000400:00000200:8.0:1619677473.361253:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.16@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.361256:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.361258:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677473.361259:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.361260:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.361261:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.361262:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677473.361265:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.361266:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677473.361268:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.361269:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677473.361269:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677473.361271:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677473.361271:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:8.0:1619677473.361272:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:8.0:1619677473.361272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)--
00000800:00000200:24.0:1619677473.361338:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:24.0:1619677473.361341:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.16@o2ib3
00000800:00000200:8.0:1619677473.361341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000400:00000200:24.0:1619677473.361343:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.16@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.361345:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.361347:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677473.361348:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.361350:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.361350:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.361351:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677473.361355:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.361356:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677473.361359:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.361360:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:24.0:1619677473.361360:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677473.361362:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677473.361362:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:24.0:1619677473.361363:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:24.0:1619677473.361363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)--
00000800:00000200:6.0:1619677473.361377:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.361379:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.361380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.361381:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.361383:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.361386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.361394:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:8.0:1619677473.361397:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.16@o2ib3
00000800:00000200:24.0:1619677473.361398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000400:00000200:8.0:1619677473.361399:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.16@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.361402:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.361403:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677473.361405:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.361406:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.361407:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.361408:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677473.361410:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.361412:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 424 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.361413:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.361414:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.361414:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.361415:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.361416:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:8.0:1619677473.361417:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c78ba2e00] (22)++
00000800:00000200:8.0:1619677473.361418:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba2e00] (23)--
00000800:00000200:8.0:1619677473.361418:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:9.2:1619677473.361421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.361429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.361430:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.361430:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.361432:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.361432:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)++
00000800:00000200:24.0:1619677473.361433:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.361433:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c78ba2e00] (23)--
00000800:00000200:24.0:1619677473.361434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.361434:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:21.0:1619677473.361435:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677473.361435:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)--
00000800:00000200:21.0:1619677473.361437:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.361440:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.361440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.361443:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.361445:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.361446:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.361447:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.361448:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.361449:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.361453:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.361455:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.361458:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.361458:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.361459:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.361461:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.361462:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.361462:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.361463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677473.361537:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:24.0:1619677473.361540:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.16@o2ib3
00000800:00000200:8.0:1619677473.361541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (22)--
00000400:00000200:24.0:1619677473.361542:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.16@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.361544:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.361546:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677473.361547:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.361548:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.361549:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.361550:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677473.361553:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.16@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.361555:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677473.361557:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.361558:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677473.361558:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677473.361560:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677473.361560:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba2e00] (21)++
00000800:00000200:24.0:1619677473.361561:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba2e00] (22)--
00000800:00000200:24.0:1619677473.361561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba2e00] (21)--
00000800:00000200:9.2:1619677473.361565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.361573:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.361575:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.361576:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.361576:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.361577:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.361580:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.361586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677473.361592:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.361594:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677473.361595:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.361595:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.361596:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.361596:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.0:1619677473.362386:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.362389:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677473.362390:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677473.362391:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.362393:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677473.362393:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.362395:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:8.0:1619677473.362395:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677473.362396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677473.362396:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.362398:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.362398:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.362398:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.362399:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677473.362400:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.362401:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.362403:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.362404:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.362404:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.362405:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.362406:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677473.362406:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.362409:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.362409:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:26.0:1619677473.362410:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.362410:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.362412:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677473.362412:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.362412:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.362413:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677473.362414:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:26.0:1619677473.362415:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.362415:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:8.0:1619677473.362415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:26.0:1619677473.362416:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.362418:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.362418:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677473.362418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.362419:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.362420:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677473.362427:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:18.0:1619677473.362427:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.362430:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:18.0:1619677473.362430:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.362431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:2.0:1619677473.362431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.362432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:18.0:1619677473.362432:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.362434:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:18.0:1619677473.362434:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.362436:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:18.0:1619677473.362437:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:22.0:1619677473.362438:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.362440:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.362440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.362441:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.362441:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.362443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.362444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:6.0:1619677473.362444:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.362445:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.362446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.362496:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.362499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677473.362499:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:10.0:1619677473.362502:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.362504:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.362506:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677473.362507:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.362508:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.362509:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.362510:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677473.362513:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.362515:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677473.362519:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.362519:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677473.362520:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677473.362522:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.362522:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677473.362523:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.362523:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:26.0:1619677473.362525:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.362525:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:10.0:1619677473.362526:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677473.362526:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677473.362527:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.362528:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.362529:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.362530:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.362531:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.362532:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.362533:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:12.0:1619677473.362535:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:26.0:1619677473.362537:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.362538:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.362538:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.362539:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:12.0:1619677473.362540:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.362540:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:10.0:1619677473.362541:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:26.0:1619677473.362542:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.362542:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677473.362542:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677473.362543:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:12.0:1619677473.362543:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:26.0:1619677473.362544:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.362544:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:26.0:1619677473.362546:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.362547:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.362548:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.362549:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.362549:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:20.0:1619677473.362561:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.362563:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677473.362563:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:20.0:1619677473.362564:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.362565:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000400:00000200:20.0:1619677473.362566:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.362566:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:21.0:1619677473.362567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.362567:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:20.0:1619677473.362570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.362772:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.362774:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.362775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.362777:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.362779:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.362781:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.362782:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.362783:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.362784:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.362785:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.362789:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.362790:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.362793:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.362793:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.362794:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.362796:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.362796:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.362797:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.362797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.362809:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.362812:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.362812:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.362814:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.362815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.362816:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.363012:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.363015:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.363015:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.363018:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.363021:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.363022:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.363023:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.363024:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.363025:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.363026:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.363030:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.363031:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.363034:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.363035:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.363035:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.363037:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.363037:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.363038:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.363040:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.363040:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.363043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.363051:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.363052:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.363054:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677473.363054:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.363055:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677473.363055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.363055:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.363057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677473.363057:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.363061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.363062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.363070:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.363072:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.363073:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.363074:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.363075:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.363075:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677473.363219:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.363221:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.363223:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.363224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.363226:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.363228:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.363229:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.363231:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.363232:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.363233:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.363236:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.363237:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.363240:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.363240:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.363241:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.363243:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.363243:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.363244:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.363245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.363256:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.363259:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.363260:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.363260:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.363262:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.363264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.363374:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.363377:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.363378:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.363380:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.363383:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.363384:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.363385:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.363387:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.363388:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.363389:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.363392:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.363394:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.363396:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.363397:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.363398:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.363399:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.363400:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.363400:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.363401:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.363407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.363411:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.363413:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.363414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.363414:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.363416:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.363419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:5.2:1619677473.363857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.363864:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.363867:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.363868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.363870:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.363872:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.363874:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.363875:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.363876:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.363878:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.363879:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.363882:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.363884:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.363887:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.363887:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.363888:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.363890:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.363890:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.363892:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.363893:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.363893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.363898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.363903:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.363904:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.363906:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.363906:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.363906:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677473.363907:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.363907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.363908:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.363909:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.363912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.363922:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.363924:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.363925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.363925:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.363926:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.363927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.364216:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.364219:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.364220:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.364220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.364222:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.364225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.364335:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.364338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.364338:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.364340:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.364342:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.364343:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.364344:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.364346:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.364347:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.364348:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.364351:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.364353:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.364355:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.364356:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.364357:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.364358:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.364359:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.364360:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.364361:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.364361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.364372:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.364374:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.364374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.364374:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.364375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.364376:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.364377:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.364379:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.364379:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.364381:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.364381:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.364382:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.364382:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.364383:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.364383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.364567:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.364569:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677473.364570:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677473.364572:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.364575:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.364576:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677473.364577:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.364579:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.364580:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.364581:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677473.364584:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.364586:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677473.364589:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.364589:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677473.364590:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677473.364592:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677473.364592:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.364593:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.364593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677473.364598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.364606:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.364609:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.364610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.364610:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.364611:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677473.364612:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.364613:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.364614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677473.364615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:24.0:1619677473.364616:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.364619:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.364621:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.364622:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.364624:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.364625:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.364626:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.364630:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.364632:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.364635:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.364636:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.364636:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.364638:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.364638:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.364639:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.364640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.364644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.364652:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.364655:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.364656:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.364656:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.364658:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.364661:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.364771:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.364774:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.364775:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.364777:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.364779:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.364781:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.364782:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.364783:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.364784:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.364785:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.364788:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.364790:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.364792:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.364793:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.364793:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.364795:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.364795:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.364796:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.364798:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.364798:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.364800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.364806:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.364808:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.364809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.364809:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.364811:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.364812:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.364813:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.364814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.364814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.364815:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677473.365070:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.365072:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.365073:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.365075:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.365076:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.365077:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.365079:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.365080:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.365081:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.365082:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.365084:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.365086:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.365088:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.365089:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.365089:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.365091:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.365091:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.365092:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.365092:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.365097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.365100:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.365102:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.365103:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.365103:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.365105:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.365107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.365181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.365187:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.365189:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.365190:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.365192:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.365195:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.365196:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.365198:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.365199:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.365200:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.365201:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.365204:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.365206:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.365209:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.365210:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.365211:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.365213:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.365213:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.365215:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.365216:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.365217:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.365222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.365228:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677473.365228:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677473.365230:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.365231:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.365231:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.365232:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.365233:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.365236:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.365236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.365239:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677473.365589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.365598:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.365600:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.365601:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.365603:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.365603:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.365606:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677473.365860:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.365863:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.365864:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.365866:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.365868:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.365869:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.365871:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.365872:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.365873:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.365874:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.365877:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.365878:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.365880:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.365881:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.365882:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.365883:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.365884:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.365884:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.365885:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.365890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.365896:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.365898:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.365899:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.365899:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.365901:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.365903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.366054:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.366057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.366057:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.366060:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.366062:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.366063:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.366064:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.366065:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.366067:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.366067:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.366071:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.366072:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.366075:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.366075:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.366076:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.366078:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.366078:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.366079:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.366079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.366091:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.366093:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.366094:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.366096:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.366097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.366098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.366224:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.366227:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.366228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.366229:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.366231:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.366233:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.366245:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.366247:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.366248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.366248:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.366249:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.366250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.366346:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.366349:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.366350:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.366352:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.366354:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.366355:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677473.366357:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.366358:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.366359:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.366360:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677473.366363:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.366365:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677473.366366:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.366367:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677473.366367:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.366369:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.366369:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.366370:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.366371:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.366375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.366381:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.366384:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.366384:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.366385:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.366386:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.366387:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677473.366945:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.366948:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.366949:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.366951:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.366952:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.366953:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.366954:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.366955:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.366956:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.366957:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.366960:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.366961:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.366963:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.366964:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.366964:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.366965:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.366966:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.366967:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.366968:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.366969:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677473.366979:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677473.366981:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677473.366982:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:26.0:1619677473.366983:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:12.0:1619677473.366983:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.366984:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.366985:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:12.0:1619677473.366985:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.366987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.366988:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.366996:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.366999:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.367000:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.367001:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.367001:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.367002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.367310:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.367312:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.367313:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.367315:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.367317:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.367319:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.367320:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.367321:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.367322:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.367323:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.367326:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.367327:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.367330:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.367331:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.367332:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.367333:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.367334:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.367335:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.367336:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.367336:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.367350:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.367352:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.367353:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.367354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.367358:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.367361:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.367362:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.367362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.367365:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.367367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.367432:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)++
00000800:00000200:22.0:1619677473.367435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)--
00000800:00000200:6.0:1619677473.367435:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.67@o2ib3
00000400:00000200:6.0:1619677473.367437:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.67@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.367439:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.367440:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677473.367442:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.367443:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.367444:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.367445:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677473.367448:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.367449:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677473.367452:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.367453:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677473.367454:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.367455:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.367456:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987d3f2000] (21)++
00000800:00000200:6.0:1619677473.367457:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987d3f2000] (22)++
00000800:00000200:6.0:1619677473.367458:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987d3f2000] (23)--
00000800:00000200:6.0:1619677473.367458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)--
00000800:00000200:9.2:1619677473.367461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.367468:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677473.367468:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)++
00000800:00000200:22.0:1619677473.367470:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987d3f2000] (23)--
00000800:00000200:22.0:1619677473.367471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)--
00000800:00000200:24.0:1619677473.367472:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:22.0:1619677473.367472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)--
00000800:00000200:8.0:1619677473.367472:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.367473:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.367475:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.367479:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.367481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677473.367485:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)++
00000800:00000200:8.0:1619677473.367487:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.367488:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.67@o2ib3
00000800:00000200:22.0:1619677473.367489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)--
00000800:00000200:8.0:1619677473.367490:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.367491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.367491:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:6.0:1619677473.367492:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.67@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.367493:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.367493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:6.0:1619677473.367495:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.367496:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677473.367497:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.367499:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.367500:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.367500:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677473.367503:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.367505:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677473.367508:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.367508:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677473.367509:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.367511:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.367511:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987d3f2000] (21)++
00000800:00000200:6.0:1619677473.367512:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987d3f2000] (22)--
00000800:00000200:6.0:1619677473.367513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)--
00000800:00000200:9.2:1619677473.367516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.367522:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.367525:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.367526:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.367526:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)++
00000800:00000200:8.0:1619677473.367526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.367528:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.67@o2ib3
00000400:00000200:24.0:1619677473.367529:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.367529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)--
00000800:00000200:24.0:1619677473.367531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677473.367531:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.67@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.367534:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.367536:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677473.367538:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.367539:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.367540:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.367541:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677473.367544:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.367545:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677473.367548:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.367549:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:22.0:1619677473.367549:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:22.0:1619677473.367551:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:22.0:1619677473.367551:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987d3f2000] (21)++
00000800:00000200:22.0:1619677473.367552:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987d3f2000] (22)--
00000800:00000200:22.0:1619677473.367553:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)--
00000800:00000200:9.2:1619677473.367557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.367563:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.367566:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.367567:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.367567:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.367569:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.367573:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677473.367595:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)++
00000800:00000200:22.0:1619677473.367598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (22)--
00000800:00000200:6.0:1619677473.367598:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.67@o2ib3
00000400:00000200:6.0:1619677473.367600:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.67@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.367603:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.367605:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677473.367606:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.367607:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.367608:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.367609:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677473.367612:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.367614:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677473.367616:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.367617:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:6.0:1619677473.367617:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677473.367618:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677473.367619:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987d3f2000] (21)++
00000800:00000200:6.0:1619677473.367620:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987d3f2000] (22)--
00000800:00000200:6.0:1619677473.367620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987d3f2000] (21)--
00000800:00000200:8.2:1619677473.367625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.367633:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.367636:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.367636:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.367638:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.367639:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.367640:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677473.367712:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.367715:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.367716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.367719:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.367721:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.367722:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.367723:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.367725:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.367726:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.367727:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.367730:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.367732:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.367734:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.367735:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.367735:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:9.2:1619677473.367735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.367737:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.367737:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.367738:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.367740:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.367740:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:8.0:1619677473.367741:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.2:1619677473.367743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.367743:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.367744:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.367746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.367746:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.367747:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.367748:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:8.0:1619677473.367748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:24.0:1619677473.367749:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.367749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.367750:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.367752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.367752:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.367754:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.367755:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.367756:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:22.0:1619677473.367936:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.367939:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.367940:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.367940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.367942:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.367945:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:13.0:1619677473.368258:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677473.368261:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677473.368263:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : GET - for me
00000400:00000200:13.0:1619677473.368265:0:19942:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.103@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:12.0:1619677473.368265:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:6.0:1619677473.368266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:13.0:1619677473.368267:0:19942:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.103@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:6.0:1619677473.368268:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.368269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:13.0:1619677473.368269:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677473.368269:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:6.0:1619677473.368272:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.368273:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677473.368274:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.368276:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:13.0:1619677473.368278:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000400:00000200:6.0:1619677473.368278:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.368279:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677473.368281:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677473.368281:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.368282:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:13.0:1619677473.368285:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.368285:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677473.368286:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:6.0:1619677473.368286:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.368287:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677473.368288:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:6.0:1619677473.368288:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.368291:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.368292:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.368293:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.368300:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.368300:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:12.0:1619677473.368301:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:6.0:1619677473.368301:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.368301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:12.0:1619677473.368304:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.368305:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.368306:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:13.0:1619677473.368308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.368308:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.0:1619677473.368314:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.368316:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.368317:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.368319:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.368320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.368321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.368573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.368579:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.368582:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.368583:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.368585:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.368587:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.368589:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.368590:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.368591:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.368592:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.368593:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.368596:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.368598:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.368600:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.368601:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.368602:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.368604:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.368604:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.368605:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.368606:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.368617:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.368620:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.368621:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.368621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.368623:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.368625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.368625:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.368627:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.368629:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.368630:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.368632:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.368633:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.368635:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.368636:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.368637:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.368638:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.368641:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.368643:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.368645:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.368646:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.368647:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.368648:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.368649:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.368650:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.368650:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.368662:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.368664:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.368665:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.368665:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.368667:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.368670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.369017:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.369020:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.369021:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.369022:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.369025:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.369026:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.369028:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.369029:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.369030:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.369031:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.369034:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.369035:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.369038:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.369039:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.369039:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.369041:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.369041:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.369042:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.369042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.369047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.369055:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.369058:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.369059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.369059:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.369061:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.369064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.369528:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.369531:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.369532:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.369534:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.369536:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.369537:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.369538:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.369540:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.369541:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.369541:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.369545:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.369546:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.369548:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.369549:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.369550:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.369551:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.369551:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.369552:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.369553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.369557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.369564:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.369566:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.369567:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.369568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.369569:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.369571:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.369765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.369771:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.369773:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.369774:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.369774:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.369776:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.369778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.369783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.369789:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.369791:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.369792:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.369792:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.369793:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.369793:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.369832:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.369835:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.369835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.369837:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.369840:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.369841:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.369842:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.369844:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.369845:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.369846:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.369849:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.369851:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.369854:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.369854:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.369855:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.369857:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.369857:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.369858:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.369859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.369870:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.369872:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.369873:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.369875:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.369876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.369877:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.370289:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.370292:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.370293:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.370297:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.370299:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.370302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.370534:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.370536:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.370537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.370539:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.370542:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.370543:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.370544:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.370545:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.370546:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.370547:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.370550:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.370552:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.370554:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.370555:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.370556:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.370557:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.370558:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.370558:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.370559:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.370563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.370572:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.370574:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.370575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.370576:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.370578:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.370580:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.370647:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.370650:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.370651:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.370653:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.370655:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.370657:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.370658:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.370659:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.370660:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.370661:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.370664:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.370666:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.370668:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.370668:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.370669:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.370670:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.370671:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.370671:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.370672:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.370677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.370683:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.370685:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.370686:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.370686:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.370688:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.370690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.370743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.370749:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.370751:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.370752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.370752:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.370754:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.370756:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.370884:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.370886:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.370887:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.370888:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.370891:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.370892:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.370893:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.370894:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.370895:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.370896:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.370899:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.370900:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.370902:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.370903:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.370904:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.370905:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.370906:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.370906:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.370907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.370911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.370914:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.370916:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.370917:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.370917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.370918:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.370920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.370932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.370935:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.370938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.370938:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.370938:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.370939:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.370940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.371196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.371204:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.371206:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.371207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.371210:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.371213:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.371214:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.371215:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.371217:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.371218:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.371219:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.371222:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.371224:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.371226:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.371227:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.371227:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.371229:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.371230:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.371231:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677473.371232:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.371232:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.371236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.371238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677473.371239:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.371241:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.371242:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:2.0:1619677473.371244:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.371245:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:10.0:1619677473.371245:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.371245:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.371246:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677473.371246:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.371247:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677473.371247:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.371248:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.371248:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.371249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677473.371249:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.371250:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677473.371250:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:10.0:1619677473.371252:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.371253:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.371254:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.371257:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.371258:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.371261:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.371262:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.371263:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.371264:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.371265:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.371266:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.371266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677473.371278:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.371281:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.371282:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.371282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.371285:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.371291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.371293:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.371299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.371300:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.371300:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.371301:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.371302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.371393:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.371396:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.371397:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.371399:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.371401:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.371403:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677473.371404:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.371405:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.371406:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.371407:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677473.371411:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.371413:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677473.371416:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.371417:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677473.371417:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677473.371419:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677473.371419:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.371420:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.371421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.371426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.371433:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.371436:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677473.371436:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.371437:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.371439:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.371442:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677473.371519:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.371522:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.371523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.371525:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.371527:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.371529:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.371530:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.371531:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.371532:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.371533:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.371537:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.371539:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.371541:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.371542:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.371542:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.371544:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.371545:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.371546:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.371547:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.371548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.371558:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.371560:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.371560:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.371561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.371562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.371562:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.371563:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.371565:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.371566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.371567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.371577:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.371580:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.371581:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.371582:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.371583:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.371584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.371666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.371675:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.371676:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.371678:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.371679:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.371680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.371682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.371693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.371700:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.371703:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.371704:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.371704:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.371705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.371706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.371839:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.371842:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.371843:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.371845:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.371847:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.371848:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.371850:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.371851:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.371852:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.371853:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.371856:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.371857:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.371860:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.371861:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.371862:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.371863:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.371864:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.371865:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.371865:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.371877:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.371879:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.371880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.371881:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.371883:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.371885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.372097:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.372099:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.372100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.372101:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.372103:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.372105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.372239:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.372242:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.372244:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.372244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.372246:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.372248:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.372249:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.372250:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.372251:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.372252:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.372256:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.372258:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.372260:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.372261:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.372261:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.372263:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.372264:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.372266:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.372267:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.372267:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.372276:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677473.372276:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.372279:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:6.0:1619677473.372279:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.372280:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.372280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:6.0:1619677473.372280:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.372281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677473.372282:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.372284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677473.372560:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.372563:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.372564:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.372566:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.372569:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.372571:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.372572:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.372574:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.372574:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.372576:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.372579:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.372581:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.372583:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.372584:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.372585:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.372586:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.372587:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.372587:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.372588:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.372593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.372601:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.372603:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.372604:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.372606:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.372607:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.372608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.372762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.2:1619677473.372765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:18.0:1619677473.372769:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677473.372771:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:18.0:1619677473.372772:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:21.0:1619677473.372773:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:2.0:1619677473.372773:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.372775:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677473.372776:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.372776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:18.0:1619677473.372777:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.372779:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.372779:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:21.0:1619677473.372780:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:18.0:1619677473.372780:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.372782:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.372782:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.372783:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.372784:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.372785:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.372785:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:21.0:1619677473.372786:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.372791:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677473.372791:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.372793:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.0:1619677473.372795:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677473.372796:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.372797:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.372797:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:21.0:1619677473.372798:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:18.0:1619677473.372798:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677473.372799:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677473.372800:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.372800:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677473.372801:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.372801:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:18.0:1619677473.372801:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677473.372802:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.2:1619677473.372802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (22)++
00000800:00000200:18.0:1619677473.372802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.0:1619677473.372803:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.372804:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.372805:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:20.0:1619677473.372806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677473.372806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:21.0:1619677473.372808:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.372810:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677473.372810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:21.0:1619677473.372811:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:21.0:1619677473.372812:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96509737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.372814:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677473.372814:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:21.0:1619677473.372815:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.372816:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677473.372816:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:20.0:1619677473.372817:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677473.372817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.372817:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.372818:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.372819:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.372820:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:21.0:1619677473.372821:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677473.372821:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.372823:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677473.372823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677473.372824:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.372825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677473.372826:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.372826:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:21.0:1619677473.372827:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677473.372828:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:21.0:1619677473.372829:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.372830:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.372830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:9.2:1619677473.372835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.372844:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.372846:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.372847:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.372848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.372849:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.372852:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677473.373182:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.373185:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.373186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.373188:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.373191:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.373192:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.373194:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.373195:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.373196:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.373197:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.373201:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.373202:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.373206:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.373206:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.373207:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.373209:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.373209:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.373210:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.373210:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.373215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.373222:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.373224:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.373225:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.373227:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.373228:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.373230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677473.373388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.373394:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.373397:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.373398:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.373400:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.373402:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.373403:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.373405:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.373407:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.373408:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.373409:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.373412:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.373414:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.373416:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.373417:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.373418:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.373420:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.373420:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.373421:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.373421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.373435:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.373437:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.373438:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.373438:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.373440:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.373442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.373451:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.373453:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.373454:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.373455:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.373455:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.373456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.373705:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.373707:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.373708:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.373710:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.373713:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.373714:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.373715:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.373717:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.373718:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.373719:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.373722:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.373724:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.373727:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.373728:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.373728:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.373730:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.373731:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.373731:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.373732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.373745:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.373747:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.373751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677473.373753:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.373755:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.373757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.373788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.373796:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.373798:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.373799:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.373799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.373801:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.373803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.373969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677473.373972:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677473.373974:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:21.0:1619677473.373977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677473.373977:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.373979:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.373980:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677473.373982:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.373983:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.373984:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.373985:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677473.373988:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.373990:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.373992:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.373993:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.373993:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.373995:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.373996:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677473.373996:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:20.0:1619677473.373998:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677473.373998:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677473.374001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677473.374004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677473.374007:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.374009:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000800:00000200:20.0:1619677473.374009:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.374009:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.374010:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000400:00000200:20.0:1619677473.374010:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.374011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.374012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677473.374012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.374013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:6.0:1619677473.374172:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.374175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.374175:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.374176:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.374178:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.374181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.374301:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.374303:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.374304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.374307:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.374309:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.374310:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.374312:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.374314:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.374315:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.374316:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.374319:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.374320:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.374323:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.374323:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.374324:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.374325:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.374326:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.374327:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.374328:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.374329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.374331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.374338:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.374340:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.374341:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.374342:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677473.374342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.374343:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.374344:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.374345:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.374345:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.374346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.374348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.374354:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.374357:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.374357:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.374358:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.374358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.374359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.374582:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.374584:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.374585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.374587:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.374588:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.374590:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.374591:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.374592:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.374593:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.374593:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.374596:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.374597:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.374599:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.374600:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.374600:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.374602:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.374602:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.374603:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.374603:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.374607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.374611:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.374613:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.374614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.374614:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.374616:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.374617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:9.2:1619677473.374819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.374825:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.374827:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.374828:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.374828:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.374830:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.374832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.374874:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.374877:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.374878:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.374879:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.374883:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.374884:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.374885:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.374886:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.374887:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.374888:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.374891:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.374893:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.374894:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.374895:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.374895:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.374896:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.374897:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.374898:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.374899:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.374900:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.374902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.374910:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.374911:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.374912:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.374914:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.374914:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.374915:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.374915:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.374916:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.374916:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.374917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677473.375052:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.375055:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.375056:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.375057:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.375060:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.375061:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.375062:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.375063:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.375064:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.375065:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.375068:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.375069:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.375071:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.375072:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.375073:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.375074:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.375075:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.375075:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.375076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.375088:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.375090:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.375091:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.375092:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.375095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.375096:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677473.375246:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.375249:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.375250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.375252:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.375254:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.375256:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.375257:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.375258:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.375259:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.375260:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.375264:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.375266:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.375268:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.375269:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.375269:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.375271:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.375271:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.375272:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.375272:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.375284:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.375287:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.375287:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.375289:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.375290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.375291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.375335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:13.0:1619677473.375381:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.375384:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.375384:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.375386:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.375388:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.375389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:10.0:1619677473.375394:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.375397:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.375398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.375399:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.375402:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.375403:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.375405:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.375406:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.375407:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.375408:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.375411:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.375412:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.375415:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.375416:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.375416:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.375418:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.375418:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.375419:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.375419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.375432:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.375435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.375435:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.375436:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.375438:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.375441:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.375450:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.375453:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.375453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.375454:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.375455:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.375455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.375736:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.375738:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.375739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.375739:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.375742:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.375744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.375891:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.375893:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.375896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.375896:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.375897:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.375899:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.375900:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.375901:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.375902:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.375903:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.375907:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.375908:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.375911:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.375912:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.375912:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.375915:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.375916:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.375917:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.375918:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.375921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677473.375923:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.375925:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.375926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:8.0:1619677473.375928:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:26.0:1619677473.375929:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.375931:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.375931:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:26.0:1619677473.375932:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:24.0:1619677473.375932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.375932:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.375934:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.375934:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:26.0:1619677473.375935:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.375936:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.375937:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.375937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677473.375940:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.375942:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.375944:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.375945:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.375946:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.375947:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.375948:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.375948:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.375949:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.375953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.375956:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.375958:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.375959:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.375959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.375960:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.375962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.376196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.376235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.376238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:18.0:1619677473.376244:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.376247:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.376248:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.376248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.376250:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.376254:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677473.376337:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677473.376340:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677473.376341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677473.376343:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677473.376345:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.376346:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677473.376347:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.376348:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.376349:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.376350:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677473.376353:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.376355:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.376356:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.376357:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.376357:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.376358:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.376359:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677473.376360:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:8.0:1619677473.376361:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677473.376362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:9.2:1619677473.376364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.376372:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.376373:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.376374:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.376375:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.376376:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677473.376377:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.376377:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677473.376377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677473.376378:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.376378:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:24.0:1619677473.376404:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.376406:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.376407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.376408:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.376410:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.376412:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.376413:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.376414:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.376415:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.376416:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.376418:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.376420:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.376422:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.376423:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.376424:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.376426:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.376426:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.376426:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.376427:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.376432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.376440:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.376442:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.376443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.376444:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.376446:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.376449:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.376451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.376459:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.376461:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.376462:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.376462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.376463:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.376464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.376709:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.376712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.376712:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.376714:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.376716:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.376718:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.376719:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.376720:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.376721:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.376722:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.376725:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.376727:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.376730:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.376731:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.376731:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.376733:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.376733:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.376734:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.376734:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.376747:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.376750:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.376750:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.376751:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.376753:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.376756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.376945:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.376948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.376948:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.376950:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.376952:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.376953:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.376954:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.376956:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.376957:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.376958:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.376961:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.376963:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.376966:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.376966:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.376967:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.376969:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.376969:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.376970:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.376970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.376982:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.376985:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.376985:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.376987:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.376989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.376990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.376990:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.376993:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677473.376994:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.376995:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.376997:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.376999:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677473.377000:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.377001:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.377002:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.377003:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.377007:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.377009:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.377011:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.377012:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677473.377013:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677473.377015:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677473.377015:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.377016:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.377017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:22.0:1619677473.377730:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.377733:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.377733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.377734:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.377736:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.377739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677473.377745:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.377748:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677473.377749:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677473.377751:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.377754:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.377756:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677473.377757:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.377759:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.377760:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.377761:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677473.377765:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.377766:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677473.377770:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.377771:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677473.377771:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677473.377773:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677473.377774:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.377775:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677473.377775:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677473.377779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.377786:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.377789:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.377790:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.377790:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.377793:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.377796:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.377805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677473.377811:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.377814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.377814:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677473.377815:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.377816:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.377816:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677473.378087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.378093:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.378095:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.378096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.378098:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.378101:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.378103:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.378104:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.378105:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.378106:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.378107:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.378111:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.378112:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.378115:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.378116:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.378117:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.378118:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.378119:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.378120:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.378120:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677473.378133:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.378135:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.378136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.378139:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.378142:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.378143:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.378144:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.378146:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.378146:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.378148:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.378151:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.378153:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.378155:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.378156:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.378157:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:5.2:1619677473.378157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677473.378158:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.378159:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.378160:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.378160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677473.378163:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677473.378164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677473.378166:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.378167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.378169:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.378171:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.378172:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:1.0:1619677473.378173:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677473.378175:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:1.0:1619677473.378175:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677473.378176:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.378176:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.378176:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.378178:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.378178:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.378179:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677473.378182:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:1.0:1619677473.378183:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.378185:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.378188:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.378189:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.378189:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.378191:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.378192:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.378192:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.378193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.378204:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.378207:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.378208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.378208:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.378210:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.378212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.378496:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.378498:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.378499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.378501:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.378503:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.378505:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.378506:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.378507:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.378508:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.378509:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.378513:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.378515:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.378517:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.378517:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.378518:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.378519:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.378520:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.378521:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.378522:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.378523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.378526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.378529:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.378531:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.378532:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.378533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.378534:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.378536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.378536:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.378538:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.378539:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:10.0:1619677473.378539:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:22.0:1619677473.379184:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.379187:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.379187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.379189:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.379192:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.379193:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.379194:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.379196:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.379197:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.379198:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.379201:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.379203:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.379205:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.379206:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.379207:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.379209:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.379209:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.379211:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.379212:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.379212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.379222:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.379224:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.379224:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.379224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.379225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.379227:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.379227:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.379229:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.379230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.379232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.379243:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.379246:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.379247:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.379247:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.379248:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.379249:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.379525:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.379528:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.379529:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.379529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.379530:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677473.379532:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.379533:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.379534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.379535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677473.379536:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.379538:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.379539:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.379541:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.379542:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.379543:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.379544:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.379547:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.379549:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.379551:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.379552:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.379552:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.379553:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.379554:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.379555:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.379555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.379559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.379566:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.379568:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.379569:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.379570:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.379571:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.379574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:9.2:1619677473.379577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.379583:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.379586:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.379586:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.379587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.379588:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.379590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.379595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.379600:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.379603:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.379604:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.379604:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.379605:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.379605:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:13.0:1619677473.379765:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.379768:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.379770:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.379772:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.379772:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.379773:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677473.379774:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.379776:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.379776:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.379778:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677473.379781:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.379783:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677473.379785:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.379786:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677473.379786:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677473.379788:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677473.379788:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.379789:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.379789:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.379794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.379800:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.379802:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.379803:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.379803:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.379805:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.379807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:17.0:1619677473.380308:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.380311:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.380312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.380314:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.380316:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.380318:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.380319:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.380321:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.380322:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.380323:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.380326:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.380327:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.380330:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.380330:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.380331:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.380333:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.380333:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.380334:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.380335:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.380340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.380349:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.380351:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.380352:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.380354:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.380355:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.380356:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.380437:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.380439:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.380440:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.380442:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.380444:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.380446:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.380447:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.380448:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.380449:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.380450:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.380454:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.380456:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.380459:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.380459:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.380460:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.380462:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.380462:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.380463:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.380463:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.380476:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.380479:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.380479:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.380480:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.380482:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.380485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.380495:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.380498:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.380498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.380499:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.380500:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.380500:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.380728:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.380730:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.380731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.380734:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.380736:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.380737:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.380738:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.380739:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.380740:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.380741:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.380745:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.380747:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.380749:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.380750:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.380751:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.380753:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.380753:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.380754:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.380754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.380766:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.380768:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.380769:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.380771:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.380771:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.380773:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.380846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.380852:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.380854:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.380855:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.380855:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.380857:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.380859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.381051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.381054:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677473.381055:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677473.381057:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.381060:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.381061:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.381062:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.381064:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.381065:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.381066:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.381069:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.381071:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.381074:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.381075:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.381075:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.381077:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.381078:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.381078:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677473.381079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.381083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.381092:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.381095:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.381096:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.381096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.381098:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.381099:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.381102:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:18.0:1619677473.381102:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:22.0:1619677473.381103:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.381103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.381105:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.381108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.381333:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.381335:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.381336:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.381339:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.381341:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.381343:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.381344:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.381345:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.381346:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.381347:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.381351:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.381352:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.381355:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.381356:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677473.381357:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.381358:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.381359:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.381360:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.381360:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:20.0:1619677473.381373:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.381376:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.381377:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.381379:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.381380:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.381381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.381540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677473.381546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.381549:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677473.381550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677473.381552:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.381554:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.381556:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677473.381557:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.381558:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.381559:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.381560:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677473.381564:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.381565:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677473.381568:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.381569:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677473.381570:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.381571:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677473.381572:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.381572:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677473.381573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:21.0:1619677473.381576:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:9.2:1619677473.381577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.381578:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.381581:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.381581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677473.381584:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:21.0:1619677473.381584:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.381585:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.381586:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.381587:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.381588:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.381588:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.381588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677473.381589:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.381590:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.381591:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.381594:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:21.0:1619677473.381594:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.381596:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.381598:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.381599:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:9.2:1619677473.381599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.381600:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.381601:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.381602:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.381603:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.381603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677473.381605:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.381608:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.381608:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.381609:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.381610:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.381610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.381615:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.381618:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.381619:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.381619:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.381621:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.381623:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.381690:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.381693:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.381694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.381695:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.381698:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.381700:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.381701:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.381702:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.381703:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.381705:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.381708:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.381709:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.381712:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.381713:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.381714:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.381715:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.381716:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.381716:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.381717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.381721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.381729:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.381732:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.381733:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.381733:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.381735:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.381738:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.381741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.381748:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.381751:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.381751:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.381752:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.381753:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.381753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.382219:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.382222:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.382223:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.382223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.382226:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.382228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.382278:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.382281:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.382282:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.382284:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.382287:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.382288:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.382290:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.382291:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.382293:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.382295:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:12.0:1619677473.382300:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:10.0:1619677473.382301:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.382302:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:12.0:1619677473.382304:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:26.0:1619677473.382305:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.382305:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677473.382306:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.382307:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.382307:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:13.0:1619677473.382308:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:26.0:1619677473.382309:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:12.0:1619677473.382309:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.382310:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.382310:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.382312:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.382312:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.382313:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.382313:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.382314:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:12.0:1619677473.382314:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.382315:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.382317:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.382318:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.382318:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:12.0:1619677473.382319:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.382319:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.382321:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.382321:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:12.0:1619677473.382322:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.382322:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.382323:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:12.0:1619677473.382324:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.382324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677473.382325:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.382325:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:12.0:1619677473.382325:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.382325:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.382326:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677473.382326:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:12.0:1619677473.382327:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:12.0:1619677473.382327:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677473.382328:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:12.0:1619677473.382328:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:26.0:1619677473.382329:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:12.0:1619677473.382329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:26.0:1619677473.382330:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.382331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.382333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677473.382336:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.382339:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.382340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.382340:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.382342:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:20.0:1619677473.382343:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:6.0:1619677473.382343:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.382345:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:6.0:1619677473.382345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.382346:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:10.0:1619677473.382346:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.382347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:20.0:1619677473.382347:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.382350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.382350:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.382350:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.382353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.382353:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677473.382469:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.382472:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.382472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.382474:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.382477:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.382478:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.382480:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.382481:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.382482:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.382483:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.382487:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.382490:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.382492:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.382493:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.382494:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.382495:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.382496:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.382497:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.382497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.382509:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.382511:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.382512:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.382513:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.382514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.382516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.382526:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.382529:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.382530:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.382530:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.382531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.382532:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.382640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.382646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.382650:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.382651:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.382653:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.382656:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.382658:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.382659:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.382660:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.382662:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.382663:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.382666:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.382668:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.382671:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.382672:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.382673:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.382675:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.382675:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.382676:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.382677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.382687:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.382690:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.382691:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.382693:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.382694:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.382695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677473.382834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.382840:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.382843:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.382844:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.382846:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.382849:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.382850:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.382852:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.382853:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.382854:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.382855:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.382858:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.382859:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.382862:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.382862:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.382863:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.382865:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.382865:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.382867:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.382868:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.382868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.382873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.382878:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.382879:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.382880:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.382881:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677473.382882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.382882:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.382882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.382883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677473.382884:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.382886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.382898:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.382900:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.382901:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.382901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.382902:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.382903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677473.383158:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.383161:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677473.383161:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677473.383163:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.383166:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.383168:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677473.383169:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.383170:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.383171:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.383172:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677473.383175:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.383177:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677473.383180:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.383181:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677473.383182:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677473.383183:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677473.383184:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.383185:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677473.383186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677473.383189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.383196:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.383199:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.383200:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.383200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.383202:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.383205:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677473.383273:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.383276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.383276:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.383278:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.383280:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.383282:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.383283:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.383284:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.383285:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.383286:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.383290:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.383291:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.383298:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.383299:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.383300:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.383302:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.383302:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.383303:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.383303:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.383316:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.383318:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.383319:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.383320:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.383323:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.383324:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.383973:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.383976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.383976:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.383977:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.383979:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.383981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.383992:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.383994:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.383995:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.383995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.383996:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.383997:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.384247:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.384250:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.384251:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.384253:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.384256:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.384257:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.384258:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.384259:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.384260:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.384261:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.384264:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.384266:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.384268:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.384268:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.384269:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.384271:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.384271:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.384272:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.384274:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.384274:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.384276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.384283:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.384286:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:10.0:1619677473.384287:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.384287:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.384288:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.384289:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677473.384289:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.384290:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677473.384290:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.384292:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.384625:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.384627:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.384628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.384630:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.384632:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.384633:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.384634:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.384635:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.384636:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.384637:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.384640:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.384641:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.384642:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.384643:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.384644:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.384645:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.384645:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.384646:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.384648:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.384648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.384651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.384659:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.384660:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.384661:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.384662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677473.384662:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.384663:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.384664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.384664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.384665:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.384665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.384667:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.384670:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.384671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.384673:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.384675:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.384676:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.384678:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.384679:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.384680:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.384681:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.384684:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.384686:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.384689:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.384690:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.384690:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.384692:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.384692:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.384693:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.384694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.384706:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.384708:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.384709:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.384710:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.384711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.384712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.385195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.385204:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.385206:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.385208:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.385210:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.385210:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.385213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.385325:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.385328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.385328:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.385330:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.385332:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.385334:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.385335:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.385336:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.385337:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.385338:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.385342:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.385343:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.385346:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.385347:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.385347:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.385349:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.385349:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.385350:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.385350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.385364:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.385366:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.385367:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.385368:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.385370:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.385373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.385392:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.385395:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.385396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.385397:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.385400:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.385402:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677473.385403:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.385404:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.385405:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.385406:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677473.385409:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.385410:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677473.385413:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.385414:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677473.385414:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.385416:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677473.385416:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.385417:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.385417:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.385432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.385440:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.385443:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.385444:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.385445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.385447:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.385450:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.385643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.385650:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.385652:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.385653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.385653:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.385655:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.385657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.385875:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.385878:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.385879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.385880:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.385882:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.385883:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.385884:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.385885:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.385886:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.385887:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.385890:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.385891:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.385894:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.385895:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.385896:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.385897:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.385898:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.385898:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.385899:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.385903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.385911:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.385914:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.385915:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.385915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.385917:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.385920:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.385925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.385932:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.385934:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.385935:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.385936:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.385937:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.385937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.386004:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.386007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.386007:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.386008:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.386010:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.386013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.386014:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.386016:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.386017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.386019:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.386022:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.386023:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.386025:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.386026:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.386027:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.386028:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.386031:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.386033:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.386036:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.386037:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.386037:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.386039:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.386039:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.386040:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.386042:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.386042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:21.0:1619677473.386052:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677473.386055:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:21.0:1619677473.386055:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.386056:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.386057:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.386057:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:21.0:1619677473.386057:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.386058:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.386059:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.386060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.386170:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.386173:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.386173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.386175:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.386178:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.386179:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.386180:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.386182:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.386183:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.386184:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.386187:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.386189:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.386191:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.386192:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.386193:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.386194:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.386195:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.386196:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.386197:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.386198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.386207:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.386208:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.386210:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.386210:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677473.386211:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.386211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.386212:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.386212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.386213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.386215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.386568:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.386570:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.386572:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.386573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.386575:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.386576:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.386578:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.386579:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.386580:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.386581:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.386585:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.386586:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.386589:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.386590:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.386590:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.386592:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.386593:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.386594:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.386594:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.386608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.386615:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.386618:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.386619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.386619:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.386621:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.386624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.387189:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.387192:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.387192:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.387195:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.387198:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.387200:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.387201:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.387203:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.387205:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.387207:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.387211:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.387213:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.387216:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.387217:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.387217:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.387219:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.387220:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.387221:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.387222:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.387223:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.387234:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.387235:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.387237:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677473.387237:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.387238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677473.387238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.387239:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.387239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.387241:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.387243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.387424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.387427:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.387428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.387429:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.387433:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.387434:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.387435:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.387437:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.387438:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.387438:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.387442:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.387443:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.387446:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.387447:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.387447:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.387449:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.387449:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.387450:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.387450:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.387455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.387463:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.387466:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.387467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.387468:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.387470:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.387473:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677473.387610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.387616:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.387618:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.387619:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.387621:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.387623:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.387624:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.387625:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.387627:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.387628:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.387629:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.387632:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.387634:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.387636:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.387637:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.387638:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.387639:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.387640:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.387640:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.387641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.387654:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.387657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.387657:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.387658:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.387660:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.387666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.387685:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.387687:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.387688:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.387688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.387690:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.387693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.387912:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.387915:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.387916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.387918:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.387920:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.387921:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.387922:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.387923:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.387924:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.387925:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.387929:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.387930:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.387933:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.387934:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.387934:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.387936:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.387936:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.387937:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.387938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.387949:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.387951:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.387952:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.387954:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.387955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.387956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.387968:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.387970:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.387971:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.387972:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.387972:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.387974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.388946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.388951:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.388953:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.388955:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.388956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.388958:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.388959:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.388961:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.388962:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.388963:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.388964:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.388967:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.388969:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.388972:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.388973:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.388973:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.388975:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.388976:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677473.388977:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677473.388977:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.388978:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.388979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677473.388980:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.388981:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677473.388982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:26.0:1619677473.388983:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677473.388983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677473.388986:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.388986:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:26.0:1619677473.388987:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677473.388988:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:26.0:1619677473.388989:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.388989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677473.388990:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.388990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677473.388990:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:26.0:1619677473.388991:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.388992:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:17.0:1619677473.388993:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.388994:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.388994:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:26.0:1619677473.388995:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677473.388996:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.388997:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677473.388998:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677473.389000:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.389000:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.389001:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.389003:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.389003:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.389004:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.389005:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677473.389005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:9.2:1619677473.389009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677473.389012:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.389015:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.389016:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677473.389016:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677473.389016:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.389017:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677473.389018:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677473.389018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:8.0:1619677473.389019:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.389020:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.389021:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.389024:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.389029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.389035:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.389037:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.389038:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.389038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.389039:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.389039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677473.389170:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.389173:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677473.389174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677473.389176:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.389179:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.389180:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677473.389181:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.389182:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.389183:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.389184:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677473.389187:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.389189:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677473.389191:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.389192:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677473.389193:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677473.389195:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677473.389196:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.389196:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677473.389197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:22.0:1619677473.389208:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.389211:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.389212:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.389212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.389214:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.389216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.389228:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.389231:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.389232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.389232:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.389233:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.389233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.389803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.2:1619677473.389806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.389810:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.389813:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.389813:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.389814:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:18.0:1619677473.389816:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.389816:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.389819:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.389820:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.389821:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.389822:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.389824:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.389825:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.389826:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.389828:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:2.0:1619677473.389829:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.389831:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:18.0:1619677473.389831:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677473.389831:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:22.0:1619677473.389832:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.389832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:18.0:1619677473.389833:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96509657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.389834:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:18.0:1619677473.389835:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.389836:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.389836:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677473.389837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677473.389839:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.389840:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:18.0:1619677473.389841:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.389841:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.389843:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.389844:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.389844:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677473.389846:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.389846:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.389846:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.389846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.389847:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.389848:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677473.389849:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.389849:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:9.2:1619677473.389849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.389849:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.389851:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.389852:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.389853:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.389855:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.2:1619677473.389856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.389857:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:20.2:1619677473.389858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:18.0:1619677473.389858:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.389858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.389859:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.389860:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.389860:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.389861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.389861:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.389862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677473.389864:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.389867:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.389868:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.389868:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.389870:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.389872:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.390093:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.390096:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.390097:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.390099:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.390101:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.390102:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.390103:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.390105:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.390106:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.390107:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.390110:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.390111:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.390113:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.390114:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.390115:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.390116:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.390117:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.390117:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.390118:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.390122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.390128:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.390130:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.390131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.390131:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.390133:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.390135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.390136:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.390139:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.390140:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.390142:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.390144:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.390145:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.390146:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.390148:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.390149:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.390150:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.390153:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.390154:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.390157:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.390158:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.390158:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.390160:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.390161:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.390161:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.390162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.390173:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.390176:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.390176:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.390178:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.390179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.390180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.390522:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.390524:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.390525:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.390527:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.390529:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.390531:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.390532:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.390533:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.390534:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.390535:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.390539:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.390541:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.390543:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.390544:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.390544:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.390546:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.390546:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.390547:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.390547:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.390552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.390555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.390557:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.390558:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.390558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.390560:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.390562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.390569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.390572:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.390575:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.390576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.390576:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.390577:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.390577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.2:1619677473.390757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.390766:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.390768:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.390769:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.390770:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.390772:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.390775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.390809:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.390811:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.390812:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.390814:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.390817:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.390819:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677473.390820:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.390821:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.390822:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.390823:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677473.390827:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.390829:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677473.390831:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.390832:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677473.390833:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677473.390835:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677473.390835:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.390836:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.390836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:17.0:1619677473.391167:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.391170:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.391170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.391172:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.391175:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.391176:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.391177:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.391179:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.391180:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.391181:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.391183:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.391185:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.391187:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.391188:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.391189:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.391190:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.391191:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.391192:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677473.391193:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.391194:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.391197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.391204:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.391206:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677473.391207:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677473.391207:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.391208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677473.391209:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.391210:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.391211:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.391213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.391214:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.391218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.391225:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.391228:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.391229:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.391229:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.391230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.391231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.391316:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.391319:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.391319:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.391321:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.391324:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.391325:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.391326:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.391328:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.391329:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.391330:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.391333:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.391335:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.391337:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.391338:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.391339:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.391340:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.391341:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.391342:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.391342:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.391354:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.391357:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.391357:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.391359:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.391361:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.391362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.391372:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.391375:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.391376:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.391377:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.391377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.391379:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.391543:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.391546:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:26.0:1619677473.391548:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.391548:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677473.391548:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.391549:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677473.391551:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.391551:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:26.0:1619677473.391553:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:21.0:1619677473.391554:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.391554:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.391554:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.391555:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.391557:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.391558:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.391558:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.391559:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677473.391559:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.391560:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.391561:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.391563:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.391563:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.391564:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677473.391564:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.391565:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677473.391565:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.391567:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677473.391568:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.391569:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.391569:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.391570:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677473.391572:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677473.391573:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.391574:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677473.391575:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677473.391575:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.391576:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:10.0:1619677473.391576:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.391577:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677473.391577:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677473.391578:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (22)++
00000800:00000200:20.0:1619677473.391578:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:10.0:1619677473.391578:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677473.391579:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677473.391580:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (23)++
00000800:00000200:20.0:1619677473.391580:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:26.0:1619677473.391581:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (24)--
00000800:00000200:20.0:1619677473.391581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677473.391582:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.391582:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.391583:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.391583:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.391583:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.391584:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.391585:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:8.2:1619677473.391585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677473.391587:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.391587:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.391590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677473.391592:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:21.0:1619677473.391593:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:10.0:1619677473.391593:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.391594:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.0:1619677473.391595:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:10.0:1619677473.391595:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:21.0:1619677473.391596:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.391597:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.391598:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:6.0:1619677473.391599:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:21.0:1619677473.391600:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677473.391601:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.391603:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.391603:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.391605:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.391608:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.391674:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.391676:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.391677:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677473.391678:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677473.391679:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.391681:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:22.0:1619677473.391682:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.391682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677473.391683:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677473.391684:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.391685:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.391686:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.391687:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.391688:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.391689:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677473.391689:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:24.0:1619677473.391691:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.391693:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.391693:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.391694:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.391695:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.391695:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.391697:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.391698:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:24.0:1619677473.391699:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.391699:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.391700:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677473.391701:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.391701:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.391701:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.391702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677473.391704:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.391705:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.391705:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.391707:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.391708:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.391708:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.391709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.391713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677473.391714:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.391716:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.391717:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.391719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.391719:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.391721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677473.391722:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.391724:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.391726:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.391726:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.391728:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.391731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.392227:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.392230:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.392231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.392231:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.392234:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.392236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.392317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.392325:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.392328:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.392329:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.392329:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.392331:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.392334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.392336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.392343:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.392345:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.392346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.392346:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.392347:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.392348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677473.392383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.392388:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.392390:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.392391:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.392393:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.392395:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.392397:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.392397:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.392399:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.392400:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.392401:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.392404:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.392406:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.392408:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.392409:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.392410:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.392411:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.392411:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.392412:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.392412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.392426:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.392429:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.392430:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.392430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.392432:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.392434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.392444:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.392447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.392447:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.392448:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.392449:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.392450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.393073:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.393075:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.393077:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.393078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.393079:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.393081:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.393082:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.393083:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.393084:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.393085:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.393088:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.393090:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.393092:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.393093:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.393094:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.393095:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.393096:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.393097:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.393098:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.393099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.393107:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677473.393109:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.393109:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.393110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677473.393111:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677473.393111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:22.0:1619677473.393112:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.393112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.393114:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.393116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.393396:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.393398:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.393399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.393401:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.393403:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.393404:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.393405:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.393407:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.393408:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.393408:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.393411:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.393413:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.393415:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.393416:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.393417:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.393418:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.393419:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.393420:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.393421:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.393422:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.393432:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.393433:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.393435:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.393435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.393435:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.393436:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.393436:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.393437:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.393439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.393440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.2:1619677473.393496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.393500:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.393502:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.393503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.393503:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.393506:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.393508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:1.0:1619677473.393653:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc6200] (21)++
00000800:00000200:1.0:1619677473.393656:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.12@o2ib3
00000800:00000200:17.0:1619677473.393657:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc6200] (22)--
00000400:00000200:1.0:1619677473.393659:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.3.12@o2ib3 : GET - for me
00000400:00000200:1.0:1619677473.393662:0:19931:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.3.12@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1619677473.393664:0:19931:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.3.12@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:1.0:1619677473.393666:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cffdc6200] (21)++
00000800:00000200:1.0:1619677473.393668:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cffdc6200] (22)++
00000800:00000200:1.0:1619677473.393669:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cffdc6200] (23)--
00000400:00000200:1.0:1619677473.393671:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.3.12@o2ib3 local destination
00000400:00000200:1.0:1619677473.393675:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.3.12@o2ib3(10.51.3.12@o2ib3:10.51.3.12@o2ib3) : REPLY try# 0
00000800:00000200:1.0:1619677473.393678:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.3.12@o2ib3
00000800:00000200:1.0:1619677473.393679:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918ce37f3780] -> 10.51.3.12@o2ib3 (2) version: 12
00000800:00000200:1.0:1619677473.393680:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918cffdc6200] (23)++
00000800:00000200:17.0:1619677473.393681:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc6200] (23)++
00000800:00000200:1.0:1619677473.393681:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cffdc6200] (24)++
00000800:00000200:17.0:1619677473.393682:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cffdc6200] (26)--
00000800:00000200:1.0:1619677473.393682:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918cffdc6200] (26)--
00000800:00000200:17.0:1619677473.393683:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc6200] (23)--
00000800:00000200:1.0:1619677473.393683:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc6200] (25)--
00000800:00000200:1.0:1619677473.393684:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc6200] (22)--
00000800:00000200:17.0:1619677473.393693:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cffdc6200] (22)++
00000800:00000200:17.0:1619677473.393694:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cffdc6200] (23)--
00000400:00000200:17.0:1619677473.393695:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.393696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc6200] (22)--
00000400:00000200:17.0:1619677473.393697:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.3.12@o2ib3: REPLY: OK
00000800:00000200:17.0:1619677473.393699:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cffdc6200] (21)--
00000800:00000200:20.2:1619677473.393757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.393794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.393801:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.393804:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.393805:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.393805:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.393807:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.393809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677473.393914:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.393916:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.393917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.393919:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.393922:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.393923:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.393924:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.393926:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.393927:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.393929:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.393934:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.393936:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.393939:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.393940:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.393940:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.393942:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.393942:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.393944:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.393945:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.393947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677473.393948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.393956:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677473.393957:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.393959:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677473.393959:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.393960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677473.393960:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:1.0:1619677473.393960:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.393961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.393961:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.393961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677473.393961:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.393962:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.393963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677473.393964:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.393966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.393967:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.393980:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.393983:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.393984:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.393984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.393985:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.393986:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.394189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.394198:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.394200:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.394201:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.394203:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.394203:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.394206:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.394208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.394216:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.394219:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.394220:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.394220:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.394221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.394222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.394249:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.394251:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.394252:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.394254:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.394256:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.394258:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.394259:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.394260:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.394261:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.394262:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.394266:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.394267:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.394269:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.394270:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.394271:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.394272:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.394273:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.394274:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.394275:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.394275:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.394278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.394284:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.394287:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.394287:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.394288:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677473.394288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.394289:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.394290:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.394291:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.394292:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677473.394292:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.395182:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.395185:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.395186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.395187:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.395189:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.395190:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.395191:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.395192:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.395193:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.395194:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.395197:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.395198:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.395200:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.395200:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.395201:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.395202:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.395203:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.395203:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.395204:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.395208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.395215:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.395217:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.395218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.395218:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.395220:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.395221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.395262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.395264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.395267:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677473.395269:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.395270:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.395272:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.395278:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.395279:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.395281:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.395282:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.395283:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.395286:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.395288:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.2:1619677473.395290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.395291:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.395291:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.395292:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.395300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.395300:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.395300:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.395301:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.395301:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.395301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:21.0:1619677473.395302:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.395302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:21.0:1619677473.395303:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.395303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.395306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677473.395309:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.395312:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.395313:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.395313:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.395315:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.395317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.395632:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.395635:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.395636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.395637:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.395640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.395641:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.395642:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.395644:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.395645:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.395646:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.395649:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.395650:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.395653:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.395654:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.395655:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.395656:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.395657:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.395657:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.395658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.2:1619677473.395659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.395666:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.395669:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.395670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677473.395671:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:18.0:1619677473.395672:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.395673:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.395674:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.395674:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.395676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.395676:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:18.0:1619677473.395676:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.395677:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.395678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:18.0:1619677473.395679:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.395680:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.395681:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.395684:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.395686:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.395689:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.395690:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677473.395690:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.395690:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.395692:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.395692:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.395693:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677473.395693:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677473.395694:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:18.0:1619677473.395694:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677473.395695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677473.395695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677473.395696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.396036:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.396039:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.396040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.396040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.396043:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.396046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.396127:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.396130:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.396131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.396132:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.396135:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.396136:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.396137:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.396139:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.396140:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.396141:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.396143:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.396145:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.396146:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.396147:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.396148:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.396149:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.396150:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.396150:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.396151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.396155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.396161:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.396163:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.396163:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.396164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.396165:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.396167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.396446:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.396448:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.396449:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.396451:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.396453:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.396455:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.396456:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.396457:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.396458:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.396459:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.396462:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.396464:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.396467:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.396469:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.396469:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.396469:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.396470:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.396472:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.396472:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677473.396472:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.396473:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.396473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:10.0:1619677473.396474:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.396475:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.396476:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.396477:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.396478:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.396479:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.396480:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.396482:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.396484:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.396486:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.396486:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.396487:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.396487:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.396488:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.396489:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.396489:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:6.0:1619677473.396489:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.396490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.396490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000400:00000200:6.0:1619677473.396490:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.396492:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:24.2:1619677473.396495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:6.0:1619677473.396495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.396498:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.396500:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.396501:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.396501:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.396503:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.396505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.396644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.396654:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.396657:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.396659:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.396659:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.396662:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.396664:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.396665:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.396667:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.396669:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.396670:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.396674:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.396677:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.396680:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.396681:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.396681:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.396683:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.396684:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.396685:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677473.396685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:22.0:1619677473.396696:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.396699:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.396700:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.396700:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.396702:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.396704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.397152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.397158:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.397161:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.397162:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.397164:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.397166:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.397168:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.397169:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.397171:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.397172:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.397173:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.397176:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.397178:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.397180:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.397181:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.397182:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.397183:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.397184:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.397185:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.397185:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.397198:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.397199:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.397200:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.397200:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.397201:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.397202:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.397203:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000400:00000200:6.0:1619677473.397203:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.397206:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.397206:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:6.0:1619677473.397206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677473.397207:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:22.0:1619677473.397207:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:6.0:1619677473.397207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.397208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:22.0:1619677473.397209:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.397210:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.397212:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.397212:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.397212:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.397214:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677473.397214:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.397216:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.397218:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.397218:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.397220:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.397222:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677473.397222:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.397225:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.397226:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.397226:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677473.397226:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.397228:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677473.397228:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.397229:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.397229:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.397230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677473.397232:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.397232:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.397233:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.397235:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.397235:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677473.397236:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677473.397237:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677473.397241:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:9.2:1619677473.397241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677473.397243:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.397244:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.397246:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.397247:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677473.397248:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677473.397249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677473.397251:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.397252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.397253:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.397255:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.397258:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677473.397946:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.397949:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.397950:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.397952:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.397954:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.397956:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.397957:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.397959:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.397959:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.397960:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.397964:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.397965:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.397968:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.397969:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.397969:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.397971:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.397972:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.397972:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.397973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.397978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.397987:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.397990:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.397990:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.397991:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.397993:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.397996:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.397997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.398005:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.398007:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.398008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.398008:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.398009:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.398010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.398556:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.398559:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.398560:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.398562:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.398564:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.398566:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677473.398567:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.398568:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.398569:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.398570:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677473.398573:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.398574:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677473.398577:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.398578:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677473.398578:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.398580:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677473.398581:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.398582:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.398582:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.398587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.398596:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.398598:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.398599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.398600:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.398602:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.398605:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.398810:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.398813:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.398814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.398816:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.398819:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.398820:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.398821:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.398823:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.398823:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.398824:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.398827:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.398829:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.398832:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.398832:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.398833:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.398835:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.398835:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.398836:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.398838:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.398838:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677473.398842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.398850:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677473.398851:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:21.0:1619677473.398852:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677473.398853:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:21.0:1619677473.398853:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.398854:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.398854:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:21.0:1619677473.398855:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.398855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.398858:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.398902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.398905:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.398906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.398908:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.398910:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.398911:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.398913:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.398914:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.398915:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.398916:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.398919:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.398921:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.398923:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.398924:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.398925:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.398927:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.398927:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.398928:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.398928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.398932:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.398935:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.398937:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.398939:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.398939:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.398940:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:20.0:1619677473.398941:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677473.398942:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.398942:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.398942:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.398943:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.398944:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.398944:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.398945:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.398948:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.398950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.398950:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.398951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.398952:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.398953:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.398954:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.398955:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.398956:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.398957:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.398957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.398968:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.398971:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.398972:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.398972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.398974:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.398977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.399022:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.399025:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.399026:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.399028:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.399031:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.399032:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.399033:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.399034:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.399035:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.399036:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.399040:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.399041:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.399044:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.399044:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.399045:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.399047:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.399047:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.399048:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.399048:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.399052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.399058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.399061:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.399062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.399062:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.399064:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.399066:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.399072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.399078:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.399081:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.399081:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.399082:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.399082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.399083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.399461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.399468:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.399470:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.399471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.399471:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.399473:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.399475:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.399900:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.399902:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677473.399903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677473.399904:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.399907:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.399908:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677473.399909:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.399911:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.399912:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.399913:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.399915:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.399917:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.399919:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.399920:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677473.399920:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677473.399922:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677473.399922:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.399923:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677473.399924:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:10.0:1619677473.399938:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.399940:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.399941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.399943:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.399945:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.399947:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.399948:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.399949:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.399950:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.399951:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.399953:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.399955:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.399957:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.399958:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.399958:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.399959:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.399960:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.399961:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.399961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.399965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.399971:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.399973:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.399974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.399974:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.399976:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.399978:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677473.400285:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.400289:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.400289:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.400291:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.400297:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.400299:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.400300:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.400301:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.400302:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.400303:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.400307:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.400309:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.400311:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.400312:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.400312:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.400314:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.400314:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.400315:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.400316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.400321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.400329:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.400331:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.400332:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.400334:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.400335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.400336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677473.400342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.400348:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.400351:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.400352:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.400352:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.400354:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.400357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.400488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.400494:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.400497:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.400498:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.400498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.400500:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.400501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.400607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.400614:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.400617:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.400618:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.400619:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.400623:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.400624:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.400625:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.400627:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.400628:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.400629:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.400633:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.400634:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.400637:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.400638:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.400639:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.400641:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.400641:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.400642:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.400643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.400656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.400659:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.400660:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.400661:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.400663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.400664:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.401094:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.401096:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.401097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.401099:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.401101:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.401103:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.401104:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.401105:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.401106:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.401107:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.401111:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.401112:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.401115:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.401116:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.401116:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.401118:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.401118:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.401120:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.401121:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.401121:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.401131:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.401131:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.401133:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.401133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.401134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.401134:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.401134:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.401136:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.401138:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.401138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.401293:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.401299:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.401300:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.401302:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.401305:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.401306:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.401307:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.401309:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.401310:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.401311:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.401314:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.401316:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.401319:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.401320:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.401321:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.401322:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.401323:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.401323:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.401324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.401337:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.401340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.401340:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.401341:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.401343:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.401345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.401814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.401820:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.401823:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.401824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.401826:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.401828:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.401830:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.401831:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.401833:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.401833:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.401835:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.401838:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.401840:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.401842:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.401843:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.401843:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.401845:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.401845:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.401847:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.401848:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.401848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.401853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.401858:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.401859:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.401861:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.401861:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677473.401862:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.401862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677473.401862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.401863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.401864:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.401867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.401879:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.401881:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.401882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.401882:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.401883:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.401884:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.402634:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.402637:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.402638:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.402640:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.402642:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.402644:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.402645:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.402647:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.402648:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.402649:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.402652:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.402654:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.402656:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.402657:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.402657:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.402659:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.402660:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.402660:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.402661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.402674:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.402676:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.402677:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.402679:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.402680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.402681:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.402692:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.402695:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.402696:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.402697:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.402697:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.402698:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.402982:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.402984:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.402987:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.402988:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.402988:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.402990:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.402991:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.402992:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.402993:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.402994:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.402998:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.403000:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.403002:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.403002:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.403003:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.403004:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.403005:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.403006:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.403006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.403017:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.403020:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.403021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.403021:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.403023:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.403025:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.403036:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.403038:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.403039:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.403039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.403040:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.403041:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.403098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.403105:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.403108:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.403109:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.403110:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.403113:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.403115:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677473.403116:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.403118:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.403119:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.403120:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677473.403123:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.403125:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677473.403128:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.403129:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677473.403130:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677473.403131:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677473.403132:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.403133:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677473.403134:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677473.403135:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677473.403140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:9.2:1619677473.403140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677473.403146:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:26.0:1619677473.403147:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:2.0:1619677473.403148:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677473.403148:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.403149:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:26.0:1619677473.403150:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.403151:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.403151:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.403153:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.403156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.403162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677473.403168:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.403171:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.403171:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677473.403171:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.403173:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.403173:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677473.403615:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.403618:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.403619:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.403620:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.403622:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.403624:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.403625:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.403626:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.403627:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.403628:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.403631:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.403633:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.403636:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.403636:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.403637:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.403639:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.403640:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.403640:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.403641:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.403645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.403653:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.403656:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.403657:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.403657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.403660:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.403663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.403665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677473.403672:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.403675:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.403676:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.403676:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.403677:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.403678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677473.403744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.403749:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.403751:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.403754:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.403754:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.403756:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.403758:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.403759:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.403760:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.403761:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.403762:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.403766:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.403767:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.403767:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.403770:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677473.403770:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.403770:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:21.0:1619677473.403771:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.403771:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.403773:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:10.0:1619677473.403773:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.403774:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.403775:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.403775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677473.403775:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.403777:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.403778:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.403779:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.403780:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677473.403780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:10.0:1619677473.403781:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.403784:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.403786:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.403788:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.403788:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.403789:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677473.403789:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:10.0:1619677473.403790:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.403791:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677473.403791:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.403792:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.403792:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:1.0:1619677473.403792:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.403793:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.403794:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677473.403794:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.403796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677473.403797:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677473.403802:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.403804:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.403805:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.403806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.403807:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.403807:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.403809:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.403809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.403809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.403810:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.403815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.403821:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.403823:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.403824:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.403824:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.403825:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.403826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.403934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.403940:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.403943:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.403944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.403946:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.403949:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.403950:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.403951:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.403952:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.403953:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.403954:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.403957:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.403958:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.403960:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.403961:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.403962:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.403963:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.403964:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.403965:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.403966:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.403970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.403977:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.403979:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.403979:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.403980:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.403981:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.403982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677473.404187:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.404190:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.404190:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.404192:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.404194:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.404196:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.404197:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.404198:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.404199:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.404200:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.404203:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.404205:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.404207:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.404208:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.404209:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.404211:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.404211:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.404212:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.404212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.404216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.404224:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.404227:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.404228:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.404228:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.404231:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.404234:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.404317:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.404320:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.404321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.404322:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.404325:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.404326:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.404327:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.404328:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.404329:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.404330:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.404332:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.404334:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.404336:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.404337:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.404338:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.404339:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.404340:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.404341:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.404341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.404354:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.404356:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.404357:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.404359:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.404360:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.404361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.404715:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.404718:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.404719:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.404721:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.404723:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.404724:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.404725:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.404727:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.404728:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.404729:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.404732:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.404734:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.404736:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.404737:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.404738:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.404739:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.404740:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.404740:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.404741:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.404745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.404752:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.404754:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.404755:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.404755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.404757:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.404759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.405220:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.405222:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.405223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.405224:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.405226:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.405229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.405403:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.405406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.405406:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.405408:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.405411:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.405412:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.405414:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.405415:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.405416:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.405417:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.405420:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.405421:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.405423:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.405424:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.405424:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.405425:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.405426:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.405427:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.405428:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.405429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.405432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.405440:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.405441:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.405442:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.405444:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.405444:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.405444:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.405445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.405446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.405446:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.405447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.405453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.405459:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.405462:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.405463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.405463:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.405463:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.405464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.406163:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.406165:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.406166:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.406168:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.406170:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.406171:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.406172:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.406174:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.406175:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.406176:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.406179:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.406180:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.406183:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.406184:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.406184:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.406186:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.406186:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.406187:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.406189:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.406189:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677473.406200:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.406202:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.406202:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.406203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.406204:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.406205:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.406205:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.406206:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.406206:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.406208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.406271:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.406274:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.406276:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.406277:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.406278:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.406280:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.406281:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.406283:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.406284:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.406285:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.406288:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.406290:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.406293:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.406293:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.406298:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.406300:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.406301:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.406302:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.406302:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.406306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.406313:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.406316:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.406317:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.406317:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.406319:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.406322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.406523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.406524:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.406526:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.406527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.406529:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.406530:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:22.0:1619677473.406531:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.406532:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.406533:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:22.0:1619677473.406533:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677473.406533:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.406534:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.406535:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.406536:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.406537:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.406537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677473.406538:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.406540:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.406542:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.406544:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.406545:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.406546:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.406547:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.406548:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.406549:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.406549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.406561:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.406563:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.406564:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.406566:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.406567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.406568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.406634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.406640:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.406643:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.406643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.406645:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.406648:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.406649:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.406650:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.406652:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.406653:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.406654:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.406657:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.406659:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.406662:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.406662:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.406663:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.406665:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.406665:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.406666:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.406666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.406678:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.406681:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.406682:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.406682:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.406684:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.406686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.406911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.406917:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.406920:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.406921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.406923:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.406925:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.406927:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.406928:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.406929:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.406930:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.406932:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.406935:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.406937:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.406939:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.406940:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.406940:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.406942:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.406943:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.406943:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.406944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.406948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.406956:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.406959:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.406960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.406960:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.406962:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.406964:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677473.407129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.407135:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.407138:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.407139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.407140:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.407143:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.407144:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.407146:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.407147:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.407148:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.407149:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.407152:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.407154:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.407157:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.407158:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.407158:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.407160:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.407161:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.407161:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.407161:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:13.0:1619677473.407174:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.407177:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.407177:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.407179:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.407181:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.407182:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.0:1619677473.407223:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.407226:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.407226:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.407228:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.407229:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.407231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.407687:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.407689:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.407690:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.407692:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.407692:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.407695:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.407705:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.407707:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.407708:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.407709:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.407709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.407710:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.408044:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.408047:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.408047:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.408049:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.408052:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.408053:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.408054:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.408056:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.408057:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.408058:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.408061:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.408062:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.408065:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.408065:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.408066:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.408068:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.408068:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.408070:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.408071:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.408071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.408082:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.408084:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.408084:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.408084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.408085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.408086:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.408087:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.408089:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.408089:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.408091:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.408130:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.408133:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.408134:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.408136:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.408139:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.408140:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.408142:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.408143:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.408144:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.408145:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.408148:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.408150:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.408153:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.408154:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.408154:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.408156:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.408156:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.408157:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.408157:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.408162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.408170:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.408173:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.408174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.408174:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.408176:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.408179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677473.408521:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677473.408523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.408524:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.408525:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.408526:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.408529:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.408530:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:24.0:1619677473.408530:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:26.0:1619677473.408531:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.408532:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.408533:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.408533:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:26.0:1619677473.408534:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677473.408534:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.408534:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.408536:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.408537:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.408539:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.408540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.408541:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.408542:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.408543:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.408544:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.408544:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.408545:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.408546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.408549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.408555:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.408558:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.408558:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.408559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.408560:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.408562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.408566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.408572:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.408574:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.408575:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.408575:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.408576:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.408576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.409466:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.409468:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.409469:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.409470:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.409472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.409474:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.409475:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.409476:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.409477:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.409478:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.409481:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.409482:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.409485:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.409486:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.409486:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.409488:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.409488:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.409489:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.409490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.409502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.409505:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.409506:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.409506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.409508:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.409511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.409601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.409604:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.409606:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.409608:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.409610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.409611:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.409613:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.409614:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.409615:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.409616:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.409617:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.409621:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.409623:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.409625:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.409626:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.409627:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.409628:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.409629:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.409629:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.409630:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.409635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.409644:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.409647:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.409648:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.409648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.409650:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.409652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.409778:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.409781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.409781:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.409783:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.409785:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.409786:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.409787:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.409788:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:6.0:1619677473.409789:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.409790:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.409791:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.409792:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.409792:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:6.0:1619677473.409795:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.409796:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.409798:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.409799:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.409800:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.409801:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:6.0:1619677473.409801:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.409801:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:26.0:1619677473.409802:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.409803:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.409803:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.409803:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677473.409804:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.409804:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.409804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:26.0:1619677473.409805:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.409809:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.409810:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.409813:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.409813:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.409814:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.409816:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.409816:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:21.0:1619677473.409816:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677473.409817:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:21.0:1619677473.409818:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:26.0:1619677473.409819:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.409819:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:21.0:1619677473.409819:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.2:1619677473.409821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:21.0:1619677473.409821:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.409822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.409823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.409825:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.409828:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.409829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.409829:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.409831:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:8.0:1619677473.409831:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.409833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.409834:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.409835:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:10.0:1619677473.409835:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:1.0:1619677473.409964:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.409967:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.409968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.409970:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.409972:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.409973:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.409974:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.409976:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.409976:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.409977:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.409980:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.409982:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.409985:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.409985:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.409986:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.409987:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.409988:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.409989:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.409989:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.409993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.410001:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.410003:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.410004:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.410006:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.410007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.410008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677473.410633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.410640:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.410642:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.410643:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.410643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.410645:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.410647:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.410775:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.410778:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677473.410779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677473.410781:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.410783:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.410785:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677473.410786:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.410787:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.410788:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.410789:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677473.410792:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.410794:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677473.410796:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.410797:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677473.410798:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677473.410799:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677473.410800:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.410800:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677473.410801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677473.410806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.410814:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.410816:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.410817:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.410817:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.410819:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.410822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.410824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.410832:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.410835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.410835:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.410836:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.410837:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.410837:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677473.410923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.410928:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.410930:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.410931:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.410933:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.410935:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.410936:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.410937:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.410938:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.410939:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.410940:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.410944:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.410946:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.410948:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.410949:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.410950:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.410951:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.410952:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.410952:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.410952:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.410965:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.410967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.410968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.410969:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.410971:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.410973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.411065:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.411068:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.411069:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.411071:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.411073:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.411074:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.411075:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.411077:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.411078:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.411079:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.411082:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.411084:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.411087:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.411088:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.411089:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.411090:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.411091:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.411092:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.411092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677473.411101:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.411104:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.411105:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.411106:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677473.411107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677473.411108:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.411110:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.411111:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.411112:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.411113:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.411114:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677473.411115:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:10.0:1619677473.411117:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.411118:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:10.0:1619677473.411118:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677473.411119:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.411119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677473.411121:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677473.411122:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.411122:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.411123:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.411125:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.411125:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677473.411126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.411126:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.411126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677473.411130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.411137:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677473.411139:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:2.0:1619677473.411140:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.411141:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.411141:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677473.411142:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:2.0:1619677473.411142:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.411142:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:22.0:1619677473.411143:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.411143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.411145:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.411148:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.411157:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.411160:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.411160:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.411160:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.411162:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.411162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.411562:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.411564:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.411565:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.411567:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.411568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.411570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.411684:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.411686:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.411687:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.411689:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.411692:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.411693:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.411694:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.411696:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.411697:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.411697:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.411701:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.411702:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.411704:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.411705:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.411706:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.411707:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.411707:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.411708:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.411709:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.411713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.411716:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.411719:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.411720:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.411720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.411721:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.411723:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.411731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.411735:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.411737:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.411738:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.411738:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.411739:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.411739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.412012:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.412015:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.412016:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.412017:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.412020:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.412022:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.412023:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.412024:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.412025:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.412026:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.412029:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.412031:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.412033:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.412034:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.412034:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.412036:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.412037:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.412038:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.412038:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.412050:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.412052:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.412053:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.412055:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.412056:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.412057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.412075:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.412077:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.412078:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.412079:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.412079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.412081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.412110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.412118:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.412121:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.412121:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.412123:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.412124:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.412127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.412728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.412735:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.412738:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.412738:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.412741:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.412743:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.412745:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.412746:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.412747:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.412748:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.412749:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.412753:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.412755:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.412757:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.412758:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.412759:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.412761:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.412762:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.412763:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.412764:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.412764:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.412770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.412776:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.412778:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.412779:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.412780:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677473.413238:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.413240:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.413241:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.413243:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.413245:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.413246:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.413247:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.413249:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.413249:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.413250:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.413253:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.413255:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.413257:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.413257:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.413258:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.413259:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.413260:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.413261:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.413262:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.413263:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.413265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.413272:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.413274:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677473.413275:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677473.413275:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.413275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.413277:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:24.0:1619677473.413277:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.413278:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677473.413279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.413279:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677473.413402:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.413404:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.413405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.413407:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.413409:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.413410:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.413411:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.413412:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.413413:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.413414:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.413416:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.413418:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.413420:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.413420:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.413421:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.413422:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.413423:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.413424:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.413424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.413428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.413432:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.413435:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.413435:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.413436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.413437:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.413439:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:1.0:1619677473.413516:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.413519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.413519:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677473.413522:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.413524:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.413526:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.413527:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.413528:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.413529:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.413530:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.413534:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.413535:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.413538:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.413539:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.413539:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.413541:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.413541:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.413542:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.413542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677473.413546:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.413549:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.413551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.413551:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.413553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.413554:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.413554:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.413555:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.413556:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.413557:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.413557:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.413558:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.413559:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.413560:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.413563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.413564:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:6.0:1619677473.413564:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.413566:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.413569:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.413569:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.413570:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.413572:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.413572:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.413573:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.413573:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.413585:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.413587:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.413588:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.413590:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.413592:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.413593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677473.413813:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.413815:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677473.413816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677473.413818:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.413821:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.413822:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.413823:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.413824:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.413825:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.413826:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.413830:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.413831:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.413834:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.413835:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.413835:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.413837:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.413838:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.413838:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677473.413839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.413843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.413852:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.413854:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.413855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.413856:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.413858:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.413861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677473.414228:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.414231:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677473.414234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.414234:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.414236:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.414236:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677473.414238:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:10.0:1619677473.414238:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.414239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677473.414239:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.414241:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.414242:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.414242:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.414243:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677473.414244:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.414245:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.414247:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.414248:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.414249:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.414249:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.414250:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677473.414251:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.414254:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.414255:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:10.0:1619677473.414255:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.414256:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677473.414257:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.414258:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677473.414258:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.414259:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:10.0:1619677473.414260:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.414261:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677473.414261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:10.0:1619677473.414261:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.414261:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.414263:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.414263:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.414264:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.414265:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.0:1619677473.414272:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677473.414274:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677473.414275:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:22.0:1619677473.414276:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.414276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:22.0:1619677473.414279:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.414280:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.414280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.414283:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.414285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.414297:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.414300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.414300:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.414301:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.414302:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.414302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.415186:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.415189:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.415190:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.415191:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.415194:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.415196:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.415197:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.415198:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.415199:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.415200:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.415204:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.415205:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.415207:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.415208:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.415208:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.415210:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.415210:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.415211:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.415212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.415216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.415220:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.415222:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.415223:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.415223:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.415225:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.415227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.415232:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.415235:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.415235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.415237:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.415240:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.415241:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.415242:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.415244:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.415245:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.415246:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.415249:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.415251:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.415253:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.415254:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.415254:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.415256:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.415257:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.415258:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.415259:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.415259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.415270:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.415271:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.415272:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.415272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.415273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.415274:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.415275:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.415276:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.415277:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.415279:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.415364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.415367:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.415370:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677473.415373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.415373:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.415376:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.415378:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.415379:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.415381:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.415382:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.415384:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.415388:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.415390:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.415392:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.415393:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.415393:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.415395:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.415396:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.415397:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.415398:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.415399:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.415402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677473.415404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.415409:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.415411:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677473.415411:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677473.415412:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.415413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677473.415413:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.415414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.415414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.415416:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.415419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677473.415809:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.415811:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.415812:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.415814:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.415816:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.415818:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.415819:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.415820:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.415822:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.415822:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.415826:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.415828:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.415830:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.415831:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.415832:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.415833:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.415834:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.415835:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.415835:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.415849:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.415852:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.415853:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.415855:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.415857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.415858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677473.415965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.415970:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.415973:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.415974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.415976:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.415978:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.415979:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.415981:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.415982:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.415983:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.415984:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.415988:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.415989:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.415992:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.415993:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.415993:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.415995:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.415995:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.415996:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.415997:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.416009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.416012:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.416013:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.416013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.416015:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.416017:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.416181:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.416184:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.416185:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.416187:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.416189:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.416191:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.416192:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.416194:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.416195:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.416196:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.416199:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.416201:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.416204:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.416205:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.416205:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.416207:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.416207:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.416208:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.416208:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.416220:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.416223:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.416224:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.416224:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.416226:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.416229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677473.416435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.416444:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.416446:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.416447:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.416449:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.416450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.416453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.416453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.416462:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.416464:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.416465:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.416466:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.416466:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.416468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.416590:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.416593:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.416594:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.416596:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.416598:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.416600:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.416601:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.416602:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.416603:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.416604:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.416607:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.416609:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.416611:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.416612:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.416612:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.416614:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.416614:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.416615:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.416616:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.416617:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.416619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.416622:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.416624:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.416625:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.416625:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.416627:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.416628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.416630:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.416632:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.416632:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.416633:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:8.0:1619677473.416790:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.416793:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.416794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.416795:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.416798:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.416799:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.416800:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.416801:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.416802:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.416803:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.416806:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.416808:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.416810:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.416811:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.416812:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.416813:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.416814:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.416815:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.416815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.416820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.416829:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.416832:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.416833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.416833:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.416835:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.416838:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677473.416994:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.416997:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.416998:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.417000:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.417002:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.417004:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.417005:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.417006:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.417007:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.417008:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.417012:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.417013:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.417016:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.417016:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.417017:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.417019:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.417019:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.417020:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.417020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.417025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.417033:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.417035:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.417036:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.417038:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.417038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.417040:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.417047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.417054:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.417057:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.417058:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.417058:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.417059:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.417060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677473.417401:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.417403:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.417404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.417406:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.417408:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.417410:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.417411:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.417412:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.417413:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.417414:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.417417:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.417418:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.417421:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.417422:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.417422:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.417424:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.417424:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.417425:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.417426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.417439:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.417441:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.417442:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.417444:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.417445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.417446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.417976:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.417978:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.417979:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.417981:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.417983:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.417985:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.417986:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.417987:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.417988:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.417989:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.417993:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.417994:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.417996:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.417997:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.417998:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.417999:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.418000:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.418000:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.418001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.418005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.418012:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.418014:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.418015:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.418015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.418017:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.418019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677473.418042:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.418045:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.418045:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677473.418047:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.418050:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.418051:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677473.418052:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.418054:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.418055:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.418056:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677473.418059:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.418060:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.418062:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.418063:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.418063:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.418065:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.418065:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.418066:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.418066:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677473.418071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.418078:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.418080:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.418081:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.418082:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.418083:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.418084:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677473.418199:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.418202:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.418203:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.418205:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.418208:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.418209:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.418210:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.418212:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.418213:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.418213:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.418217:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.418218:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.418220:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.418221:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.418221:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.418223:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.418223:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.418224:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.418224:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.418225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:9.2:1619677473.418229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677473.418233:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.418235:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677473.418235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:21.0:1619677473.418236:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.418237:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.418238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:21.0:1619677473.418238:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.418238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:10.0:1619677473.418238:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.418240:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.418241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.418242:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:8.0:1619677473.418269:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feaea6400] (21)++
00000800:00000200:8.0:1619677473.418272:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.25@o2ib3
00000800:00000200:24.0:1619677473.418273:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feaea6400] (22)--
00000400:00000200:8.0:1619677473.418275:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.25@o2ib3 : GET - for me
00000400:00000200:8.0:1619677473.418279:0:19938:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.25@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:8.0:1619677473.418281:0:19938:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.25@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:8.0:1619677473.418283:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feaea6400] (21)++
00000800:00000200:8.0:1619677473.418284:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feaea6400] (22)--
00000400:00000200:8.0:1619677473.418286:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.25@o2ib3 local destination
00000800:00000200:8.0:1619677473.418287:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919fce371c00] -> 10.51.5.25@o2ib3 (2) version: 12
00000800:00000200:8.0:1619677473.418288:0:19938:0:(o2iblnd.c:1251:kiblnd_query()) peer_ni 10.51.5.25@o2ib3 ffff919fce371c00, alive 0 secs ago
00000400:00000200:8.0:1619677473.418292:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.25@o2ib3(10.51.5.25@o2ib3:10.51.5.25@o2ib3) : REPLY try# 0
00000800:00000200:8.0:1619677473.418298:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.25@o2ib3
00000800:00000200:8.0:1619677473.418299:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919fce371c00] -> 10.51.5.25@o2ib3 (2) version: 12
00000800:00000200:8.0:1619677473.418300:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919feaea6400] (21)++
00000800:00000200:8.0:1619677473.418300:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feaea6400] (22)++
00000800:00000200:8.0:1619677473.418302:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919feaea6400] (23)--
00000800:00000200:8.0:1619677473.418303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feaea6400] (22)--
00000800:00000200:24.0:1619677473.418316:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feaea6400] (22)++
00000800:00000200:24.0:1619677473.418318:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feaea6400] (23)--
00000400:00000200:24.0:1619677473.418318:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.418319:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feaea6400] (22)--
00000400:00000200:24.0:1619677473.418320:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.25@o2ib3: REPLY: OK
00000800:00000200:24.0:1619677473.418322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feaea6400] (21)--
00000800:00000200:26.0:1619677473.418369:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.418372:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.418373:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.418374:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.418376:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.418377:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.418378:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.418379:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.418380:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.418381:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.418384:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.418385:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.418387:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.418388:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.418388:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.418390:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.418390:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.418391:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.418392:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.418395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.418399:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.418401:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.418402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.418402:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.418404:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.418406:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.418412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.418415:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.418417:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.418418:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.418418:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.418419:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.418419:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.0:1619677473.418810:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.418813:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.418815:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.418816:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.418817:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.418819:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.418820:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.418822:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.418823:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.418824:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.418827:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.418829:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.418831:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.418832:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.418832:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.418834:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.418835:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.418836:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.418837:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.418838:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.418848:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677473.418848:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.418851:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:6.0:1619677473.418851:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.418852:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.418852:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.418852:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677473.418852:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.418854:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.418857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.418910:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.418912:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.418913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.418915:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.418917:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.418918:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.418919:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.418921:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.418922:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.418923:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.418926:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.418927:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.418930:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.418930:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.418931:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.418933:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.418933:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.418934:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.418934:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.418946:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.418948:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.418949:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.418951:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.418951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.418953:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.418966:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.418969:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.418970:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.418971:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.418971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.418972:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.419932:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.419935:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.419936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.419938:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.419940:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.419942:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.419943:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.419944:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.419945:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.419946:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.419950:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.419951:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.419954:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.419955:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.419956:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.419957:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.419958:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.419958:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.419959:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.419972:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.419975:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.419976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.419976:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677473.419978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:6.0:1619677473.419978:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.419981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677473.419985:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.419988:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.419989:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.419991:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677473.419992:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:18.0:1619677473.419994:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.419995:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677473.419995:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.419995:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:18.0:1619677473.419997:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.419998:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.419999:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.420000:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.420000:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.420002:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677473.420002:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677473.420003:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.420004:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.420006:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.420006:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677473.420011:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677473.420012:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677473.420014:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677473.420015:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.420017:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.420017:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.420018:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.420018:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677473.420019:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.420019:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677473.420021:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.420021:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677473.420022:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677473.420022:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.420022:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:18.0:1619677473.420023:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677473.420023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677473.420024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677473.420027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.2:1619677473.420028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677473.420033:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.420036:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:17.0:1619677473.420036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:10.0:1619677473.420036:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.420037:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.420039:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.420039:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.420040:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.420040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677473.420042:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:17.0:1619677473.420042:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.420045:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.420218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.420224:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.420227:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.420228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.420228:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.420230:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.420232:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.420238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.420243:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.420246:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.420247:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.420247:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.420248:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.420248:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.420278:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.420281:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.420281:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.420283:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.420286:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.420287:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.420288:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.420289:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.420290:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.420291:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.420298:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.420300:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.420302:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.420303:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.420304:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.420305:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.420306:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.420307:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.420307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.420320:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.420322:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.420323:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.420325:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.420326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.420327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.420582:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.420584:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.420585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.420587:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.420590:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.420591:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.420592:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.420593:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.420594:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.420595:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.420598:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.420600:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.420603:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.420603:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.420604:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.420606:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.420606:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.420607:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.420607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.420619:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.420622:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.420623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.420623:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.420626:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.420628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.420770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.420776:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.420779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.420779:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.420781:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.420784:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.420785:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.420787:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.420788:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.420789:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.420790:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.420793:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.420794:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.420797:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.420797:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.420798:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.420800:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.420800:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.420802:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.420802:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.420803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.420808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677473.420812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:22.0:1619677473.420813:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.420813:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.420815:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677473.420816:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.420816:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677473.420817:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.420817:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.420817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.420819:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.420819:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677473.420822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677473.420824:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.420826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677473.420829:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:2.0:1619677473.420829:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.420832:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677473.420832:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.420833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.420833:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.420834:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.420834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:2.0:1619677473.420834:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.420836:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.420839:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.420840:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.420842:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.420847:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.420850:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.420853:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.420854:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.420855:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.420857:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.420858:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.420860:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.420861:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.420864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.420873:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.420877:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.420878:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.420879:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.420881:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.420883:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.421375:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.421378:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.421379:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.421381:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.421383:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.421385:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.421387:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.421389:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.421390:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.421391:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.421395:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.421398:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.421401:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.421402:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.421402:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.421403:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.421404:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:10.0:1619677473.421405:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.421407:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:10.0:1619677473.421407:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.421408:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:10.0:1619677473.421409:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.421410:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96509521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.421411:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.421412:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.421413:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.421414:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:9.2:1619677473.421414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677473.421414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.421416:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.421416:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.421417:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.421418:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.421418:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.421421:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.421421:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:10.0:1619677473.421422:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.421422:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.421424:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.421424:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.421425:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677473.421425:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:8.0:1619677473.421425:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.421426:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.421426:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.421427:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:8.0:1619677473.421427:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.421428:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:24.2:1619677473.421429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:21.0:1619677473.421429:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677473.421429:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.421430:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677473.421430:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.421431:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.0:1619677473.421431:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677473.421432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677473.421432:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000400:00000200:21.0:1619677473.421433:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.421434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:8.0:1619677473.421434:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.421434:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.421436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:8.0:1619677473.421436:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.421439:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.421826:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.421829:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.421830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.421832:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.421834:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.421836:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.421837:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.421838:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.421839:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.421840:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.421843:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.421844:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.421847:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.421848:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.421848:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.421850:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.421850:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.421851:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.421852:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.421853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.421863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.421863:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.421865:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.421865:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.421866:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.421866:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.421866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.421868:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.421870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.421870:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.422150:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.422153:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.422154:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.422156:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.422158:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.422160:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677473.422161:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.422162:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.422163:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.422164:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677473.422168:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.422170:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677473.422171:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.422172:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677473.422173:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.422174:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.422174:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.422175:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.422176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.422180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.422186:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.422188:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.422188:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.422189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.422190:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.422192:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.0:1619677473.422650:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.422652:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.422654:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.422655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:26.0:1619677473.422656:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:21.0:1619677473.422657:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.422658:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677473.422659:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:21.0:1619677473.422660:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.422660:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:21.0:1619677473.422662:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.422663:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.422663:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.422664:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.422665:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.422667:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.422668:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.422668:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677473.422670:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.422671:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.422671:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.422672:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:21.0:1619677473.422674:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.422675:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.422675:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677473.422676:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.422677:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.422678:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.0:1619677473.422678:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.422679:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.422680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:26.0:1619677473.422681:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.422681:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.422682:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.422684:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.422684:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.422685:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.422686:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.422687:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.422689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.422691:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.422693:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.422694:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.422695:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.422696:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:22.0:1619677473.422696:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.422699:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:22.0:1619677473.422699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.422699:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677473.422700:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.422700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.422702:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.422702:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.422702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.422703:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677473.422704:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.422732:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.422734:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.422735:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.422737:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.422739:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.422740:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.422741:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.422743:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.422743:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.422744:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.422747:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.422749:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.422750:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.422751:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.422751:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.422752:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.422753:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.422753:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.422754:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.422758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.422764:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.422766:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.422766:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.422767:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.422768:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.422769:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.423308:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.423310:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.423312:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.423313:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.423315:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.423316:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.423317:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.423319:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.423319:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.423320:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.423323:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.423324:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.423326:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.423327:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.423328:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.423329:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.423330:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.423330:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.423331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.423336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.423340:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.423342:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.423343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.423343:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.423344:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.423346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:6.0:1619677473.423365:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.423367:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.423368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.423370:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.423372:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.423373:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.423375:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.423376:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.423377:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.423378:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.423381:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.423383:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.423386:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.423386:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.423387:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.423389:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.423390:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.423390:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.423391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.423404:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.423406:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.423407:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.423408:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.423409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.423410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.423614:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.423617:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.423617:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.423620:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.423622:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.423624:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.423625:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.423626:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.423627:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.423628:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.423632:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.423634:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.423637:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.423638:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.423638:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.423640:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.423640:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.423641:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.423641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.423646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.423655:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.423657:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.423658:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.423659:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.423661:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.423664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677473.423847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.423854:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.423856:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.423857:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.423857:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.423859:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.423861:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.423874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.423880:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.423882:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.423884:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.423885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.423887:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.423889:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.423890:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.423891:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.423892:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.423893:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.423897:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.423899:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.423901:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.423902:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.423902:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.423904:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.423904:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.423905:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.423906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.423910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.423918:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.423920:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.423921:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.423922:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.423923:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.423925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677473.424077:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.424080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.424080:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.424082:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.424083:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.424085:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.424086:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.424087:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.424088:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.424089:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.424093:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.424094:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.424096:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.424097:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.424097:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.424098:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.424099:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.424100:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.424101:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.424102:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.424104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.424112:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.424113:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.424114:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.424116:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.424116:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.424117:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.424117:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.424117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.424118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.424118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.424125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.424132:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.424134:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.424135:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.424135:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.424135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.424136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.424274:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.424277:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.424278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.424280:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.424282:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.424284:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677473.424285:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.424287:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.424288:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.424289:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677473.424292:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.424298:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677473.424301:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.424302:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677473.424303:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677473.424305:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677473.424305:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.424306:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.424306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:13.0:1619677473.424320:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.424323:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.424324:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.424326:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.424329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.424330:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.424335:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.424336:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.424337:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.424338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.424339:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.424832:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.424835:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.424836:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.424838:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.424840:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.424842:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.424843:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.424844:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.424845:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.424846:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.424849:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.424851:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.424854:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.424855:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.424855:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.424857:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.424857:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.424858:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.424859:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677473.424860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.424866:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.424869:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.424870:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677473.424871:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:2.0:1619677473.424871:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677473.424873:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.424874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:2.0:1619677473.424874:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.424875:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.424876:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677473.424877:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.424878:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.424879:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.424880:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.424881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:2.0:1619677473.424881:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.424885:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.424887:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677473.424889:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677473.424890:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.424891:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677473.424892:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.424892:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677473.424893:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.424893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677473.424894:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.424894:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677473.424895:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.424895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677473.424895:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.424896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.424908:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.424911:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.424913:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.424915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677473.424915:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.424918:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.425059:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.425062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.425062:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.425065:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.425067:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.425068:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.425070:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.425071:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.425072:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.425073:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.425076:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.425078:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.425080:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.425081:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.425082:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.425083:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.425083:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.425085:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.425086:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.425087:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.425089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.425094:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.425096:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.425097:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.425097:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.425098:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.425099:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.425101:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:8.0:1619677473.425101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.425102:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.425103:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:17.0:1619677473.425315:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.425318:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.425318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.425320:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.425322:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.425324:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.425325:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.425326:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.425327:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.425328:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.425332:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.425334:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.425336:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.425337:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.425338:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.425339:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.425340:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.425341:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677473.425342:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.425343:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.425347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.425354:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.425356:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677473.425357:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677473.425357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.425358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677473.425360:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.425360:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.425362:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.425363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.425365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.425460:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.425463:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.425463:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.425465:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.425468:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.425469:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.425471:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.425472:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.425473:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.425474:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.425477:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.425478:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.425481:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.425482:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.425483:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.425485:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.425485:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.425486:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.425488:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.425488:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.425500:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.425500:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.425502:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677473.425503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.425503:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.425504:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.425505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.425505:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.425507:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.425510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.425520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.425526:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.425529:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.425529:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.425531:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.425533:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.425535:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.425536:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.425537:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.425538:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.425539:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.425542:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.425543:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.425546:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.425547:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.425547:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.425549:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.425549:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.425550:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.425550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.425563:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.425565:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.425566:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.425566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.425568:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.425570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.425785:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.425788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.425788:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.425790:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.425792:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.425794:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.425795:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.425796:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.425797:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.425798:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.425801:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.425803:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.425806:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.425807:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.425807:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.425809:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.425809:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.425810:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.425811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.425822:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.425824:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.425825:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.425827:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.425828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.425830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.425845:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.425847:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.425848:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.425849:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.425850:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.425851:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.425854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.425857:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.425860:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:24.0:1619677473.425860:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677473.425861:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.425862:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677473.425863:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.425863:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.425864:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.425865:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.425866:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.425867:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:24.0:1619677473.425867:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:26.0:1619677473.425868:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.425869:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.425870:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.425871:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.425874:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.425876:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.425879:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.425879:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.425880:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.425882:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.425882:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.425883:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.425883:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.425888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.425896:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.425899:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.425900:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.425900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.425902:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.425905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677473.426481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.426489:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.426492:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.426493:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.426494:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.426495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.426497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.426919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.426924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.426927:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.426929:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.426929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.426931:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.426933:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677473.426934:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.426936:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.426937:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.426938:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677473.426941:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.426943:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677473.426946:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.426947:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677473.426948:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677473.426949:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677473.426950:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.426951:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677473.426952:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677473.426953:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:9.2:1619677473.426956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677473.426957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.426961:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677473.426964:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677473.426965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.426965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:10.0:1619677473.426965:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.426967:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.426968:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:10.0:1619677473.426969:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.426970:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.426971:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.426973:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.426975:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.426976:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.426977:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:10.0:1619677473.426977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:26.0:1619677473.426978:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.426978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.426980:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.426981:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.426982:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:9.2:1619677473.426984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:26.0:1619677473.426986:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.426988:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.426990:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.426991:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.426992:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.426993:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.426993:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.426994:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.426994:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.426994:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.426995:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.426995:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677473.426996:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.426997:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.426997:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.427001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.427005:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.427007:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.427008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.427008:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.427010:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.427012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.427025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.427028:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.427030:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.427031:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.427031:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.427032:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.427032:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.427337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.427340:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.427340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.427342:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.427345:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.427346:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.427348:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.427349:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.427350:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.427351:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.427354:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.427355:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.427358:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.427359:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.427359:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.427361:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.427361:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.427362:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.427363:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.427375:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.427377:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.427378:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.427380:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.427381:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.427382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.427408:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.427410:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.427411:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.427413:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.427416:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.427417:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.427418:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.427419:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.427421:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.427422:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.427425:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.427426:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.427428:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.427429:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.427430:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.427431:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.427432:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.427432:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.427433:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.427437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.427443:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.427446:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.427447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.427447:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.427448:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.427450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.427457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.427463:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.427465:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.427466:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.427466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.427467:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.427467:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.428608:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.428611:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.428612:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.428614:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.428615:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.428617:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.428618:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.428619:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.428620:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.428621:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.428623:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.428625:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.428627:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.428627:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.428628:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.428630:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.428630:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.428631:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.428631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.428636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.428640:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.428642:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.428643:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.428643:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.428644:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.428646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:6.0:1619677473.428835:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.428838:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.428839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.428841:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.428843:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.428844:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.428845:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.428847:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.428848:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.428849:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.428851:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.428853:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.428855:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.428856:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.428857:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.428858:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.428859:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.428860:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.428861:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.428862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.428872:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.428874:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.428874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.428874:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.428875:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.428877:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.428877:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.428879:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.428881:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.428881:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.428889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.428892:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.428894:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.428895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.428897:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.428898:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677473.428900:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.428901:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.428902:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.428903:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.428906:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.428908:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.428911:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.428912:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677473.428913:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677473.428914:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677473.428915:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.428916:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.428917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:10.0:1619677473.429619:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.429622:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.429623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.429624:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.429627:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.429628:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.429629:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.429630:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.429632:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.429633:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.429636:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.429638:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.429641:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.429641:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.429642:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.429644:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.429644:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.429645:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.429645:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.429658:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.429661:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.429662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.429663:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.429665:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.429668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.429724:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.429726:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677473.429727:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677473.429729:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.429731:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.429733:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.429734:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.429735:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.429736:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.429737:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.429740:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.429741:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.429745:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.429745:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.429746:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.429748:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.429748:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.429749:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677473.429751:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677473.429751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:9.2:1619677473.429754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.429762:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.429764:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000800:00000200:2.0:1619677473.429765:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677473.429766:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677473.429766:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:18.0:1619677473.429767:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.429767:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.429768:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000400:00000200:2.0:1619677473.429769:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.429772:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.429774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.429782:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.429784:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.429785:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.429785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.429786:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.429787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677473.429989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.429997:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.430000:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.430001:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.430002:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.430004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.430006:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.430010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.430017:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.430020:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.430021:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.430022:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.430023:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.430024:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677473.430246:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.430248:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.430249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.430251:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.430254:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.430256:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.430257:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.430258:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.430259:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.430260:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.430263:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.430264:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.430267:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.430268:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.430268:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.430270:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.430270:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.430271:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.430272:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.430276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.430284:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.430286:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.430288:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.430288:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.430290:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.430300:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677473.430334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.430341:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.430343:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.430344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.430346:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.430349:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.430350:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677473.430350:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.430351:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.430353:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:1.0:1619677473.430353:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.430354:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677473.430355:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.430356:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.430356:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.430359:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.430359:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.430361:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677473.430361:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.430362:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.430363:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.430364:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677473.430364:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:26.0:1619677473.430365:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.430365:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:26.0:1619677473.430366:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.430367:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.430367:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.430368:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677473.430369:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.430369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677473.430371:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.430374:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.430375:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.430375:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (23)++
00000800:00000200:26.0:1619677473.430377:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (24)--
00000800:00000200:26.0:1619677473.430377:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.430378:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.430379:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.430381:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:22.0:1619677473.430384:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000800:00000200:6.0:1619677473.430384:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (24)++
00000400:00000200:22.0:1619677473.430385:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.430385:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000400:00000200:6.0:1619677473.430386:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.430387:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.430388:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.430390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.430390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.430391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.430402:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.430404:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.430405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.430406:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.430407:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.430407:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.430520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.430527:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.430530:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.430530:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.430532:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.430535:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.430537:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.430538:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.430539:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.430540:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.430542:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.430545:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.430547:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.430550:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.430551:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.430552:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.430554:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.430554:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.430555:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.430556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.430567:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.430570:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.430571:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.430573:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.430574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.430575:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.431047:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.431050:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.431051:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.431052:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.431055:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.431056:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.431058:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.431059:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.431060:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.431061:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.431064:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.431066:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.431068:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.431069:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.431070:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.431071:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.431072:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.431072:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.431073:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.431077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.431082:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.431084:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.431085:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.431086:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.431087:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.431089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.431147:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.431150:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.431151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.431153:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.431155:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.431157:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.431158:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.431159:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.431160:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.431161:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.431165:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.431167:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.431169:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.431170:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.431170:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.431172:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.431172:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.431173:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.431174:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.431186:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.431188:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.431189:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.431190:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.431192:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.431192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677473.431373:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.431376:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677473.431377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677473.431378:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.431381:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.431382:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.431384:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.431385:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.431386:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.431387:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.431391:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.431392:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.431395:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.431396:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.431396:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.431398:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.431398:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.431399:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.431399:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677473.431412:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.431415:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.431416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.431416:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.431418:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.431421:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.431431:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.431434:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.431435:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.431436:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.431439:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.431440:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.431441:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.431443:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.431444:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.431445:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.431447:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.431449:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.431451:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.431451:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.431452:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.431453:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.431453:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.431455:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.431456:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.431456:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677473.431459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.431467:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.431468:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.431469:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.431471:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:10.0:1619677473.431471:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.431472:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.431472:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.431473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.431473:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.431474:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677473.432162:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.432165:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.432166:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.432168:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.432169:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.432171:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.432172:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.432173:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.432174:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.432175:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.432178:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.432179:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.432181:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.432182:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.432183:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.432184:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.432185:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.432185:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.432186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.432197:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.432200:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.432201:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.432202:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.432205:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.432206:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.432543:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.432545:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.432546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.432547:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.432550:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.432551:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.432552:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.432553:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.432554:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.432555:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.432558:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.432559:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.432561:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.432562:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.432562:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.432564:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.432564:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.432565:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.432566:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.432566:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.432570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.432574:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.432576:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.432577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.432577:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.432579:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.432580:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.432581:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.432582:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.432582:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.432583:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:10.0:1619677473.432643:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.432646:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.432646:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.432648:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.432650:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.432651:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.432652:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.432653:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.432654:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.432655:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.432657:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.432659:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.432661:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677473.432661:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.432661:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.432662:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.432663:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.432664:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677473.432664:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.432664:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677473.432665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.432666:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.432666:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.432666:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677473.432669:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.432669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:24.0:1619677473.432670:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:22.0:1619677473.432671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677473.432672:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.432673:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:24.0:1619677473.432674:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.432674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677473.432676:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.432676:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:24.0:1619677473.432677:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677473.432677:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.432677:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677473.432678:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.432678:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.432680:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677473.432680:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.432680:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.432681:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.432681:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677473.432682:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677473.432682:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.432683:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96509471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.432685:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677473.432685:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.432685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:22.0:1619677473.432686:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.432686:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677473.432687:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.432688:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.432689:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.432690:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.432691:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.432691:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677473.432691:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.432692:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.432692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.432693:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.432695:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.432696:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:9.2:1619677473.432696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677473.432697:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.432699:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.432699:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.432700:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.432700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677473.432702:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.432705:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.432705:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.432706:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.432707:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.432709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.432713:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.432715:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.432716:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.432718:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.432718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.432720:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.432740:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.432743:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.432744:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.432745:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.432745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.432747:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.432995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.433004:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.433007:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.433007:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.433008:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.433010:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.433013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.433022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.433029:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.433032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.433032:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.433033:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.433034:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.433034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:13.0:1619677473.433921:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.433923:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.433926:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.433928:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.433928:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.433930:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:13.0:1619677473.433931:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.433932:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.433933:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.433934:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:13.0:1619677473.433937:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.433939:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677473.433941:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.433942:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:13.0:1619677473.433942:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:13.0:1619677473.433944:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:13.0:1619677473.433944:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.433946:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.433946:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:10.0:1619677473.433950:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:9.2:1619677473.433950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.433952:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.433953:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.433954:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.433957:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:10.0:1619677473.433957:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.433958:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:24.0:1619677473.433959:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:10.0:1619677473.433959:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.433960:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.433960:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.433960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677473.433961:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.433962:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.433962:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.433965:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.433966:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.433967:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.433968:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.433969:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.433970:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.433971:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.433972:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.433972:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.433973:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.433977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.433980:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.433983:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.433983:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.433984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.433985:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.433987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.433990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.433993:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.433995:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.433996:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.433996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.433997:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.433997:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:6.0:1619677473.434190:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.434193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.434193:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.434196:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.434198:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.434199:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.434200:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.434202:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.434203:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.434204:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.434207:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.434209:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.434211:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.434212:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.434213:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.434214:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.434215:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.434215:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.434216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.434228:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.434231:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.434232:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.434233:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.434234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.434235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.434362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.434374:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.434377:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.434377:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.434379:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.434380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.434383:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677473.434444:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.434446:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.434447:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.434449:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.434451:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.434453:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.434454:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.434455:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.434457:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.434457:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.434461:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.434463:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.434466:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.434466:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.434467:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.434469:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.434469:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.434470:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.434470:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.434483:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.434486:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.434487:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.434487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.434489:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.434493:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.434610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.434616:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.434619:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.434620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.434622:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.434624:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.434625:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.434627:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.434628:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.434629:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.434630:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.434633:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.434635:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.434637:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.434638:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.434639:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.434640:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.434641:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.434641:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.434642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.434654:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.434656:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.434657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.434657:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.434660:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.434662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.435184:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.435187:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.435188:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.435189:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.435192:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.435194:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.435195:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.435196:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.435197:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.435198:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.435200:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.435202:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.435205:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.435206:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.435206:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.435208:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.435208:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.435209:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.435209:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.435221:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.435224:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.435224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.435225:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.435227:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.435230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.435368:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.435370:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.435371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677473.435373:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677473.435375:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.435376:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.435377:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.435379:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.435380:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.435381:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.435384:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.435385:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.435388:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.435389:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.435389:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.435391:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.435391:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.435393:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677473.435394:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677473.435394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:6.0:1619677473.435405:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.435407:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.435409:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.435409:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.435411:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.435413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.435419:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677473.435422:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:22.0:1619677473.435422:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.435423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.435423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677473.435424:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.435426:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.435426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.435427:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.435427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.436017:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.436019:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677473.436020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677473.436022:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.436024:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.436025:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677473.436026:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.436028:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.436029:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.436030:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677473.436032:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.436034:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.436037:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.436038:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677473.436038:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677473.436040:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677473.436041:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.436041:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.436041:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677473.436046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.436054:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.436057:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.436058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.436058:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.436060:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.436063:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677473.436121:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.436124:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.436125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.436127:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.436130:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.436131:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.436132:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.436134:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.436135:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.436136:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.436139:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.436141:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.436143:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.436144:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.436145:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.436146:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.436147:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.436147:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.436148:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.436152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.436155:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.436157:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.436158:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.436158:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.436160:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.436163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.2:1619677473.436329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.436338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.436340:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.436342:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.436343:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.436345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.436346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677473.436465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.436468:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.436469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.436471:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.436473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.436475:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.436476:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.436477:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.436479:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.436480:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.436482:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.436484:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.436487:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.436487:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.436488:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.436490:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.436490:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.436491:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.436491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.436496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.436504:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.436506:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.436507:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.436508:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.436509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.436511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677473.436519:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.436522:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.436522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.436524:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.436527:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.436528:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.436529:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.436531:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.436532:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.436532:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.436536:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.436537:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.436540:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.436540:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.436541:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.436543:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.436543:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.436544:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.436545:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.436546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.436556:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.436557:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.436558:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.436558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.436559:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.436559:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.436559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.436561:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.436562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.436563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.436635:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.436638:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.436639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.436641:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.436643:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.436644:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.436646:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.436647:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.436648:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.436649:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.436652:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.436654:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.436657:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.436657:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.436658:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.436660:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.436660:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.436661:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.436661:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.436666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.436677:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.436681:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.436683:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.436683:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.436686:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.436690:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.437367:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.437369:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.437370:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.437372:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.437374:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.437375:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.437377:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.437378:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.437379:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.437380:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.2:1619677473.437381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:10.0:1619677473.437383:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.437384:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.437386:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.437387:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.437387:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677473.437387:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677473.437389:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.437389:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.437390:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.437390:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677473.437390:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.437391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.437392:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:9.2:1619677473.437395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:2.0:1619677473.437395:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.437397:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.437398:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.437400:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.437401:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.437402:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:2.0:1619677473.437402:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.437404:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.437405:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.437405:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.437406:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677473.437407:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.437408:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.437410:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.437410:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.437411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677473.437411:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.437413:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.437414:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.437415:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.437416:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.437416:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.437419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.437421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.437428:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677473.437428:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677473.437430:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:17.0:1619677473.437430:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677473.437431:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677473.437431:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.437432:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677473.437432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.437434:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.437436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.437440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.437447:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.437450:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.437451:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.437451:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.437452:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.437453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.437728:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.437731:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.437732:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.437734:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.437736:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.437737:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.437738:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.437740:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.437741:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.437742:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.437745:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.437747:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.437749:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.437750:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.437750:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.437752:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.437752:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.437753:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.437753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.437758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.437761:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.437764:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.437765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.437765:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.437767:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.437768:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.438065:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.438068:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.438069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.438071:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.438073:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.438075:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.438076:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.438077:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.438078:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.438079:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.438083:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.438085:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.438087:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.438088:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.438088:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.438090:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.438091:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.438091:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.438092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.438104:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.438106:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.438107:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.438109:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.438110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.438111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.438671:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.438674:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.438675:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.438677:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.438679:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.438681:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.438682:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.438684:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.438684:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.438686:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.438689:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.438690:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.438693:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.438694:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.438695:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.438696:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.438697:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.438697:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.438698:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.438703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.438712:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.438715:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.438716:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.438716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.438718:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.438720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.438723:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.438725:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.438727:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.438728:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.438728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.438729:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.438730:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677473.438883:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.438886:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677473.438888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677473.438889:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.438892:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.438893:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.438894:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.438895:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.438896:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.438897:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.438900:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.438901:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.438902:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.438903:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.438904:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.438905:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.438905:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.438906:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.438906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677473.438916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.438922:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.438924:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.438924:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.438925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.438926:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.438927:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.439151:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.439153:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.439154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.439155:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.439157:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.439160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.439239:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.439242:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.439244:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.439245:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.439247:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.439249:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.439250:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.439251:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.439252:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.439253:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.439257:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.439259:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.439261:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.439262:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.439263:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.439264:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.439265:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.439265:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.439266:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.439278:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.439280:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.439281:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.439281:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.439283:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.439285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.439323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.439329:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.439332:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.439332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.439335:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.439337:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.439338:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.439339:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.439341:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.439342:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.439343:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.439346:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.439348:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.439350:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.439351:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.439352:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.439353:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.439354:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.439355:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.439356:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.439357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.439362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.439368:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.439368:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.439369:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677473.439370:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.439370:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.439371:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.439372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.439372:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.439374:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.439376:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.439441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.439448:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.439450:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.439451:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.439452:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.439453:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.439455:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.439463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.439464:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.439467:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.439468:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:8.0:1619677473.439469:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:26.0:1619677473.439470:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.439471:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.439472:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.439472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.439472:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.439473:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:26.0:1619677473.439474:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:8.0:1619677473.439474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:26.0:1619677473.439475:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.439476:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.439477:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.439478:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.439480:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.439482:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.439484:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.439485:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.439485:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.439487:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.439487:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.439488:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.439489:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.439490:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:20.2:1619677473.439491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:24.2:1619677473.439492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.439495:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.439497:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:2.0:1619677473.439497:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:24.0:1619677473.439498:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.439498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.439500:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.439500:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.439501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677473.439502:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.439503:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:2.0:1619677473.439503:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.439505:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000400:00000200:2.0:1619677473.439505:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.439506:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:2.0:1619677473.439506:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677473.439507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000400:00000200:2.0:1619677473.439508:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.439509:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.439510:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.439511:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.439514:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.439515:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.439518:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.439518:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.439519:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.439521:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.439522:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.439523:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.439524:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.439524:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.439527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.439530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.439536:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.439537:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.439537:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677473.439538:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.439538:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.439539:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.439540:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.439540:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.439541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.439541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677473.439602:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677473.439604:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677473.439604:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.439605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.439607:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:22.0:1619677473.439607:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.439608:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677473.439608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:26.0:1619677473.439610:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.439610:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.439611:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.439612:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.439612:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.439614:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:22.0:1619677473.439614:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.439616:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.439616:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677473.439616:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:22.0:1619677473.439617:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.439618:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.439619:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677473.439619:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:26.0:1619677473.439620:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677473.439620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.439620:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.439622:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677473.439622:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.439625:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:26.0:1619677473.439626:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.439626:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:9.2:1619677473.439627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677473.439628:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.439630:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.439631:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.439631:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.439631:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.439632:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.439633:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.439633:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.439634:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:18.0:1619677473.439634:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.439635:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.439635:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677473.439635:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.439635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677473.439636:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.439637:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677473.439637:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.439638:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.439638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.439639:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.439640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677473.439647:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.439649:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.439650:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.439651:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:21.0:1619677473.439652:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.439653:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.439653:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:21.0:1619677473.439654:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.439655:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.439655:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.439657:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.439662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.439664:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.439666:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.439667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.439667:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.439668:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.439669:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.439671:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.439673:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.439674:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.439675:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.439675:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.439677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.439813:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.439816:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.439816:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.439819:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.439822:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.439823:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.439825:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.439826:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.439827:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.439828:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.439831:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.439832:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.439835:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.439836:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.439836:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.439838:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.439839:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.439840:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.439840:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.439852:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.439854:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.439855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.439856:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.439858:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.439860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.440215:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.440218:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.440219:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.440221:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.440223:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.440224:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.440225:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.440227:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.440228:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.440228:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.440231:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.440232:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.440235:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.440236:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.440236:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.440238:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.440238:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.440239:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.440239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677473.440251:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.440253:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.440255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.440255:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.440257:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.440260:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.440268:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.440270:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.440271:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.440271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.440272:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.440273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.440723:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.440725:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.440726:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.440728:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.440731:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.440732:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.440733:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.440734:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.440735:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.440736:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.440739:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.440740:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.440742:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.440743:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.440743:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.440745:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.440745:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.440746:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.440746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.440751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.440755:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.440757:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.440758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.440758:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.440760:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.440761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.440771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.440773:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.440775:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.440776:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.440776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.440777:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.440777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.441202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.441236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.441242:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.441245:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.441246:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.441246:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.441247:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.441250:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677473.441732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.441735:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.441736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.441736:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.441739:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.441742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.441905:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.441907:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.441908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.441910:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.441912:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.441914:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.441915:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.441916:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.441917:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.441918:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.441922:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.441923:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.441926:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.441927:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.441927:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.441929:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.441929:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.441931:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.441932:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.441932:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.441935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.441944:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.441945:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.441946:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.441947:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.441947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677473.441947:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.441948:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.441949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677473.441950:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.441953:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.442009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.442012:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.442013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.442015:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.442017:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.442019:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.442020:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.442021:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.442023:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.442024:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.442026:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.442028:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.442030:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.442031:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.442032:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.442033:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.442034:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.442034:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.442035:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.442048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.442051:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.442051:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.442053:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.442054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.442056:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.442080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.442088:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.442091:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.442092:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.442094:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.442097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.442098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677473.442099:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.442101:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.442102:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.442104:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.442107:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.442108:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677473.442110:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.442111:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.442112:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.442113:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677473.442116:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.442117:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677473.442120:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.442121:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677473.442122:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677473.442124:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.442124:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.442125:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677473.442126:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677473.442126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677473.442130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.442137:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:1.0:1619677473.442139:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:26.0:1619677473.442140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.442140:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.442141:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.442141:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.442142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.442142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:10.0:1619677473.442143:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.442147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677473.442293:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.442297:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.442301:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:26.0:1619677473.442303:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.442305:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.442306:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.442307:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.442308:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.442310:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.442311:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.442313:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.442315:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.442317:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.442318:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.442319:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.442320:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.442321:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.442321:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.442322:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.442327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.442331:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.442333:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.442334:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.442334:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.442336:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.442338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.442616:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.442618:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.442619:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.442620:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.442622:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.442624:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.442625:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.442626:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.442627:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.442628:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.442631:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.442632:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.442634:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.442634:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.442635:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.442636:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.442637:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.442638:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.442639:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.442640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.442643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.442649:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.442651:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.442652:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.442653:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677473.442653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.442654:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.442655:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.442655:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.442656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677473.442656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.443249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.443255:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.443258:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.443259:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.443261:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.443263:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.443264:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.443266:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.443267:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.443268:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.443269:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.443273:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.443274:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.443277:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.443278:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.443279:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.443281:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.443282:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.443283:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.443283:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.443303:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.443306:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.443307:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.443309:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.443310:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.443311:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.443312:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)++
00000800:00000200:12.0:1619677473.443312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.443313:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.443313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.443314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.443396:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.443399:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.443399:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.443401:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.443403:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.443405:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.443406:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.443407:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.443408:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.443409:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677473.443409:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.443411:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.443412:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677473.443413:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.443414:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.443415:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:8.0:1619677473.443417:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.443418:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677473.443418:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677473.443419:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.443419:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:8.0:1619677473.443419:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.443421:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:8.0:1619677473.443421:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.443421:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.443422:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:8.0:1619677473.443422:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677473.443423:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.443424:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.443424:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677473.443427:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:8.0:1619677473.443431:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.443432:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.443435:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.443436:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.443436:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.443437:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677473.443437:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677473.443438:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.443438:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677473.443439:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:8.0:1619677473.443439:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677473.443440:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:8.0:1619677473.443440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677473.443441:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677473.443441:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.443441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677473.443442:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.443444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.443445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.443445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.2:1619677473.443446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677473.443453:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677473.443454:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677473.443456:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677473.443457:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677473.443457:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677473.443458:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.443458:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000400:00000200:2.0:1619677473.443458:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.443459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.443460:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:2.0:1619677473.443460:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.443463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.443553:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.443556:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.443557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.443559:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.443561:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.443563:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.443564:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.443565:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.443566:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.443567:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.443570:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.443572:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.443574:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.443575:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.443576:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.443577:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.443578:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.443579:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.443580:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.443581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.443591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.443593:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.443593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.443593:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.443594:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.443595:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.443596:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.443598:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.443599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.443600:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.444032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.444038:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.444040:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.444041:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.444042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.444043:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.444046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.444052:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.444055:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.444057:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.444058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.444059:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.444061:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677473.444063:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.444064:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.444065:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.444066:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.444070:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.444072:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.444074:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.444075:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677473.444076:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677473.444077:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677473.444078:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.444079:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.444079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.2:1619677473.444110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.444116:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.444119:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.444119:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.444122:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.444125:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.444126:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.444127:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.444129:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.444130:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.444131:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.444134:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.444135:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.444138:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.444139:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.444139:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.444141:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.444142:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.444142:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.444142:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.444155:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.444158:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.444159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.444159:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.444161:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.444163:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.444184:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.444187:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.444187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.444189:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.444192:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.444193:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677473.444195:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.444196:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.444197:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.444198:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677473.444201:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.444203:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677473.444206:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.444207:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677473.444208:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677473.444210:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677473.444210:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.444211:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.444211:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:6.0:1619677473.444240:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.444243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.444243:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.444244:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.444247:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.444250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.444421:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.444424:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.444425:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.444427:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.444430:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.444431:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.444433:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.444434:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.444435:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.444436:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.444439:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.444441:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.444444:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.444444:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.444445:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.444447:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.444447:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.444448:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.444450:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.444450:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.444461:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.444463:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.444464:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.444464:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.444464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.444466:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.444466:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.444467:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677473.444467:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.444469:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.444578:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.444580:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.444581:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.444583:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.444585:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.444586:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.444588:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.444589:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.444590:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.444590:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.444593:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.444594:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.444596:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.444597:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.444597:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.444599:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.444599:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.444600:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.444600:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.444605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.444609:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.444611:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.444612:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.444613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.444614:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.444616:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.444650:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.444652:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.444653:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.444654:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.444657:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.444658:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.444659:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.444660:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.444661:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.444662:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.444665:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.444666:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.444669:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.444670:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.444670:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.444672:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.444672:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.444673:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.444673:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.444685:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.444687:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.444688:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.444689:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.444691:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.444693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.445098:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.445100:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.445101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.445103:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.445106:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.445108:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.445109:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.445110:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.445111:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.445112:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.445116:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.445117:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.445120:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.445120:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.445121:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.445123:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.445123:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.445124:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.445124:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.445136:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.445138:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.445139:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.445141:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.445142:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.445143:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.445438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.445445:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.445448:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.445449:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.445449:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.445451:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.445453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.445864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.445870:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.445872:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.445873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.445873:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.445875:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.445877:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677473.445877:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.445880:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.445882:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.445883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.445885:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.445886:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.445888:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.445889:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.445890:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.445891:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.445895:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.445896:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.445899:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.445899:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.445900:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.445901:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.445902:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.445903:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.445903:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.445915:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.445918:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.445919:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.445919:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.445921:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.445923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.445935:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.445937:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.445938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.445939:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.445939:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.445940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.446307:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.446310:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.446311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.446311:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.446314:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.446316:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.446708:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.446710:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.446711:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.446713:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.446716:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.446717:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.446718:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.446720:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.446720:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.446721:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.446724:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.446726:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.446729:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.446729:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.446730:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.446732:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.446732:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.446733:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.446734:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.446735:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:21.0:1619677473.446745:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.446747:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:26.0:1619677473.446748:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:21.0:1619677473.446748:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.446750:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.446750:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:21.0:1619677473.446750:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.446751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.446752:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.446753:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.446769:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.446771:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.446772:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.446773:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.446774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.446774:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.447360:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.447363:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.447364:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.447365:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.447368:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.447370:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.447371:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.447372:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.447373:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.447374:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.447377:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.447378:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.447380:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.447381:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.447381:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.447383:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.447383:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.447384:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.447385:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.447389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.447396:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.447398:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.447399:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.447399:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.447401:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.447403:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.447407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.447413:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.447415:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.447416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.447416:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.447417:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.447417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.447449:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.447452:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.447452:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.447454:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.447457:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.447458:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.447460:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.447461:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.447462:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.447463:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.447466:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.447468:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.447470:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.447471:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.447472:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.447474:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.447474:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.447475:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.447475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.447487:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.447489:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.447490:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.447492:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.447493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.447495:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.448043:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.448046:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.448047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.448047:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.448050:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.448052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.448075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.448082:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.448084:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.448085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.448087:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.448090:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.448091:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.448093:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.448094:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.448096:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.448097:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.448100:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.448102:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.448105:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.448106:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.448107:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.448109:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.448110:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.448111:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.448111:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.448123:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.448125:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.448126:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.448128:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.448130:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.448131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.448644:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.448647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.448647:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.448649:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.448651:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.448653:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.448654:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.448655:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.448656:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.448657:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.448660:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.448662:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.448664:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.448665:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.448666:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.448667:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.448668:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.448669:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.448669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.448682:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.448684:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.448688:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.448689:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.448689:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.448692:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677473.448710:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.448712:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677473.448713:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677473.448715:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.448717:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.448718:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.448720:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.448721:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.448722:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.448723:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.448726:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.448727:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.448730:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.448731:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.448731:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.448733:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.448734:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677473.448734:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677473.448735:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677473.448739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.448749:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.448753:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.448755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.448755:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.448758:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.448763:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.448782:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.448785:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.448786:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.448788:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.448790:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.448791:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.448792:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.448794:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.448795:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.448796:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.448798:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.448800:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.448802:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.448802:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.448803:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.448805:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.448805:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.448806:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.448806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.448810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:5.2:1619677473.448811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677473.448814:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.448817:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.448818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.448818:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.448818:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677473.448820:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.448822:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:1.0:1619677473.448823:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.448824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.448828:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.448831:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.448833:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.448835:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.448837:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.448839:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.448841:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.448846:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.448849:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.448853:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.448854:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.448855:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.448857:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.448858:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.448859:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.448860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.448871:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.448873:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.448874:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.448875:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.448877:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.448879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.448996:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.448998:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.448999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.449001:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.449003:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.449004:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.449005:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.449006:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.449007:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.449008:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.449011:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.449013:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.449016:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.449016:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.449017:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.449019:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.449019:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.449020:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.449020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.449032:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.449035:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.449035:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.449037:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.449038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.449039:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.449223:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.449226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.449226:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.449228:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.449231:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.449232:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.449233:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.449235:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.449236:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.449236:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.449240:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.449241:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.449244:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.449245:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.449246:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.449247:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.449248:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.449249:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.449249:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.449261:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.449264:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.449265:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.449265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.449268:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.449271:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.449286:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.449286:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677473.449289:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.449289:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677473.449289:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.449290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.449291:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000400:00000200:24.0:1619677473.449292:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.449292:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.449292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:24.0:1619677473.449299:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.449301:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.449302:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.449303:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.449304:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.449305:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.449309:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.449310:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.449313:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.449314:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.449314:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.449316:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.449316:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.449317:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.449318:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.449323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.449334:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.449339:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.449341:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.449341:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.449344:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.449345:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.449346:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (23)++
00000800:00000200:2.0:1619677473.449347:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (24)--
00000800:00000200:2.0:1619677473.449349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.449350:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.449351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.449470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.449475:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.449478:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.449478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.449480:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.449482:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.449484:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677473.449485:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.449486:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.449487:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.449488:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677473.449492:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.449493:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677473.449496:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.449497:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677473.449497:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677473.449499:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677473.449500:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.449500:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.449501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:26.0:1619677473.449507:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.449510:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.449511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.449513:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.449513:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:26.0:1619677473.449515:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.449515:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.449516:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.449516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677473.449517:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.449519:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.449519:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.449520:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.449521:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.449522:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.449522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677473.449526:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.449527:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.449530:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.449531:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.449532:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.449534:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.449534:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.449535:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.449535:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.449547:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.449549:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.449550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.449550:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.449553:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.449556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.449565:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.449568:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.449568:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.449569:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.449570:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.449570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:12.0:1619677473.449659:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.449662:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.449664:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.449666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.449666:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.449668:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677473.449669:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.449670:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.449671:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.449672:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677473.449675:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.449677:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677473.449680:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.449681:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677473.449682:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677473.449683:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677473.449685:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.449686:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677473.449687:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677473.449688:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677473.449689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.449697:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.449699:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.449700:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677473.449701:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:2.0:1619677473.449701:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.449703:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.449704:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677473.449704:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.449705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677473.449706:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:10.0:1619677473.450046:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.450049:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.450049:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.450051:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.450054:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.450055:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.450056:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.450058:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.450059:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.450060:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.450063:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.450065:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.450067:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.450068:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.450069:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.450070:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.450070:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.450071:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.450071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.450075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.450080:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.450082:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:6.0:1619677473.450082:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.450083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.450083:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.450085:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.450085:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.450086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.450086:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.450087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:6.0:1619677473.450088:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.450091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.2:1619677473.450093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.450095:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.450098:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.450099:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.450099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.450100:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.450101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.450555:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.450558:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.450559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.450561:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.450563:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.450564:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.450566:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.450567:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.450568:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.450569:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.450573:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.450574:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.450577:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.450578:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.450578:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.450580:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.450581:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.450582:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.450583:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.450583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.450594:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.450594:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.450596:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.450596:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.450596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.450597:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.450597:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.450599:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.450600:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.450602:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.450896:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.450899:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677473.450900:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677473.450902:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.450904:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.450906:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.450907:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.450908:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.450909:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.450910:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.450913:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.450915:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.450918:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.450918:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.450919:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.450921:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.450921:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.450922:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677473.450923:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677473.450928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.450936:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.450938:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.450939:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.450941:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.450941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.450944:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.451122:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.451124:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.451125:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.451127:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.451129:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.451130:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.451132:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.451133:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.451134:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.451134:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.451137:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.451139:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.451141:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.451141:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.451142:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.451143:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.451143:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.451144:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.451144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.451148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.451152:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.451154:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.451155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.451155:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.451157:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.451159:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.0:1619677473.451191:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.451193:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.451196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.451196:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.451198:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.451199:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.451201:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.451202:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.451203:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.451204:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.451207:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.451209:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.451212:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.451212:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.451213:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.451215:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.451215:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.451217:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.451218:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.451218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677473.451220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.451227:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.451230:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.451230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.451231:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.451232:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:24.0:1619677473.451233:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677473.451234:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.451235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677473.451236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.451236:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.451239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.451245:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.451248:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.451249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.451249:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.451250:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.451250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.451667:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.451670:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.451670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.451672:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.451674:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.451676:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.451677:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.451678:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.451679:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.451680:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.451684:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.451685:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.451687:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.451688:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.451689:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.451690:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.451690:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.451691:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.451691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.451696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.451702:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.451703:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.451704:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.451705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.451706:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.451707:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.451875:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.451878:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.451878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.451879:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.451881:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.451884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.452145:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.452148:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.452149:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.452151:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.452153:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.452154:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.452155:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.452156:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.452157:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.452158:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.452161:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.452162:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.452164:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.452165:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.452166:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.452167:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.452168:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.452169:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.452170:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.452171:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.452173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.452179:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.452181:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.452182:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.452182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.452183:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677473.452184:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.452185:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677473.452186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.452186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.452187:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.453015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.453018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.453018:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.453020:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.453022:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.453024:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.453025:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.453027:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.453028:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.453029:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.453032:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.453034:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.453036:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.453037:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.453038:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.453039:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.453040:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.453041:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.453041:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.453054:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.453057:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.453057:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.453059:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.453061:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.453061:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.453062:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.453064:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.453064:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.453064:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.453064:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (24)--
00000800:00000200:20.0:1619677473.453065:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.453066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.453424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677473.453433:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677473.453438:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677473.453439:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677473.453444:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.453447:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.453450:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677473.453452:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.453455:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.453457:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.453459:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677473.453465:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.453468:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677473.453472:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.453473:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677473.453474:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677473.453476:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.453477:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677473.453478:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677473.453479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677473.453483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.453491:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.453494:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.453495:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.453495:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.453498:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.453501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.453504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.453510:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.453513:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.453513:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.453514:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.453515:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.453515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:5.2:1619677473.453546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.453553:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.453557:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.453559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.453562:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.453565:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.453567:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.453570:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.453572:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.453573:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.453575:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.453580:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.453583:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.453586:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.453587:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.453588:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.453590:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.453591:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.453593:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.453593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.453604:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.453607:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.453608:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.453608:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.453610:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.453612:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.453997:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.453999:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.454000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.454001:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.454003:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.454005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.454065:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.454068:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.454068:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.454070:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.454072:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.454074:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.454075:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.454077:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.454078:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.454079:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.454082:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.454084:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.454087:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.454088:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.454088:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.454090:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.454090:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.454091:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.454092:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.454103:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.454106:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.454106:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.454107:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.454109:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.454111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.454139:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.454142:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.454143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.454144:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.454147:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.454148:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.454149:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.454151:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.454152:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.454152:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.454155:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.454156:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.454158:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.454159:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.454159:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.454161:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.454161:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.454163:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.454164:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.454164:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.454168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.454173:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.454175:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.454176:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.454177:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.454177:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:8.0:1619677473.454178:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.454179:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.454180:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:8.0:1619677473.454180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.454181:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:26.0:1619677473.454343:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.454346:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.454347:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.454348:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.454350:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.454352:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.454353:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.454354:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.454355:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.454356:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.454359:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.454360:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.454363:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.454363:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.454364:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.454366:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.454366:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.454367:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.454367:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.454380:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.454383:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.454385:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.454385:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.454387:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.454393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.454394:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.454397:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.454398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.454398:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.454399:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.454400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.454568:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.454571:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.454571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.454573:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.454576:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.454577:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.454579:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.454580:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.454581:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.454582:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.454585:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.454586:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.454589:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.454590:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.454590:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.454592:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.454592:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.454593:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.454593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.454605:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.454608:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.454609:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.454610:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.454612:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.454613:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.455249:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.455252:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.455253:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.455255:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.455257:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.455259:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.455260:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.455262:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.455263:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.455264:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.455267:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.455269:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.455271:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.455272:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.455272:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.455274:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.455274:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.455275:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.455275:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.455280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.455283:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.455285:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.455286:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.455286:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.455288:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.455290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:13.0:1619677473.455743:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.455745:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.455748:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.455750:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.455750:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.455752:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677473.455754:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.455755:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.455756:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.455757:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677473.455760:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.455762:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677473.455764:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.455765:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677473.455765:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677473.455767:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677473.455768:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.455769:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.455769:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677473.455774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.455783:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.455786:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.455786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.455787:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.455789:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.455792:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.455799:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.455801:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.455802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.455803:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.455805:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.455810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.455811:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.455813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.455813:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677473.455815:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.455818:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.455819:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677473.455820:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.455822:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.455822:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.455824:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677473.455827:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.455829:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677473.455832:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.455832:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677473.455833:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677473.455835:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677473.455835:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.455836:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677473.455837:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677473.455838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677473.455841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677473.455848:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:18.0:1619677473.455849:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677473.455850:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677473.455851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.455852:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:18.0:1619677473.455852:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.455853:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.455853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.455855:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.455858:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.455866:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.455869:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.455870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.455872:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.455874:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.455876:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.455877:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.455879:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.455880:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.455881:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.455884:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.455886:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.455889:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.455890:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.455890:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.455892:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.455893:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.455893:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.2:1619677473.455893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:8.0:1619677473.455893:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677473.455894:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.0:1619677473.455895:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.455897:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.455898:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.455899:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.455899:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:8.0:1619677473.455901:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:18.0:1619677473.455902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.455902:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:8.0:1619677473.455903:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.455904:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.455905:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.455906:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.455907:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.2:1619677473.455908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:2.0:1619677473.455909:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.455911:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.455913:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.455914:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677473.455916:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:2.0:1619677473.455916:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96509364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.455917:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.455917:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.455918:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:8.0:1619677473.455918:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:2.0:1619677473.455918:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.455919:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.455919:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.455920:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.455920:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677473.455921:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.455921:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:2.0:1619677473.455921:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677473.455923:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.455923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677473.455925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.455925:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677473.455926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.455927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677473.455927:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677473.455929:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.455930:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.455931:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.455931:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677473.455931:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:18.0:1619677473.455932:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.455933:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.455933:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677473.455934:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.455935:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:24.0:1619677473.455936:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.455936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.455936:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:2.0:1619677473.455937:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677473.455938:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.455939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677473.455939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677473.455940:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677473.455949:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.455951:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.455952:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.455954:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.455954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.455957:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.456077:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.456080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.456080:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.456082:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.456085:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.456086:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.456087:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.456088:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677473.456089:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.456090:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.456090:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.456091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677473.456091:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:6.0:1619677473.456094:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.456096:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.456097:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.456098:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677473.456099:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:6.0:1619677473.456100:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.456101:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.456102:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.456102:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:6.0:1619677473.456102:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.456102:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.456104:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.456104:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:6.0:1619677473.456104:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:12.0:1619677473.456105:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:10.0:1619677473.456105:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.456106:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000400:00000200:6.0:1619677473.456107:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.456108:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:6.0:1619677473.456109:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.2:1619677473.456110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:13.0:1619677473.456111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.456111:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.456112:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.456113:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000400:00000200:12.0:1619677473.456113:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.456113:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.456113:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677473.456115:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:12.0:1619677473.456115:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:6.0:1619677473.456115:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.456115:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677473.456116:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.456116:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96509361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.456116:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:6.0:1619677473.456116:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:12.0:1619677473.456117:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.456117:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:24.0:1619677473.456119:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:12.0:1619677473.456119:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.456120:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:12.0:1619677473.456124:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.456126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:12.0:1619677473.456126:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.2:1619677473.456128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:12.0:1619677473.456128:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.456129:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:12.0:1619677473.456129:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:12.0:1619677473.456130:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677473.456131:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677473.456131:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:20.0:1619677473.456132:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.456132:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:12.0:1619677473.456132:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.456133:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:24.0:1619677473.456134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:20.0:1619677473.456134:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.456134:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.0:1619677473.456134:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:21.0:1619677473.456135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677473.456135:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.456136:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:20.0:1619677473.456137:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.456137:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:9.2:1619677473.456138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.456144:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.456146:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.456147:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.456148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.456150:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.456152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:13.0:1619677473.456199:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.456201:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.456204:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.456205:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.456206:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.456208:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677473.456209:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.456211:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.456212:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.456213:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677473.456217:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.456218:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677473.456221:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.456222:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677473.456222:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677473.456224:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677473.456225:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.456226:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.456226:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.456230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.456238:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.456241:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.456242:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.456242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.456245:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.456248:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.456392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.456398:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.456401:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.456402:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.456403:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.456406:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.456408:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.456409:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.456410:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.456411:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.456412:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.456415:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.456417:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.456420:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.456421:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.456421:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.456423:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.456424:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.456425:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.456425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677473.456592:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.456595:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.456596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.456598:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.456600:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.456602:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677473.456603:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.456605:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.456606:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.456606:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677473.456609:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.456611:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677473.456614:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.456614:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:22.0:1619677473.456615:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677473.456617:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:22.0:1619677473.456617:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.456618:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.456618:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.2:1619677473.456624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.456632:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.456635:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.456636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.456636:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.456638:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.456642:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.456907:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.456910:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677473.456910:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677473.456911:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677473.456913:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.456914:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.456916:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.456916:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.456918:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677473.456919:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.456921:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.456922:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.456923:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677473.456924:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677473.456926:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.456927:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.456927:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.456928:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.456929:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677473.456929:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677473.456932:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677473.456933:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.456933:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677473.456933:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677473.456935:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677473.456935:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677473.456936:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)++
00000800:00000200:6.0:1619677473.456937:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (24)--
00000800:00000200:22.0:1619677473.456938:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.456938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677473.456939:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:22.0:1619677473.456939:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677473.456939:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:8.2:1619677473.456941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677473.456942:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:22.0:1619677473.456942:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.456943:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.456943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677473.456947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677473.456949:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.456951:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.456952:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.456952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.456954:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:17.0:1619677473.456955:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.456957:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:17.0:1619677473.456957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677473.456958:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.456959:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.456961:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.456964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677473.456971:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.456974:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677473.456975:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677473.456977:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.456980:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.456981:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677473.456983:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.456984:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.456985:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.456986:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677473.456989:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.456990:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677473.456993:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.456994:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:6.0:1619677473.456994:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677473.456996:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:6.0:1619677473.456997:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.456998:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677473.456999:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677473.456999:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677473.457002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.457008:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:22.0:1619677473.457009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677473.457011:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:10.0:1619677473.457011:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.457012:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:22.0:1619677473.457012:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:10.0:1619677473.457012:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.457013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:10.0:1619677473.457015:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.457018:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:12.0:1619677473.457047:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.457049:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.457052:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.457054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.457055:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.457056:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:12.0:1619677473.457057:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.457058:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.457060:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.457061:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:12.0:1619677473.457064:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.457066:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677473.457068:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.457069:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:12.0:1619677473.457070:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677473.457072:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.457072:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.457074:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677473.457075:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677473.457076:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.457086:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.457087:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.457088:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.457089:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.457089:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677473.457090:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677473.457090:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.457091:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.457092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677473.457093:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:26.0:1619677473.457437:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.457440:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.457441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.457443:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.457445:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.457446:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.457448:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.457449:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.457450:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.457451:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.457453:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.457455:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.457457:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.457458:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.457458:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.457460:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.457460:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.457461:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.457462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.457466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.457473:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.457475:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.457476:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.457476:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.457478:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.457480:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.457656:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.457659:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.457660:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.457661:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.457663:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.457664:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.457665:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.457666:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.457667:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.457668:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.457670:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.457672:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.457674:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.457674:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.457675:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.457676:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.457677:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.457677:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.457678:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.457682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.457684:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.457687:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.457687:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.457688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.457689:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.457691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.457813:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.457816:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.457817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.457818:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.457820:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.457823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.458231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.458237:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.458240:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.458241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.458243:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.458245:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.458246:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.458248:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.458249:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.458250:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.458251:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.458254:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.458256:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.458259:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.458259:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.458260:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.458262:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.458262:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.458264:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.458265:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.458265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.458270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.458275:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.458275:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.458277:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.458278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677473.458278:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.458279:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677473.458279:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.458280:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.458281:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.458283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.458397:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.458400:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.458400:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.458403:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.458405:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.458406:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.458408:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.458409:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.458410:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.458411:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.458414:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.458416:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.458419:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.458419:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.458420:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.458421:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.458422:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.458422:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.458423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.458428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.458434:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.458436:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.458436:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.458437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.458438:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.458439:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677473.458471:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.458474:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.458474:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.458477:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.458480:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.458481:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.458482:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.458483:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.458484:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.458485:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.458489:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.458490:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.458492:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.458493:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.458494:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.458495:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.458496:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.458497:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.458498:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.458499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.458509:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.458510:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.458511:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.458511:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.458512:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.458514:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.458514:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.458516:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.458517:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.458519:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.458556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.458562:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.458565:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.458566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.458568:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.458571:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.458573:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.458574:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.458575:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.458577:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.458578:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.458581:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.458582:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.458585:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.458586:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.458586:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.458589:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.458590:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.458591:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.458591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.458595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.458602:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.458604:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.458604:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.458605:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.458607:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.458608:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.458616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.458623:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.458625:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.458626:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.458627:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.458627:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.458627:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:26.0:1619677473.458640:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.458642:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.458643:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.458645:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.458647:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.458648:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.458649:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.458650:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.458651:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.458652:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.458655:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.458657:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.458659:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.458659:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.458660:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.458662:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.458662:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.458663:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.458665:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.458665:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.458668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.458672:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.458674:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.458675:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.458675:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.458677:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.458678:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.458679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.458680:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.458680:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.458681:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:1.0:1619677473.458865:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.458868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.458868:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.458871:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.458873:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.458875:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.458876:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.458877:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.458878:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.458880:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.458883:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.458884:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.458887:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.458888:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.458888:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.458890:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.458890:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.458891:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.458891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.458896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.458904:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.458907:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.458908:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.458910:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.458911:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.458914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.458916:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.458918:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.458919:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677473.458919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:10.0:1619677473.458921:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.458923:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.458925:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.458926:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.458927:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:10.0:1619677473.458927:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.458928:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.458929:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677473.458930:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.458931:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.458932:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677473.458932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.458933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:10.0:1619677473.458933:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.458935:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.458938:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.458939:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.458939:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.458941:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.458941:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.458942:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.458943:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.458955:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.458957:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.458958:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.458958:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.458961:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.458964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.458972:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.458975:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.458976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.458976:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.458978:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.458978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.459182:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.459184:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.459185:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.459187:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.459190:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.459191:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.459192:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.459194:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.459195:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.459196:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.459199:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.459200:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.459203:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.459204:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.459204:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.459206:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.459206:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.459207:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.459208:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.459219:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.459222:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.459222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.459223:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.459225:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.459227:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.459364:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.459367:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.459368:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.459370:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.459373:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.459374:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.459375:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.459377:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.459378:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.459378:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.459381:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.459382:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:21.0:1619677473.459383:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:10.0:1619677473.459384:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.459385:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.459385:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:21.0:1619677473.459386:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:10.0:1619677473.459386:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.459387:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:21.0:1619677473.459388:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.459388:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.459388:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:20.0:1619677473.459389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.459391:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.2:1619677473.459392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:21.0:1619677473.459393:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.459394:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.459395:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000400:00000200:21.0:1619677473.459396:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.459397:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.459398:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:21.0:1619677473.459398:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.459399:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.459399:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.459402:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:21.0:1619677473.459402:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.459404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:21.0:1619677473.459404:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.459407:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.459408:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.459408:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.459410:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.459410:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.459411:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.459412:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.459424:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.459426:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.459427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.459427:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.459429:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.459432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.459440:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.459442:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.459443:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.459443:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.459444:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.459444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.459529:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.459531:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.459533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.459533:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.459535:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.459537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.459602:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.459605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.459605:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:8.0:1619677473.459607:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.459610:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.459611:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.459613:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.459614:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.459615:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.459616:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.459620:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.459621:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.459624:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.459624:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.459625:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.459627:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.459627:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.459628:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.459629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677473.459633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.459641:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.459644:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.459645:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.459645:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.459648:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.459651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.459961:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.459964:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.459964:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.459967:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.459969:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.459970:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.459972:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.459973:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.459974:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.459975:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.459979:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.459980:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.459983:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.459984:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.459984:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.459986:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.459986:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.459987:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.459987:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.459999:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.460001:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.460002:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.460004:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.460005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.460006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.460019:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.460021:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.460022:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.460023:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.460024:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.460024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.460346:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.460349:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.460350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.460352:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.460354:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.460355:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.460357:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.460358:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.460359:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.460360:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.460364:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.460365:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.460367:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.460368:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.460369:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.460370:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.460371:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.460372:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.460372:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.460377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.460380:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.460382:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.460383:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.460383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.460385:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.460387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.460396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.460399:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.460401:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.460402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.460402:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.460403:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.460404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:9.2:1619677473.460472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.460481:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.460484:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.460485:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.460485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.460488:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.460491:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.461026:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.461029:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.461029:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.461031:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.461034:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.461035:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.461037:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.461038:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.461039:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.461040:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.461043:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.461045:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.461048:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.461049:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.461049:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.461051:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.461052:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.461052:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.461053:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.461058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.461066:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.461068:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.461069:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677473.461070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:21.0:1619677473.461071:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.461073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.461075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:2.0:1619677473.461076:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.461079:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.461080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.461082:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.461084:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.461086:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.461087:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.461089:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.461090:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.461091:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.461095:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.461097:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.461100:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.461101:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.461101:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.461103:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.461104:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.461105:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.461106:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.461107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.461112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677473.461117:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677473.461118:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.461120:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:12.0:1619677473.461120:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.461121:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.461121:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.461122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677473.461123:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.461126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.461127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.461459:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:6.0:1619677473.461460:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677473.461462:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.461463:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:6.0:1619677473.461463:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.461464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677473.461466:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.461468:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.461470:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.461470:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.461472:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:6.0:1619677473.461472:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.461473:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.461473:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.461475:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.461475:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.461476:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.461477:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.461478:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:6.0:1619677473.461478:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.461482:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.461483:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.461485:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.461487:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:6.0:1619677473.461488:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.461489:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.461489:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677473.461490:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:6.0:1619677473.461490:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.461491:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:6.0:1619677473.461491:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.461492:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:6.0:1619677473.461492:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677473.461493:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:6.0:1619677473.461493:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.461494:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:6.0:1619677473.461494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677473.461495:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.461498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.461501:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.461504:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.461505:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.461505:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.461507:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.461507:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:24.0:1619677473.461509:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.0:1619677473.461509:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.461510:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.461512:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.461513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.461514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.461653:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.461655:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.461656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.461657:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.461659:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.461661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.462402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.462404:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.462405:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.462407:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.462410:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.462411:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.462412:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.462414:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.462415:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.462416:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.462419:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.462420:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.462423:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.462424:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.462424:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.462426:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.462426:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.462427:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.462427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.462433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.462441:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.462443:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.462445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.462445:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.462447:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.462450:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.462727:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.462729:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.462730:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.462732:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.462734:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.462735:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.462737:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.462738:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.462739:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.462740:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.462743:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.462744:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.462746:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.462747:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.462748:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.462749:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.462749:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.462751:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.462752:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.462752:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.462755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.462761:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.462763:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.462764:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.462764:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.462765:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677473.462766:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.462767:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.462768:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677473.462768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.462769:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.462774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.462780:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.462782:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.462783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.462783:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.462784:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.462784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.462945:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.462948:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.462949:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.462950:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.462952:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.462953:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.462954:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.462956:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.462957:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.462957:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.462960:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.462961:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.462963:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.462964:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.462965:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.462966:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.462967:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.462968:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.462969:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.462969:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.462972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.462974:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.462977:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.462977:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.462978:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.462979:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.462981:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.462982:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.462983:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.462984:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.462985:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:5.2:1619677473.463004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.463010:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.463013:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.463014:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.463016:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.463019:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.463020:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.463022:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.463023:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.463024:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.463025:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.463028:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.463030:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.463032:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.463033:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.463034:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.463035:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.463036:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.463036:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.463037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.463049:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.463052:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.463053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.463053:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.463055:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.463057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.463085:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.463087:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.463088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.463090:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.463093:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.463094:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677473.463096:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.463097:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.463098:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.463099:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677473.463102:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.463104:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677473.463106:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.463107:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677473.463108:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677473.463110:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677473.463111:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.463111:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.463112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677473.463127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.463135:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.463138:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.463139:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.463139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.463141:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.463143:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677473.463394:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.463397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.463397:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.463398:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.463400:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.463403:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.463771:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.463773:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.463776:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.463778:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.463778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.463780:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:26.0:1619677473.463781:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:21.0:1619677473.463781:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.463782:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.463784:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:21.0:1619677473.463784:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.463785:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677473.463785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.463788:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.463789:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.463790:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.463791:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677473.463792:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.463793:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.463795:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.463795:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.463796:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.463796:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.463796:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:26.0:1619677473.463797:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677473.463798:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.463799:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.463799:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.463800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:26.0:1619677473.463801:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.463803:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677473.463804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677473.463806:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.463806:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.463807:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.463809:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.463809:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.463810:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.463811:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:8.0:1619677473.463811:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:26.0:1619677473.463812:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677473.463813:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.463814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.463814:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.463816:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.463819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.463824:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.463825:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677473.463825:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.463825:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:22.0:1619677473.463827:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.463827:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677473.463827:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:26.0:1619677473.463828:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677473.463828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677473.463829:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.463830:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.463832:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.463833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.463834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:6.0:1619677473.463834:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677473.463835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.463835:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.463837:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.463838:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.463839:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.463843:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.463844:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.463847:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.463848:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.463848:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.463850:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.463850:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.463851:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.463851:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.463863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.463866:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.463867:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.463868:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.463869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.463871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.463964:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.463966:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.463967:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.463969:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.463972:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.463973:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.463975:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.463976:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.463977:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.463978:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.463981:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.463983:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.463985:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.463986:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.463987:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.463988:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.463989:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.463989:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.463990:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.463994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.463997:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.463999:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.464000:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.464000:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.464002:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.464004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.464015:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.464017:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.464019:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.464020:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.464022:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.464023:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.464024:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.464025:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.464026:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.464027:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.464030:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.464031:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.464034:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.464035:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.464035:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.464037:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.464037:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.464038:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.464039:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.464061:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.464064:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.464065:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.464065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.464067:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.464070:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.464080:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.464083:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.464084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.464084:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.464085:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.464086:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.465388:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.465391:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.465392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.465393:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.465396:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.465397:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.465398:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.465400:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.465400:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.465401:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.465404:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.465406:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.465408:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.465409:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.465410:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.465411:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.465412:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.465413:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.465414:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.465415:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.465425:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.465427:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.465427:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.465428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.465429:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.465429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.465430:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.465432:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.465436:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.465437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.465440:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.465443:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.465443:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.465445:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.465448:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.465449:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.465451:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.465452:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.465453:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.465454:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677473.465455:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677473.465457:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.465458:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.465458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.465459:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.465459:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:22.0:1619677473.465461:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.465462:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.465463:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.465463:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677473.465464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.465465:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.465465:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.465467:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.465468:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.465469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677473.465472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.465481:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677473.465482:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677473.465483:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677473.465484:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.465485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677473.465485:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.465485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677473.465486:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.465487:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.465489:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.465489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.465490:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:21.0:1619677473.465491:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.465491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.465492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677473.465669:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.465672:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.465673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.465675:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.465677:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.465678:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.465680:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.465681:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.465682:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.465683:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.465686:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.465688:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.465690:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.465690:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.465691:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.465692:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.465693:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.465694:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.465694:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.465698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.465702:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.465704:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.465705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.465705:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.465707:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.465709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.465717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.465720:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.465722:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.465723:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.465723:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.465723:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.465724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.465909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.465915:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.465918:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.465919:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.465921:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.465924:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.465925:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.465927:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.465928:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.465929:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.465930:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.465933:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.465934:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.465936:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.465937:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.465938:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.465940:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.465940:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.465941:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.465942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.465946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.465953:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.465956:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.465957:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.465957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.465958:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.465959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.465967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.465974:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.465977:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.465978:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.465978:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.465978:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.465979:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.465988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.465993:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.465996:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.465997:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.465998:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.466000:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.466001:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.466002:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.466003:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.466004:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.466005:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.466009:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.466010:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.466013:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.466014:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.466014:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.466016:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.466017:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.466017:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.466018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.466030:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.466032:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.466033:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.466035:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.466037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.466038:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.466047:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.466050:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.466051:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.466051:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.466052:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.466053:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.466902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.466905:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.466906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.466908:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.466910:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.466911:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.466913:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.466914:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.466915:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.466917:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.466920:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.466921:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.466924:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.466925:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.466925:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.466927:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.466927:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.466928:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.466928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.466942:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.466944:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.466945:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.466946:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.466948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.466949:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.466961:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.466964:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.466965:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.466965:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.466966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.466967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.467318:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.467321:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.467322:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.467323:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.467325:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.467327:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.467328:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.467329:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.467330:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.467331:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.467334:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.467335:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.467337:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.467338:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.467338:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.467340:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.467340:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.467341:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.467341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.467346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.467350:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.467352:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.467353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.467353:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.467355:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.467357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.467475:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.467478:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:21.0:1619677473.467478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:10.0:1619677473.467479:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677473.467481:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:26.0:1619677473.467482:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.467484:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.467484:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.467484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:26.0:1619677473.467486:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.467487:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.467487:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.467489:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.467490:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.467490:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:26.0:1619677473.467491:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:21.0:1619677473.467492:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.467494:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.467495:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677473.467496:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.467498:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:21.0:1619677473.467498:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677473.467500:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.467501:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.467502:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.467503:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:21.0:1619677473.467503:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.467504:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.467504:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.467505:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.0:1619677473.467505:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.467508:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.467509:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:9.2:1619677473.467509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677473.467510:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.467511:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.467512:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.467512:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.467513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677473.467516:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.467518:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:9.2:1619677473.467518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:24.0:1619677473.467520:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.467520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.467522:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.467522:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.467522:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.467524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.467524:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.467525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677473.467525:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:8.0:1619677473.467525:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.467526:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.467526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:8.0:1619677473.467528:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:22.0:1619677473.467529:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.467531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.467532:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.467704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.467711:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.467713:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.467714:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.467717:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.467719:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.467720:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.467722:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.467723:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.467724:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.467725:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.467729:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.467730:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.467733:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.467734:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.467734:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.467736:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.467737:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.467737:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.467738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.467750:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.467752:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.467753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.467753:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.467756:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.467758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.467800:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.467802:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677473.467803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677473.467805:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.467808:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.467809:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677473.467810:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.467812:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.467813:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.467813:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677473.467817:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.467818:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677473.467821:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.467821:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677473.467822:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677473.467824:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677473.467824:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.467825:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677473.467826:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677473.467827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:9.2:1619677473.467830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.467837:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677473.467837:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677473.467839:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677473.467839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:2.0:1619677473.467840:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:1.0:1619677473.467840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677473.467841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.467841:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.467843:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.467846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.467847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.467854:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.467857:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.467858:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.467858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.467859:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.467860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:10.0:1619677473.468070:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.468072:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.468074:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.468075:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.468077:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.468079:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.468080:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.468081:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.468082:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.468083:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.468086:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.468088:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.468090:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.468091:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.468092:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.468093:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.468093:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.468095:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.468096:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.468096:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.468098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.468102:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.468104:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.468105:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.468106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.468107:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.468109:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:24.0:1619677473.468109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.468111:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.468112:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.468113:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:8.0:1619677473.468193:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:8.0:1619677473.468196:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.3.50@o2ib3
00000800:00000200:24.0:1619677473.468197:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000400:00000200:8.0:1619677473.468198:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.468202:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.468203:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677473.468204:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.468205:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.468207:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.468208:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.468211:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.468212:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.468212:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.468214:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.468214:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:24.0:1619677473.468217:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.468217:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677473.468218:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.0:1619677473.468218:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:24.0:1619677473.468219:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.468219:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:24.0:1619677473.468221:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.468222:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.468222:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.468222:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677473.468223:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:8.0:1619677473.468224:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (22)--
00000800:00000200:8.0:1619677473.468225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (21)--
00000400:00000200:24.0:1619677473.468226:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.468226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.468227:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.468230:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.468231:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.468231:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.468233:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.468234:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.468234:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.468235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677473.468235:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.468237:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.468238:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.468239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:20.0:1619677473.468240:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.468241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.468243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677473.468247:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.468250:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.468251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.468251:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.468254:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.468257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677473.468269:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:24.0:1619677473.468272:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000800:00000200:8.0:1619677473.468272:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000400:00000200:8.0:1619677473.468274:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.468277:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.468278:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.468280:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.468281:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.468282:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.468282:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.468285:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.468287:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.468289:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.468290:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.468291:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.468292:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.468300:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (22)++
00000800:00000200:8.0:1619677473.468300:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (23)--
00000800:00000200:8.0:1619677473.468301:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000800:00000200:8.0:1619677473.468302:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:8.0:1619677473.468304:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000800:00000200:24.0:1619677473.468305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000400:00000200:8.0:1619677473.468306:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.468308:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.468309:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677473.468310:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.468311:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.468312:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.468313:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677473.468316:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.468317:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.468317:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677473.468320:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677473.468320:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677473.468321:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.468321:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677473.468321:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677473.468321:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.468323:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.468323:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677473.468323:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:8.0:1619677473.468324:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (22)--
00000800:00000200:8.0:1619677473.468324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (21)--
00000800:00000200:22.0:1619677473.468326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.468330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677473.468334:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:24.0:1619677473.468337:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000800:00000200:17.0:1619677473.468338:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:8.0:1619677473.468338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000400:00000200:24.0:1619677473.468339:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.468342:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.468342:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:24.0:1619677473.468343:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677473.468343:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.468343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:24.0:1619677473.468344:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.468346:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.468346:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.468347:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.468347:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677473.468350:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.468352:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677473.468352:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677473.468353:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.468354:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.468354:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.468355:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.468355:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:24.0:1619677473.468357:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda11a00] (22)++
00000800:00000200:24.0:1619677473.468359:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (23)--
00000800:00000200:24.0:1619677473.468359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000800:00000200:9.2:1619677473.468361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.468369:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.468370:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.468371:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.468373:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (22)++
00000400:00000200:8.0:1619677473.468373:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.468374:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda11a00] (23)--
00000800:00000200:24.0:1619677473.468375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000800:00000200:24.0:1619677473.468376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.468376:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.468377:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:24.0:1619677473.468379:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000800:00000200:8.0:1619677473.468380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000400:00000200:24.0:1619677473.468381:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.468382:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.468384:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677473.468385:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.468386:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.468387:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.468387:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677473.468390:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.468392:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677473.468394:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.468395:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:24.0:1619677473.468395:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677473.468397:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677473.468398:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:24.0:1619677473.468398:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (22)--
00000800:00000200:24.0:1619677473.468398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (21)--
00000800:00000200:6.0:1619677473.468410:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.468413:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.468414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.468414:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.468416:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.468419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.468435:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.468437:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.468438:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.468438:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.468440:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.468440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.468442:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:8.0:1619677473.468445:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000800:00000200:24.0:1619677473.468446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000400:00000200:8.0:1619677473.468447:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.468450:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.468451:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.468453:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.468454:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.468455:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.468456:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.468458:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.468460:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.468462:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.468463:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.468463:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.468465:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.468466:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:8.0:1619677473.468467:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (22)--
00000800:00000200:8.0:1619677473.468467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (21)--
00000800:00000200:6.0:1619677473.468489:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.468491:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.468492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.468493:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.468495:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.468498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.468508:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:22.0:1619677473.468509:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.468511:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000800:00000200:22.0:1619677473.468512:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.468512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000800:00000200:22.0:1619677473.468513:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.468513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:24.0:1619677473.468514:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.468515:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.468515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:24.0:1619677473.468517:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.468519:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677473.468520:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.468521:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.468522:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.468523:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677473.468526:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.468528:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677473.468528:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (22)++
00000800:00000200:8.0:1619677473.468529:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000800:00000200:24.0:1619677473.468531:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677473.468531:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.468532:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:24.0:1619677473.468532:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:24.0:1619677473.468534:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:8.0:1619677473.468534:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.468535:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677473.468536:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.468538:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.468539:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.468540:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.468541:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (23)++
00000800:00000200:24.0:1619677473.468542:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (24)--
00000800:00000200:24.0:1619677473.468542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (23)--
00000800:00000200:24.0:1619677473.468544:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000400:00000200:8.0:1619677473.468544:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.468546:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677473.468548:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.468549:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677473.468549:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:9.2:1619677473.468550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.0:1619677473.468551:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677473.468551:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:8.0:1619677473.468553:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda11a00] (23)++
00000800:00000200:8.0:1619677473.468555:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (24)--
00000800:00000200:10.0:1619677473.468556:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:8.0:1619677473.468556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (23)--
00000800:00000200:8.0:1619677473.468557:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (22)++
00000800:00000200:24.0:1619677473.468559:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (23)++
00000800:00000200:10.0:1619677473.468559:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:8.0:1619677473.468559:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000800:00000200:26.0:1619677473.468560:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:24.0:1619677473.468560:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda11a00] (24)--
00000400:00000200:10.0:1619677473.468560:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.468561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (23)--
00000800:00000200:24.0:1619677473.468562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000400:00000200:10.0:1619677473.468562:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.468562:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.468565:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:8.0:1619677473.468565:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.468566:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677473.468567:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.468569:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.468570:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.468571:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677473.468574:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.468575:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677473.468577:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.468578:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677473.468578:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677473.468580:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677473.468580:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:8.0:1619677473.468581:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (22)--
00000800:00000200:8.0:1619677473.468581:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (21)--
00000800:00000200:26.0:1619677473.468639:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.468642:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.468642:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.468644:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.468646:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.468648:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.468649:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.468650:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.468651:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.468652:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.468655:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.468656:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.468660:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.468660:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.468661:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.468662:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.468663:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.468664:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.468664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.468677:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.468680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.468680:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.468681:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.468683:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.468686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.468706:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda11a00] (21)++
00000800:00000200:24.0:1619677473.468709:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.50@o2ib3
00000800:00000200:8.0:1619677473.468710:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (22)--
00000400:00000200:24.0:1619677473.468712:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.50@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.468712:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:24.0:1619677473.468714:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.468715:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.468716:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:24.0:1619677473.468716:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:24.0:1619677473.468717:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.468718:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.468719:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.468720:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.468720:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677473.468720:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.468722:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.468723:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.468724:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.468725:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.468726:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677473.468726:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.468726:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.468728:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.468729:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:24.0:1619677473.468729:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677473.468730:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.468731:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677473.468731:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda11a00] (21)++
00000800:00000200:24.0:1619677473.468732:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda11a00] (22)--
00000800:00000200:10.0:1619677473.468732:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.468733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda11a00] (21)--
00000800:00000200:10.0:1619677473.468735:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.468736:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.468736:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.468738:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.468738:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.468739:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.468739:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.468752:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.468754:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.468755:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.468757:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.468757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.468760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.468936:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.468939:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.468940:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.468941:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.468944:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.468946:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.468947:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.468948:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.468949:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.468950:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.468953:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.468954:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.468957:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.468958:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.468958:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.468960:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.468960:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.468961:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.468961:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.468983:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.468986:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.468987:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.468987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.468989:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.468993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.469265:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.469268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.469268:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.469270:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.469272:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.469274:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.469275:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.469276:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.469277:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.469278:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.469281:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.469283:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.469285:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.469286:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.469286:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.469288:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.469289:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.469289:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.469290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.469304:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.469306:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.469307:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.469309:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.469311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.469311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.469388:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.469390:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.469391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.469392:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.469394:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.469396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.469487:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.469490:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.469490:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.469492:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.469495:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.469496:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.469497:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.469498:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.469499:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.469500:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.469503:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.469505:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.469507:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.469508:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.469508:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.469509:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.469510:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.469511:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.469511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.469516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.469520:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.469522:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.469523:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.469524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.469525:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.469527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.469543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.469546:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.469549:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.469551:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.469552:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.469554:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.469555:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.469556:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.469558:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.469559:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.469560:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.469563:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.469565:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.469568:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.469568:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.469569:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.469571:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.469571:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.469572:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.469573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.469577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.469586:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.469588:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.469589:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.469589:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.469591:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.469593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677473.470160:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.470163:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677473.470164:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677473.470166:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.470169:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.470171:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677473.470172:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.470174:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.470175:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.470176:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677473.470180:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.470182:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677473.470185:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.470186:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677473.470187:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677473.470189:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677473.470189:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.470191:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677473.470192:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677473.470192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677473.470195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.470202:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:2.0:1619677473.470203:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:26.0:1619677473.470205:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677473.470205:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:2.0:1619677473.470205:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000400:00000200:26.0:1619677473.470206:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.470206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677473.470206:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:26.0:1619677473.470208:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.470212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.470220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677473.470226:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.470229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.470229:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677473.470230:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.470231:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.470231:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677473.470423:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.470425:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.470426:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.470427:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.470429:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.470431:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.470432:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.470433:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.470434:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.470435:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.470438:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.470439:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.470441:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.470442:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.470443:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.470444:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.470445:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.470446:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.470446:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.470450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.470454:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.470456:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.470457:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.470457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.470459:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.470461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.470470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.470474:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.470476:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.470477:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.470477:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.470477:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:8.0:1619677473.470478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:6.0:1619677473.470909:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.470912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.470912:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.470914:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.470916:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.470918:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.470919:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.470920:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.470921:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.470922:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.470925:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.470927:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.470929:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.470930:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.470931:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.470933:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.470933:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.470933:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.470934:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.470947:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.470950:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.470950:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.470952:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.470953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.470954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.470969:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.470972:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.470973:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.470974:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.470974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.470975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.471348:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.471351:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.471352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.471354:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.471356:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.471357:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.471359:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.471360:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.471361:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.471362:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.471365:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.471367:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.471369:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.471370:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.471371:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.471372:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.471373:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.471374:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.471375:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.471376:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.471378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.471381:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.471383:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.471384:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.471384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.471387:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.471388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.471389:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.471391:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.471392:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.471393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:22.0:1619677473.471457:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.471460:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.471461:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.471461:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.471463:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.471467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677473.471593:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.471596:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.471597:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.471599:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.471601:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.471602:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.471604:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.471605:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.471606:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.471607:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.471611:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.471612:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.471615:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.471616:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.471616:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.471618:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.471619:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.471619:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.471619:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.471624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.471633:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.471635:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.471636:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.471637:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.471639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.471640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.471832:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.471835:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.471836:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.471838:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.471840:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.471842:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.471843:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.471845:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.471846:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.471847:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.471850:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.471852:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.471855:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.471855:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.471856:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.471858:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.471858:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.471859:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.471860:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.471871:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.471874:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.471874:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.471876:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.471877:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.471879:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.472002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.472008:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.472011:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.472012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.472014:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.472016:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.472017:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.472019:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.472020:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.472021:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.472022:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.472025:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.472026:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.472029:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.472029:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.472030:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.472032:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.472032:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.472033:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.472033:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.472045:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.472047:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.472048:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.472048:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.472051:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.472052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.472459:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.472462:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.472462:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.472464:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.472466:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.472468:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.472469:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.472470:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.472471:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.472472:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.472475:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.472476:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.472479:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.472480:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.472480:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.472482:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.472482:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.472483:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.472485:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.472485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.472496:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.472497:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.472497:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.472498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.472499:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.472499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.472500:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.472501:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.472503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.472504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.472609:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.472612:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.472613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.472615:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.472618:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.472619:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.472621:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.472622:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.472623:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.472624:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.472628:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.472629:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.472632:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.472633:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.472634:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.472635:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.472636:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.472637:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.472637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.472642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.472650:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677473.472651:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:17.0:1619677473.472653:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.472653:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677473.472654:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:17.0:1619677473.472654:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.472655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:17.0:1619677473.472656:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.472658:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.472659:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:26.0:1619677473.472660:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.472661:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.472663:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.472664:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.472665:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.472666:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.472669:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.472671:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.472673:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.472673:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.472674:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.472675:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.472676:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.472676:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.472677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.472681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.472684:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.472686:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.472687:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.472687:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.472689:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.472691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.472734:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.472737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.472737:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.472739:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.472741:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.472742:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.472743:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.472744:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.472745:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.472746:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.472749:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.472751:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.472753:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.472753:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.472754:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.472755:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.472756:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.472757:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.472758:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.472759:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.472761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.472767:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.472769:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.472770:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.472770:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.472771:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.472772:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.472773:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.472774:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.472774:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.472776:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677473.472826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.472833:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.472836:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.472837:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.472839:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.472842:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.472844:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.472845:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.472846:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.472848:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.472849:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.472853:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.472856:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.472858:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.472858:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.472859:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.472861:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.472862:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.472862:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.472863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.472867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.472874:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.472876:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.472877:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.472878:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.472879:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.472881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.473008:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.473010:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.473011:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.473013:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.473015:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.473016:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.473017:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.473019:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.473020:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.473022:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.473023:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.473025:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.473026:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.473027:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.473028:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.473030:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.473031:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.473032:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677473.473032:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:26.0:1619677473.473033:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.473033:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:26.0:1619677473.473034:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.473035:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.473035:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:26.0:1619677473.473036:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.473037:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.473038:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.473038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677473.473039:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.473040:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.473042:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.473044:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.473045:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.473045:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.473047:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.473047:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.473048:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.473048:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:22.0:1619677473.473049:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.473052:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.473053:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.473053:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.473056:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.473058:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:12.0:1619677473.473060:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.473063:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.473064:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.473065:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.473068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.473069:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.473100:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.473103:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.473104:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.473106:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.473108:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.473109:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.473110:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.473111:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.473112:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.473113:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.473116:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.473117:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.473120:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.473120:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.473121:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.473122:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.473122:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.473123:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.473124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:13.0:1619677473.473136:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.473139:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.473140:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.473142:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.473143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.473145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.473343:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.473346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.473346:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.473347:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.473349:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.473352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.473360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:22.0:1619677473.473361:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.473363:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.473364:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.473365:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.473366:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.473366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677473.473366:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.473368:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.473370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.473371:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.473374:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.473376:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677473.473377:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.473378:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.473379:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.473380:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677473.473384:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.473386:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677473.473389:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.473390:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677473.473391:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677473.473392:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677473.473393:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.473394:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677473.473395:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677473.473395:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677473.473398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677473.473401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:26.0:1619677473.473405:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:18.0:1619677473.473407:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:26.0:1619677473.473408:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.473409:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.473409:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:10.0:1619677473.473409:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:18.0:1619677473.473410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:26.0:1619677473.473411:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.473411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:26.0:1619677473.473415:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.473420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.473426:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.473428:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.473429:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.473429:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.473430:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.473431:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677473.473846:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.473848:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.473849:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.473851:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.473853:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.473854:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.473855:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.473856:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.473857:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.473858:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.473861:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.473863:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.473865:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.473865:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.473866:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.473868:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.473868:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.473869:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.473869:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.473873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.473880:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.473883:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.473884:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.473884:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.473885:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.473887:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.473891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.473896:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.473899:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.473900:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.473900:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.473900:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.473901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.473949:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.473952:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.473952:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.473954:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.473956:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.473957:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.473958:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.473959:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.473960:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.473961:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.473963:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.473965:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.473966:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.473967:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.473968:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:20.2:1619677473.473969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677473.473969:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.473969:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.473971:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.473972:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.473972:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.473974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:2.0:1619677473.473974:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677473.473977:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:2.0:1619677473.473977:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.473978:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677473.473979:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.473980:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.473980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:2.0:1619677473.473980:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.473982:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.473983:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.473984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:2.0:1619677473.473984:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:26.0:1619677473.473985:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:2.0:1619677473.473985:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.473987:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000400:00000200:2.0:1619677473.473987:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.473988:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:2.0:1619677473.473988:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.473989:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000400:00000200:2.0:1619677473.473989:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.473992:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.473994:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.473997:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.473998:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.473999:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.474001:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.474001:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.474002:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.474003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.0:1619677473.474089:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.474092:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.474094:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.474094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.474097:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.474098:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.474099:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.474101:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.474102:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.474103:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.474106:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.474108:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.474110:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.474111:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.474111:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.474113:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.474114:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.474115:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.474116:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.474117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677473.474127:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.474130:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.474131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.474131:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:6.0:1619677473.474131:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.474133:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:6.0:1619677473.474133:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:20.0:1619677473.474134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.474135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.474135:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.474856:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.474859:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.474860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.474862:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.474864:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.474866:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.474867:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.474868:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.474869:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.474870:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.474874:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.474875:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.474878:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.474878:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.474879:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.474881:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.474881:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.474882:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.474882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.474895:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.474897:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.474898:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.474900:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.474901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.474902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.474971:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.474974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.474974:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.474976:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.474979:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.474981:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.474982:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.474984:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.474985:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.474986:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.474990:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.474991:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.474994:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.474995:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.474995:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.474997:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.474998:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.474999:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.475000:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.475001:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.475003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.475012:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.475014:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.475015:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.475015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.475016:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677473.475016:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.475017:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.475018:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677473.475018:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.475022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.475106:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.475109:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.475110:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.475111:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.475114:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.475115:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.475116:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.475117:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.475118:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.475119:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.475122:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.475124:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.475126:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.475126:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.475127:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.475128:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.475128:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:10.0:1619677473.475129:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.475129:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.475130:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:8.0:1619677473.475130:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677473.475131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677473.475133:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000800:00000200:24.2:1619677473.475134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:8.0:1619677473.475135:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.475137:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000400:00000200:8.0:1619677473.475137:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:24.0:1619677473.475138:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.475138:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.475139:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.475140:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.475141:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.475141:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.475142:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677473.475144:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.475145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:8.0:1619677473.475146:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.475148:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677473.475150:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.475151:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677473.475152:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677473.475154:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677473.475154:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677473.475155:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.2:1619677473.475156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.475156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:24.0:1619677473.475160:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.2:1619677473.475160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677473.475162:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.475163:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.475163:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.475164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.475165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:1.0:1619677473.475169:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.475172:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.475173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.475173:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.475175:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.475178:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.475213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.475220:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.475222:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.475223:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.475223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.475224:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.475225:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.475651:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.475653:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.475654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.475655:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.475657:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.475660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.476007:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.476010:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.476011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.476012:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.476015:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.476016:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.476017:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.476019:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.476020:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.476020:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.476023:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.476025:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.476027:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.476028:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.476028:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.476030:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.476030:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.476031:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.476031:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.476036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.476039:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.476041:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.476042:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677473.476042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677473.476043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.476044:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.476046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:21.0:1619677473.476048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.476051:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.476053:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.476054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.476056:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.476057:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.476059:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.476060:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.476061:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.476062:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.476065:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.476067:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.476069:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.476070:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.476071:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.476073:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.476073:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.476074:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.476075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.476089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.476097:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.476100:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.476101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.476101:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.476103:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.476105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677473.476470:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.476473:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.476474:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.476475:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.476478:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.476479:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.476480:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.476481:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.476482:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.476483:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.476486:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.476488:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.476491:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.476491:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.476492:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.476494:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.476494:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.476494:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.476495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677473.476502:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.476506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.476506:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677473.476508:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677473.476508:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.476510:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.476511:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.476511:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.476513:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.476513:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:21.0:1619677473.476514:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677473.476514:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.476516:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677473.476516:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.476517:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.476518:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.476521:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.476523:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.476525:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.476526:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.476526:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.476527:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677473.476528:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.476528:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.476529:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677473.476530:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677473.476530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677473.476531:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.476531:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.476532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.476533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.476533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.476539:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.476542:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.476542:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.476543:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.476544:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.476546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.476552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.476559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.476561:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.476562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.476562:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.476563:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.476563:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.476718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.476724:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.476727:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.476728:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.476730:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.476732:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.476733:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.476735:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.476736:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.476737:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.476738:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.476741:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.476743:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.476745:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.476746:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.476747:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.476749:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.476749:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.476751:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.476752:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.476752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.476757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.476762:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.476763:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.476764:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677473.476765:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.476765:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.476766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.476766:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.476766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677473.476769:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.476771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.476780:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.476782:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.476783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.476784:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.476785:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.476785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.477008:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.477011:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.477012:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.477014:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.477016:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.477017:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.477018:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.477020:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.477021:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.477022:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.477025:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.477026:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.477028:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.477029:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.477030:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.477031:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.477032:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.477032:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.477033:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.477037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.477040:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.477042:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.477043:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.477043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.477045:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.477046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:6.0:1619677473.477317:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.477320:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.477321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.477321:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.477324:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.477327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.477595:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.477598:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.477600:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.477600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.477602:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.477603:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.477605:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.477606:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.477607:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.477608:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.477611:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.477612:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.477615:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.477615:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.477616:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.477618:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.477618:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.477619:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.477619:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.477631:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.477633:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.477634:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.477634:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.477636:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.477639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.477649:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.477651:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.477652:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.477652:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.477653:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.477654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.477729:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.477732:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.477732:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.477735:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.477737:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.477738:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.477740:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.477741:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.477742:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.477743:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.477746:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.477748:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.477750:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.477751:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.477752:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.477753:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.477754:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.477754:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.477755:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.477767:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.477773:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.477774:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.477775:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.477777:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.477779:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.477931:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.477934:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.477935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.477937:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.477939:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.477940:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.477941:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.477942:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.477943:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.477944:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.477946:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.477948:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.477950:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.477950:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.477951:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.477952:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.477952:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.477953:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.477955:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.477955:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.477958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.477962:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.477964:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.477965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.477965:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.477967:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.477968:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.477969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.477970:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.477971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.477972:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:26.0:1619677473.477999:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.478001:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.478002:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.478004:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.478006:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.478007:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.478008:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.478009:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.478010:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.478011:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.478013:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.478015:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677473.478016:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677473.478017:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.478018:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.478018:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.478019:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:26.0:1619677473.478020:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.478020:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.478020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.478021:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.478021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.478022:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.478024:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.478025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677473.478026:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.478027:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.478028:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.478029:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.478030:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.478031:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677473.478031:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.478034:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677473.478034:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:22.0:1619677473.478034:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.478034:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:24.0:1619677473.478035:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.478035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.478036:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:24.0:1619677473.478037:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.478037:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.478038:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.478039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.478039:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.478040:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.478041:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:10.0:1619677473.478041:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677473.478042:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677473.478042:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.478043:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.478043:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677473.478043:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.478044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:10.0:1619677473.478044:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.478045:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.478049:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.478051:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.478054:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.478055:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.478055:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677473.478056:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677473.478057:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677473.478058:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677473.478058:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.478058:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:20.0:1619677473.478059:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.478059:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:20.0:1619677473.478061:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.478062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.478064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.478070:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.478073:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.478074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.478074:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.478077:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.478079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.478292:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.478297:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.478298:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:26.0:1619677473.478301:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.478304:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.478306:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677473.478307:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.478308:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.478309:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.478310:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677473.478313:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.478314:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677473.478317:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.478318:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677473.478318:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677473.478320:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677473.478321:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.478321:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.478322:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.478327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.478336:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.478339:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.478340:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.478340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.478343:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.478345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.478349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.478356:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.478358:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.478359:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.478359:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.478361:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.478361:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.478787:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.478790:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.478791:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.478792:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.478795:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.478796:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.478797:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.478799:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.478800:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.478801:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.478804:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.478805:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.478807:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.478808:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.478808:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.478809:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.478810:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.478811:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.478811:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.478815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.478818:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.478821:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.478822:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.478822:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.478823:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.478825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.478835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.478837:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.478839:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.478840:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.478840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.478841:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.478841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.478841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.478848:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677473.478850:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677473.478851:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.478852:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677473.478853:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.478854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677473.478854:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677473.478856:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.478857:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.478858:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.478859:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.478860:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.478860:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.478862:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.478862:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.478864:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.478864:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.478865:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.478866:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.478867:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.478869:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.478871:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.478874:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677473.478875:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.478875:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.478876:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.478877:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.478878:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.478878:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.478879:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677473.478880:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.478880:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677473.478881:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.478882:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.2:1619677473.478884:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (23)++
00000800:00000200:8.0:1619677473.478888:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (24)--
00000800:00000200:8.0:1619677473.478888:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.478889:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.478890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677473.478892:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)++
00000800:00000200:17.0:1619677473.478895:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (24)--
00000800:00000200:1.0:1619677473.478895:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (24)++
00000400:00000200:17.0:1619677473.478896:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.478896:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (24)--
00000400:00000200:1.0:1619677473.478897:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.478898:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:1.0:1619677473.478899:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.478902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.478902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.478903:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.478947:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.478950:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.478951:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.478953:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.478955:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.478956:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.478958:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.478959:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.478960:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.478961:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.478964:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.478966:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.478968:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.478968:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.478969:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.478970:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.478971:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.478972:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.478972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.478977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.478983:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.478985:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.478986:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.478986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.478988:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.478991:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:13.0:1619677473.479124:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.479126:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.479129:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.479130:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.479131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.479133:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677473.479134:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.479135:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.479136:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.479137:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677473.479141:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.479143:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677473.479146:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.479146:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677473.479147:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677473.479149:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677473.479149:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.479150:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.479151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677473.479155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.479162:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.479165:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.479166:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.479166:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.479168:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.479171:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.479350:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.479353:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.479354:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.479354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.479357:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.479359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.479427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:18.0:1619677473.479435:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677473.479438:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:10.0:1619677473.479438:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:2.0:1619677473.479438:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:26.0:1619677473.479441:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:18.0:1619677473.479441:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.479441:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:18.0:1619677473.479444:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.479444:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.479445:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.479446:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.479447:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.479448:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677473.479449:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.479450:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.479450:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.479451:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.479452:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.479453:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.479454:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677473.479456:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.479458:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.479458:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.479460:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677473.479461:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.479462:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.479462:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.479463:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.479464:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.479464:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:18.0:1619677473.479465:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:10.0:1619677473.479465:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:18.0:1619677473.479466:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:10.0:1619677473.479466:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:18.0:1619677473.479467:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:10.0:1619677473.479467:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.479468:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.479470:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.479470:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677473.479470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.479477:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.479480:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.479480:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.479481:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.479482:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:2.0:1619677473.479483:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.479484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.479485:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:2.0:1619677473.479485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677473.479486:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.479486:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677473.479492:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.479495:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.479496:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.479496:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.479499:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.479501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677473.479541:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.479544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.479544:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.479546:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.479548:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.479550:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.479551:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.479552:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.479553:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.479554:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.479557:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.479559:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.479562:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.479562:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.479563:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.479565:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.479565:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.479566:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.479567:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.479568:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.479571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.479578:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677473.479580:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677473.479580:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.479581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.479582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677473.479583:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.479583:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.479585:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.479586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.479588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.479617:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.479619:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.479620:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.479622:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.479624:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.479625:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.479627:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.479628:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.479629:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.479630:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.479633:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.479634:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.479637:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.479637:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.479639:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.479640:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.479640:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.479641:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.479642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.479645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.479649:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.479651:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.479652:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.479653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.479654:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.479656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.480354:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.480356:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.480357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.480358:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.480361:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.480362:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.480363:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.480364:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.480365:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.480366:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.480369:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.480370:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.480372:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.480372:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.480373:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.480374:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.480375:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.480375:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.480376:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.480380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.480384:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.480386:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.480387:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.480387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.480388:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.480390:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.0:1619677473.480399:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.480401:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677473.480404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.480404:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.480406:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.480408:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677473.480409:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.480411:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.480412:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.480413:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.480416:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.480418:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.480421:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.480422:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677473.480422:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677473.480424:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677473.480425:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.480426:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.480426:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:6.0:1619677473.480473:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.480476:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.480476:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.480478:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.480481:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.480482:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.480483:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.480485:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.480486:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.480487:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.480490:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.480492:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.480494:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.480495:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.480495:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.480497:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.480498:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.480499:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.480500:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.480500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.480511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.480513:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.480513:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.480514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.480515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.480515:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.480516:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.480518:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.480518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.480520:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.481168:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.481171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.481171:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.481172:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.481174:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.481177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.481253:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.481256:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.481257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.481259:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.481261:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.481262:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.481263:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.481265:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.481266:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.481267:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.481270:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.481271:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.481273:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.481274:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.481274:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.481276:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.481276:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.481277:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.481279:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.481279:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.481281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.481285:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.481287:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.481288:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.481289:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.481290:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.481292:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.481292:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.481299:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.481301:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.481302:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:5.2:1619677473.481513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.481519:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.481523:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.481524:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.481526:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.481528:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.481530:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.481531:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.481532:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.481533:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.481534:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.481537:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.481539:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.481541:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.481542:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.481543:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.481544:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.481545:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.481545:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.481546:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.481558:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.481562:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.481563:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.481563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.481565:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.481567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.481880:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.481883:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.481885:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.481886:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.481887:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.481889:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.481890:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.481891:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.481892:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.481893:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.481896:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.481898:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.481901:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.481902:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.481902:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.481904:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.481904:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.481905:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.481906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.481910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.481917:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.481919:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.481920:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.481920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.481923:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.481925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.482013:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.482016:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.482017:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.482020:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.482022:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.482024:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.482025:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.482027:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.482028:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.482029:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.482032:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.482034:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.482037:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.482038:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.482038:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.482040:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.482040:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.482041:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.482042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.482054:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.482056:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.482057:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.482058:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.482059:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.482061:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.482866:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.482869:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.482870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.482872:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.482875:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.482876:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.482878:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.482879:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.482880:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.482881:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.482884:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.482886:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.482889:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.482890:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.482891:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.482892:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.482893:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.482894:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.482895:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.482895:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.482907:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.482910:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677473.482910:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.482910:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.482911:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.482912:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677473.482913:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.482913:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.482914:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.482916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.483171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.483174:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.483176:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.483178:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.483179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.483181:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.483182:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.483184:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.483186:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.483187:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.483188:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.483192:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.483195:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.483198:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.483199:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.483199:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.483201:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.483201:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.483203:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.483204:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.483205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.483208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677473.483210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.483215:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677473.483216:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677473.483217:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.483218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.483219:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677473.483219:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.483220:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.483220:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677473.483220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.483222:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.483222:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.483223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.483224:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.483225:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:6.0:1619677473.483226:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.483229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.483292:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.483296:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.483298:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677473.483301:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.483303:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.483305:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.483306:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.483307:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.483308:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.483309:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.483312:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.483314:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.483316:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.483317:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.483318:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.483320:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.483320:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.483321:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.483323:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.483323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.483333:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.483336:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.483337:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.483337:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.483337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.483339:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677473.483340:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.483340:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.483341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.483342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.483352:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.483355:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.483356:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.483356:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.483358:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.483358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.483395:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.483398:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.483399:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.483400:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.483403:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.483405:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.483406:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.483408:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.483409:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.483410:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.483413:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.483415:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.483417:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.483418:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.483418:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.483420:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.483421:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.483422:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.483422:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.483426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.483434:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.483437:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.483438:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.483438:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.483440:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.483443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.483445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.483452:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.483454:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.483455:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.483455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.483457:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.483457:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.483545:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.483548:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.483548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.483550:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.483553:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.483555:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.483556:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.483558:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.483559:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.483560:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.483563:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.483565:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.483567:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.483568:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.483569:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.483570:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.483571:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677473.483572:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:22.0:1619677473.483572:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.483572:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677473.483575:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.483575:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.483578:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.483580:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.483582:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.483583:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.483584:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.483584:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:26.0:1619677473.483585:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.483586:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:21.0:1619677473.483586:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.483587:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.483589:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.483589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677473.483591:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.483592:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.483593:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.483596:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.483597:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.483597:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.483599:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.483599:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.483600:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.483601:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.483604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:20.0:1619677473.483604:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677473.483607:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:20.0:1619677473.483607:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.483608:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.483609:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.483609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677473.483610:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:21.0:1619677473.483610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.483610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.483611:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.483613:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.483615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.483623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.483627:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.483630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.483630:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.483630:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.483631:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.483632:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.483806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.483813:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.483816:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.483818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.483820:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.483822:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.483823:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.483825:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.483826:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.483827:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.483828:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.483832:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.483833:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.483837:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.483838:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.483839:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.483840:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.483841:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.483843:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.483844:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.483844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.483850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677473.483855:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677473.483855:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677473.483857:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.483858:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.483858:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.483858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.483859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677473.483860:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.483863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.483864:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.483877:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.483880:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.483881:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.483881:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.483882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.483883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.484056:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.484058:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.484059:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.484061:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.484063:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.484065:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.484066:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.484067:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.484068:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.484069:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.484072:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.484074:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.484077:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.484078:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.484078:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.484080:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.484080:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.484081:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.484082:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.484093:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.484096:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.484097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.484097:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.484099:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.484103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.484951:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.484953:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.484955:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.484955:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.484957:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.484960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.485221:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.485224:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.485224:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.485226:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.485229:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.485230:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.485231:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.485232:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.485233:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.485234:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.485237:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.485238:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.485240:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.485241:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.485242:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.485243:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.485243:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.485244:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.485245:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.485249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.485252:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.485255:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.485256:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.485256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.485257:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.485259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:1.0:1619677473.485760:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.485762:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.485763:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.485765:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.485768:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.485770:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677473.485771:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.485772:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.485773:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.485774:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677473.485777:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.485779:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677473.485782:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.485783:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677473.485783:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.485785:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677473.485785:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.485786:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.485786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.485792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.485799:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.485801:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.485802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.485803:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.485805:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.485807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.485890:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.485893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.485893:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.485895:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.485897:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.485899:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.485900:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.485901:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.485902:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.485903:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.485907:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.485908:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.485911:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.485912:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.485913:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.485914:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.485915:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.485916:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.485916:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.485928:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.485930:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.485931:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.485933:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.485934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.485935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677473.486104:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.486106:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.486107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.486109:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.486109:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:24.0:1619677473.486112:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.486113:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:24.0:1619677473.486113:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677473.486113:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:24.0:1619677473.486114:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.486116:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.486116:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.486117:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.486118:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677473.486118:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.486121:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677473.486122:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.486123:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.486123:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.486125:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677473.486125:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.486126:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.486129:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.486129:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.486130:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.486132:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.486132:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.486134:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:10.0:1619677473.486134:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.486135:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.486135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677473.486135:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677473.486138:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.486138:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:9.2:1619677473.486138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677473.486139:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677473.486141:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677473.486141:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677473.486142:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677473.486142:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:2.0:1619677473.486147:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.486148:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.486150:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677473.486150:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.486151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.486151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677473.486151:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.486152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677473.486153:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.486157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.486160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.486168:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.486170:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.486172:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.486172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.486173:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.486173:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677473.486289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.486296:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.486299:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.486300:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.486302:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.486305:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.486306:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.486307:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.486309:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.486310:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.486311:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.486314:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.486316:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.486318:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.486319:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.486320:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.486322:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.486322:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.486323:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.486323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.2:1619677473.486334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:22.0:1619677473.486336:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.486338:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.486339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.486340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.486340:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:22.0:1619677473.486342:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.486343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677473.486343:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.486344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.486346:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.486349:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.486350:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.486351:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.486352:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.486353:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.486354:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677473.486357:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:2.0:1619677473.486358:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.486359:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.486360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677473.486360:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677473.486361:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.486362:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.486362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677473.486362:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.486363:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.486363:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.486365:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.486366:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.486367:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.486368:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.486369:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.486371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.486374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.486380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.486381:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.486381:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677473.486382:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.486382:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.486383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.486384:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.486384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.486385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.486385:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677473.486804:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.486808:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.486808:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.486810:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.486813:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.486814:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.486815:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.486817:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.486818:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.486819:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.486822:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.486824:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.486826:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.486827:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.486828:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.486830:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.486830:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.486831:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.486831:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.486835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.486843:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.486846:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.486847:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.486847:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.486850:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.486854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:10.0:1619677473.486906:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.486908:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.486909:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.486911:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.486913:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.486915:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.486916:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.486917:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.486918:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.486919:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.486922:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.486923:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.486926:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.486926:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.486927:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.486928:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.486929:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.486929:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.486930:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.486934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.486939:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.486941:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.486942:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.486942:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.486944:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.486946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:12.0:1619677473.487344:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.487347:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.487349:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677473.487351:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.487352:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.487352:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677473.487354:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.487355:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.487356:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.487357:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677473.487360:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.487362:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 352 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677473.487364:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.487365:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677473.487366:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677473.487367:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677473.487368:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.487369:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.487370:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.487374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.487383:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.487386:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.487387:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.487387:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.487389:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.487393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677473.487458:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.487460:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.487461:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.487463:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.487466:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.487467:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.487468:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.487469:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.487471:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.487472:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.487475:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.487477:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.487479:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.487480:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.487481:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.487482:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.487483:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.487484:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.487485:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.487486:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.487496:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.487498:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.487498:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.487498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.487499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.487500:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.487501:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.487503:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.487504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.487505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.487559:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.487562:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.487562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.487563:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.487565:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.487568:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.487743:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.487746:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.487747:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.487749:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.487751:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.487753:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.487754:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.487755:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.487757:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.487758:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.487761:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.487763:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.487766:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.487766:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.487767:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.487769:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.487769:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.487770:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.487770:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.487792:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.487795:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.487796:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.487796:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.487798:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.487801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.487887:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.487889:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.487890:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.487892:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.487895:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.487896:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.487897:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.487898:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.487899:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.487900:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.487903:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.487905:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.487907:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.487907:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.487908:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.487909:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.487909:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.487911:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.487912:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.487912:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.487915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.487918:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.487920:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.487921:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.487921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.487923:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.487925:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:24.0:1619677473.487925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.487927:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.487928:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.487929:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:8.0:1619677473.488313:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.488315:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.488316:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.488318:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677473.488320:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.488321:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677473.488322:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.488323:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.488324:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.488325:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.488328:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677473.488328:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.488329:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.488330:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.488331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.488332:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.488333:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.488333:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.0:1619677473.488334:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:10.0:1619677473.488335:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.488336:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677473.488336:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677473.488336:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677473.488337:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.488337:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.488338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:10.0:1619677473.488339:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.488340:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.488341:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.488343:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.488345:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.488348:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.488349:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.488349:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.488350:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677473.488351:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.488351:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677473.488352:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677473.488352:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.488352:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677473.488353:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.488355:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:21.0:1619677473.488356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.488358:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.488364:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.488367:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.488368:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.488368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.488370:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.488373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.488380:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.488383:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.488384:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:6.0:1619677473.488386:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677473.488387:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.488389:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.488389:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.488390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.488390:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677473.488390:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677473.488391:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.488392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:1.0:1619677473.488392:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.488393:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.488394:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.488395:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.488398:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.488400:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.488402:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.488403:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.488404:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.488405:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.488406:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.488407:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.488407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677473.488419:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.488421:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.488422:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.488422:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.488424:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.488427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.488980:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.488983:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.488983:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.488985:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.488988:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.488990:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.488991:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.488992:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.488993:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.488994:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:6.0:1619677473.488994:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.488997:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.488997:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:26.0:1619677473.489000:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.489000:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.489003:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:6.0:1619677473.489003:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.489005:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677473.489005:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677473.489006:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000400:00000200:6.0:1619677473.489006:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.489007:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.489008:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.489009:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.489009:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:6.0:1619677473.489009:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.489010:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000400:00000200:6.0:1619677473.489010:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.489011:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.2:1619677473.489014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:6.0:1619677473.489014:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.489016:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.489017:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:6.0:1619677473.489018:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.489019:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:6.0:1619677473.489019:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:24.0:1619677473.489020:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.489020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:6.0:1619677473.489020:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.489021:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:24.0:1619677473.489022:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.489022:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.489022:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.489023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677473.489024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:21.0:1619677473.489035:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.489037:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.489038:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.489039:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.489040:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.489042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.2:1619677473.489044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.489047:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.489050:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.489050:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.489051:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.489052:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.489053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.0:1619677473.489057:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.489059:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.489060:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.489061:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.489062:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.489063:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.489139:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.489141:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.489142:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.489144:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.489146:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.489148:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.489149:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.489150:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.489151:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.489152:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.2:1619677473.489156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677473.489156:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.489158:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677473.489158:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.489160:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.489160:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.0:1619677473.489161:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677473.489161:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:20.0:1619677473.489163:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.489163:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.489163:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677473.489164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677473.489164:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.489164:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677473.489165:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.489167:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.489168:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.489168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:20.0:1619677473.489169:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.489170:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.489172:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677473.489174:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:20.0:1619677473.489175:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.489176:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.489177:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.489177:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677473.489178:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.489180:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.489180:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.489181:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.489182:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677473.489183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.489183:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.489184:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.489185:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.489185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.489190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.489198:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.489200:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.489201:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.489201:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.489203:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.489206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.489212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.489219:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.489222:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.489223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.489223:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.489224:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.489225:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677473.489351:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.489354:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.489355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.489356:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.489359:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.489361:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677473.489362:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.489363:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.489364:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.489365:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677473.489368:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.489369:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677473.489371:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.489372:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.489372:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677473.489373:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677473.489374:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.489375:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.489375:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677473.489375:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.489375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.489376:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677473.489377:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.489377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.489380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.489396:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.489399:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.489401:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.489402:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.489404:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.489405:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.489407:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.489408:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.489409:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.489410:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.489412:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.489414:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.489416:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.489417:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.489418:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.489419:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.489420:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.489421:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.489422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.489432:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.489435:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.489436:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.489436:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.489438:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.489440:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677473.489522:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.489524:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.489525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.489527:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.489529:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.489531:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677473.489532:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.489533:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.489534:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.489535:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.489539:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.489541:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.489543:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.489544:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.489544:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.489546:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.489547:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.489547:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.489548:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677473.489559:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.489562:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.489563:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.489563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.489565:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.489567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.489581:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.489584:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.489584:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.489585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.489585:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.489586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.489815:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.489817:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.489818:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.489820:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.489823:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.489824:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:24.0:1619677473.489825:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.489827:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.489828:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.489829:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677473.489832:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.489834:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677473.489836:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.489837:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:24.0:1619677473.489838:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677473.489839:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677473.489840:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.489840:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.489841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677473.490163:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.490165:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.490167:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.490168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.490170:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.490172:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677473.490173:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.490174:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.490175:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.490176:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.490180:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.490181:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.490184:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.490185:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677473.490186:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677473.490188:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677473.490188:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.490190:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677473.490191:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677473.490192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.490202:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677473.490204:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677473.490205:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.490206:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677473.490848:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.490850:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.490851:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.490853:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.490855:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.490856:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.490857:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.490858:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.490859:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.490860:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.490863:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.490865:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.490867:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.490867:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.490868:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.490869:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.490870:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.490871:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.490871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.490876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.490880:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.490882:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.490883:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.490883:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.490885:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.490887:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.491021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.491027:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.491030:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.491031:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.491033:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.491035:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.491037:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.491038:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.491039:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.491040:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.491042:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.491045:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.491047:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.491050:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.491051:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.491051:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.491053:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.491054:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.491055:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.491055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677473.491408:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.491411:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.491412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.491413:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.491416:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.491418:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.491419:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.491421:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.491422:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.491423:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.491427:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.491430:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.491433:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.491434:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.491434:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.491436:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.491437:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.491437:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.491438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.491451:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.491453:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.491454:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.491455:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.491457:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.491457:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.491458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.491460:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.491461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.491461:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.491463:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.491466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.491919:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.491921:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.491922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.491924:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.491926:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.491928:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677473.491929:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.491930:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.491930:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:24.0:1619677473.491931:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.491932:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.491933:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.491934:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:24.0:1619677473.491935:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.491936:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.491937:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.491938:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.491940:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.491940:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:24.0:1619677473.491941:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:10.0:1619677473.491941:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.491942:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:10.0:1619677473.491942:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.491943:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.491944:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677473.491944:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677473.491944:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677473.491945:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.491946:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:10.0:1619677473.491946:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.491947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677473.491948:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.491949:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.491950:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.491951:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.491952:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.491952:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.491953:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.491954:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.491957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677473.491958:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.491960:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677473.491960:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:20.0:1619677473.491962:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.491962:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:20.0:1619677473.491963:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.491963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677473.491964:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:8.0:1619677473.491964:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677473.491965:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.491966:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:20.0:1619677473.491966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677473.491967:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.491968:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.491969:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.491971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677473.492917:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.492920:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.492921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.492923:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.492926:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.492927:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.492928:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.492930:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.492930:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.492931:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.492934:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.492936:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.492939:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.492939:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.492940:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.492942:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.492942:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.492943:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.492943:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.492962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.492965:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.492966:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.492966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.492968:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.492971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.492985:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.492988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.492988:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.492990:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.492992:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.492994:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.492995:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.492996:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.492997:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.492998:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.493001:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.493003:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.493006:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.493006:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.493007:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.493008:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.493009:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.493010:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.493010:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.493023:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.493025:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.493026:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.493027:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.493028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.493030:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.493063:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.493066:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.493066:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.493068:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.493071:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.493072:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.493073:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.493075:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.493076:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.493077:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.493080:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.493082:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.493085:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.493085:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.493086:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.493088:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.493088:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.493089:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.493090:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.493094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.493101:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.493104:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.493105:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.493106:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.493108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.493108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677473.493126:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.493128:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.493129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.493130:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.493132:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.493134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.493179:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.493181:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.493182:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.493184:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.493186:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.493187:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.493188:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.493190:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.493190:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.493191:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.493194:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.493196:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.493198:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.493199:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.493199:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.493201:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.493201:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.493202:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.493204:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.493204:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.493206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.493212:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.493214:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.493215:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.493215:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.493217:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.493217:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.493218:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.493219:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.493219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.493221:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.493226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.493232:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.493235:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.493235:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.493235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.493236:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.493237:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677473.493457:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.493460:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.493462:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.493463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.493465:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.493466:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.493467:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.493469:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.493470:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.493471:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.493474:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.493476:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.493478:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.493479:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.493480:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.493481:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.493482:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.493483:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.493484:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.493485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:10.0:1619677473.493490:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.493493:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.493494:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:20.0:1619677473.493494:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:26.0:1619677473.493495:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677473.493496:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:10.0:1619677473.493496:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.493496:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.493497:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:20.0:1619677473.493497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.493498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:10.0:1619677473.493498:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.493499:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.493499:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:10.0:1619677473.493500:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:6.0:1619677473.493500:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.493501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677473.493501:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.493502:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:10.0:1619677473.493503:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.493504:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.493505:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:6.0:1619677473.493505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677473.493507:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.493509:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677473.493509:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.493510:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.493512:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.493512:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.493513:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.493513:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677473.493513:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.493516:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.493516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.493517:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.493518:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.493518:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.493518:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:26.0:1619677473.493519:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.493519:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.493519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.493520:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.493521:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677473.493521:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.493522:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.493523:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.493523:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:26.0:1619677473.493524:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677473.493524:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677473.493524:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.493526:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.2:1619677473.493526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.493527:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.493528:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.493528:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.0:1619677473.493530:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.493533:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.493533:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.493534:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.493536:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.493537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:5.2:1619677473.493603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.493609:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.493612:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.493613:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.493615:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.493617:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.493619:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.493620:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.493621:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.493622:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.493623:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.493627:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.493628:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.493631:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.493632:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.493632:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.493634:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.493634:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.493635:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.493635:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.493648:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.493650:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.493651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.493651:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.493653:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.493655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.493722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.493773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.493782:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.493784:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.493785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.493786:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.493788:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.493791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.493793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677473.493800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.493803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.493803:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.493804:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.493805:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.493805:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:10.0:1619677473.493944:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.493947:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.493948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.493950:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.493952:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.493953:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.493955:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.493956:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.493957:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.493958:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.493960:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.493962:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.493964:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.493964:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.493965:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.493966:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.493967:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.493967:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.493968:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.493972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.493978:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.493980:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.493981:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.493981:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.493983:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.493985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.494176:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.494178:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.494179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.494181:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.494183:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.494185:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.494186:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.494187:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.494188:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.494189:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.494192:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.494194:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.494196:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.494197:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.494197:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.494199:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.494200:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.494200:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.494200:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.494205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.494213:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.494216:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.494217:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.494217:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.494220:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.494223:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.494351:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.494353:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.494354:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.494356:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.494358:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.494359:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.494360:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.494361:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.494362:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.494363:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.494366:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.494367:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.494369:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.494370:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.494371:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.494372:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.494372:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.494373:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.494375:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.494375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.494378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.494381:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.494383:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.494384:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.494385:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.494386:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.494388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.494391:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.494393:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.494394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.494395:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.494396:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:26.0:1619677473.494397:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.494397:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.494399:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.494401:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.494403:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.494404:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.494405:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.494406:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.494407:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.494410:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.494411:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.494414:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.494415:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.494415:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.494417:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.494417:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.494418:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.494418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.494431:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.494434:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.494435:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.494435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.494437:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.494440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.494540:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.494542:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.494543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.494545:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.494546:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.494548:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.494549:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.494550:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.494551:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.494552:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.494555:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.494556:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.494559:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.494559:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.494560:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.494562:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.494562:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.494563:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.494564:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.494565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.494575:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.494575:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.494576:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.494577:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.494577:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.494578:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.494578:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.494580:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.494581:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.494582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.494591:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.494594:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.494595:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.494596:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.494596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.494597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.494967:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.494970:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.494970:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.494972:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.494975:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.494976:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.494977:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.494979:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.494980:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.494981:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.494984:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.494986:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.494989:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.494989:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.494990:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.494992:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.494992:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.494993:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.494993:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.494998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.495006:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.495008:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.495010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.495010:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.495011:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:2.0:1619677473.495012:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.495014:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.495015:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677473.495015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:26.0:1619677473.495017:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.495020:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.495021:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.495022:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.495024:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.495025:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.495026:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.495029:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.495030:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.495032:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.495033:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.495033:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.495035:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.495035:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.495036:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.495038:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.495038:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:13.0:1619677473.495050:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677473.495051:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.495052:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:13.0:1619677473.495053:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677473.495053:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677473.495054:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.495054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:13.0:1619677473.495057:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.495058:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.495060:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.495122:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.495125:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.495126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.495127:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.495130:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.495131:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.495132:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.495134:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.495135:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.495135:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.495139:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.495140:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.495142:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.495143:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.495143:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.495145:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.495145:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.495146:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.495146:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.495150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.495153:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.495155:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.495156:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.495156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.495158:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.495160:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.495168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.495171:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.495174:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.495174:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.495174:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.495175:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.495176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.495195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.495201:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.495204:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.495205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.495207:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.495210:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.495211:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.495212:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.495213:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.495215:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.495216:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.495219:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.495219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677473.495220:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.495222:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.495222:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.495223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677473.495223:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:6.0:1619677473.495223:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.495224:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677473.495225:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.495226:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.495226:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.495227:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.495228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:6.0:1619677473.495228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.495232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.495239:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.495241:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.495242:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.495243:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.495244:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.495246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.495259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.495266:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.495269:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.495269:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.495270:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.495270:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.495271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:12.0:1619677473.495961:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.495964:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677473.495966:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.111@o2ib7 : GET - for me
00000800:00000200:13.0:1619677473.495968:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677473.495968:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.111@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677473.495970:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.111@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677473.495971:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.495972:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677473.495973:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677473.495975:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:12.0:1619677473.495977:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677473.495978:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.495978:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.495979:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:12.0:1619677473.495980:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.495980:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.495981:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.495981:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.495983:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.495983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:10.0:1619677473.495983:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.495984:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.495985:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.495987:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.495987:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.495988:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.495991:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.495993:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:13.0:1619677473.495995:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677473.495995:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.495996:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.495996:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:13.0:1619677473.495997:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.495998:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.495998:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.495998:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.495999:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:13.0:1619677473.496000:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: REPLY: OK
00000800:00000200:10.0:1619677473.496000:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:12.0:1619677473.496001:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.496003:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.2:1619677473.496004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.496007:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.496009:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.496010:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.496010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.496012:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.496014:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.496239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.496245:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.496248:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.496249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.496251:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.496253:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.496254:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.496256:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.496257:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.496258:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.496259:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.496262:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.496264:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.496266:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.496267:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.496267:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.496269:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.496270:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.496271:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.496271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.496276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.496284:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.496286:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.496287:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.496288:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.496290:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.496292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677473.496396:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.496399:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.496400:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.496401:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.496405:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.496406:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.496407:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.496408:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.496409:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.496410:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.496413:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.496415:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.496417:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.496418:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.496419:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.496421:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.496421:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.496423:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.496424:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.496424:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677473.496426:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.496434:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.496437:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677473.496437:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.496438:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.496438:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.496439:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.496440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677473.496440:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.496440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677473.496443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.496730:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.496732:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.496733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.496735:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.496736:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.496738:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.496739:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.496740:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.496741:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.496742:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.496745:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.496746:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.496748:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.496749:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.496750:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.496751:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.496751:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.496752:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.496753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.496757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.496760:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.496762:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.496763:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.496763:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.496765:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.496767:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.496796:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.496798:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677473.496799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677473.496801:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.496804:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.496805:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677473.496806:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.496807:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.496808:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.496809:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677473.496812:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.496814:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677473.496816:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.496817:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677473.496818:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677473.496819:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677473.496820:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.496820:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677473.496821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677473.496825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.496833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.496836:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.496837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.496837:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.496839:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.496842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.496925:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.496927:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.496928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.496929:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.496931:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.496934:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.496940:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.496942:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.496943:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.496945:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.496947:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.496948:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.496949:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.496951:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.496952:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.496953:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.496956:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.496958:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.496960:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.496961:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.496962:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.496963:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.496964:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.496964:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.496965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.496977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.496979:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.496980:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.496982:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.496983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.496985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.497228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.497231:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.497233:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:21.0:1619677473.497236:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.497236:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.497238:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.497240:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.497241:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.497243:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.497244:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.497245:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.497248:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.497250:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.497252:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.497253:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.497254:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.497255:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.497256:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.497257:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.497257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677473.497269:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.497271:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.497272:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.497272:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.497274:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.497277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677473.497347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.497351:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677473.497351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677473.497353:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.497355:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.497357:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677473.497358:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.497359:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.497360:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.497361:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677473.497364:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.497366:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677473.497368:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.497369:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677473.497369:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677473.497371:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677473.497372:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.497372:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677473.497373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677473.497378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.497387:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.497390:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.497391:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.497391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.497393:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.497396:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677473.497652:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.497655:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.497657:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.497660:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.497660:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.497661:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677473.497663:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.497664:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.497665:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.497666:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677473.497669:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.497671:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677473.497674:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.497675:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:12.0:1619677473.497675:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:12.0:1619677473.497677:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:12.0:1619677473.497678:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.497678:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.497679:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677473.497684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.497692:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.497695:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.497696:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.497696:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.497698:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.497701:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677473.498058:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.498061:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.498062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.498064:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.498066:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.498067:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.498069:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.498070:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.498071:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.498072:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.498075:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.498077:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.498079:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.498080:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.498081:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.498082:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.498083:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.498083:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.498084:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.498097:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.498099:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.498100:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.498100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.498103:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.498106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.498114:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.498117:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.498118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.498118:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.498119:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.498120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.498292:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.498298:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.498299:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:26.0:1619677473.498301:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.498304:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.498305:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.498306:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.498308:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.498309:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.498309:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.498313:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.498314:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.498316:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.498316:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.498317:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.498318:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.498319:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.498320:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.498322:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.498322:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.498324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.498329:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.498331:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.498332:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.498332:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.498334:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.498336:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.498336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.498338:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.498339:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.498340:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:5.2:1619677473.498407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.498413:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.498416:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.498417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.498418:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.498421:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.498422:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.498424:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.498425:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.498426:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.498427:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.498430:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.498432:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.498434:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.498435:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.498436:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.498437:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.498438:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.498439:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.498440:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.498441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.498446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.498451:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.498452:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677473.498453:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.498453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.498454:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.498455:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.498456:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.498456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.498458:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.498460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.498474:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.498477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.498477:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.498478:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.498479:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.498480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.498516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.498518:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.498519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.498521:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.498523:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.498525:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.498526:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.498527:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.498528:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.498529:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.498533:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.498534:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.498537:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.498538:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.498538:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.498540:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.498540:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.498541:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.498541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.498554:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.498556:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.498557:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.498559:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.498559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.498561:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.498695:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.498698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.498698:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.498700:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.498702:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.498703:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.498705:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.498706:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.498707:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.498707:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.498710:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.498712:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.498713:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.498714:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.498714:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.498716:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.498717:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.498717:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.498718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.498729:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.498732:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.498732:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.498734:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.498735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.498736:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.498995:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.498998:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.498999:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.499001:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.499003:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.499004:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.499006:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.499007:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.499008:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.499009:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.499013:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.499015:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.499018:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.499019:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.499019:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.499021:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.499021:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.499022:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.499022:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.499034:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.499037:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.499038:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.499038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.499041:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.499044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.499204:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.499207:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.499208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.499208:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.499210:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.499213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.499304:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.499306:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.499308:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.499310:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.499312:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.499314:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.499315:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.499316:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.499317:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.499318:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.499321:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.499323:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.499325:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.499326:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.499327:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.499329:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.499329:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.499330:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.499331:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.499343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.499346:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.499347:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.499347:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:6.0:1619677473.499347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.499350:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.499350:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:22.0:1619677473.499353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:13.0:1619677473.499353:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.499354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.499356:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.499357:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677473.499359:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.499361:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.499362:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.499363:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677473.499366:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.499368:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677473.499370:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.499370:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677473.499371:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677473.499372:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677473.499373:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.499374:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677473.499376:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677473.499376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677473.499378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.499384:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.499387:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.499388:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.499388:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.499390:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.499390:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677473.499392:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:26.0:1619677473.499393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:12.0:1619677473.499393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.499394:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677473.499738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.499744:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.499747:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.499749:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.499750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.499752:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.499753:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677473.499753:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:21.0:1619677473.499755:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.499756:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.499756:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:21.0:1619677473.499757:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.499758:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.499760:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677473.499760:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.499762:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.499763:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.499764:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:21.0:1619677473.499765:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677473.499765:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.499767:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.499768:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.499768:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.499769:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.499769:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:10.0:1619677473.499769:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.0:1619677473.499771:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.499771:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.499772:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.499773:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000400:00000200:10.0:1619677473.499773:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.499774:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.499777:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677473.499777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677473.499778:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.499779:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.499781:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.499781:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.499782:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.499783:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677473.499786:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:9.2:1619677473.499787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.0:1619677473.499788:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.499789:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.499790:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.499791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.499792:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677473.499795:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.499797:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:8.2:1619677473.499798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:18.0:1619677473.499799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.499799:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.499801:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.499804:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677473.499806:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.499808:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.499809:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.499810:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677473.499810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.499812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.499870:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.499873:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.499874:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.499876:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.499878:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.499880:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.499881:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.499882:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.499883:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.499884:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.499888:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.499889:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.499891:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.499892:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.499892:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.499894:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.499894:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.499895:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.499896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.499899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.499902:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.499905:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.499905:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.499905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.499908:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.499909:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.499919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.499922:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.499925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.499925:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.499926:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.499926:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:8.0:1619677473.499927:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.0:1619677473.500299:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.500301:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.500302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.500304:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.500306:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.500308:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.500309:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.500310:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.500311:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.500312:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.500315:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.500317:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.500320:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.500321:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.500322:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.500323:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.500324:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.500325:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.500325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.500330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.500339:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.500342:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.500343:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.500343:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.500346:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.500349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.500720:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.500723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.500723:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.500725:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.500727:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.500728:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.500729:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.500731:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.500731:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.500732:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.500735:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.500737:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.500738:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.500739:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.500740:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.500741:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.500741:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.500742:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.500743:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.500747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.500751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.500753:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.500754:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.500754:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.500756:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.500758:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:6.0:1619677473.500846:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.500848:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.500849:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.500850:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.500852:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.500855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.500942:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.500944:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.500945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.500947:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.500949:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.500951:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.500951:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.500953:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.500954:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.500955:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.500959:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.500960:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.500963:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.500964:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.500964:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.500966:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.500966:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.500968:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.500969:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.500969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.500979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.500981:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.500981:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.500982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.500983:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.500983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.500984:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.500986:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.500987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.500988:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.500999:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.501002:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.501003:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.501004:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.501004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.501005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:12.0:1619677473.501639:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.501642:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.501644:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.501647:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.501647:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.501648:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677473.501649:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.501651:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.501652:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.501653:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677473.501656:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.501658:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677473.501661:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.501662:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677473.501662:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677473.501664:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677473.501664:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.501665:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.501666:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.501671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.501679:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.501682:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.501683:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.501683:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.501685:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.501688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677473.501824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.501826:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.501827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.501829:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.501831:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.501833:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.501834:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.501835:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.501837:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.501837:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.501841:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.501842:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.501845:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.501846:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.501846:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.501848:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.501848:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.501849:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.501850:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.501854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.501862:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.501865:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.501866:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.501866:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.501868:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.501872:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677473.501962:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.501965:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.501967:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.501968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.501969:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.501970:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.501972:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.501973:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.501974:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.501975:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.501978:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.501980:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.501982:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.501983:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.501983:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.501985:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.501986:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.501986:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.501987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.501991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.501999:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.502001:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.502002:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.502003:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677473.502004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.502006:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.502137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.502143:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.502146:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.502147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.502149:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.502151:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.502152:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.502154:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.502155:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.502156:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.502157:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.502160:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.502161:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.502164:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.502165:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.502166:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.502168:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.502169:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.502170:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.502170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.502181:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.502184:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.502185:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.502187:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.502187:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.502189:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677473.502471:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.502474:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.502475:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.502477:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.502479:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.502480:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.502482:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.502483:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.502484:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.502485:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.502488:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.502489:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.502492:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.502493:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.502493:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.502495:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.502495:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.502496:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.502496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.502509:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.502511:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.502512:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.502514:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.502515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.502516:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.502536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.502541:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.502544:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.502546:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.502546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.502548:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.502550:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677473.502551:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.502552:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.502553:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.502554:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677473.502558:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.502559:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.502561:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.502562:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.502563:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.502565:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.502565:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.502566:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.502567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.502571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.502579:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.502581:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.502582:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.502584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.502584:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.502586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677473.502676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.502682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.502685:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.502686:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.502689:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.502691:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.502693:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.502694:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.502695:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.502696:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.502697:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.502701:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.502703:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.502705:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.502706:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.502707:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.502708:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.502709:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.502710:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.502710:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.502722:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.502724:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.502725:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.502726:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.502728:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.502730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.502812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.502814:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.502815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.502816:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.502818:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.502821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.502832:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.502834:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.502835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.502835:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.502836:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.502837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.502898:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.502901:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.502902:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.502903:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.502906:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.502907:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.502908:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.502909:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.502910:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.502911:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.502914:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.502916:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.502919:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.502919:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.502920:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.502922:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.502922:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.502923:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.502924:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.502925:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.502935:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.502938:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.502938:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.502938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.502939:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.502940:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.502940:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.502941:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.502941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.502943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.503091:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.503094:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.503095:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.503096:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.503099:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.503100:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.503101:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.503102:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.503103:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.503104:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.503107:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.503108:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.503110:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.503111:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.503111:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.503113:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.503113:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.503114:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.503114:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.503118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.503123:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.503125:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.503126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.503126:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.503128:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.503131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.0:1619677473.503132:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.503134:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.503135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.503136:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.503139:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.503140:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.503141:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.503142:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.503143:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.503144:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.503147:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.503148:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.503151:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.503152:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.503152:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.503154:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.503155:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.503155:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.503156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.503168:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.503170:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.503171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.503172:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.503174:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.503176:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.503683:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.503686:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.503687:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.503689:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.503691:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.503693:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.503694:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.503695:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.503696:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.503697:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.503700:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.503701:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.503704:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.503705:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.503705:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.503707:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.503708:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.503709:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.503710:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.503711:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.503722:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.503724:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.503725:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.503725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.503726:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677473.503726:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.503727:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.503728:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.503728:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.503730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.503914:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.503917:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677473.503917:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:10.0:1619677473.503919:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.503921:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.503923:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.503924:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.503925:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.503926:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.503927:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.503930:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.503931:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.503934:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.503935:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.503936:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.503937:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.503938:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677473.503938:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677473.503939:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677473.503944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.503953:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.503956:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.503957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.503957:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.503959:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.503962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677473.504145:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.504147:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.504149:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.504150:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.504153:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.504154:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.504155:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.504156:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.504157:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.504158:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.504161:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.504162:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.504164:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.504164:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.504165:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.504167:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.504167:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.504168:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.504168:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.504172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.504178:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.504181:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.504182:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.504182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.504183:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.504186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.504191:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.504193:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.504194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.504196:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.504198:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.504199:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.504200:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.504202:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.504202:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.504203:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.504206:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.504208:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.504210:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.504211:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.504211:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.504213:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.504214:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.504214:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.504215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.504227:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.504229:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.504230:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.504232:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.504233:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.504234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.504339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.504346:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.504349:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.504349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.504352:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.504355:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.504356:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.504357:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.504359:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.504360:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.504361:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.504364:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.504366:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.504368:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.504369:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.504370:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.504372:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.504372:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.504373:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.504374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.504378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.504385:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.504388:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.504388:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.504389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.504390:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.504391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.504396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.504403:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.504405:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.504406:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.504406:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.504406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.504407:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.504485:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.504488:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.504489:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.504489:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.504491:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.504494:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.504709:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.504712:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.504713:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.504715:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.504717:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.504719:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677473.504720:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.504721:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.504722:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.504723:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677473.504727:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.504728:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.504730:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.504731:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677473.504732:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677473.504733:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677473.504734:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.504734:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.504735:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.2:1619677473.506056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.506059:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.506062:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677473.506064:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.506065:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.506067:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.506068:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.506070:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.506071:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.506073:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.506074:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.506077:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.506079:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.506082:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.506083:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.506083:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.506085:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.506086:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.506087:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.506087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.506092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.506101:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.506104:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.506105:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.506105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.506107:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.506109:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.506117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.506124:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.506127:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.506128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.506128:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.506130:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.506130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677473.506254:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.506256:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.506257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.506259:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.506262:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.506263:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.506264:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.506265:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.506267:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.506268:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.506271:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.506272:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.506275:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.506276:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.506277:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.506278:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.506279:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.506280:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.506281:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.506282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677473.506284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.506292:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.506294:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677473.506298:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.506299:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:8.0:1619677473.506299:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.506301:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677473.506301:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.506302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677473.506302:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.506306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.506317:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.506319:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.506320:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.506322:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.506324:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.506326:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.506327:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.506328:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.506329:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.506330:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.506333:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.506335:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.506337:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.506338:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.506338:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.506340:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.506340:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.506342:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.506343:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.506343:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.506346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.506349:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.506351:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.506352:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.506353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.506354:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.506356:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.506356:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.506358:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.506358:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.506359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:6.0:1619677473.506416:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.506419:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.506420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.506422:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.506424:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.506425:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.506427:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.506429:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.506430:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.506431:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.506435:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.506438:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.506441:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.506442:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.506442:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.506444:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.506445:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.506445:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.506446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.506459:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.506461:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.506462:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.506464:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677473.506464:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.506464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.506466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.506467:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.506468:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.506468:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.506470:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.506473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.506482:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.506485:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.506486:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.506486:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.506487:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.506488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.506657:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.506660:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.506661:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.506663:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.506665:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.506666:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677473.506667:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.506669:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.506670:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.506671:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677473.506674:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.506676:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677473.506678:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.506679:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:26.0:1619677473.506680:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677473.506681:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677473.506682:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.506683:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.506684:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.506684:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:12.0:1619677473.506695:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.506697:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:10.0:1619677473.506697:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:12.0:1619677473.506698:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.506699:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:12.0:1619677473.506700:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.506700:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.506701:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677473.506702:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.506703:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677473.506716:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677473.506720:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677473.506721:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.506722:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677473.506723:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.506724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:24.0:1619677473.507421:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.507424:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.507425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.507427:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677473.507428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:24.0:1619677473.507429:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.507430:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.507432:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.507433:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.507434:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.507434:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677473.507435:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.507437:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.507438:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:24.0:1619677473.507439:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677473.507440:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.507441:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.507442:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.507444:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677473.507445:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.507446:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.507446:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677473.507446:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.507447:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.507448:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:1.0:1619677473.507448:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.507449:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677473.507449:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677473.507450:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.507451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677473.507452:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.507454:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.507457:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.507457:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.507458:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.507460:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.507461:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.507461:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.507462:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.507463:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.507466:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.507467:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.507469:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.507470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.507471:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.507472:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.507473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:6.0:1619677473.507473:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.507474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.507474:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.507476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.507491:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.507493:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.507494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.507495:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.507496:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.507496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.507503:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.507506:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.507507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.507509:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.507512:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.507513:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.507515:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.507516:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.507517:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.507518:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.507521:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.507523:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.507526:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.507526:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.507527:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.507529:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.507529:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.507530:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.507530:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.507542:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.507544:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.507545:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.507546:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.507549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.507550:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.507609:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.507611:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.507612:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.507614:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.507616:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.507618:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.507619:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.507620:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.507621:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.507622:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.507625:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.507626:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.507629:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.507630:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.507630:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.507632:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.507632:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.507633:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.507633:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:20.0:1619677473.507645:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.507648:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.507649:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.507650:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.507651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.507653:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.507666:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.507668:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.507669:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.507670:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.507671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.507672:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.507797:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.507799:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.507802:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.507803:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.507804:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.507806:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.507807:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.507808:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.507809:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.507810:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.507813:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.507814:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.507817:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.507817:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.507818:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.507820:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.507820:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.507821:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.507822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.507833:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.507835:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.507836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.507836:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.507838:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.507840:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.507969:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.507972:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.507972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.507974:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.507977:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.507978:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.507979:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.507980:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.507981:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.507982:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.507986:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.507987:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.507990:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.507991:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.507991:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.507993:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.507994:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.507995:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.507996:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.507996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.508006:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.508007:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.508008:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.508009:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677473.508010:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.508010:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.508011:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.508011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.508012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.508014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.508178:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.508181:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.508182:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.508184:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.508187:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.508188:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.508189:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.508191:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.508192:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.508193:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.508197:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.508199:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.508202:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.508203:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.508204:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.508205:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.508206:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.508206:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.508207:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.508219:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677473.508222:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677473.508222:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000400:00000200:22.0:1619677473.508223:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.508223:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677473.508224:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.508225:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.508226:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.508229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.508230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.508230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.508243:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.508245:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.508246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.508247:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.508248:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.508248:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.508518:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.508521:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.508522:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.508523:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.508526:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.508528:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.508529:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.508531:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.508531:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.508532:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.508535:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.508537:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.508540:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.508541:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.508541:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.508543:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.508543:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.508544:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.508544:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.508557:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.508560:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.508561:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.508561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.508563:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.508566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.508649:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.508652:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.508652:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.508655:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.508658:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.508659:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.508660:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.508662:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.508663:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.508664:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.508667:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.508668:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.508671:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.508671:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.508672:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.508674:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.508674:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.508675:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.508675:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.508680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.508688:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.508691:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.508692:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.508692:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.508694:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.508697:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.508708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.508716:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.508718:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.508719:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.508719:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.508720:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.508721:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.508835:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.508838:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.508839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.508841:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.508844:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.508845:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.508846:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.508847:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.508848:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.508849:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.508852:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.508854:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.508856:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.508856:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.508857:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.508858:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.508859:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.508859:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.508860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.508864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.508867:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.508869:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.508870:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.508870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.508872:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.508874:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.508895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.508898:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.508900:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.508901:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.508901:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.508902:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.508902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.509442:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.509444:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.509445:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.509447:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.509449:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.509450:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.509451:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.509452:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.509453:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.509454:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.509457:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.509458:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.509460:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.509461:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.509461:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.509462:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.509463:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.509464:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.509464:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.509469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.509476:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.509478:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.509479:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.509479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.509481:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.509483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.509636:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.509639:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.509640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.509642:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.509644:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.509645:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.509647:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.509648:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.509649:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.509650:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.509654:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.509655:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.509658:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.509658:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.509659:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.509660:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.509661:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.509662:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.509662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.509674:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.509677:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.509677:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.509679:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.509680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.509681:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.509688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.509696:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.509700:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.509702:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.509705:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.509708:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.509710:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.509712:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.509715:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.509716:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.509718:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.509723:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.509726:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.509730:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.509731:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.509732:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.509734:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.509735:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.509737:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.509738:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.509739:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.509740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.509744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677473.509748:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.509751:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677473.509752:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:17.0:1619677473.509752:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.509752:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.509754:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.509755:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.509756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677473.509757:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677473.509759:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677473.510504:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.510507:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.510508:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.510508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.510510:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.510513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.511452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.511458:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.511461:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.511462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.511464:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.511467:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.511468:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.511469:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.511471:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.511472:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.511473:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.511476:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.511478:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.511480:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.511480:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.511481:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.511483:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.511483:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.511484:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.511485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.511490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.511497:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.511499:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.511500:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.511501:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.511502:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.511503:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.511853:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.511856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.511856:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.511858:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.511860:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.511861:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.511862:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.511864:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.511865:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.511866:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.511868:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.511870:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.511872:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.511873:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.511874:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.511876:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.511876:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.511877:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.511877:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.511889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.511892:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.511892:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.511894:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.511895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.511896:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.511918:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.511921:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.511922:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.511924:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.511926:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.511927:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.511928:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.511930:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.511931:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.511932:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.511935:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.511937:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.511939:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.511939:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.511940:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.511942:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.511942:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.511943:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.511943:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.511948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.511953:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.511955:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.511956:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.511956:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.511958:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.511959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:5.2:1619677473.512142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.512148:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.512151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.512151:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.512153:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.512155:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.512157:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.512158:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.512160:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.512161:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.512162:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.512165:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.512167:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.512169:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.512170:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.512171:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.512172:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.512173:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.512174:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.512174:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.512186:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.512188:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.512189:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.512189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.512191:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.512193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.512396:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.512398:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.512399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.512400:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.512402:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.512405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.512525:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.512528:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.512528:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.512530:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.512533:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.512534:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.512535:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.512537:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.512538:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.512539:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.512542:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.512543:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.512546:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.512547:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.512547:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.512549:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.512550:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.512551:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.512551:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.512555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.512563:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.512566:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.512567:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.512567:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.512569:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.512573:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677473.512974:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.512976:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.512978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.512979:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.512982:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.512983:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.512985:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.512986:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.512987:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.512988:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.512991:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.512992:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.512995:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.512996:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.512996:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.512998:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.512998:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.512999:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.513001:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.513001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.513011:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.513014:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.513014:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.513014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.513015:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.513016:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.513017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.513017:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.513017:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.513020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.513285:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.513288:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.513289:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.513291:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.513298:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.513299:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.513300:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.513302:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.513303:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.513304:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.513307:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.513308:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.513311:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.513312:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.513312:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.513314:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.513315:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.513315:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.513316:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.513321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.513330:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.513333:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.513334:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.513334:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.513336:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.513340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.513367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.513369:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.513370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.513372:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.513373:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677473.513374:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.513375:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677473.513376:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.513377:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677473.513377:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.513379:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.513380:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.513380:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.513381:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.513383:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.513385:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.513386:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.513387:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.513387:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.513388:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.513389:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.513390:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.513391:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.513391:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.513392:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.513393:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677473.513394:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.513396:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.513396:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.513396:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.513397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:10.0:1619677473.513399:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.513400:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.513401:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.513402:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.513403:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.513404:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.513404:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.0:1619677473.513407:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.513409:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.513410:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.513411:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.513412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.513414:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.513416:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.513418:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.513419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.513420:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.513422:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.513424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.513429:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.513431:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.513432:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.513433:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.513434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.513435:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.513631:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.513633:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.513635:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.513636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.513638:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.513639:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.513641:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.513642:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.513643:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.513644:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.513647:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.513649:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.513651:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.513652:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.513653:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.513655:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.513655:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.513656:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.513657:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.513668:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.513671:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.513672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.513672:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.513674:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.513676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.513690:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.513692:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.513693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.513694:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.513695:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.513695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.513946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (20)++
00000800:00000200:26.0:1619677473.513973:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.513976:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.513977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.513978:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.513982:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.513983:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.513984:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.513985:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.513986:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677473.513986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (21)++
00000800:00000200:9.2:1619677473.513986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:26.0:1619677473.513987:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.513990:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.513991:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.513993:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.513994:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:2.0:1619677473.513994:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677473.513995:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.513996:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.513996:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.513997:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:2.0:1619677473.513997:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.513998:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.513999:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.513999:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:18.0:1619677473.513999:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.514001:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.2:1619677473.514002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:2.0:1619677473.514005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677473.514006:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:9.2:1619677473.514007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:8.0:1619677473.514009:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.514010:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.514010:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.514012:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:8.0:1619677473.514012:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.514014:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:8.0:1619677473.514014:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:18.0:1619677473.514015:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.514015:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.514016:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:18.0:1619677473.514017:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.514019:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.514019:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.514020:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.514020:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677473.514027:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.514029:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.514030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.514032:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.514034:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.514036:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.514038:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96509060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.514039:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.514040:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.514041:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.514044:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.514046:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.514049:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.514050:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.514050:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.514052:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.514052:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.514053:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.514053:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:24.0:1619677473.514055:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.514055:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677473.514055:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.514056:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.514057:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.514058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677473.514059:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.514062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677473.514065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.514068:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.514068:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.514069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.514069:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.514070:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.514071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677473.514071:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.514072:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677473.514074:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.514075:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.514077:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.514079:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.514079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.514080:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.514081:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.514135:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.514138:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677473.514139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677473.514141:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.514143:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.514145:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677473.514146:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.514148:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.514149:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.514149:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677473.514153:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.514154:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677473.514156:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.514157:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677473.514157:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.2:1619677473.514157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (20)++
00000800:00000200:24.0:1619677473.514159:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677473.514160:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.514161:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677473.514162:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677473.514162:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.2:1619677473.514165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.514171:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.514173:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.514174:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.514174:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677473.514175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000400:00000200:26.0:1619677473.514177:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.514177:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677473.514177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.514178:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:26.0:1619677473.514179:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677473.514201:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.514204:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.514205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.514205:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.514207:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.514210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.514337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (20)++
00000800:00000200:9.2:1619677473.514372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.2:1619677473.514375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (21)++
00000800:00000200:18.0:1619677473.514381:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.514383:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.514384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.514385:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.514387:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.514390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.514417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.2:1619677473.514423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (20)++
00000800:00000200:2.0:1619677473.514424:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.514427:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.514428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.514429:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.514432:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.514437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677473.514460:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.514463:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.2:1619677473.514463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (21)++
00000400:00000200:22.0:1619677473.514464:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.514464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.514467:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.514470:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.514496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.514505:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.514508:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:6.0:1619677473.514508:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:17.0:1619677473.514509:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.514509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.2:1619677473.514510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f3800] (22)++
00000400:00000200:17.0:1619677473.514511:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.514511:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.514512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.514512:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.514514:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.514515:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.514518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:13.0:1619677473.514521:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.514524:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.514525:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.514526:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.514529:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.514534:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.514550:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.2:1619677473.514550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:12.0:1619677473.514553:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.514554:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.514555:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.514555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.514556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.0:1619677473.514558:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.514561:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.514561:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.514563:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.514564:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.514567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.514715:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.514718:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.514719:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.514721:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.514723:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.514724:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.514725:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.514727:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.514728:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.514729:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.514732:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.514733:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.514735:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.514736:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.514736:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.514738:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.514738:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.514739:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.514739:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.514743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.514750:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.514752:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.514753:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.514753:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.514755:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.514757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.514763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.514769:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.514772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.514772:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.514772:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.514773:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.514774:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.514959:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.514962:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.514964:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.514965:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.514967:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.514968:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.514969:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.514971:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.514972:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.514973:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.514976:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.514978:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.514980:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.514981:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.514981:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.514983:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.514983:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.514984:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.514985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.514989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.514997:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.514999:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.514999:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.515001:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677473.515002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.515003:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677473.515024:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.515027:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.515028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.515030:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.515032:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.515033:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.515035:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.515036:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.515037:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.515038:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.515041:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.515043:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.515045:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.515046:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.515047:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.515049:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.515049:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.515050:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.515051:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.515052:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677473.515054:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.515057:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.515058:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.515060:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.515062:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.515062:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:24.0:1619677473.515063:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.515064:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677473.515064:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.515064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677473.515065:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.515065:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.515065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.515066:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.515067:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.515067:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.515068:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.515068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:24.0:1619677473.515069:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677473.515070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:24.0:1619677473.515072:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.515074:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.515077:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.515078:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.515079:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.515081:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.515081:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.515082:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.515083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.515087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.515095:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.515098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.515098:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.515099:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.515101:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.515104:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677473.515478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.515484:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.515487:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.515488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.515490:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.515492:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.515493:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.515495:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.515496:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.515497:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.515498:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.515502:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.515503:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.515506:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.515507:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.515508:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.515510:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.515511:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.515512:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.515512:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677473.516174:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.516177:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.516178:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.516178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.516181:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.516184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.516524:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.516526:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677473.516527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677473.516529:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.516531:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.516533:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677473.516534:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.516536:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.516537:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.516538:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677473.516541:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.516543:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.516546:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.516547:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677473.516548:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677473.516550:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677473.516550:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.516551:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677473.516552:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677473.516552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677473.516556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677473.516563:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:2.0:1619677473.516564:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:22.0:1619677473.516565:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677473.516565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677473.516566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:2.0:1619677473.516566:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.516568:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.516568:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.516570:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.516573:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677473.516706:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.516709:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.516710:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.516712:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.516714:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.516715:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.516717:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.516718:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.516719:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.516720:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.516723:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.516724:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.516727:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.516727:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.516728:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.516730:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.516730:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.516731:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.516731:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.516735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.516742:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.516745:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.516746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.516746:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.516747:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.516749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.516826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.516832:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.516835:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.516836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.516838:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.516840:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.516841:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.516843:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.516844:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.516845:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.516846:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.516849:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.516851:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.516853:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.516854:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.516855:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.516856:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.516857:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.516858:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.516859:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.516860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.516864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.516870:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.516871:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.516872:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.516872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677473.516873:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.516873:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677473.516874:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.516875:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.516877:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.516879:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.517070:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.517073:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.517073:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.517076:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.517078:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.517079:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.517081:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.517082:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.517083:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.517084:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.517087:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.517088:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.517091:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.517092:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.517093:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.517094:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.517095:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.517096:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.517096:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.517109:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.517111:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.517112:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.517114:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.517115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.517116:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677473.517117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.517119:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.517120:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.517122:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.517124:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.517126:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.517127:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.517128:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.517129:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.517130:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.517133:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.517134:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.517137:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.517138:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.517138:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.517140:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.517140:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.517141:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.517143:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.517143:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677473.517146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.517155:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677473.517156:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677473.517158:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.517159:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.517159:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.517159:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677473.517160:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.517160:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677473.517161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.517163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.517270:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.517273:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.517274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.517276:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.517278:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.517280:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.517281:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.517282:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.517284:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.517284:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.517288:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.517289:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.517292:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.517292:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.517298:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.517299:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.517300:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.517301:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.517301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.517314:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.517316:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.517317:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.517319:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.517320:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.517321:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.517367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.517375:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.517378:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.517379:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.517380:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.517382:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.517386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.517389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.517391:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.517393:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.517394:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.517394:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.517395:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.517395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677473.517858:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.517861:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.517862:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.517863:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.517866:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.517868:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.517869:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.517870:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.517871:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.517872:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.517875:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.517876:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.517879:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.517880:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.517881:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.517882:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.517883:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.517883:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.517884:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.517896:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.517899:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.517899:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.517900:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.517902:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.517905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.517919:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.517922:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.517923:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.517923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.517924:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.517924:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.518201:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.518204:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.518205:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.518207:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.518209:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.518211:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.518213:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.518214:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.518215:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.518216:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.518220:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.518221:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.518224:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.518225:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.518225:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.518227:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.518227:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677473.518227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.518228:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.518229:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677473.518235:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.518237:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.518239:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.518239:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.518241:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:2.0:1619677473.518241:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.518243:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.518244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677473.518244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:6.0:1619677473.518245:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.518247:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.518250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.518578:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.518581:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.518584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.518584:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.518586:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.518587:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.518589:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.518590:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.518591:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.518592:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.518595:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.518597:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.518600:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.518601:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.518601:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.518603:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.518604:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.518605:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.518605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.518610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.518617:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.518620:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.518621:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.518621:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.518623:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.518626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.518798:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.518801:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.518802:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.518804:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.518806:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.518808:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.518809:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.518810:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.518811:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.518812:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.518815:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.518817:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.518819:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.518820:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.518820:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.518822:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.518823:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.518823:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.518824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.518835:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.518837:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.518838:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.518840:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.518841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.518842:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.518859:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.518861:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.518862:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.518863:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.518864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.518865:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.518997:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.519000:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.519001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.519003:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.519006:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.519007:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677473.519009:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.519010:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.519011:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.519012:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677473.519015:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.519017:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677473.519019:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.519020:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677473.519021:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677473.519022:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677473.519023:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.519023:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.519024:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677473.519482:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.519484:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.519485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.519487:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.519489:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.519491:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.519492:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.519493:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.519494:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.519495:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.519497:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.519499:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.519501:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.519502:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.519502:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.519504:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.519504:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.519505:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.519505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.519510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.519517:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.519519:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.519520:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.519520:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.519522:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.519524:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.519705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.519712:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.519715:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.519715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.519718:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.519720:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.519721:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.519723:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.519725:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.519726:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.519727:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.519730:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.519732:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.519734:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.519734:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.519735:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.519737:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.519738:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.519739:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677473.519740:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.519740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.519743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.519746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.519752:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.519753:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.519753:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.519754:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677473.519754:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.519755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.519756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.519756:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.519757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677473.519757:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.519766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.519773:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.519775:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.519776:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.519777:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.519777:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.519777:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:6.0:1619677473.519869:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.519872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.519872:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.519873:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.519875:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.519878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.520067:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.520069:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.520070:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.520072:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.520074:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.520075:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.520077:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.520078:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.520079:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.520080:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.520082:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.520084:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.520087:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.520088:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.520088:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.520090:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.520090:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.520091:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.520091:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.520096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.520099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:17.0:1619677473.520104:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677473.520104:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677473.520107:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677473.520107:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.520108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:1.0:1619677473.520108:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.520109:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.520110:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.520111:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.520113:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.520114:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:2.0:1619677473.520115:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.520116:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.520118:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.520119:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.520120:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.520123:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.520125:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.520127:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.520128:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.520129:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.520130:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.520131:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.520132:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.520132:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677473.520136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.520143:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.520146:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.520147:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.520147:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.520149:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.520150:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.520155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.520162:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.520165:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.520165:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.520166:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.520167:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.520167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677473.520195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.520201:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.520203:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.520204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.520206:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.520208:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.520210:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.520211:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.520213:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.520214:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.520215:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.520218:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.520221:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.520224:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.520225:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.520225:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.520227:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.520227:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.520228:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.520229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.520242:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.520246:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.520247:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.520250:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.520252:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.520253:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677473.520473:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.520476:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.520477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.520479:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.520482:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.520483:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.520484:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.520486:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.520487:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.520488:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.520491:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.520492:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.520494:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.520495:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.520496:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.520498:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.520498:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.520499:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.520499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.520512:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.520514:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.520515:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.520517:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.520518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.520520:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.521582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.521588:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.521591:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.521592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.521594:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.521596:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.521597:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.521598:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.521599:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677473.521600:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.521601:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.521602:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:1.0:1619677473.521602:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677473.521603:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677473.521606:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.521607:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.521608:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677473.521610:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.521611:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677473.521612:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677473.521612:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.521612:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:8.0:1619677473.521613:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96509018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.521614:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677473.521614:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.521615:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.521615:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:8.0:1619677473.521616:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.521616:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677473.521617:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.521620:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.521622:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.521625:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.521626:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.521626:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.521628:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.521629:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.521629:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677473.521629:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:8.0:1619677473.521630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.521632:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.521633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.521633:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677473.521634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:6.0:1619677473.521635:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.521637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.521641:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.521644:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.521645:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.521645:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.521648:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.521649:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.521650:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.521651:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.521652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.521653:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.521654:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.521654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.521795:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.521798:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.521799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.521801:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.521803:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.521804:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.521806:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.521807:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.521808:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.521809:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.521812:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.521814:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.521815:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.521816:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.521817:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.521818:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.521819:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.521819:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.521820:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.521824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.521827:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.521829:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.521830:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.521830:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.521832:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.521834:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:6.0:1619677473.521897:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.521900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.521900:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.521901:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.521903:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.521905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.522558:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.522561:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.522562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.522564:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.522566:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.522568:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.522569:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.522571:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.522571:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.522572:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.522575:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.522577:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.522580:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.522580:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.522581:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.522582:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.522583:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.522584:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:10.0:1619677473.522585:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677473.522585:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.522586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677473.522587:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.522588:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.522590:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.522592:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.522594:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.522595:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.522596:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.522597:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677473.522597:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.522598:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:10.0:1619677473.522598:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.522600:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677473.522601:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.522601:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677473.522601:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.522601:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.522601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.522602:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677473.522603:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:22.0:1619677473.522604:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.522606:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.522606:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677473.522607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.522607:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.522609:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.522609:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.522611:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.522612:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.522613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677473.522622:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.522625:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.522625:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.522626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.522626:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.522627:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.522628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677473.522628:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.522629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.522630:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.522661:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.522664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.522664:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.522666:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.522668:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.522670:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677473.522671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677473.522671:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.522673:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:10.0:1619677473.522673:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.522674:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.522674:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.522674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.522676:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.522678:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.522680:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677473.522682:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.522683:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677473.522683:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677473.522684:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96509012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.522684:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.522684:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677473.522686:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.522686:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.522687:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.522687:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.522687:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677473.522688:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.522689:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.522689:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.522691:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.522693:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.522695:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.522696:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.522696:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.522698:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.522698:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.522700:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:26.0:1619677473.522702:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.522702:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.522703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.522704:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677473.522704:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.522705:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.522706:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.522707:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.522708:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.522708:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.522709:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.522713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.522714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.522715:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.522715:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.522716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.522717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.522717:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.522718:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.522720:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.522721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.522722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.522728:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.522731:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.522732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.522732:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.522733:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.522733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.522830:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.522833:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.522834:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.522836:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.522839:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.522840:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677473.522841:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.522843:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.522844:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.522845:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677473.522848:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.522850:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.522852:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.522853:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677473.522854:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677473.522855:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677473.522856:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.522857:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.522857:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:24.0:1619677473.523207:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.523209:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677473.523210:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677473.523212:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.523215:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.523217:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.523218:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96509010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.523219:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.523220:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.523221:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.523225:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.523226:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.523229:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.523230:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.523230:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.523232:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.523232:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.523233:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.523233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677473.523245:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.523248:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.523249:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.523249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.523251:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.523254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.523594:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.523596:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.523597:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.523597:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.523600:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.523602:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.524068:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.524070:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.524071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.524073:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.524076:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.524077:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.524078:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.524079:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.524080:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.524081:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.524084:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.524085:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.524088:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.524089:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.524090:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.524091:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.524092:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.524092:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.524093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.524098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.524106:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.524109:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.524110:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.524112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.524112:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.524115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.524118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.524125:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.524128:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.524129:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.524130:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.524130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.524131:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.524180:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.524183:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.524184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.524186:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.524188:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.524190:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.524191:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.524193:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.524193:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.524195:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.524198:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.524199:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.524202:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.524202:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.524203:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.524205:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.524206:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.524207:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.524207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.524219:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.524221:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.524222:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.524223:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.524224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.524225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.524738:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.524741:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.524742:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.524744:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.524747:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.524749:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.524750:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96509004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.524751:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.524752:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.524753:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.524756:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.524758:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.524761:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.524761:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.524762:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.524764:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.524764:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.524765:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.524765:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.524777:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.524780:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.524781:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.524782:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.524784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.524785:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.524882:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.524884:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.524885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.524887:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.524890:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.524891:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.524892:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.524893:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.524894:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.524895:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.524898:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.524899:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.524901:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.524902:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.524902:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.524904:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.524904:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.524905:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.524905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.524909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.524916:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.524919:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.524920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.524920:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.524922:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.524924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.524973:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.524975:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.524976:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.524977:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.524980:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.524981:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.524982:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96509002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.524983:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.524984:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.524985:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.524988:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.524989:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.524992:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.524993:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.524994:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.524996:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.524996:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.524997:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.524999:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.524999:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677473.525002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.525011:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677473.525013:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:20.0:1619677473.525014:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.525014:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.525015:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.525016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:20.0:1619677473.525016:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.525016:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.0:1619677473.525017:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.525019:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.525603:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.525606:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.525607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.525607:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.525609:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.525612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.525886:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.525889:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.525889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.525892:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.525894:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.525895:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.525896:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96509000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.525898:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.525899:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.525900:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.525903:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.525905:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.525907:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.525908:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.525909:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.525910:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.525911:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.525912:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.525912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.525924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.525926:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.525927:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.525928:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.525929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.525930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677473.525943:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:20.0:1619677473.525945:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677473.525946:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677473.525947:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:20.0:1619677473.525948:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.525949:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.525949:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000400:00000200:24.0:1619677473.525950:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.525950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.525951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:24.0:1619677473.525952:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.525953:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.525955:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.525956:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.525957:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.525958:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.525961:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.525962:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.525964:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.525964:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.525965:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.525966:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.525967:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.525967:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.525968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677473.525972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.525978:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.525980:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.525980:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.525981:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.525982:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.525984:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.525994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.525999:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.526002:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.526002:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.526002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.526003:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.526004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.526371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.526377:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.526380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.526380:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.526382:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.526385:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.526386:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.526387:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.526389:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.526389:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.526390:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.526393:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.526395:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.526397:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.526398:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.526399:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.526400:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.526401:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.526401:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.526402:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.526415:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.526417:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.526418:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.526418:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.526420:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.526422:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:6.0:1619677473.526422:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.526425:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.526426:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.526428:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.526430:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.526431:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.526433:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.526434:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.526435:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.526436:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.526439:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.526441:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.526443:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.526444:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.526444:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.526446:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.526446:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.526447:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.526448:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.526452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.526461:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.526463:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.526464:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.526465:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.526467:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.526468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677473.526898:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.526900:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.526901:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.526903:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.526906:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.526907:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.526908:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.526909:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.526910:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.526911:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.526914:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.526916:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.526918:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.526919:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.526920:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.526921:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.526922:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.526923:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.526923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.526928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.526936:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.526939:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.526940:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.526940:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.526942:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.526945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.527466:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.527469:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.527470:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.527472:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677473.527472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:26.0:1619677473.527474:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.527476:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.527477:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.527478:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.527479:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.527480:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.527480:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677473.527483:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.527485:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:26.0:1619677473.527486:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677473.527487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677473.527489:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.527489:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.527491:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.527491:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677473.527491:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677473.527492:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.527494:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.527494:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677473.527494:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.527495:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677473.527495:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.527496:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677473.527496:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.527497:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677473.527498:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677473.527499:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677473.527501:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.527502:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.527504:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.527505:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.527506:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.527507:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.527508:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.527509:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:2.0:1619677473.527509:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677473.527511:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.527512:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.527513:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.527513:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.527516:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.527517:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677473.527517:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.527519:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.527520:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677473.527521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.527521:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677473.527521:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677473.527522:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.527522:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.527523:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.527524:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.527524:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.527525:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.527525:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.527526:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.527526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677473.527526:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:22.0:1619677473.527527:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677473.527527:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677473.527528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677473.527529:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.527530:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.527532:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.527533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.527536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677473.527536:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.527538:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.527539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.527539:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.527541:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.527544:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677473.527544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.527547:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:26.0:1619677473.527548:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:17.0:1619677473.527548:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.527548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.527550:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:26.0:1619677473.527551:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.527551:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:17.0:1619677473.527553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:26.0:1619677473.527554:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.527557:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.527558:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.527559:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.527561:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.527562:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.527563:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.527566:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.527568:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.527570:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.527571:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.527572:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.527573:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.527573:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.527575:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.527576:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.527577:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.527579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.527582:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.527584:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.527585:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.527585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.527587:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.527589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.527589:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.527591:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.527592:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.527593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.527598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.527601:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.527603:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.527604:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.527604:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.527605:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:8.0:1619677473.527605:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.527675:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.527678:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.527678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.527679:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.527681:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.527684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.528102:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.528105:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.528106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.528108:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.528110:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.528111:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.528112:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.528114:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.528115:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.528116:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.528119:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.528120:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.528123:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.528124:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.528124:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.528126:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.528127:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.528128:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.528128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.528140:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.528142:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.528143:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.528144:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.528145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.528147:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.528857:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.528860:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677473.528861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677473.528862:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.528865:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.528866:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677473.528868:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.528869:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.528870:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.528871:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677473.528874:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.528876:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677473.528879:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.528879:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677473.528880:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677473.528882:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677473.528882:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.528883:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.528883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677473.528889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.528897:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.528899:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.528901:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.528901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.528903:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.528905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677473.529341:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.529344:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.529345:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.529345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.529347:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.529350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.529636:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.529638:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.529639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.529641:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.529643:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.529644:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.529646:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.529647:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.529648:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.529649:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.529652:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.529653:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.529656:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.529656:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.529657:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.529659:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.529659:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.529660:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.529661:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.529662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.529672:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.529672:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.529673:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.529674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.529674:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.529675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.529675:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.529677:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.529678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.529679:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677473.529721:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.529724:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.529725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.529727:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.529729:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.529731:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.529732:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.529733:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.529734:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.529735:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.529739:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.529741:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.529743:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.529744:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.529745:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.529746:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.529747:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.529748:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.529748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.529760:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.529763:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.529764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.529764:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.529766:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.529769:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.530075:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.530078:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.530079:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.530081:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.530083:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.530084:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.530085:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.530087:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.530088:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.530088:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.530091:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.530093:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.530095:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.530095:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.530096:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.530098:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.530098:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.530099:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.530099:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.530103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.530110:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.530112:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.530113:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.530113:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.530115:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.530117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.530124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.530130:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.530132:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.530133:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.530133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.530134:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.530135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.530185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.530191:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.530194:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.530195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.530197:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.530200:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.530201:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.530202:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.530204:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.530205:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.530206:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.530209:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.530210:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.530212:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.530213:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.530214:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.530215:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.530216:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.530217:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.530217:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.530221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.530229:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.530231:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.530231:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.530232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.530233:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.530234:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.530239:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.530242:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.530243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.530245:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.530247:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.530249:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.530250:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.530251:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.530252:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.530253:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.530256:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.530258:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.530261:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.530261:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.530262:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.530264:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.530264:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.530265:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.530265:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.530270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.530277:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.530280:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.530281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.530281:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.530283:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.530287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677473.531063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.531068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.531071:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.531072:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.531074:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.531077:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.531078:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.531079:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.531081:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.531082:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.531083:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.531086:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.531087:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.531090:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.531091:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.531091:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.531093:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.531093:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.531094:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.531094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.531108:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.531110:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.531111:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.531112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.531114:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.531116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.531124:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.531127:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.531128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.531128:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.531129:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.531130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.531204:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.531207:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.531208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677473.531210:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677473.531212:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.531213:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.531214:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.531216:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.531217:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.531218:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.531221:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.531223:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.531224:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.531225:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.531225:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.531227:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.531227:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.531228:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677473.531230:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677473.531230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:9.2:1619677473.531232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.531241:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.531242:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.531242:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.531244:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.531244:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677473.531244:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677473.531245:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.531246:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.531246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.531247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677473.531337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.531340:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.531340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.531341:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.531343:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.531348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.531349:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.531350:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.531351:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.531351:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.531352:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.531353:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.531354:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.531354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.531355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.531356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:22.0:1619677473.531357:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.531359:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.531360:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.531361:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.531363:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.531364:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.531367:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.531369:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.531371:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.531372:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.531373:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.531374:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.531375:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.531375:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.531376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.531388:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.531390:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.531391:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.531393:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.531394:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.531395:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.532268:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.532271:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.532272:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.532274:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.532277:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.532278:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.532279:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.532280:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.532282:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.532283:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.532286:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.532288:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.532290:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.532291:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.532292:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.532300:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.532301:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.532302:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.532303:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.532304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677473.532309:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.532311:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.532312:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.532313:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.532315:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.532317:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.532318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.532319:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.532320:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.532321:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677473.532352:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.532354:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.532355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.532357:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.532360:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.532361:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.532363:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.532364:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.532365:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.532366:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.532369:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.532370:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.532373:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.532373:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.532374:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.532375:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.532376:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.532377:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.532377:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.532389:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.532392:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.532393:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.532393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.532396:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.532398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.532481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677473.532487:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.532490:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677473.532491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677473.532493:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.532495:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.532497:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677473.532498:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.532499:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.532501:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.532501:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677473.532505:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.532507:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677473.532510:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.532511:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677473.532512:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677473.532513:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677473.532514:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.532514:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677473.532515:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677473.532519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.532526:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.532529:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.532530:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.532531:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.532533:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.532536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.532592:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.532595:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.532596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.532598:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.532600:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.532601:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.532602:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.532604:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.532605:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.532606:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.532609:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.532611:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.532614:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.532615:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.532615:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.532617:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.532618:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.532618:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.532619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.532623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.532632:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.532635:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.532635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.532636:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.532638:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.532642:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.532758:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.532760:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.532761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.532762:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.532764:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.532767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.533034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.533040:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.533042:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.533044:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.533045:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.533047:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.533049:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.533050:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.533052:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.533053:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.533054:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.533057:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.533059:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.533062:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.533063:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.533064:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.533065:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.533066:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.533067:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.533068:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.533069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.533074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.533080:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.533082:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.533083:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.533083:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677473.533575:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.533578:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.533579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.533581:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.533583:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.533585:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.533586:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.533588:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.533589:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.533590:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.533593:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.533594:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.533597:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.533597:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.533598:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.533600:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.533600:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.533601:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.533601:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.533615:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.533617:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.533618:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.533620:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.533621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.533622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.533632:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.533635:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.533636:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.533637:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.533637:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.533639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.533640:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.533642:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.533645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.533645:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.533647:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.533649:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.533650:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.533651:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.533652:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.533653:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.533657:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.533658:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.533661:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.533662:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.533662:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.533664:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.533664:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.533666:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.533667:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.533667:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677473.533677:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.533678:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.533680:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:6.0:1619677473.533680:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.533681:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.533681:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:6.0:1619677473.533681:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.533682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677473.533683:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.533686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.533765:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.533768:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.533769:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.533771:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.533773:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.533775:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.533776:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.533777:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.533778:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.533779:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.533783:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.533785:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.533788:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.533789:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.533789:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.533791:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.533791:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.533792:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.533792:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.533798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.533806:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.533809:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.533809:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.533811:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.533812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.533815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.533922:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.533925:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.533925:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.533927:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.533930:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.533931:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.533933:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.533934:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.533935:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.533936:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.533939:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.533940:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.533943:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.533943:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.533944:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.533946:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.533946:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.533947:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.533947:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:20.0:1619677473.533958:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.533961:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.533962:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.533964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.533964:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.533967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.534135:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.534138:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.534139:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.534141:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.534144:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.534145:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.534146:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.534148:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.534148:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.534149:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.534152:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.534154:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.534157:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.534158:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.534158:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.534160:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.534160:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.534162:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.534163:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.534163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.534166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.534175:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.534176:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.534177:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.534178:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677473.534178:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.534179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677473.534179:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.534179:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677473.534181:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.534184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.534265:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.534268:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.534269:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.534269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.534271:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.534274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.534283:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.534286:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.534287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.534287:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.534288:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.534289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.534346:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.534349:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.534351:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.534351:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.534353:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.534355:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.534356:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.534357:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.534359:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.534360:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.534363:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.534364:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.534367:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.534368:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.534368:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.534370:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.534371:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.534371:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.534372:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.534376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.534384:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.534386:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.534387:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.534389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.534389:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677473.534391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.534395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.534403:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.534405:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.534406:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.534407:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677473.534407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.534409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677473.534749:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.534752:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.534753:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.534755:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.534758:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.534759:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.534760:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.534762:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.534763:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.534764:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.534767:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.534768:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.534771:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.534772:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.534773:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.534775:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.534775:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.534776:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.534776:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.534780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.534789:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677473.534789:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677473.534792:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677473.534792:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.534793:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677473.534793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.534794:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.534794:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.534795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:10.0:1619677473.534796:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.534798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:10.0:1619677473.534799:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.534800:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.534802:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.534803:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.534803:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.534806:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.534808:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.534810:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.534810:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.534811:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.534812:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.534813:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.534813:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.534814:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.534818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.534825:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.534827:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.534828:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.534828:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.534830:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.534832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.535127:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.535130:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.535130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.535132:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.535135:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.535136:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.535137:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.535139:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.535140:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.535141:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.535144:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.535145:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.535147:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.535148:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.535149:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.535150:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.535151:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.535152:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.535152:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.535164:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.535166:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.535167:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.535168:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.535169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.535171:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.535393:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.535396:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.535396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.535398:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.535401:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.535402:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.535403:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.535405:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.535406:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.535407:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.535410:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.535412:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.535415:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.535416:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677473.535416:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.535418:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.535419:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.535419:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.535420:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:20.0:1619677473.535433:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.535435:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.535436:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.535438:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.535439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.535440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.535813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.535819:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.535822:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.535822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.535824:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.535827:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.535829:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.535830:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.535831:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.535832:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.535833:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.535837:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.535838:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.535841:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.535842:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.535842:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.535844:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.535844:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.535846:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.535847:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.535847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.535852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.535857:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.535858:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.535860:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.535860:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677473.535861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.535861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677473.535862:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.535862:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677473.535864:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.535866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.535993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.536002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.536005:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.536006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.536006:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.536008:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.536011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.536018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.536025:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.536028:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.536029:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.536029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.536030:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.536031:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.536291:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.536296:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.536299:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.536300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.536302:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.536304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.536421:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.536423:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.536424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.536426:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.536429:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.536431:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677473.536432:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.536433:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.536435:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.536435:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677473.536439:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.536440:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677473.536443:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.536444:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677473.536445:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677473.536447:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677473.536447:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.536448:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.536448:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.536453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.536461:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.536464:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.536465:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.536465:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.536467:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.536471:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.536728:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.536731:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.536731:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.536733:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.536737:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.536738:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.536739:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.536741:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.536742:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.536743:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.536746:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.536747:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.536750:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.536751:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.536751:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.536753:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.536753:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.536754:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.536755:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.536767:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.536769:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.536770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.536771:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.536773:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.536775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.536864:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.536866:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.536867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.536869:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.536871:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.536872:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.536873:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.536875:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.536876:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.536877:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.536879:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.536881:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.536883:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.536884:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.536884:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.536886:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.536886:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.536887:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.536888:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.536889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.536899:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.536901:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.536901:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.536902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.536903:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.536903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.536904:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.536906:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.536907:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.536908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.536919:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.536921:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.536922:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.536923:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.536924:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.536925:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677473.537072:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.537075:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.537075:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.537078:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.537079:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677473.537080:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.537082:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677473.537082:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677473.537082:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677473.537084:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.537085:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.537087:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.537087:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.537088:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677473.537090:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.537092:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.537092:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677473.537094:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677473.537094:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.537095:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.537096:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677473.537097:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.537098:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:10.0:1619677473.537098:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.537099:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.537100:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.537101:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.537101:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:10.0:1619677473.537101:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.537102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:10.0:1619677473.537103:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.537106:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.537107:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.537107:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.2:1619677473.537107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677473.537109:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.537109:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.537110:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.537111:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677473.537115:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.537117:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.537118:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.537120:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.537121:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:22.0:1619677473.537122:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.537122:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677473.537125:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.537126:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.537126:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.537128:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.537131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.537438:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.537441:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677473.537442:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677473.537444:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.537446:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.537448:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677473.537449:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.537451:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.537452:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.537453:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677473.537457:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.537458:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677473.537462:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.537463:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677473.537463:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677473.537465:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677473.537465:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.537466:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.537467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677473.537480:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.537483:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.537484:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.537485:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.537488:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.537489:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:26.0:1619677473.537701:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.537704:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.537705:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.537706:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.537709:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.537711:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677473.537712:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.537713:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.537714:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.537715:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677473.537718:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.537719:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677473.537721:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.537722:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677473.537723:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677473.537724:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677473.537725:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.537725:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.537726:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677473.538084:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.538087:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.538088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.538088:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.538090:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.538093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.538187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.0:1619677473.538188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.538190:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.538192:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.538193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:18.0:1619677473.538193:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:21.0:1619677473.538195:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.538196:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:18.0:1619677473.538196:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.538196:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:21.0:1619677473.538198:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.538199:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.538199:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677473.538201:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.538202:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677473.538202:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.538203:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:21.0:1619677473.538205:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677473.538205:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.538207:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:18.0:1619677473.538207:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.538208:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.538209:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.0:1619677473.538211:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.538212:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.538212:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:18.0:1619677473.538213:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.538214:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.538215:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:18.0:1619677473.538215:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.0:1619677473.538216:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.538216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:18.0:1619677473.538218:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.538219:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.538219:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:9.2:1619677473.538220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677473.538221:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.538222:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.538223:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.538224:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677473.538227:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.538230:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.538231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.538231:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.538233:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.538234:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:8.0:1619677473.538236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677473.538237:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.538237:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.538239:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.538242:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.538243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677473.538245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:20.2:1619677473.538288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:13.0:1619677473.538290:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.538292:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.538299:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.538299:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:13.0:1619677473.538301:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.538303:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:24.0:1619677473.538652:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.538655:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.538656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.538657:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.538659:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.538661:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.538662:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.538663:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.538664:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.538665:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.538668:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.538669:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.538672:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.538673:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.538673:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.538675:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.538676:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.538676:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.538677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.538682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.538690:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.538692:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.538693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.538694:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.538696:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.538699:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.539004:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.539007:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.539008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.539009:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.539012:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.539013:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.539014:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.539016:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.539017:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.539017:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.539020:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.539022:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.539024:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.539025:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.539025:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.539026:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.539027:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.539028:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.539028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.539032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.539038:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.539040:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.539041:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.539042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.539043:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.539046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.539082:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.539084:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.539085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.539087:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.539089:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.539091:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.539092:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.539094:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.539095:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.539096:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.539099:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.539101:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.539103:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.539104:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.539104:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.539106:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.539106:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.539107:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.539108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.539119:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.539122:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.539122:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.539124:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.539125:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.539127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.539266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.539272:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.539275:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.539275:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.539277:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.539280:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.539282:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677473.539283:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.539284:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.539285:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.539287:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677473.539290:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.539292:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677473.539304:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.539305:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677473.539306:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677473.539308:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677473.539308:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.539309:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.539310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677473.539314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.539322:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.539325:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.539326:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.539326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.539328:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.539331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.539333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.539339:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.539342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.539342:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.539343:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.539344:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.539344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.539814:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.539816:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.539817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.539819:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.539822:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.539823:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.539824:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.539825:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.539826:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.539827:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.539830:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.539832:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.539835:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.539835:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.539836:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.539838:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.539838:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.539839:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.539839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.539844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.539851:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.539854:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.539855:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.539855:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.539857:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.539861:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.539901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.539906:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.539909:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.539909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.539912:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.539914:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.539915:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.539916:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.539918:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.539919:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.539920:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.539923:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.539925:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.539927:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.539927:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.539928:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.539930:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.539930:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.539931:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.539932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.539936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.539943:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.539944:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.539945:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.539946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.539947:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.539948:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.539956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.539963:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.539965:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.539966:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.539966:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.539966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.539967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.539987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.539990:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.539991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.539993:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.539995:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.539996:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.539998:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.539999:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.540000:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.540001:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.540004:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.540006:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.540008:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.540009:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.540010:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.540012:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.540012:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.540013:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.540013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.540025:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.540027:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.540028:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.540030:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.540031:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.540032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.540356:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.540358:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.540359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.540360:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.540362:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.540365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.540374:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.540377:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.540377:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.540378:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.540380:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.540380:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.540404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.540412:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.540414:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.540415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.540415:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.540417:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.540420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.540477:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.540479:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.540480:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.540482:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.540484:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.540485:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.540486:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.540488:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.540489:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.540489:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.540492:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.540494:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.540496:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.540497:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.540497:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.540499:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.540499:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.540500:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.540500:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.540505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.540508:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.540510:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.540511:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.540511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.540513:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.540515:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.540631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677473.540637:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.540640:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.540641:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.540643:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.540645:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.540647:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677473.540648:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.540649:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.540651:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.540652:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677473.540655:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.540657:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677473.540660:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.540661:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677473.540662:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677473.540664:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677473.540664:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.540665:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.540666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677473.540670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.540676:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.540679:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.540680:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.540680:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.540683:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.540686:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:22.0:1619677473.541029:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.541032:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.541032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.541035:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.541037:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.541038:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.541039:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.541041:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.541042:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.541043:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.541045:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.541047:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.541049:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.541050:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.541051:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.541052:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.541053:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.541053:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.541054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.541066:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.541068:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.541069:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.541071:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.541072:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.541073:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.541608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.541617:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.541620:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.541621:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.541621:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.541623:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.541626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.541629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.541630:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.541632:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.541633:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.541633:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.541634:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.541635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677473.541683:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.541686:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.541687:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.541687:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.541689:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.541692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.541868:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.541870:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.541871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.541872:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.541875:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.541876:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.541877:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.541878:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.541879:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.541880:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.541885:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.541887:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.541890:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.541891:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.541892:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.541894:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.541894:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.541895:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.541896:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.541897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.541907:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.541909:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.541909:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.541910:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.541911:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.541911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.541912:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.541915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677473.541919:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.541923:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.541924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.541926:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.541927:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.541927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.541928:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.541929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.541942:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.541944:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.541945:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.541947:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.541949:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.541950:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.541952:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.541954:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.541955:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.541956:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.541958:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.541960:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677473.541961:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.541962:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.541963:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.541964:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.541966:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.541966:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677473.541967:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:10.0:1619677473.541967:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.541968:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677473.541969:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.541969:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.541970:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677473.541970:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.541971:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.541973:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.541974:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:10.0:1619677473.541974:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.541975:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.541976:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.541976:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.541978:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.541979:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.541980:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.541981:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.541982:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:22.0:1619677473.541983:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.541983:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.541984:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.541985:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:22.0:1619677473.541986:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.541987:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.541987:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.541987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.541988:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.541988:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.541988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.541989:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.541992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:12.0:1619677473.541996:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677473.541998:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677473.541999:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:26.0:1619677473.542000:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.542001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:12.0:1619677473.542001:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.542001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:12.0:1619677473.542004:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.542005:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.542007:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.542008:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)++
00000800:00000200:13.0:1619677473.542008:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.542008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.542009:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.542010:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.0:1619677473.542363:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.542365:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.542368:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.542368:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.542370:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.542371:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.542373:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.542374:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.542375:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.542376:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.542379:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.542381:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.542384:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.542385:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.542385:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.542387:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.542388:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.542388:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.542389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.542393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.542401:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.542403:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.542404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.542404:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.542407:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.542409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.542749:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.542752:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.542753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.542754:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.542757:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.542758:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.542759:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.542760:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.542761:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.542762:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.542765:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.542767:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.542769:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.542770:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.542771:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.542772:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.542773:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.542774:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.542774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.542786:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.542788:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.542788:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.542790:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.542791:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.542793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.542993:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.542996:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.542997:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.542999:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.543001:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.543003:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.543004:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.543006:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.543007:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.543008:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.543013:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.543015:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.543018:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.543019:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.543020:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.543022:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.543022:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.543023:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.543023:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.543036:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.543038:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677473.543040:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:22.0:1619677473.543041:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.543042:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.543043:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.543044:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.543045:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.543046:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677473.543048:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.543049:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.543051:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.543052:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.543055:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.543056:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.543056:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.543057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.543058:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677473.543061:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.543062:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677473.543063:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677473.543065:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677473.543065:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.543066:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677473.543066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677473.543071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.543078:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.543081:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.543082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.543082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.543084:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.543087:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677473.543255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.543264:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.543266:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.543267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.543268:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.543270:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.543273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.543780:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.543783:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.543783:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.543786:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.543788:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.543789:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.543791:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.543792:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.543793:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.543794:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.543797:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.543799:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.543801:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.543802:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.543803:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.543805:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.543805:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.543806:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.543806:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.543820:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.543822:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.543823:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.543824:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.543826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.543827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.543840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.543848:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.543852:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.543853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.543857:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.543860:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.543863:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.543865:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.543867:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.543869:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.543871:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.543876:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.543879:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.543882:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.543884:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.543885:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.543887:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.543888:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.543889:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.543889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.543900:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.543903:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.543904:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.543904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.543906:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.543909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.543921:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.543924:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.543925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.543925:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.543926:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.543927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.544104:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.544107:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.544108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.544109:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.544112:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.544113:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.544115:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.544116:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.544117:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.544118:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.544122:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.544123:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.544126:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.544127:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.544127:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.544129:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.544129:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.544130:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.544131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.544142:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.544145:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.544146:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.544146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.544148:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.544151:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.544162:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.544165:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.544166:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.544166:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.544167:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.544167:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.544442:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.544445:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.544446:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.544448:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.544451:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.544452:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.544453:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.544455:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.544456:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.544456:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.544459:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.544461:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.544464:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.544465:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.544465:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.544467:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.544467:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.544469:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.544470:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.544470:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.544484:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.544486:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.544487:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.544488:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.544492:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.544495:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.544496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.544497:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.544499:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.544502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.544629:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.544632:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.544633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.544634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.544637:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.544640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.544641:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.544642:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.544642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.544644:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.544647:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.544648:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.544649:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.544650:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.544652:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.544652:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.544655:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.544657:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.544659:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.544660:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.544661:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.544662:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.544663:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.544663:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.544664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.544676:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.544678:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.544679:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.544680:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.544681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.544683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.545193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.545200:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.545203:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.545203:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.545205:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.545208:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.545209:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.545211:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.545212:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.545213:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.545214:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.545217:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.545219:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.545221:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.545222:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.545223:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.545225:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.545225:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.545226:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.545227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.545232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.545241:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.545243:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.545244:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.545244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.545246:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.545249:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.545252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.545260:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.545262:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.545263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.545264:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.545265:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.545265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.545501:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.545504:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.545505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.545507:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.545509:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.545510:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.545511:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.545513:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.545514:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.545515:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.545518:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.545519:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.545521:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.545522:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.545523:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.545525:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.545525:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.545527:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.545528:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.545528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.545539:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.545539:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.545541:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.545541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.545542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.545542:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.545542:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.545544:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.545545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.545546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.545557:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.545560:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.545561:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.545561:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.545562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.545563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.545733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677473.545738:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.545740:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.545741:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677473.545741:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:8.0:1619677473.545743:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.545744:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.545745:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.545745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:8.0:1619677473.545746:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.545748:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.545748:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.545750:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96508907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.545751:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:8.0:1619677473.545751:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.545752:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.545753:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.545756:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.545757:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.545760:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.545761:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.545761:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.545763:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.545764:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.545765:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.545767:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.545767:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.545769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.545777:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.545780:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.545780:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.545781:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.545781:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.545782:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.545783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.545784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677473.545784:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.545787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.546527:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.546529:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.546530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.546533:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.546535:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.546536:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.546537:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.546538:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.546540:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.546541:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.546544:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.546546:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.546548:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.546549:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.546550:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.546551:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.546552:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.546552:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.546553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.546566:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.546568:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.546569:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.546571:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.546572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.546573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.546587:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.546590:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.546590:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.546591:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.546593:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.546596:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.546881:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.546884:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.546885:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.546887:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.546890:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.546891:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.546892:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.546894:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.546895:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.546895:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.546898:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.546900:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.546903:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.546904:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.546904:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.546906:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.546906:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.546907:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.546907:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.546919:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.546922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.546922:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.546923:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.546925:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.546928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.547375:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.547377:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.547378:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.547380:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.547382:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.547383:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.547384:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.547386:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.547387:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.547388:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.547391:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.547392:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.547395:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.547395:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.547396:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.547398:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.547398:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.547398:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.547399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.547412:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.547415:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.547415:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.547417:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.547419:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.547419:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.547455:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.547458:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.547458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.547461:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.547464:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.547465:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.547467:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.547468:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.547469:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.547470:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.547474:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.547476:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.547479:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.547480:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.547480:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.547482:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.547482:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.547483:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.547484:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.547488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.547497:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.547500:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.547501:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.547502:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.547504:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.547507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677473.547555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.547560:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.547563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.547563:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677473.547565:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.547568:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.547569:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.547570:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.547572:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.547573:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.547573:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.547576:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.547578:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.547580:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.547581:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.547581:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.547583:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.547584:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.547584:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.547585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.547589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.547596:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.547598:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.547598:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.547599:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.547600:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.547601:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.547839:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.547842:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.547843:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.547845:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.547848:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.547849:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.547850:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.547851:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.547852:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.547853:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.547856:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.547858:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.547860:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.547861:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.547862:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.547864:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.547864:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.547865:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.547865:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.547877:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.547880:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.547881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.547881:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.547884:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.547887:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.547895:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.547898:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.547898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.547899:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.547900:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.547900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.548129:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.548132:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.548133:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.548135:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.548137:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.548139:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.548140:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.548141:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.548142:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.548143:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.548146:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.548147:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.548150:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.548150:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.548151:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.548152:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.548153:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.548154:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.548155:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.548156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.548159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.548162:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.548165:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.548166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.548166:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.548168:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:8.0:1619677473.548168:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.548170:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:8.0:1619677473.548170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.548171:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.548172:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:6.0:1619677473.548209:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.548212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.548212:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.548214:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.548216:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.548218:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.548219:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.548220:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.548221:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.548222:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.548225:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.548227:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.548229:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.548230:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.548231:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.548232:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.548233:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.548233:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.548234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.548246:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.548248:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.548249:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.548250:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.548251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.548253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.548307:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.548310:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.548310:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.548311:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.548313:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.548316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.548364:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.548367:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.548368:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.548369:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.548372:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.548374:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.548375:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.548376:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.548377:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.548378:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.548381:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.548382:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.548385:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.548386:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.548386:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.548388:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.548388:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.548389:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.548390:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.548401:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.548404:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.548405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.548405:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.548407:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.548410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.548770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.548776:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.548779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.548779:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.548781:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.548784:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.548785:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.548787:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.548788:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.548789:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.548790:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.548793:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.548795:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.548798:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.548798:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.548799:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.548801:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.548801:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.548802:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.548802:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.548814:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.548817:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.548818:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.548818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.548820:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.548822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.549229:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.549231:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.549232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.549234:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.549236:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.549238:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.549239:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.549240:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.549242:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.549243:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.549246:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.549248:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.549250:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.549251:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.549252:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.549254:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.549254:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.549255:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.549256:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.549257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.549267:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.549267:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.549268:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.549269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.549270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.549270:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.549271:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.549272:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.549274:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.549275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.549285:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.549288:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.549289:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.549289:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.549290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.549291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.549363:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.549366:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.549367:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.549369:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.549372:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.549373:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.549375:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.549376:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.549377:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.549378:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.549381:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.549383:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.549386:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.549387:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.549387:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.549389:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.549389:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.549390:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.549390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.549395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.549403:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.549406:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.549407:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.549407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.549409:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.549413:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.549471:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.549473:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.549474:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.549475:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.549478:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.549479:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.549481:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.549482:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.549483:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.549484:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.549487:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.549488:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.549491:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.549492:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.549493:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.549494:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.549495:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.549496:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.549496:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.549510:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.549513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.549513:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.549514:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.549516:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.549519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677473.549975:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.549978:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.549978:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677473.549980:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.549983:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.549984:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.549985:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.549987:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.549988:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.549989:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.549992:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.549994:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.549997:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.549997:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.549998:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.550000:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.550000:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.550001:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677473.550002:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.550003:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:22.0:1619677473.550013:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.550016:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677473.550016:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:22.0:1619677473.550017:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.550017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.550018:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677473.550018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:10.0:1619677473.550018:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:22.0:1619677473.550019:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.550019:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677473.550021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.550021:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.550022:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.550023:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.550026:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.550027:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.550029:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.550030:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.550031:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.550032:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.550036:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.550038:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.550040:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.550040:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.550041:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.550042:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.550043:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.550044:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.550044:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.550048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.550052:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.550054:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.550055:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.550055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.550057:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.550059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:6.0:1619677473.550096:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.550099:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.550100:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.550102:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.550104:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.550105:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.550106:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.550108:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.550109:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.550110:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.550113:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.550115:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.550117:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.550118:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.550119:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.550121:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.550121:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.550122:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.550122:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.550134:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.550137:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.550137:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.550139:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.550140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.550141:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.550505:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.550508:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.550509:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.550509:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.550511:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.550514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.550523:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.550526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.550526:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.550527:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.550528:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.550528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.550543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.550549:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.550552:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.550553:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.550555:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.550557:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.550558:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.550560:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.550562:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.550563:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.550564:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.550567:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.550569:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.550572:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.550573:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.550573:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.550575:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.550576:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.550577:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.550577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677473.550577:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.550581:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.550582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.550585:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.550587:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.550589:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.550590:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.550591:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.550593:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.550594:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.550599:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.550601:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.550604:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.550605:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.550605:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.550607:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.550608:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.550609:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.550610:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.550610:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.550614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.550621:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677473.550622:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677473.550623:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.550623:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677473.550624:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677473.550624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677473.550625:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.550627:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.550628:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.550629:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677473.550928:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.550931:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.550932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.550934:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.550936:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.550938:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.550939:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.550940:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.550941:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.550942:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.550946:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.550947:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.550950:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.550950:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.550951:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.550953:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.550953:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.550954:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.550954:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.550966:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.550968:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.550969:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.550971:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.550972:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.550973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.551539:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.551541:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.551542:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.551544:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.551546:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.551547:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.551548:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.551550:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.551551:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.551551:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.551554:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.551556:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.551558:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.551559:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.551559:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.551561:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.551561:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.551562:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.551562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.551567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.551571:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.551573:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.551574:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.551574:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.551576:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.551577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.551587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.551589:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.551591:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.551592:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.551592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.551593:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.551594:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.551759:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.551761:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.551762:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.551763:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.551765:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.551767:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.551768:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.551769:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.551770:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.551771:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.551774:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.551775:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.551778:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.551779:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.551779:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.551781:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.551781:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.551783:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.551784:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.551784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.551797:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.551799:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.551799:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.551800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.551806:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.551808:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.551809:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.551810:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.551812:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.551814:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.551948:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.551951:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.551951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.551953:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.551955:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.551957:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.551958:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.551959:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.551960:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.551961:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.551964:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.551966:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.551968:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.551969:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.551969:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.551971:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.551971:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.551972:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.551972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.551984:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.551987:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.551988:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.551989:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.551990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.551992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.552189:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.552192:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.552194:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.552194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.552197:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.552199:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.552200:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.552201:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.552202:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.552204:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.552207:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.552209:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.552210:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.552212:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.552212:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (23)++
00000800:00000200:6.0:1619677473.552212:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677473.552213:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (24)++
00000800:00000200:22.0:1619677473.552214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.552214:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.552216:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677473.552216:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:6.0:1619677473.552216:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.552217:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.552218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.552218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.552229:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.552231:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.552232:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.552232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.552233:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.552236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.552381:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.552384:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.552384:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.552387:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.552389:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.552391:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.552392:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.552393:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.552394:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.552395:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.552398:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.552400:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.552403:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.552404:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.552405:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.552406:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.552407:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.552407:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.552408:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.552413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.552422:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.552424:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.552425:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.552427:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.552428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.552430:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.552432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.552440:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.552443:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.552444:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.552444:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.552445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.552446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.552825:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.552828:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.552829:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.552831:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.552833:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.552834:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.552835:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.552836:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.552838:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.552838:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.552842:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.552843:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.552846:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.552847:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.552847:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.552849:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.552849:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.552850:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.552851:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.552851:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.552861:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.552864:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.552864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.552864:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.552865:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.552866:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.552867:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.552869:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.552870:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.552871:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.552882:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.552886:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.552887:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.552888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.552888:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.552889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.552894:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.552897:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.552898:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.552900:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.552902:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.552904:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.552905:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.552907:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.552907:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.552909:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.552911:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.552913:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.552915:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.552916:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.552917:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.552918:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.552918:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.552919:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.552920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.552924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.552930:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.552932:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.552933:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.552933:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.552935:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.552937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.552998:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.553001:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.553002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.553003:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.553006:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.553007:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.553008:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.553009:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.553010:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.553011:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.553015:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.553017:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.553019:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.553020:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.553021:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.553023:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.553023:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.553024:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.553024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.553036:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.553039:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.553040:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.553040:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.553042:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.553045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.553057:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.553060:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.553060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.553061:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.553062:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.553063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.553080:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.553083:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.553084:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.553086:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.553088:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.553089:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.553090:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.553092:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.553093:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.553093:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.553097:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.553098:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.553100:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.553101:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.553102:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.553103:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.553103:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.553104:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.553104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.553108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.553112:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.553114:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.553115:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.553116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.553117:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.553119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:5.2:1619677473.553451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.553457:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.553460:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.553461:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.553463:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.553466:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.553467:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.553468:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.553470:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.553471:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.553472:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.553475:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.553477:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.553480:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.553480:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.553481:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.553483:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.553483:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.553484:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.553484:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.553498:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.553500:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.553501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.553501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.553503:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.553505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.553513:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.553516:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.553517:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.553517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.553518:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.553519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.553632:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.553635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.553635:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.553637:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.553639:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.553641:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.553642:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.553643:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.553644:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.553645:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.553648:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.553650:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.553652:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.553653:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.553654:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.553655:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.553656:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.553656:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.553657:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.553669:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.553672:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.553672:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.553674:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.553675:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.553676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.553856:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.553858:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:22.0:1619677473.553859:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:10.0:1619677473.553859:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677473.553861:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.553862:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.553863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:26.0:1619677473.553864:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.553865:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677473.553865:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.553866:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.553867:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.553868:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.553869:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.553869:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677473.553870:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677473.553870:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.553872:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.553873:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.553874:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677473.553876:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.553878:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:22.0:1619677473.553880:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.553882:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.553882:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.553883:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677473.553883:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677473.553885:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.553885:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.553886:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:22.0:1619677473.553886:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.553886:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677473.553887:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677473.553888:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:26.0:1619677473.553888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:22.0:1619677473.553889:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677473.553889:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.553890:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.553891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677473.553891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.553899:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.553901:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:18.0:1619677473.553902:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.553903:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.553903:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:2.0:1619677473.553903:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.553904:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:18.0:1619677473.553905:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.553905:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:18.0:1619677473.553909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677473.554149:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.554152:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.554153:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.554153:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.554155:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.554159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.554343:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.554346:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.554347:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.554348:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.554351:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.554353:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.554354:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.554355:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.554356:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.554357:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.554360:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.554361:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.554364:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.554365:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.554365:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.554367:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.554367:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.554368:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.554368:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.554381:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.554384:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.554384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.554385:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.554387:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.554390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.554648:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.554651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.554651:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.554653:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.554656:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.554657:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.554658:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.554659:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.554660:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.554661:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.554665:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.554666:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.554669:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.554669:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.554670:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.554672:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.554672:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.554673:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.554673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.554685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.554687:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.554688:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.554690:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.554691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.554692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677473.554834:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.554836:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677473.554837:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677473.554839:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.554842:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.554844:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677473.554845:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.554846:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.554847:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.554848:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.554852:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.554854:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.554856:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.554857:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677473.554857:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677473.554859:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677473.554859:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.554860:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677473.554860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:10.0:1619677473.554947:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.554950:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.554951:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.554952:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.554955:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.554956:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.554957:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.554959:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.554959:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.554960:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.554963:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.554965:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.554967:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.554967:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.554968:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.554969:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.554970:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.554971:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.554972:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.554972:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.554975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.554979:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.554981:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.554982:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.554982:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.554984:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.554985:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.554986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.554987:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.554988:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.554989:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.555140:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.555143:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.555145:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.555145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.555147:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.555149:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.555150:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.555152:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.555153:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.555154:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.555157:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.555159:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.555162:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.555163:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.555163:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.555165:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.555166:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.555167:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.555168:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.555169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677473.555171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.555177:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677473.555178:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.555180:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677473.555180:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.555180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.555181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677473.555181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.555182:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.555184:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.555187:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.555192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.555198:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.555201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.555201:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.555201:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.555202:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.555203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.555556:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.555559:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.555560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.555562:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.555565:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.555566:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.555567:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.555569:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.555570:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.555571:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.555574:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.555576:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.555578:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.555578:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.555579:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.555581:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.555581:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.555582:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:2.0:1619677473.555582:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:22.0:1619677473.555583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:2.0:1619677473.555585:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677473.555586:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677473.555588:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.555591:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.555592:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677473.555593:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.555594:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.555595:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:2.0:1619677473.555595:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.555596:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.555598:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.555598:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.555600:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.555601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.555601:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.555602:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.555603:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677473.555605:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.555606:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677473.555607:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677473.555609:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677473.555609:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.555611:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677473.555611:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:6.0:1619677473.555626:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.555629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.555629:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.555630:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.555632:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.555634:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.555895:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.555897:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.555898:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.555898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.555901:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.555903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.556177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.556184:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.556186:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.556187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.556190:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.556192:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.556194:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.556195:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.556196:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.556197:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.556199:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.556202:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.556204:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.556207:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.556208:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.556209:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.556211:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.556211:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.556213:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.556214:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.556215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.556220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677473.556226:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677473.556226:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.556228:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.556229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677473.556230:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:2.0:1619677473.556230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677473.556231:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.556234:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.556236:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.556237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.556432:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.556435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.556435:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.556437:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.556439:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.556440:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.556442:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.556443:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.556444:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.556445:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.556448:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.556449:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.556452:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.556452:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.556453:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.556455:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.556455:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.556456:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.556457:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.556458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.556468:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.556470:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.556470:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.556471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677473.556471:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:22.0:1619677473.556472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.556473:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.556474:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.556474:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.556475:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:21.0:1619677473.556476:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.556476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677473.556477:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.556479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677473.556480:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.556481:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.556482:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.556483:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.556485:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.556486:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:20.0:1619677473.556488:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:10.0:1619677473.556489:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.556491:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.556491:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:20.0:1619677473.556492:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.556493:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.556493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.556494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.556494:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.556494:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.556495:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.556497:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.556497:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.556498:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.556498:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.556503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.556506:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.556508:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.556509:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.556509:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.556511:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.556513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.0:1619677473.556579:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.556582:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.556584:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.556585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.556586:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.556588:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.556589:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.556591:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.556592:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.556593:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.556596:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.556598:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.556601:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.556602:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.556602:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.556604:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.556605:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.556605:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.556606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.556610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.556616:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.556619:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.556620:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.556620:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.556622:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.556624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.556656:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.556658:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.556659:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.556661:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.556663:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.556665:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.556666:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.556668:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.556668:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.556670:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.556672:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.556674:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.556677:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.556677:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.556678:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.556680:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.556680:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.556681:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.556681:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.556705:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.556707:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.556708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.556708:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.556711:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.556713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.556723:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.556726:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.556727:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.556727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.556728:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.556728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.557452:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.557455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.557455:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.557457:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.557459:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.557461:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.557462:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.557463:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.557464:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.557465:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.557468:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.557470:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.557472:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.557473:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.557474:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.557475:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.557476:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.557476:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.557477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.557489:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.557492:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.557492:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.557494:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.557495:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.557496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.557690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.557697:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.557700:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.557700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.557702:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.557705:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.557707:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.557708:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.557710:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.557710:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.557712:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.557715:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.557717:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.557719:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.557720:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.557720:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.557721:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.557722:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.557723:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677473.557724:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.557724:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.557728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.557730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.557736:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.557737:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677473.557738:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.557738:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.557738:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.557739:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.557740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.557740:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.557741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677473.557741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.557934:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.557937:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.557938:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.557941:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.557944:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.557945:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.557946:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.557947:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.557948:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.557949:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.557951:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677473.557952:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.557954:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:10.0:1619677473.557954:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.557955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677473.557956:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677473.557957:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.557957:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.557957:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:24.0:1619677473.557959:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.557959:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.557959:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.557960:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.557960:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677473.557961:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.557962:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96508846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.557963:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.557964:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.557964:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:9.2:1619677473.557964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:24.0:1619677473.557965:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.557967:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.557967:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.557968:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.557971:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.557971:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.557971:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.557972:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.557973:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.557973:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.557974:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:8.0:1619677473.557975:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.557977:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.557978:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.557978:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.557978:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.557979:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.557980:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.557981:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.557983:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.557984:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.557984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677473.557994:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.557997:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.557997:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.557997:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.557998:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.557999:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.558000:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677473.558000:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.558001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.558002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.558616:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.558618:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.558619:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.558621:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.558623:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.558624:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.558625:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.558627:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.558628:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.558629:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.558632:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.558633:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.558636:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.558637:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.558637:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.558639:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.558639:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.558640:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.558640:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.558653:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.558656:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.558656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.558657:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.558659:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.558662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.558699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.558705:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.558708:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.558709:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.558711:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.558713:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.558715:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.558716:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.558717:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.558718:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.558719:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.558722:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.558723:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.558726:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.558727:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.558727:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.558729:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.558729:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.558730:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.558731:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.558732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.558737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.558742:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.558742:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.558745:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.558745:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677473.558746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.558746:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.558746:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.558747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677473.558748:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.558750:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.558888:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.558891:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.558892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.558894:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.558896:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.558898:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.558899:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.558901:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.558901:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.558903:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.558906:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.558908:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.558910:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.558911:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.558912:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.558914:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.558914:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.558915:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.558915:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.558920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.558928:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.558931:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.558932:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.558932:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.558934:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.558937:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.559201:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.559203:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.559204:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.559206:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.559209:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.559210:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.559211:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.559213:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.559214:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.559215:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.559218:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.559220:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.559222:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.559223:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.559223:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.559225:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.559225:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.559226:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.559226:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.559238:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.559241:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.559241:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.559243:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.559245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.559246:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.559506:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.559509:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.559510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.559511:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.559514:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.559515:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.559516:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.559518:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.559519:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.559520:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.559523:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.559524:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.559526:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.559527:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.559527:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.559529:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.559529:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.559530:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.559530:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.559535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.559538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.559541:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.559542:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.559542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.559543:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.559546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.559552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.559556:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.559558:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.559559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.559559:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.559560:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.559560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.559670:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.559672:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.559673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.559674:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.559676:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.559679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.560107:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.560110:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.560111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.560113:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.560114:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.560116:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.560117:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.560118:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.560119:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.560120:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.560124:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.560126:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.560128:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.560129:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.560129:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.560131:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.560131:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.560132:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.560132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.560145:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.560147:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.560148:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.560150:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.560151:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.560152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.560554:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.560557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.560557:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.560560:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.560562:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.560563:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.560564:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.560565:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.560566:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.560568:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.560571:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.560572:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677473.560572:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677473.560574:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.560574:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.560576:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.560577:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677473.560578:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677473.560578:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:17.0:1619677473.560579:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.560579:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:17.0:1619677473.560581:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.560581:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.560581:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:17.0:1619677473.560582:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.560583:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.560584:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.560585:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:17.0:1619677473.560586:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.560586:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.2:1619677473.560587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.560588:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.560590:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.560591:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.560591:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.560593:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.560594:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.560594:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.560595:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:21.0:1619677473.560596:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.560598:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.560599:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.560601:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.560602:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.560603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677473.560608:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.560611:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.560612:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.560612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.560614:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.560616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.560627:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.560629:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.560630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.560630:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.560631:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.560632:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.560679:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.560682:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.560683:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.560685:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.560687:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.560689:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:10.0:1619677473.560690:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.560691:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.560692:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.560693:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:10.0:1619677473.560696:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.560698:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677473.560700:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.560700:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:10.0:1619677473.560701:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677473.560703:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677473.560703:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.560704:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.560704:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.560709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.560715:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.560717:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.560718:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.560718:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.560720:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.560722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.561031:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.561033:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.561034:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.561036:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.561038:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.561040:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.561041:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.561042:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.561043:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.561044:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.561048:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.561050:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.561052:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.561053:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.561053:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.561055:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.561056:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.561056:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.561057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.561068:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.561070:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.561071:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.561072:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.561073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.561075:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.561101:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.561104:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.561104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.561106:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.561108:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.561110:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.561111:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.561112:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.561113:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.561114:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.561117:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.561119:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.561121:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.561122:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.561123:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.561124:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.561124:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.561126:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.561127:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.561127:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.561129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.561133:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.561135:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.561136:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.561137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.561138:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.561140:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.561140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.561142:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.561143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.561144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:26.0:1619677473.561468:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.561471:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.561472:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.561473:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.561475:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.561476:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.561477:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.561478:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.561479:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.561480:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.561483:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.561484:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.561486:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.561486:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.561487:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.561489:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.561489:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.561490:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.561490:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.561495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.561502:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.561504:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.561505:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.561505:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.561506:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.561508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.561514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.561520:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.561522:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.561523:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.561523:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.561524:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.561525:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.561535:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.561537:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.561538:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.561540:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.561542:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.561543:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.561544:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.561546:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.561546:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.561547:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.561550:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.561551:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.561554:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.561555:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.561555:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.561557:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.561557:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.561559:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.561560:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.561560:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.561573:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.561575:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.561575:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.561576:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.561582:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.561585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.561585:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.561586:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.561588:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.561591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.562119:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.562122:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.562123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.562128:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.562130:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.562132:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.562133:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.562134:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.562135:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.562136:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.562140:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.562141:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.562143:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.562144:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.562145:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.562147:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.562147:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.562148:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.562149:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.562152:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.562154:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.562156:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.562157:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.562158:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.562159:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.562159:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.562159:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.562160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.562161:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677473.562162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.562162:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.562164:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.562165:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.562167:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.562167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.562169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.562180:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.562183:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.562185:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.562185:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.562186:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.562187:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.562188:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.562191:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.562192:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.562194:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.562196:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.562198:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.562199:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.562200:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.562201:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.562203:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.562206:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.562207:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.562209:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.562210:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.562211:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.562212:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.562213:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.562213:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.562214:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.562218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.562221:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.562223:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.562224:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.562224:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.562226:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.562228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.0:1619677473.562549:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677473.562551:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677473.562552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677473.562554:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677473.562556:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.562557:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.562558:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.562559:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.562560:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.562561:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.562564:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.562566:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.562569:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.562569:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.562570:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.562571:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.562572:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677473.562573:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677473.562573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677473.562579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.562588:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.562591:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.562592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.562593:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.562595:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.562598:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677473.562851:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.562854:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.562854:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.562856:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.562859:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.562861:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.562862:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.562863:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.562864:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.562865:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.562868:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.562870:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.562872:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.562873:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.562874:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.562875:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.562876:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.562877:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.562877:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.562889:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.562892:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.562893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:20.2:1619677473.562893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:6.0:1619677473.562893:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.562895:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.562898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677473.562900:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.562903:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.562904:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.562906:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.562908:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.562910:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.562911:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.562912:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.562913:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.562914:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.562918:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.562919:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.562922:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.562922:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.562923:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.562924:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.562925:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.562926:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.562927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.562933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.562941:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.562943:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.562944:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.562944:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.562946:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.562948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.562949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.562956:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.562959:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.562960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.562960:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.562961:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.562962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.563024:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.563026:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.563027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.563029:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.563031:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.563033:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.563034:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.563035:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.563037:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.563038:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.563041:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.563042:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.563045:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.563045:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.563046:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.563048:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.563048:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.563049:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.563049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.563062:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.563064:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.563065:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.563066:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.563067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.563068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.563211:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.563214:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.563214:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.563217:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.563219:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.563220:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.563221:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.563223:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.563224:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.563225:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.563228:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.563230:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.563232:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.563233:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.563234:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.563235:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.563236:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.563237:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.563237:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.563241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.563247:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.563250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.563250:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.563250:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.563253:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.563255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677473.563289:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.563291:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677473.563293:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677473.563300:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.563302:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.563303:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.563305:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.563306:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.563307:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.563308:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.563311:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.563313:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.563315:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.563316:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.563316:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.563318:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.563318:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.563319:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677473.563320:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677473.563324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.563332:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.563335:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.563335:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.563335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.563337:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.563338:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.563342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.563350:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.563352:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.563353:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.563353:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.563354:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.563354:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677473.563455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.563460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.563463:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.563464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.563465:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.563468:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.563469:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.563470:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.563472:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.563472:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.563473:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.563476:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.563478:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.563480:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.563481:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.563482:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.563482:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677473.563483:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.563484:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.563484:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677473.563485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677473.563485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677473.563485:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.563487:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.563489:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.563491:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.563492:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.563494:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.563494:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.563496:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.563499:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.563501:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.563504:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.563505:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.563505:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (23)++
00000800:00000200:10.0:1619677473.563507:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (24)--
00000800:00000200:10.0:1619677473.563508:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677473.563508:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (24)++
00000800:00000200:10.0:1619677473.563509:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.563511:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.563511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677473.563511:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000800:00000200:22.0:1619677473.563512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.563512:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.563514:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.563517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.563523:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.563526:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.563526:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.563527:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.563529:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.563532:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.563532:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.563533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.563534:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.563537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.563554:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.563556:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.563557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.563558:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.563559:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.563559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.563566:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.563568:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.563570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.563571:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.563574:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.563576:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.563577:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.563578:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.563579:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.563580:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.563583:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.563585:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.563588:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.563589:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.563589:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.563591:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.563591:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.563592:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.563592:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:22.0:1619677473.563614:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.563617:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.563618:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.563618:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.563620:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.563623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.563833:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.563835:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.563836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.563837:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.563840:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.563843:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.563844:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.563844:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.563846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.563847:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.563847:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.563848:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.563849:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:22.0:1619677473.563850:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.563850:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.563851:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:6.0:1619677473.563851:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.563852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:22.0:1619677473.563853:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.563855:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.563855:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.563857:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.563860:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.563862:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.563864:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.563865:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.563866:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.563867:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.563868:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.563868:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.563869:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.563881:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.563883:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.563884:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.563886:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.563887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.563888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.564227:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.564230:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.564230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.564233:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.564235:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.564237:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.564238:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.564239:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.564240:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.564241:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.564244:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.564246:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.564249:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.564250:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.564250:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.564252:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.564252:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.564254:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.564255:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.564255:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.564258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.564267:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.564268:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:18.0:1619677473.564269:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.564270:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.564270:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:2.0:1619677473.564270:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.564271:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:18.0:1619677473.564272:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.564272:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677473.564276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.564278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677473.564285:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.564288:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.564288:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.564289:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.564290:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.564291:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677473.564457:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.564460:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.564461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.564463:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.564466:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.564467:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.564469:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.564470:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.564471:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.564472:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.564475:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.564476:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.564479:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.564480:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.564480:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.564482:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.564483:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.564483:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.564484:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.564488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.564497:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.564500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.564500:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.564501:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.564503:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.564506:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677473.564771:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.564774:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.564775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.564777:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.564780:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.564781:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.564782:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.564783:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.564785:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.564785:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.564789:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.564790:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.564792:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.564793:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.564793:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.564795:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.564795:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.564796:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.564797:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.564798:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.564801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.564804:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.564806:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.564807:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.564808:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.564809:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.564811:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.564812:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.564813:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.564814:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.564815:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:17.0:1619677473.564834:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.564837:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.564838:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.564840:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.564842:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.564844:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677473.564845:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.564846:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.564847:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.564848:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677473.564852:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.564853:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677473.564856:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.564857:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677473.564858:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677473.564859:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.564860:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.564861:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.564861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.564866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.564873:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.564875:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.564876:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.564877:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.564879:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.564882:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677473.564900:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.564903:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.564904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.564906:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.564908:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.564909:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.564910:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.564912:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.564913:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.564914:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.564917:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.564919:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.564921:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.564922:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.564922:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.564924:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.564924:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.564925:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.564926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.564938:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.564940:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.564941:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.564943:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.564943:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.564945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.565217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:18.0:1619677473.565223:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677473.565226:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:2.0:1619677473.565227:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:18.0:1619677473.565229:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.565231:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.565232:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677473.565234:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.565235:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.565236:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.565237:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677473.565240:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.565242:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.565244:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.565245:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677473.565246:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677473.565247:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677473.565248:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:18.0:1619677473.565249:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:18.0:1619677473.565249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:20.0:1619677473.565261:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.565263:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.565264:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.565266:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.565267:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.565268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.565288:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.565290:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.565291:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.565292:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.565293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.565294:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.565790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.565793:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.565794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.565795:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.565798:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.565800:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.565801:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.565802:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.565803:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.565804:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.565807:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.565808:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.565811:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.565812:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.565813:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.565814:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.565815:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.565816:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.565817:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.565818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.565828:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.565830:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.565830:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.565830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.565831:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.565831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.565832:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.565834:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.565835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.565836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.565845:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.565848:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.565849:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.565849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.565852:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.565854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.566442:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.566444:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.566445:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:10.0:1619677473.566446:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.566447:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:26.0:1619677473.566449:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.566449:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.566451:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.566452:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.566454:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.566455:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.566456:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.566458:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.566459:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.566460:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677473.566461:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.566461:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.566463:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.566463:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.566464:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677473.566468:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.566468:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.566469:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.566469:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.566470:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.566471:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.566472:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.566472:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.566473:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677473.566473:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.566474:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677473.566474:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.566475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677473.566476:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677473.566476:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.566476:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.566477:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.566479:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.566479:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:6.0:1619677473.566486:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.566489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.566489:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.566490:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.566492:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:6.0:1619677473.566492:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.566495:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:6.0:1619677473.566495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.566496:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677473.566496:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.566607:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.566610:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.566610:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.566612:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.566615:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.566616:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.566617:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.566619:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.566620:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.566620:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.566624:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.566625:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.566627:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.566627:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.566628:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.566629:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.566630:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.566631:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.566631:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.566635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.566639:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.566641:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.566642:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.566642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.566643:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.566645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.566651:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.566654:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:24.2:1619677473.566655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:6.0:1619677473.566655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.566657:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.566658:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:22.0:1619677473.566659:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.566660:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.0:1619677473.566661:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.566662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:22.0:1619677473.566662:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.566662:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000400:00000200:22.0:1619677473.566663:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.566663:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:22.0:1619677473.566664:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.566664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:22.0:1619677473.566665:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.566668:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.566670:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.566672:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.566673:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.566674:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.566675:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.566675:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.566676:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.566677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.566688:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.566690:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.566691:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.566693:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.566694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.566695:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.2:1619677473.566821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.566824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.566826:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.566827:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.566827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.566829:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.566831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:21.0:1619677473.567086:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.567088:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.567090:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.567091:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.567093:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.567095:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.567096:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.567097:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.567098:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.567099:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.567103:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.567104:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.567107:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.567108:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.567108:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.567110:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.567110:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.567112:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.567112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.567123:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.567126:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.567127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.567127:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.567129:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.567131:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.567529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.567536:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.567538:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.567540:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:22.0:1619677473.567541:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:2.0:1619677473.567541:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.567543:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.567544:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.567544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:2.0:1619677473.567545:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.567547:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.567547:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.567548:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.567550:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.567551:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.567552:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.567554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:2.0:1619677473.567555:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.567556:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677473.567557:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.567558:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.567559:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.567559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677473.567559:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.567560:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.567560:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677473.567560:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.567561:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.567562:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.567563:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.567564:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.567564:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.567568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.567572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.567577:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.567578:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.567579:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.567579:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:18.0:1619677473.567581:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.567581:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.567582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.567583:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:2.0:1619677473.567583:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677473.567584:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.567586:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.567587:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.567588:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.567590:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.567591:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.567592:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.567595:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.567596:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.567599:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.567600:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.567600:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.567602:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.567603:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.567603:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.567604:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677473.567703:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.567706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.567706:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.567708:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.567710:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.567712:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.567713:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.567714:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.567715:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.567716:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.567719:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.567721:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.567724:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.567725:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.567725:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.567727:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:6.0:1619677473.567727:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.567727:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.567728:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.567729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677473.567730:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.567731:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.567733:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.567735:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.567737:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.567738:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.567739:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.567740:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.567740:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:26.0:1619677473.567741:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:20.0:1619677473.567742:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.567743:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.567745:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677473.567745:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.567746:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.567747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.567748:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.567750:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.567751:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.567751:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.567753:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.567753:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.567754:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.567755:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.567759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.567763:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.567765:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.567766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.567766:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.567768:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.567770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.568464:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.568466:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.568467:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.568469:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.568471:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.568472:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.568473:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.568475:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.568476:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.568477:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.568479:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.568481:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.568483:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.568484:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.568484:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.568486:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.568486:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.568487:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.568488:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.568492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.568499:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.568501:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.568502:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.568502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.568504:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.568506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.568534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:5.2:1619677473.568534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.0:1619677473.568541:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:1.0:1619677473.568541:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.0:1619677473.568544:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:17.0:1619677473.568544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.568544:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677473.568546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.568548:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.568549:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.568551:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.568552:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.568554:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677473.568554:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:21.0:1619677473.568555:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.568556:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.568557:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.568558:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.568559:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.568559:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.568561:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677473.568561:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:21.0:1619677473.568566:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677473.568567:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.568569:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677473.568570:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.0:1619677473.568573:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.568574:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677473.568574:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.568575:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677473.568575:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.568575:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677473.568577:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677473.568577:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677473.568578:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:1.0:1619677473.568578:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.0:1619677473.568579:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:1.0:1619677473.568579:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.0:1619677473.568580:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:1.0:1619677473.568580:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:12.0:1619677473.568591:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.568593:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.568594:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.568596:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.568598:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.568599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:22.0:1619677473.568602:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.568604:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.568605:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.568605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.568607:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.568609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.568620:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.568623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.568623:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.568625:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.568627:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.568629:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.568630:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.568632:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.568633:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.568634:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.568637:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.568639:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.568642:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.568642:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.568643:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.568645:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.568646:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.568647:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.568647:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.568658:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.568660:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.568661:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.568663:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.568664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.568665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.568677:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.568679:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.568680:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.568681:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.568681:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.568682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.568689:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.568692:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.568693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.568695:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.568697:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.568699:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.568700:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.568701:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.568702:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.568703:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.568707:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.568709:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.568712:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.568713:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.568713:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.568715:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.568715:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.568716:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.568716:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.568739:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.568741:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.568742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.568743:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.568745:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.568747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.568756:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.568758:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.568759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.568759:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.568760:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.568761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.568825:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.568828:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.568829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.568830:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.568833:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.568835:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.568836:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.568837:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.568838:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.568839:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.568843:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.568844:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.568847:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.568847:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.568848:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.568850:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.568850:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.568852:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.568853:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.568853:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677473.568863:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.568865:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677473.568866:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:22.0:1619677473.568867:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.568867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.568868:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.568869:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677473.568869:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.568869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.568872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.569123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.569129:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.569131:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.569132:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.569132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.569134:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.569136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.569350:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.569353:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677473.569355:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.569355:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.569357:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.569359:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.569360:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.569362:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.569363:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.569364:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.569367:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.569369:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.569371:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.569371:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.569372:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.569374:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.569374:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.569376:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677473.569377:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677473.569378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677473.569381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.569388:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.569390:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677473.569390:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.569390:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.569391:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677473.569391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.569391:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.569393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.569394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677473.569395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.569403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.569410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.569412:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.569414:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.569414:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677473.569415:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.569416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677473.569484:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.569487:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.569488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.569489:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.569492:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.569493:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.569494:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.569495:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.569496:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.569497:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.569500:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.569502:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.569504:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.569505:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.569505:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.569507:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.569507:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.569508:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.569510:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.569510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.569520:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.569522:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.569522:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.569523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.569524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.569524:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.569525:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.569527:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.569527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.569529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.569554:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.569557:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.569558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.569558:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.569561:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.569563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.569682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.569685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.569687:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.569690:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.569691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.569691:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.569693:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.569694:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.569695:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.569696:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.569697:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.569701:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.569702:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.569705:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.569706:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.569706:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.569708:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.569709:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.569709:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.569710:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.569714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.569722:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.569725:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.569726:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.569726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.569728:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.569731:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.569741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.569748:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.569750:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.569751:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.569751:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.569752:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.569753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677473.570511:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.570514:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.570515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:22.0:1619677473.570516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677473.570517:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.570519:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.570520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.570521:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.570521:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.570523:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677473.570524:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.570524:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.570525:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:22.0:1619677473.570525:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677473.570526:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:22.0:1619677473.570527:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.570527:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.570529:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.570530:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.570530:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.570531:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677473.570532:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.570537:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.570537:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.570540:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.570541:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.570542:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.570543:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.570544:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.570544:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.570544:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.570545:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.570545:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:26.0:1619677473.570546:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:22.0:1619677473.570546:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.570546:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:26.0:1619677473.570547:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96508780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.570547:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:10.0:1619677473.570547:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:22.0:1619677473.570548:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.570548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:26.0:1619677473.570549:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.570550:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.570551:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.570552:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.570553:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677473.570554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:26.0:1619677473.570555:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.570555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677473.570556:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.570557:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.570558:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.570558:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.570559:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.570561:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.570562:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.570562:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677473.570562:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.570563:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.570563:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.570563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.0:1619677473.570564:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.570565:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.570566:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.570566:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:21.0:1619677473.570566:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.570567:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.570567:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677473.570567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.570569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.570572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.570580:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.570583:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.570583:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.570586:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.570586:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.570590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.570593:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.570596:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.570597:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.570598:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.570601:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.570602:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.570603:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.570605:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.570606:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.570607:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.570610:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.570611:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.570613:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.570613:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.570614:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.570615:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.570616:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.570616:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.570617:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.570621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.570625:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.570627:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.570628:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.570629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.570630:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.570632:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:21.0:1619677473.570795:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.570798:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.570800:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.570800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.570802:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.570804:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.570805:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.570807:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.570807:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.570809:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.570812:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.570814:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.570816:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.570817:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.570817:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.570819:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.570820:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.570820:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.570821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.570833:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.570835:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.570836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.570836:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.570838:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.570840:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677473.570968:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.570970:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677473.570971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677473.570973:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.570975:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.570977:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677473.570978:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.570980:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.570981:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.570982:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677473.570985:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.570987:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.570989:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.570990:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.570990:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.570991:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.570992:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.570993:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677473.570994:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677473.570995:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677473.570998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.571007:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677473.571007:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.571008:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677473.571008:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.571008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.571009:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677473.571010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.571010:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.571011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677473.571012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.571264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.571267:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.571268:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.571268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.571270:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.571273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.571281:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.571284:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.571285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.571285:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.571286:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.571287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.571288:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.571291:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.571291:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.571298:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.571300:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.571301:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.571303:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.571304:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.571305:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.571306:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.571309:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.571311:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.571313:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.571314:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.571315:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.571316:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.571317:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.571318:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.571319:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.571320:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.571331:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.571333:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.571334:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.571334:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.571334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.571336:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677473.571337:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.571337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.571338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.571340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.571399:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.571401:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.571402:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.571404:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.571406:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.571407:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.571409:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.571410:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.571411:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.571412:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.571414:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.571416:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.571419:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.571419:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.571420:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.571422:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.571422:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.571423:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.571423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.571435:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.571437:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.571438:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.571440:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.571441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.571442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677473.571643:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.571646:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.571647:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.571648:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.571651:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.571653:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.571654:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.571655:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.571656:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.571657:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.571661:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.571663:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.571666:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.571666:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.571667:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.571669:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.571670:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.571670:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.571671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.571675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.571684:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.571687:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.571688:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.571688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.571690:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.571694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.571851:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.571854:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.571855:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.571857:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.571860:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.571861:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.571863:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.571864:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.571865:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.571866:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.571869:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.571871:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.571872:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.571873:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.571874:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.571875:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.571875:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.571875:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.571876:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.571880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.571886:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.571888:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.571889:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.571890:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.571891:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.571892:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.571898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.571904:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.571906:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.571907:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.571907:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.571908:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.571908:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.572014:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.572016:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.572017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.572019:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.572020:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.572021:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.572022:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.572023:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.572024:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.572025:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.572028:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.572029:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.572031:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.572031:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.572032:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.572033:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.572034:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.572035:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.572036:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.572037:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.572039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.572042:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.572044:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.572045:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.572046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.572047:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.572049:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.572049:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.572051:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.572052:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.572053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.572072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.572075:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.572077:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.572078:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.572078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.572079:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.572080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.572266:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.572268:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.572269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.572271:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.572274:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.572275:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.572276:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.572278:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.572279:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.572280:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.572283:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.572285:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.572287:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.572288:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.572288:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.572290:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.572291:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.572292:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.572292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.572305:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.572307:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.572308:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.572310:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.572311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.572312:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.572333:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.572336:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.572336:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.572337:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.572338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.572339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.573175:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.573177:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.573178:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.573180:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.573182:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.573184:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.573185:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.573186:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.573187:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.573188:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.573191:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.573192:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.573195:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.573196:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.573196:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.573198:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.573199:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.573199:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.573200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677473.573212:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.573213:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:8.0:1619677473.573214:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677473.573215:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:24.0:1619677473.573216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677473.573216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:8.0:1619677473.573217:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.573217:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.573219:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.573220:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.573221:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.573222:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.573222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:8.0:1619677473.573224:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.573225:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.573226:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.573229:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.573231:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.573233:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.573234:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.573235:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.573236:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.573237:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.573237:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.573238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.573250:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.573252:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.573253:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.573254:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.573256:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.573259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.573305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.573308:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.573309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.573311:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.573312:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.573314:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.573315:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.573316:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.573317:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.573318:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.573321:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.573323:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.573325:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.573326:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.573327:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.573328:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.573329:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.573330:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.573331:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.573332:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.573342:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.573344:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.573345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.573345:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.573346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.573347:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.573347:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.573349:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.573350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.573351:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.573448:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.573450:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.573451:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.573453:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.573456:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.573457:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.573458:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.573459:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.573460:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.573461:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.573464:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.573466:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.573468:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.573469:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.573470:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.573471:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.573472:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.573472:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.573473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.573477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.573480:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.573483:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.573484:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.573484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.573485:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.573487:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:6.0:1619677473.573521:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.573523:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.573524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.573525:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.573527:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.573530:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.573532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.573539:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.573541:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.573542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.573544:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.573546:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.573547:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.573548:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.573550:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.573551:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.573552:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.573555:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.573557:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.573559:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.573560:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.573561:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.573562:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.573563:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.573564:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.573564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.573576:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.573579:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.573580:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.573580:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.573582:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.573584:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.573668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.573674:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.573677:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.573678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.573680:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.573682:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.573684:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.573685:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.573687:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.573688:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.573689:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.573692:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.573694:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.573697:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.573698:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.573699:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.573700:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.573701:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.573702:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.573702:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677473.573709:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.573712:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.573713:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.573715:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.573716:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:10.0:1619677473.573717:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.573718:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:13.0:1619677473.573719:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:10.0:1619677473.573719:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.573720:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.573720:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.573721:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.573723:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.573723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:10.0:1619677473.573723:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.573726:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.573727:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.573728:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.573730:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.573731:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.573732:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.573733:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.573734:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.573735:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.573735:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.573739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.573745:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.573748:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.573749:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.573749:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.573750:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.573753:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.573758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.573764:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.573767:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.573768:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.573768:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.573768:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.573769:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.574159:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.574162:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.574163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.574164:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.574167:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.574169:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.574170:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.574171:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.574172:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.574173:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.574176:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.574178:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.574179:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.0:1619677473.574180:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.574181:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.574182:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.574182:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.574183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677473.574183:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.574184:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.574184:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.574185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677473.574185:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.574187:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.574188:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.574190:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.574191:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.574193:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.574194:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.574197:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.574199:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.574199:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.574200:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.574201:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.574202:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.574204:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.574205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.574205:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.574205:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.574206:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.574207:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.574208:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.574208:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.574208:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.574209:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.574209:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.574209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677473.574210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.574211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.574221:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.574223:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.574224:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.574226:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.574227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.574228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.574238:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.574241:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.574242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.574244:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.574247:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.574248:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677473.574250:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.574251:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.574252:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.574253:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677473.574256:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.574257:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677473.574261:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.574261:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677473.574262:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677473.574264:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677473.574264:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.574265:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.574265:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:12.0:1619677473.574277:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.574280:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.574281:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.574283:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.574285:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.574286:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.574302:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.574306:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.574308:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.574309:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.574310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.574312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.574801:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.574804:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.574804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.574806:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.574808:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.574810:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.574811:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.574812:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.574813:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.574814:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.574817:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.574819:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.574821:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.574822:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.574823:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.574824:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.574824:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.574825:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.574826:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:13.0:1619677473.574838:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.574840:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.574841:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.574842:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.574845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.574845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.575043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.575050:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677473.575050:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677473.575053:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677473.575053:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:24.0:1619677473.575054:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.575054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677473.575055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:6.0:1619677473.575057:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.575058:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.575061:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:6.0:1619677473.575061:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.575062:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.575063:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.575065:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.575066:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.575067:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.575070:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.575072:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.575074:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.575075:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.575076:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.575077:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.575078:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.575079:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.575079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.575092:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.575094:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.575095:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.575096:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.575097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.575099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.575187:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.575190:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.575191:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.575191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.575193:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.575196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.575586:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.575589:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.575590:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.575591:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.575594:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.575595:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.575596:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.575597:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677473.575598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677473.575598:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.575599:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.575602:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.575603:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:18.0:1619677473.575605:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677473.575605:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.575606:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.575606:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:18.0:1619677473.575608:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:10.0:1619677473.575608:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.575608:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:2.0:1619677473.575608:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677473.575609:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:18.0:1619677473.575610:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.575610:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000400:00000200:18.0:1619677473.575613:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.2:1619677473.575614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:18.0:1619677473.575615:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.575616:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.575617:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.575618:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.575618:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000400:00000200:18.0:1619677473.575619:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.575620:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.575621:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.575621:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.575623:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.575624:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.575626:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.575626:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:18.0:1619677473.575628:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.575628:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.575629:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.575631:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.575631:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.575632:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.575633:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.575638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.575645:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.575647:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.575648:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.575648:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.575650:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.575651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.576106:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.576109:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.576109:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.576111:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.576113:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.576115:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.576116:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.576117:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.576118:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.576119:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677473.576119:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.576122:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.576122:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677473.576123:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.576125:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.576125:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.576127:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.576128:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677473.576128:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.576129:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.576129:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:26.0:1619677473.576130:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.576131:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.576131:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.576131:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677473.576132:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.576132:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677473.576133:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.576133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:26.0:1619677473.576137:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.576138:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.576141:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.576142:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.576143:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.576144:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.576144:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677473.576145:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.576145:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.576146:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677473.576146:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.576147:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.576149:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.576150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.576151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.576158:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.576160:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.576161:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.576162:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.576162:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:22.0:1619677473.576164:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.576164:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.576165:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.576166:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.576166:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.576167:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.576168:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.576424:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.576427:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.576427:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.576429:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.576432:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.576433:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.576434:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.576436:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.576437:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.576437:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.576440:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.576442:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.576444:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.576445:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.576445:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.576447:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.576447:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.576448:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.576449:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.576461:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.576464:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.576464:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.576466:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.576468:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.576469:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.576845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.576852:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.576854:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.576855:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.576855:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.576857:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.576859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677473.576935:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.576937:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.576940:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.576940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.576942:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.576944:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.576945:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.576946:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.576947:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.576948:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.576951:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.576952:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.576955:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.576956:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.576957:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.576958:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.576959:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.576960:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.576961:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.576962:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.576971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677473.576972:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.576973:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.576974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677473.576974:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.576975:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.576975:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677473.576975:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.576977:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.576980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.576994:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.576997:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.576997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.576999:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.577002:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.577003:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.577004:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.577005:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.577006:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.577007:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.577010:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.577012:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.577014:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.577015:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.577016:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.577017:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.577018:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.577019:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.577020:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.577020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.577030:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.577031:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.577032:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677473.577033:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.577033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.577034:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.577034:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.577035:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.577037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.577038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.577199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:22.0:1619677473.577231:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.577233:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:9.2:1619677473.577234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677473.577234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.577235:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.577237:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.577240:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:22.0:1619677473.577240:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.577242:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.577243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.577244:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.577246:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.577248:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677473.577248:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.577251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.577251:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.577252:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.577253:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.577254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.577865:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.577868:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.577869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.577871:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.577873:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.577875:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.577876:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.577877:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.577878:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.577879:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.577882:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.577884:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.577886:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.577887:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.577887:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.577889:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.577889:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.577890:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.577891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677473.577899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.577902:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.577903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677473.577903:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.577905:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:8.0:1619677473.577905:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.577906:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.577907:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.577908:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.577908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677473.577909:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677473.577911:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677473.577911:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.577912:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.577913:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.577914:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.577917:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.577919:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.577922:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.577923:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.577923:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.577925:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.577925:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.577927:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.577928:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.577929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677473.577938:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.577941:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.577941:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.577942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.577942:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.577943:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.577944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677473.577944:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.577945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.577947:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:13.0:1619677473.578061:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.578063:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.578065:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.578068:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.578068:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.578069:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677473.578071:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.578072:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.578073:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.578074:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677473.578077:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.578078:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677473.578081:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.578082:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677473.578082:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677473.578084:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677473.578084:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.578085:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.578086:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.578089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.578096:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.578099:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.578100:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.578100:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.578102:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.578104:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.578115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677473.578121:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.578123:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677473.578124:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.578124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.578125:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.578126:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677473.578199:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.578201:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.578202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.578203:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.578206:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.578207:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.578208:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.578209:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.578210:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.578211:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.578214:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.578216:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.578219:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.578219:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.578220:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.578222:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.578222:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.578223:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.578223:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.578235:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.578237:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.578238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.578239:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.578241:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.578244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.578675:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.578678:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.578679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.578680:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.578683:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.578684:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.578686:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.578687:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.578688:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.578689:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.578692:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.578693:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.578696:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.578697:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.578697:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.578699:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.578699:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.578700:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.578700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.578713:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.578716:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.578717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.578717:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.578720:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.578723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.578788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.578794:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.578797:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.578798:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.578800:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.578802:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.578804:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.578805:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.578806:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.578807:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.578808:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.578811:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.578813:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.578815:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.578816:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.578817:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.578818:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.578819:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.578820:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.578821:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.578821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.578827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.578832:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.578834:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.578835:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.578836:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.578836:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677473.578836:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.578837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677473.578838:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.578838:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.578841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.578853:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.578856:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.578857:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.578858:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.578859:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.578859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.578873:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.578875:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.578876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.578877:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.578879:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.578881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.578938:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.578940:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.578941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.578943:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.578944:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.578946:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.578947:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.578948:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.578949:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.578950:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.578953:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.578954:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.578957:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.578958:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.578958:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.578960:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.578960:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.578961:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.578961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677473.578963:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.578965:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.578966:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.578968:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.578971:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.578972:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.578973:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.578973:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:26.0:1619677473.578975:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.578975:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:26.0:1619677473.578976:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.578976:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.578977:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677473.578978:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.578979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677473.578981:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.578981:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.578983:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.578985:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.578986:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.578987:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.578988:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.578989:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.578990:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.578990:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.578994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.579000:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.579003:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.579004:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.579004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.579006:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.579008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.579314:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.579317:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.579317:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677473.579319:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.579321:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.579322:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.579323:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.579325:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.579326:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.579327:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.579330:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.579332:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.579333:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.579334:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.579334:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.579336:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.579336:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.579337:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.579337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.579341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.579345:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.579348:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.579349:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.579349:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.579350:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.579352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.579825:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.579828:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.579829:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.579831:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.579833:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.579834:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.579836:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.579837:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.579838:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.579839:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.579842:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.579843:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.579845:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.579846:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.579847:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.579849:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.579849:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.579850:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.579850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.579863:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.579866:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.579866:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.579868:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.579869:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.579871:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.579880:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.579883:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.579884:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.579884:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.579885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.579886:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.580005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.580011:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.580014:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.580015:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.580017:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.580019:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.580020:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.580022:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.580023:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.580024:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.580025:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.580029:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.580030:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.580033:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.580034:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.580034:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.580036:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.580037:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.580038:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.580039:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.580040:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.580043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.580045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.580050:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677473.580051:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677473.580052:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.580053:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677473.580053:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677473.580054:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677473.580054:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.580055:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.580056:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.580059:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677473.580258:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.580260:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.580263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.580263:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.580265:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.580266:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.580268:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.580269:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.580270:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.580271:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.580274:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.580276:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.580278:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.580279:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.580280:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.580281:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.580282:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.580283:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.580284:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.580295:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.580301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.580302:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.580303:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.580305:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.580307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.580314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677473.580316:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.580319:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.580320:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.580320:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677473.580320:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.580321:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.580321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.580322:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.580324:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.580325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.580326:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.580328:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.580329:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.580330:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.580331:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.580332:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.580335:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.580337:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.580339:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.580340:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.580341:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.580343:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.580343:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.580344:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.580346:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.580346:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.580349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677473.580351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.580354:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.580356:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.580357:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.580358:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677473.580358:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.580360:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.580361:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.580361:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.580363:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.580365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677473.580563:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.580566:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.580567:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.580568:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.580571:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.580573:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.580574:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.580575:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.580576:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.580577:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.580581:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.580583:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.580585:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.580586:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.580587:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.580589:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.580589:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.580590:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.580590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.580595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.580602:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.580604:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.580606:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.580606:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.580608:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.580611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.580618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.580625:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.580628:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.580629:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.580629:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.580630:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.580631:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.580704:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.580706:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.580707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.580709:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.580711:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.580712:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.580714:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.580715:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.580716:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.580717:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.580721:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.580722:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.580724:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.580725:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.580726:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.580727:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.580728:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.580730:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.580731:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.580731:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.580741:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.580742:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.580743:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.580744:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.580744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.580744:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.580745:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.580746:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.580747:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.580748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.580868:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.580870:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.580871:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.580872:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.580874:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.580877:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.580894:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.580897:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.580898:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.580900:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.580902:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.580903:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.580905:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.580906:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.580907:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.580908:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.580911:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.580913:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.580915:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.580915:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.580916:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.580918:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.580918:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.580919:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.580920:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.580921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.580923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.580927:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.580929:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.580930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.580930:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.580932:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.580933:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.580934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:10.0:1619677473.580935:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.580936:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.580937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:26.0:1619677473.580946:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.2:1619677473.580946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.580948:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:24.0:1619677473.580949:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.580949:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.580951:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.580951:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.580952:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.580952:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:26.0:1619677473.580953:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.580953:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.580953:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:26.0:1619677473.580955:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.580956:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.580957:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.580958:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.580959:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.580962:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.580963:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.580966:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.580967:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.580967:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.580969:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.580969:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.580970:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.580972:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.580972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.580982:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.580985:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.580985:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677473.580986:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.580986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.580987:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.580988:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677473.580988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.580989:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.580992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.581109:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.581112:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.581113:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.581114:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.581117:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.581118:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.581119:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.581121:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.581122:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.581122:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.581125:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.581127:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.581129:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.581130:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.581130:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.581132:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.581133:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.581133:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.581134:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.581138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.581147:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.581150:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.581150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.581151:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.581153:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.581157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677473.581424:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.581427:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677473.581428:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677473.581430:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.581432:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.581434:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.581435:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.581436:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.581437:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.581438:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.581441:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.581442:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.581445:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.581446:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.581446:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.581448:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.581449:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.581449:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677473.581450:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677473.581455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.581463:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.581465:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.581466:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.581468:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.581469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.581471:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.581744:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.581746:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.581747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.581749:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.581751:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.581752:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.581754:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.581755:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.581756:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.581757:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.581761:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.581762:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.581765:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.581766:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.581766:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.581768:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.581768:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.581769:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.581770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.581782:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.581784:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.581785:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.581787:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.581788:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.581789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.582076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.582083:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.582085:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.582086:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.582086:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677473.582087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:8.0:1619677473.582088:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.582090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677473.582093:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:9.2:1619677473.582096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677473.582096:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.582097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.582098:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.582101:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.582102:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:2.0:1619677473.582102:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.582103:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.582105:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.582106:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.582106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.582107:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:2.0:1619677473.582107:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.582108:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.582108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:2.0:1619677473.582108:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.582110:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.582112:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.582116:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.582116:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.582117:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.582119:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.582120:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.582121:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677473.582122:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677473.582123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677473.582128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:12.0:1619677473.582133:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677473.582134:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677473.582136:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:12.0:1619677473.582136:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677473.582137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:12.0:1619677473.582137:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.582137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:12.0:1619677473.582139:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.582140:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.582141:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677473.582565:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.582568:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.582568:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677473.582571:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.582573:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.582574:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677473.582576:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.582577:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.582578:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.582579:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677473.582582:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.582584:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677473.582587:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.582588:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677473.582589:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677473.582591:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677473.582591:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.582592:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.582592:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.582597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.582605:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.582608:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.582609:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.582609:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.582611:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.582614:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:22.0:1619677473.582650:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.582653:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.582654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.2:1619677473.582655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:22.0:1619677473.582656:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.582658:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.582660:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.582661:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677473.582662:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:22.0:1619677473.582663:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.582664:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.582665:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.582665:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.582665:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:22.0:1619677473.582669:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677473.582669:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.582672:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677473.582672:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.582673:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:22.0:1619677473.582674:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.582675:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:18.0:1619677473.582675:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.582676:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677473.582676:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.582677:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:18.0:1619677473.582677:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.582678:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:18.0:1619677473.582678:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.582679:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.582680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:18.0:1619677473.582681:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.582683:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.582685:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.582686:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.582687:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.582688:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.582689:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.582690:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.0:1619677473.582691:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677473.582691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.0:1619677473.582694:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.582694:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.582694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:21.0:1619677473.582697:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.582697:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.582699:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677473.582701:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.582704:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.582705:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.582705:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.582707:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.582709:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.582725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677473.582727:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.582730:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.582731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.582731:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.582733:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677473.582734:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.582736:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677473.582737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677473.582737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.582738:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.582739:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.582739:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:13.0:1619677473.582827:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.582830:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.582831:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.582834:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.582834:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.582835:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677473.582836:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.582838:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.582839:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.582839:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:13.0:1619677473.582842:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.582844:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677473.582846:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.582847:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:13.0:1619677473.582847:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:13.0:1619677473.582849:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:13.0:1619677473.582849:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.582850:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.582850:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.582854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.582861:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.582863:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.582864:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.582864:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.582866:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.582868:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.583062:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.583064:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.583065:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.583067:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.583069:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.583071:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.583072:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.583073:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.583074:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.583075:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.583078:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.583079:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.583082:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.583083:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.583084:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.583085:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.583086:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.583087:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.583088:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.583089:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.583099:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.583101:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.583102:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.583102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.583103:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677473.583103:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.583104:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.583105:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.583105:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.583108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.583545:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.583547:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.583548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.583550:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.583552:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.583553:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.583554:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.583555:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.583556:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.583557:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.583561:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.583562:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.583565:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.583565:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.583566:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.583568:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.583568:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.583568:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.583569:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.583582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.583585:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.583586:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.583587:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:5.2:1619677473.583587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677473.583589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.583590:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677473.583593:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.583596:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.583597:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.583599:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.583600:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:17.0:1619677473.583601:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.583602:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677473.583603:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677473.583604:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.583604:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000400:00000200:17.0:1619677473.583604:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.583605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.583606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:17.0:1619677473.583606:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.583607:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.583608:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.583611:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.583612:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.583615:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.583616:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.583616:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.583618:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.583619:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.583619:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.583620:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.583632:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.583634:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.583635:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.583635:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.583638:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.583640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.583703:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.583706:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.583707:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.583709:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.583712:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.583713:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.583714:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.583715:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.583716:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.583717:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.583721:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.583722:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.583725:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.583726:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.583726:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.583728:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.583728:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.583729:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.583729:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.583734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.583742:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.583744:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.583745:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.583746:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.583747:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.583747:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.583749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.583750:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.583750:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.0:1619677473.583751:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.583753:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.583754:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.583755:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.583756:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.583757:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.583758:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.583761:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.583763:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.583765:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.583766:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.583767:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.583768:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.583769:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.583769:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.583770:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.583782:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.583784:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.583785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.583786:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.583788:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.583791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.583800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.583802:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.583803:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.583803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.583804:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.583805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.583816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.583824:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.583827:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.583828:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.583829:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.583830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.583832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677473.583917:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.583920:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.583921:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.583923:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.583925:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.583927:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677473.583928:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.583929:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.583930:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.583931:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677473.583935:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.583936:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677473.583938:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.583939:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677473.583939:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677473.583941:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677473.583941:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.583943:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.583944:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.583944:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.583957:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.583958:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.583959:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.583960:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.584044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.584050:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.584052:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.584053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.584053:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.584055:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.584057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.584221:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.584224:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.584225:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.584226:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.584228:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.584229:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.584230:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.584231:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.584232:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.584233:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.584236:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.584237:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.584239:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.584240:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.584241:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.584242:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.584242:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.584243:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.584243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.584247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.584254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.584256:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.584257:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.584257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.584258:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.584260:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.584597:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.584600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.584600:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.584602:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.584604:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.584606:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.584607:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.584609:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.584610:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.584611:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.584614:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.584616:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.584618:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.584619:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.584620:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.584621:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.584622:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.584623:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.584624:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.584625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.584636:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.584636:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.584637:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.584638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.584638:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.584639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.584639:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.584641:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.584642:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.584643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.584755:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.584757:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.584758:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.584759:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.584761:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.584764:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.584843:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.584845:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.584848:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.584848:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.584851:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.584852:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.584854:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.584855:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.584856:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.584857:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.584861:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.584862:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.584865:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.584866:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.584867:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.584869:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.584869:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.584870:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.584871:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.584875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.584882:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.584884:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.584885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.584885:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.584887:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.584890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.584946:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.584948:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.584949:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.584951:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.584953:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.584954:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.584955:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.584957:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.584957:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.584959:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.584962:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.584964:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.584966:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.584967:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.584967:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.584969:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.584969:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.584970:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.584970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.584974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.584977:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.584980:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.584981:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.584981:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.584983:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.584984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.585077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.585083:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.585086:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.585087:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.585089:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.585091:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.585093:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.585094:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.585096:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.585097:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.585098:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.585102:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.585104:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.585107:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.585107:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.585108:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.585110:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.585111:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.585112:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.585112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677473.585499:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.585502:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.585503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.585504:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.585507:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.585509:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.585510:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.585512:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.585513:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.585514:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.585517:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.585519:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.585522:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.585522:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.585523:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.585525:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.585525:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.585526:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.585526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.585540:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.585542:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.585543:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.585544:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.585545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.585547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.585734:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.585737:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.585738:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.585740:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.585742:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.585743:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.585745:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.585746:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.585747:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.585748:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.585751:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.585753:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.585755:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.585756:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.585756:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.585758:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.585758:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.585759:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.585759:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.585764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.585768:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.585770:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.585771:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.585772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.585773:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.585775:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.0:1619677473.585787:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.585790:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.585790:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.585792:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.585794:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.585796:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.585797:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.585798:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.585799:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.585800:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.585803:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.585804:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.585807:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.585808:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.585808:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.585810:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.585810:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.585812:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.585813:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.585813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.585816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.585825:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.585826:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.585827:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677473.585828:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677473.585828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.585829:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.585829:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.585830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677473.585831:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.585835:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.585859:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.585862:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.585863:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.585864:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.585866:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.585868:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.585869:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.585871:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.585872:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.585872:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.585876:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.585877:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.585880:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.585881:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.585881:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.585883:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.585883:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.585884:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.585885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.585907:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.585910:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.585911:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.585911:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.585913:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.585916:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.586303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.586309:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.586311:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.586313:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.586314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.586316:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.586318:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.586319:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.586320:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.586321:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.586322:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.586326:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.586327:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.586330:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.586331:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.586331:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.586333:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.586333:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.586334:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.586335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.586340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.586349:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.586351:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.586352:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.586352:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.586354:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.586357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677473.586402:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.586405:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.586405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.586407:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.586409:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.586411:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.586412:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.586413:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.586414:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.586415:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.586418:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.586420:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.586422:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.586422:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.586423:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.586425:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.586425:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.586426:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.586426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.586439:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.586441:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.586442:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.586444:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.586445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.586446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.586511:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.586514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.586514:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.586515:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.586517:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.586520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.586889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.586895:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.586897:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.586898:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.586898:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.586900:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.586903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.586931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.586937:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.586939:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.586940:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.586940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.586941:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.586942:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.587489:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.587492:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.587492:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.587494:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.587497:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.587498:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.587500:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.587501:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.587502:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.587503:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.587507:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.587509:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.587511:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.587512:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.587513:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.587515:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.587515:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.587516:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.587517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.587530:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.587532:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.587533:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.587535:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.587536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.587537:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.587547:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.587550:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.587551:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.587551:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.587552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.587553:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.587828:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.587830:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.587831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.587834:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.587836:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.587838:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.587839:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.587840:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.587842:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.587843:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.587846:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.587848:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.587851:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.587852:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.587852:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.587854:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.587855:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.587855:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.587856:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.587868:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.587870:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.587871:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.587872:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.587874:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.587876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.588052:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.588054:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.588055:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.588057:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.588060:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.588061:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.588062:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.588064:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.588065:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.588065:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.588069:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.588070:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.588073:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.588074:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.588074:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.588076:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.588076:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.588078:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.588079:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.588079:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:21.0:1619677473.588090:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677473.588092:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:21.0:1619677473.588092:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.588093:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.588094:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.588095:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:21.0:1619677473.588095:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.588095:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.588096:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.588098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.588192:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.588195:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.588195:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.588198:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.588199:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:10.0:1619677473.588201:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.588202:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:10.0:1619677473.588202:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.588204:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.588205:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.588205:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.588206:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.588207:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.588207:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.588208:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677473.588209:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.588210:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.588211:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.588212:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.588213:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.588218:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677473.588220:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.588221:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677473.588223:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.588224:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.588225:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.588226:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677473.588226:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.588227:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.588228:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677473.588228:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.588229:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.588229:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:10.0:1619677473.588229:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.588230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:10.0:1619677473.588230:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.588231:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.588233:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.588233:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.588234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677473.588237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677473.588244:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677473.588245:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677473.588246:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:8.0:1619677473.588246:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.588247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.588247:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:8.0:1619677473.588247:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.588248:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:21.0:1619677473.588248:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.588249:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.588250:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677473.588250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677473.588251:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.588251:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.2:1619677473.588255:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677473.588259:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.588260:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.588261:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.588263:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.588264:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.588264:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.588264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.588265:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.588349:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.588351:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.588352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:5.2:1619677473.588352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677473.588354:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.588356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.588358:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:17.0:1619677473.588358:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677473.588359:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.588361:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677473.588361:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.588361:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677473.588362:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.588364:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677473.588365:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.588367:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.588368:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677473.588368:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677473.588369:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:17.0:1619677473.588370:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.588371:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.588372:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.588373:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.588373:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:17.0:1619677473.588373:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.588374:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.588376:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:17.0:1619677473.588377:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.588378:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.588379:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:17.0:1619677473.588379:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.588380:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.588381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:17.0:1619677473.588381:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.588382:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.588383:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.588384:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.588385:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.588385:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.588386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677473.588390:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.588392:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.588393:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.588394:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.588396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.588396:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.588397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.588397:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.588398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.588400:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677473.588400:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.588401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.588401:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.588402:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.588404:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.588406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.588417:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.588419:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.588420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.588421:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.588422:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.588422:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677473.588497:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.588499:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.588500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.588502:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.588504:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.588506:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677473.588507:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.588508:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.588510:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.588510:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677473.588514:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.588515:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677473.588518:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.588519:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677473.588519:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677473.588521:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677473.588522:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.588523:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677473.588524:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.588524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677473.588527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.588534:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:17.0:1619677473.588536:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:10.0:1619677473.588536:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.588537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.588538:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:10.0:1619677473.588538:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.588539:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.588539:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:10.0:1619677473.588540:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.588543:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.588683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.588690:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.588692:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.588693:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.588693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.588695:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.588697:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.588896:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.588899:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.588899:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.588900:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.588902:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.588905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.588916:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.588919:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.588919:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.588920:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.588921:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.588921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.588990:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.588993:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.588993:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.588995:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.588998:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.588999:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.589000:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.589002:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.589003:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.589004:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.589006:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.589008:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.589010:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.589011:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.589011:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.589013:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.589013:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.589014:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.589014:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.589018:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.2:1619677473.589019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.589021:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.589023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.589023:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.589025:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.589026:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.589027:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.589027:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.589028:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.589029:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.589030:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96508674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.589032:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.589033:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.589034:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.589036:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.589037:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.589038:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677473.589038:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.589039:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.589040:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.589040:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.589041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.589042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.589043:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.589044:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.589045:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.589046:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.589047:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.589047:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.589048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.589059:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.589062:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.589063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.589063:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.589066:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.589068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.589201:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.589204:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.589204:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.589206:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.589208:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.589210:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.589211:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.589212:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.589213:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.589214:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.589217:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.589219:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.589221:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.589222:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.589222:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.589224:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.589225:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.589225:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.589226:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.589238:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.589240:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.589241:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.589243:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.589243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.589245:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.589396:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.589398:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.589399:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.589401:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.589404:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.589405:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677473.589407:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.589408:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.589409:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.589410:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677473.589413:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.589415:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677473.589418:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.589418:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677473.589419:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677473.589421:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677473.589421:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.589422:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.589422:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.590207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.590214:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.590217:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.590218:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.590218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.590219:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.590219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.590221:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:24.0:1619677473.590222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.590222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.590224:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.590226:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.590228:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.590229:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.590230:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.590231:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.590232:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.590236:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.590238:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.590240:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.590241:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.590242:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.590243:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.590244:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.590245:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.590245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.590257:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.590259:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.590260:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.590261:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.590263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.590264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.590275:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.590278:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.590279:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.590279:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.590280:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.590281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.590716:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.590718:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.590719:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.590721:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.590723:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.590725:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.590726:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.590728:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.590729:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.590730:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.590733:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.590735:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.590738:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.590739:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.590739:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.590741:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.590741:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.590743:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.590744:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.590744:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.590756:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.590758:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.590759:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.590759:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.590760:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.590760:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.590761:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.590762:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.590762:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.590765:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.590805:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.590808:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.590809:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.590811:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.590813:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.590814:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.590816:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.590817:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.590818:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.590819:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.590823:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.590825:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.590827:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.590828:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.590828:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.590830:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.590830:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.590831:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.590831:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.590836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.590840:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.590842:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.590843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.590843:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.590845:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.590847:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:6.0:1619677473.590856:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.2:1619677473.590859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:6.0:1619677473.590859:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.590860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.590860:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.590862:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000400:00000200:6.0:1619677473.590862:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.590864:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.590865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:6.0:1619677473.590865:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.590866:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.590866:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.590867:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.591111:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.591114:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.591115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.591117:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.591119:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.591121:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.591122:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.591123:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.591124:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.591125:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.591129:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.591130:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.591133:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.591133:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.591134:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.591136:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.591136:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.591137:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.591137:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.591150:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.591152:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.591152:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.591154:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.591155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.591156:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.591375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.591381:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.591384:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.591385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.591387:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.591390:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.591391:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.591392:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.591394:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.591395:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.591396:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.591399:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.591401:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.591402:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.591403:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.591404:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.591405:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.591406:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.591408:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.591409:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.591409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.591412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.591415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.591421:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.591422:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677473.591423:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.591423:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.591423:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.591424:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.591425:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.591425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.591426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.591426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677473.591428:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.591431:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.591432:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.591434:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.591436:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.591438:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.591439:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.591440:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.591441:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.591442:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.591446:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.591447:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.591450:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.591451:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.591452:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.591453:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.591454:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.591455:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.591455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.591460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.591469:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.591472:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.591473:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.591473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.591475:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.591479:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677473.591685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.591691:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.591694:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.591695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.591697:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.591699:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.591700:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.591702:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.591703:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.591704:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.591705:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.591709:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.591710:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.591713:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.591714:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.591715:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.591717:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.591718:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.591719:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.591719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.591731:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.591733:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.591734:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.591736:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.591739:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.591740:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.591750:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.591753:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.591754:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.591755:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.591755:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.591756:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.591940:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.591943:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.591944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.591946:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.591948:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.591950:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.591951:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.591953:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.591954:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.591955:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.591959:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.591962:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.591964:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.591965:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.591966:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.591968:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.591968:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.591970:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.591971:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.591971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.591981:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.591981:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.591983:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.591984:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.591984:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.591985:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.591985:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.591986:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.591988:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.591989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.591999:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.592002:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.592003:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.592005:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.592007:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.592009:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.592010:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.592011:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.592013:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.592014:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.592017:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.592019:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.592021:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.592022:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.592023:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.592025:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.592025:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.592026:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.592026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.592031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.592039:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.592042:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.592043:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.592043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.592046:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.592049:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.592234:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.592238:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.592238:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.592240:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677473.592241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:26.0:1619677473.592243:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.592245:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.592246:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.592246:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677473.592247:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.592249:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.592249:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677473.592251:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677473.592251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.592252:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.592255:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677473.592255:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.592256:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:21.0:1619677473.592257:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.592258:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.592260:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.592260:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.592261:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.592261:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:21.0:1619677473.592261:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.592262:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.592263:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.592264:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.592264:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.592265:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:21.0:1619677473.592265:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.592267:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.592270:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.592270:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.592271:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.592273:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.592273:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.592274:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.592274:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677473.592276:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:6.0:1619677473.592278:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.592279:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:8.2:1619677473.592279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.592280:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:6.0:1619677473.592280:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.592281:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.592281:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.592282:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.592284:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.592284:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.592286:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:17.0:1619677473.592287:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:10.0:1619677473.592287:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.592288:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677473.592289:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.592290:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.592290:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.592290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:10.0:1619677473.592291:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:6.0:1619677473.592293:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:17.0:1619677473.592300:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.592301:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677473.592302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.592304:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:17.0:1619677473.592305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.592306:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.592306:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.592307:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.592308:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:17.0:1619677473.592309:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.592309:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.592309:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:17.0:1619677473.592310:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:10.0:1619677473.592310:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:17.0:1619677473.592311:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:10.0:1619677473.592311:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:1.0:1619677473.592311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.592312:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.2:1619677473.592314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.592318:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.592321:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:24.0:1619677473.592322:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.592322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.592324:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.592326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.592501:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.592504:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.592504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.592505:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.592507:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.592510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.592518:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.592521:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.592522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.592522:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.592523:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.592524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.593002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.593005:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.593005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.593007:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.593010:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.593011:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.593012:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.593013:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.593014:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.593015:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.593019:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.593021:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.593023:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.593024:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.593024:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.593026:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.593027:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.593027:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.593027:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.593040:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.593042:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.593043:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.593045:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.593046:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.593047:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.593352:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.593355:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.593356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.593357:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.593360:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.593361:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.593363:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.593364:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.593365:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.593366:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.593369:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.593371:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.593374:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.593375:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.593375:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.593377:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.593377:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.593378:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.593378:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.593392:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.593394:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.593395:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.593396:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.593398:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.593401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.593563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.593570:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.593572:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.593573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.593575:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.593578:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.593579:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.593581:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.593582:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.593583:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.593584:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.593587:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.593589:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.593592:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.593592:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.593593:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.593595:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.593595:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.593596:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.593596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.593609:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.593611:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.593612:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.593612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.593615:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.593617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.593649:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.593652:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.593652:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.593655:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.593657:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.593658:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.593660:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.593661:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.593662:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.593663:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.593666:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.593668:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.593670:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.593671:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.593672:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.593673:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.593674:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.593674:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.593675:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.593686:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.593689:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.593689:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.593691:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.593693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.593694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.593846:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.593849:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.593849:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.593851:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.593854:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.593855:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.593856:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.593858:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.593859:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.593860:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.593863:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.593864:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.593867:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.593868:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.593868:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.593870:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.593871:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.593871:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.593872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.593884:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.593886:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.593887:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.593888:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.593889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.593890:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.593901:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.593904:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.593905:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.593906:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.593906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.593907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.594259:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.594262:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.594263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.594265:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.594267:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.594268:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.594270:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.594271:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.594272:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.594273:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.594276:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.594278:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.594280:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.594281:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.594282:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.594283:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.594283:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.594284:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.594285:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.594289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.594295:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.594301:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.594303:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.594304:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.594305:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.594307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.594334:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.594337:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.594338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.594339:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.594342:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.594343:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.594344:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.594345:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.594346:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.594347:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.594349:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.594351:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.594353:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.594353:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.594354:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.594355:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.594356:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.594356:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.594357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.594362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.594367:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.594369:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.594369:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.594370:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.594371:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.594373:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:20.2:1619677473.594430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.594433:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.594435:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677473.594437:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.594439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.594440:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.594442:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.594443:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.594444:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.594445:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.594446:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.594449:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.594450:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.594453:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.594454:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.594454:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.594456:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.594457:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.594457:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.594458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:22.0:1619677473.594482:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.594484:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.594486:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.594486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.594487:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.594491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.594526:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.594529:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.594529:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.594530:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.594532:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.594534:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677473.594658:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.594661:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.594662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.594664:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.594667:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.594668:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677473.594669:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.594671:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.594672:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.594673:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677473.594676:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.594678:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677473.594680:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.594681:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677473.594682:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677473.594684:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.594684:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.594685:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.594685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.594690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.594696:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:22.0:1619677473.594698:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:10.0:1619677473.594699:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.594700:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.594700:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.594701:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:6.0:1619677473.594702:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000400:00000200:10.0:1619677473.594703:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.594704:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.594704:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.594705:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:10.0:1619677473.594706:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:22.0:1619677473.594707:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.594707:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.594709:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:9.2:1619677473.594709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:22.0:1619677473.594710:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.594712:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.594713:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.594714:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:26.0:1619677473.594715:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:6.0:1619677473.594716:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.594718:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.594718:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:6.0:1619677473.594718:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677473.594719:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:22.0:1619677473.594719:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.594720:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:6.0:1619677473.594720:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.594721:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:6.0:1619677473.594721:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.594722:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677473.594722:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.594723:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.594727:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677473.594727:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.594728:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:6.0:1619677473.594728:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.594729:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:6.0:1619677473.594730:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.594731:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.594732:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:6.0:1619677473.594732:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.594733:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (22)++
00000800:00000200:22.0:1619677473.594734:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)--
00000800:00000200:6.0:1619677473.594734:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.594735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000800:00000200:6.0:1619677473.594735:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.594735:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.594736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.594736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677473.594737:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:22.0:1619677473.594739:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:6.0:1619677473.594739:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:6.0:1619677473.594740:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000400:00000200:22.0:1619677473.594741:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.594742:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.594743:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.594745:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:12.0:1619677473.594745:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:22.0:1619677473.594746:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.594747:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.594748:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677473.594748:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:22.0:1619677473.594749:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677473.594749:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.594750:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.594752:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:12.0:1619677473.594752:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.594752:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:13.0:1619677473.594753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:22.0:1619677473.594754:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.594754:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96508641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.594755:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.594755:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.594756:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677473.594756:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.594758:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.594758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:6.0:1619677473.594758:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.594759:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:12.0:1619677473.594762:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677473.594763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:12.0:1619677473.594763:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:6.0:1619677473.594763:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.594764:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.594765:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677473.594766:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:12.0:1619677473.594766:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677473.594766:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677473.594767:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.594767:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:22.0:1619677473.594768:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:12.0:1619677473.594768:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:22.0:1619677473.594769:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)++
00000800:00000200:12.0:1619677473.594769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:22.0:1619677473.594770:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)--
00000800:00000200:6.0:1619677473.594770:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.594771:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000800:00000200:6.0:1619677473.594771:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677473.594771:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677473.594772:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:22.0:1619677473.594774:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:6.0:1619677473.594774:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677473.594774:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)++
00000800:00000200:6.0:1619677473.594776:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198623bcc00] (24)++
00000400:00000200:22.0:1619677473.594777:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.594777:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (25)--
00000800:00000200:6.0:1619677473.594778:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (24)--
00000800:00000200:6.0:1619677473.594778:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)++
00000400:00000200:22.0:1619677473.594780:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.594780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677473.594780:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000400:00000200:22.0:1619677473.594782:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677473.594782:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.594783:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96508639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.594785:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.594787:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.594788:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.594788:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:9.2:1619677473.594789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677473.594791:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:22.0:1619677473.594792:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.594792:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.594792:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.594793:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.594794:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.594795:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677473.594795:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:6.0:1619677473.594795:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:6.0:1619677473.594796:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96508638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.594798:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.594798:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:6.0:1619677473.594798:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.594799:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.594799:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.594800:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.594801:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.594803:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.594804:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.594805:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:9.2:1619677473.594805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000400:00000200:6.0:1619677473.594805:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.594806:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677473.594807:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677473.594808:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677473.594809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.594809:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)++
00000800:00000200:22.0:1619677473.594810:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (25)--
00000800:00000200:22.0:1619677473.594811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (24)--
00000800:00000200:6.0:1619677473.594811:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.594812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)++
00000800:00000200:18.0:1619677473.594812:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677473.594813:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198623bcc00] (24)--
00000800:00000200:6.0:1619677473.594813:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677473.594813:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677473.594814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000800:00000200:22.0:1619677473.594815:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:18.0:1619677473.594815:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677473.594815:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677473.594816:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)++
00000800:00000200:2.0:1619677473.594816:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677473.594817:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:18.0:1619677473.594817:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:6.0:1619677473.594817:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)--
00000800:00000200:18.0:1619677473.594818:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677473.594818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000400:00000200:22.0:1619677473.594819:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677473.594819:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.594819:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:9.2:1619677473.594821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677473.594821:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000400:00000200:22.0:1619677473.594822:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.594823:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.594824:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677473.594826:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.594827:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:22.0:1619677473.594828:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.594829:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.594830:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677473.594830:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.594831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.594831:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.594833:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.594834:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.594834:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.594836:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677473.594836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677473.594837:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677473.594839:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.594840:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.594841:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.594842:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677473.594845:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677473.594845:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.594846:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677473.594847:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677473.594847:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677473.594849:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677473.594849:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.594850:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)++
00000800:00000200:6.0:1619677473.594850:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:6.0:1619677473.594850:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677473.594851:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)--
00000800:00000200:22.0:1619677473.594852:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000800:00000200:6.0:1619677473.594852:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:22.0:1619677473.594853:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:6.0:1619677473.594853:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (22)++
00000800:00000200:22.0:1619677473.594854:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:6.0:1619677473.594854:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198623bcc00] (24)++
00000800:00000200:6.0:1619677473.594855:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (25)--
00000400:00000200:22.0:1619677473.594856:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.594856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (24)--
00000800:00000200:6.0:1619677473.594856:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)++
00000800:00000200:6.0:1619677473.594857:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000400:00000200:22.0:1619677473.594859:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677473.594859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677473.594859:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.594861:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677473.594862:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.594863:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.594864:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.594865:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677473.594867:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:6.0:1619677473.594867:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.594869:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:6.0:1619677473.594869:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677473.594870:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677473.594870:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.594870:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96508634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.594872:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.594873:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:21.0:1619677473.594873:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:6.0:1619677473.594873:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.594874:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.594875:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677473.594878:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.594879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677473.594880:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.594881:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.594882:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:6.0:1619677473.594882:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.594883:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677473.594883:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677473.594884:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677473.594885:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:6.0:1619677473.594885:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677473.594886:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)++
00000800:00000200:6.0:1619677473.594886:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)++
00000800:00000200:22.0:1619677473.594887:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (26)--
00000800:00000200:6.0:1619677473.594887:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (26)--
00000800:00000200:22.0:1619677473.594888:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000800:00000200:6.0:1619677473.594888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (25)--
00000800:00000200:6.0:1619677473.594889:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:22.0:1619677473.594891:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)++
00000800:00000200:9.2:1619677473.594891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677473.594891:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:22.0:1619677473.594892:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198623bcc00] (24)--
00000800:00000200:22.0:1619677473.594893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000400:00000200:6.0:1619677473.594893:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.594894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:22.0:1619677473.594895:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:9.2:1619677473.594895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677473.594895:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.594897:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.594897:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:10.0:1619677473.594898:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:6.0:1619677473.594898:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.594900:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:6.0:1619677473.594900:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.594901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:6.0:1619677473.594901:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.594902:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.594902:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677473.594902:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:22.0:1619677473.594904:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.594904:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.594905:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:22.0:1619677473.594906:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677473.594906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:6.0:1619677473.594906:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677473.594906:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.594908:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.594908:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.594909:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.594909:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677473.594911:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.594911:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677473.594914:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.594916:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.594917:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:2.0:1619677473.594919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.594920:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.594921:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.594921:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677473.594921:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677473.594921:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677473.594922:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.594924:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.594924:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)++
00000800:00000200:6.0:1619677473.594924:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677473.594924:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)++
00000800:00000200:6.0:1619677473.594925:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (25)--
00000800:00000200:22.0:1619677473.594926:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198623bcc00] (24)++
00000800:00000200:6.0:1619677473.594926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (24)--
00000800:00000200:22.0:1619677473.594927:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)--
00000800:00000200:22.0:1619677473.594927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (24)--
00000800:00000200:6.0:1619677473.594927:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (24)++
00000800:00000200:22.0:1619677473.594928:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)++
00000800:00000200:6.0:1619677473.594928:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:22.0:1619677473.594929:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:9.2:1619677473.594929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677473.594931:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.594932:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.594933:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.594934:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.594935:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:6.0:1619677473.594935:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677473.594936:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.594937:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.594938:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:6.0:1619677473.594938:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.594939:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.594939:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:6.0:1619677473.594939:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677473.594942:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.594942:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677473.594943:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96508630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.594944:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.594945:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.594946:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.594947:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:6.0:1619677473.594947:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677473.594951:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.594952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.594952:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.594953:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677473.594953:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:6.0:1619677473.594954:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677473.594956:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.594956:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677473.594956:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677473.594957:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.594957:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)++
00000800:00000200:6.0:1619677473.594958:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (25)--
00000800:00000200:22.0:1619677473.594959:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.594959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (24)--
00000800:00000200:22.0:1619677473.594960:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)++
00000800:00000200:22.0:1619677473.594960:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (25)--
00000800:00000200:6.0:1619677473.594960:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)++
00000800:00000200:22.0:1619677473.594961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (24)--
00000800:00000200:6.0:1619677473.594961:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:22.0:1619677473.594962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.594963:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.594964:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.594964:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677473.594965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677473.594966:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.594967:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.594968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.594968:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)++
00000800:00000200:22.0:1619677473.594969:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198623bcc00] (24)--
00000400:00000200:6.0:1619677473.594969:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:22.0:1619677473.594970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000400:00000200:6.0:1619677473.594970:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.594971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.594972:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:8.0:1619677473.594972:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:6.0:1619677473.594972:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.594973:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000400:00000200:6.0:1619677473.594973:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.594974:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677473.594975:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.594976:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.594976:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.594977:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.594979:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.594979:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.594980:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.594981:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677473.594982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677473.594983:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.594984:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.594984:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.594985:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:22.0:1619677473.594986:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677473.594986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677473.594986:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:22.0:1619677473.594987:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.594988:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.594988:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677473.594989:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)++
00000800:00000200:6.0:1619677473.594990:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)--
00000800:00000200:6.0:1619677473.594990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000800:00000200:24.0:1619677473.594992:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677473.594992:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.594992:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:6.0:1619677473.594993:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:22.0:1619677473.594994:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:9.2:1619677473.594994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:24.0:1619677473.594996:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.594996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:6.0:1619677473.594996:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.594997:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.594998:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.594998:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.594998:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.594998:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677473.594999:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:6.0:1619677473.594999:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.595000:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.595001:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:6.0:1619677473.595001:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677473.595002:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.595003:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:6.0:1619677473.595003:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.595004:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:6.0:1619677473.595004:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.595005:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.595005:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677473.595006:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)++
00000800:00000200:22.0:1619677473.595007:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198623bcc00] (24)++
00000400:00000200:26.0:1619677473.595008:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.595008:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (25)--
00000800:00000200:22.0:1619677473.595009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (24)--
00000800:00000200:22.0:1619677473.595010:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000400:00000200:6.0:1619677473.595010:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.595011:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.595012:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677473.595012:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677473.595013:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.595014:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.595014:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:18.0:1619677473.595014:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:22.0:1619677473.595015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.595015:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.595016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677473.595016:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:6.0:1619677473.595016:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:6.0:1619677473.595016:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677473.595017:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)++
00000800:00000200:2.0:1619677473.595017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:22.0:1619677473.595018:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198623bcc00] (24)--
00000800:00000200:6.0:1619677473.595018:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:22.0:1619677473.595019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (23)--
00000800:00000200:21.0:1619677473.595019:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:18.0:1619677473.595019:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.595019:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)++
00000800:00000200:6.0:1619677473.595019:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (24)--
00000800:00000200:22.0:1619677473.595020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)--
00000800:00000200:6.0:1619677473.595020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000800:00000200:21.0:1619677473.595021:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.595022:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.595022:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.595024:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:18.0:1619677473.595024:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:8.2:1619677473.595024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.595025:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:18.0:1619677473.595026:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.595027:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.595027:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.595028:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:18.0:1619677473.595028:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.595029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.595029:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.595029:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.595030:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677473.595031:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.595034:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:18.0:1619677473.595035:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.595037:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677473.595041:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.595041:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:18.0:1619677473.595042:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677473.595043:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:22.0:1619677473.595045:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000800:00000200:18.0:1619677473.595045:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677473.595045:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:6.0:1619677473.595045:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:18.0:1619677473.595046:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677473.595047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:6.0:1619677473.595047:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677473.595050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:6.0:1619677473.595050:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.595051:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677473.595052:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.595054:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.595055:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.595056:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.595057:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:6.0:1619677473.595059:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.595061:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:6.0:1619677473.595061:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.595062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.595062:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.595064:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.595065:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.595065:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677473.595066:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.595068:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677473.595068:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677473.595069:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:6.0:1619677473.595070:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (22)--
00000800:00000200:6.0:1619677473.595071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)--
00000800:00000200:9.2:1619677473.595074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677473.595077:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:26.0:1619677473.595080:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:22.0:1619677473.595080:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:6.0:1619677473.595081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000800:00000200:26.0:1619677473.595082:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:22.0:1619677473.595082:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.595083:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.595084:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.595085:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.595085:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.595087:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677473.595088:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.595089:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.595090:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.595091:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.595092:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:22.0:1619677473.595094:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.595096:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677473.595099:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.595099:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677473.595100:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.595101:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.595102:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:22.0:1619677473.595103:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (22)--
00000800:00000200:22.0:1619677473.595103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)--
00000800:00000200:9.2:1619677473.595107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.595114:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.595117:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.595118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.595118:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.595120:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.595123:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.595136:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:22.0:1619677473.595139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000800:00000200:6.0:1619677473.595139:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000400:00000200:6.0:1619677473.595141:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.595144:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.595146:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677473.595147:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.595148:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.595149:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.595150:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677473.595153:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.595154:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677473.595156:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.595157:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677473.595158:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677473.595160:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677473.595160:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:6.0:1619677473.595162:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198623bcc00] (22)++
00000800:00000200:6.0:1619677473.595163:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (23)--
00000800:00000200:6.0:1619677473.595163:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000800:00000200:9.2:1619677473.595165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.595172:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:22.0:1619677473.595173:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)++
00000800:00000200:10.0:1619677473.595174:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.595175:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:22.0:1619677473.595175:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198623bcc00] (23)--
00000800:00000200:22.0:1619677473.595176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000400:00000200:10.0:1619677473.595176:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.595177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)--
00000400:00000200:10.0:1619677473.595178:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.595181:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:1.0:1619677473.595363:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.595366:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.595367:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.595369:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.595372:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.595374:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.595375:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.595376:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.595377:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.595378:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.595381:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.595383:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.595386:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.595387:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.595387:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.595389:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.595389:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.595390:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.595390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677473.595404:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.595406:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.595408:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.595408:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.595410:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.595412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.595514:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.595516:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.595517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.595519:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.595521:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.595523:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.595524:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.595525:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.595526:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.595528:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.595531:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.595533:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.595536:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.595536:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.595537:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.595539:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.595539:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.595540:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.595540:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.595552:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.595555:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.595555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.595556:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.595558:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.595561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.595748:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.595750:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.595751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.595753:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.595756:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.595757:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.595758:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.595759:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.595760:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.595761:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.595765:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.595767:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.595769:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.595770:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.595771:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.595772:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.595773:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.595774:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.595775:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.595775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.595785:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.595787:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.595787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.595787:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.595788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.595790:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.595790:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.595792:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.595793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.595794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.595901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.595908:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.595910:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.595911:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.595911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.595913:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.595915:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.595964:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.595967:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.595968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.595969:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.595972:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.595973:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.595974:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.595975:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.595976:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.595977:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.595980:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.595982:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.595984:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.595984:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.595985:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.595986:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.595987:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.595988:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.595989:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.595990:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677473.595992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.595995:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.595997:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.595998:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.595998:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.596000:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.596002:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677473.596002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:26.0:1619677473.596004:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.596005:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.596006:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:17.0:1619677473.596083:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.596086:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.596087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.596089:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.596092:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.596093:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677473.596094:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.596096:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.596097:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.596098:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677473.596102:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.596104:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677473.596107:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.596107:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677473.596108:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677473.596110:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.596110:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.596111:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.596112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.596116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.596122:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.596125:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.596126:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.596126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.596128:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.596132:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.596136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.596142:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.596144:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.596145:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.596145:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.596146:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.596147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677473.596254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe9285400] (21)++
00000800:00000200:24.0:1619677473.596257:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.52@o2ib3
00000800:00000200:8.0:1619677473.596257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9285400] (22)--
00000400:00000200:24.0:1619677473.596259:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.2.52@o2ib3 : GET - for me
00000400:00000200:24.0:1619677473.596263:0:19939:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.2.52@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:24.0:1619677473.596265:0:19939:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.2.52@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:24.0:1619677473.596267:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe9285400] (21)++
00000800:00000200:24.0:1619677473.596268:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe9285400] (22)--
00000400:00000200:24.0:1619677473.596270:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.2.52@o2ib3 local destination
00000400:00000200:24.0:1619677473.596275:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.2.52@o2ib3(10.51.2.52@o2ib3:10.51.2.52@o2ib3) : REPLY try# 0
00000800:00000200:24.0:1619677473.596277:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.2.52@o2ib3
00000800:00000200:24.0:1619677473.596280:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919fedd18d80] -> 10.51.2.52@o2ib3 (2) version: 12
00000800:00000200:24.0:1619677473.596280:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919fe9285400] (21)++
00000800:00000200:24.0:1619677473.596281:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe9285400] (22)++
00000800:00000200:24.0:1619677473.596283:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919fe9285400] (23)--
00000800:00000200:24.0:1619677473.596283:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9285400] (22)--
00000800:00000200:8.0:1619677473.596298:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe9285400] (22)++
00000800:00000200:8.0:1619677473.596300:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe9285400] (23)--
00000400:00000200:8.0:1619677473.596301:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.596302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9285400] (22)--
00000400:00000200:8.0:1619677473.596302:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.2.52@o2ib3: REPLY: OK
00000800:00000200:8.0:1619677473.596305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe9285400] (21)--
00000800:00000200:6.0:1619677473.596315:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.596318:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.596319:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.596319:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.596321:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.596324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.596379:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.596382:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677473.596383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677473.596385:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.596387:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.596389:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677473.596390:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.596391:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.596392:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.596393:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677473.596397:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.596399:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677473.596401:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.596402:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677473.596403:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677473.596404:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677473.596405:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.596406:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677473.596406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677473.596411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.596419:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.596422:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.596422:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.596423:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.596425:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.596428:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.596639:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.596642:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.596643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.596645:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.596647:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.596649:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.596650:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.596651:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.596652:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.596653:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.596657:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.596658:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.596660:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.596661:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.596661:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.596663:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.596663:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.596664:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.596665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.596677:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.596679:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.596680:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.596681:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.596683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.596684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.596696:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.596699:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.596700:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.596701:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.596701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.596702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.596995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.597002:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.597005:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.597006:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.597006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.597008:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.597010:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.597068:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.597071:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.597072:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.597073:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.597075:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.597077:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.597078:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.597079:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.597080:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.597081:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.597084:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.597085:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.597088:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.597089:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.597090:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.597092:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.597092:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.597093:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.597093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.597106:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.597109:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.597110:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.597110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.597113:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.597116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.597238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.597244:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.597247:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.597248:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.597251:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.597254:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.597255:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.597256:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.597257:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.597259:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.597260:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.597263:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.597265:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.597267:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.597268:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.597268:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.597270:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.597270:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.597271:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.597272:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.597276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.597284:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.597286:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.597287:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.597288:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.597289:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.597292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677473.597469:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.597472:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.597473:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.597475:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.597478:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.597479:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.597481:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.597482:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.597483:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.597484:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.597488:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.597489:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.597489:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.597491:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.597492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677473.597492:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.597493:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.597494:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677473.597494:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.597495:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.597496:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677473.597496:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.597497:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.597498:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:6.0:1619677473.597498:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.0:1619677473.597499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677473.597499:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.597500:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.597501:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.597502:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.597505:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.597507:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.597510:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.597511:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677473.597511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.597511:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.597512:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.597513:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.597513:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677473.597514:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.597514:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677473.597514:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.597514:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:22.0:1619677473.597515:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.597515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.597515:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.597516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677473.597517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.597518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.597527:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.597529:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.597530:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.597531:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.597531:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.597532:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.597533:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.597534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.597534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.597535:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.597536:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.597536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677473.597570:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677473.597573:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:2.0:1619677473.597573:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:18.0:1619677473.597576:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.597578:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.597580:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677473.597582:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.597583:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.597584:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.597586:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677473.597589:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.597590:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677473.597593:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.597594:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677473.597595:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677473.597597:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677473.597597:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677473.597598:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:18.0:1619677473.597599:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:9.2:1619677473.597603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.597609:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.597612:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.597613:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.597613:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.597615:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.597619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.597620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.597626:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.597628:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.597629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.597629:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:2.0:1619677473.597629:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:8.0:1619677473.597630:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.597631:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677473.597632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000800:00000200:2.0:1619677473.597632:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000400:00000200:2.0:1619677473.597634:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.597637:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.597638:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677473.597640:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.597641:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.597642:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.597642:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677473.597645:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.597647:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677473.597650:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.597650:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:2.0:1619677473.597651:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677473.597652:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677473.597653:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677473.597654:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:2.0:1619677473.597654:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:22.0:1619677473.597666:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.597668:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.597669:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.597669:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.597672:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.597674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.597685:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.597688:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.597688:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.597688:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.597689:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.597690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.598392:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.598395:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.598396:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:22.0:1619677473.598396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677473.598397:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677473.598398:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.598399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.598400:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.598400:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.598402:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677473.598403:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.598404:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.598405:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.598406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677473.598406:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.598406:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.598409:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.598411:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.598413:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.598414:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.598414:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.598416:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.598417:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.598417:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.598418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.598431:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.598434:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.598435:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.598438:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:13.0:1619677473.598438:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.598439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:24.0:1619677473.598441:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:13.0:1619677473.598442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677473.598442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.598443:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.598446:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.598448:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.598449:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.598450:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.598451:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.598452:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.598456:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.598457:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.598460:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.598461:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.598461:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.598463:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.598464:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.598464:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.598465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.598477:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.598479:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.598480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.598481:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.598483:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.598485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.598513:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.598516:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.598516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.598518:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.598521:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.598522:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.598523:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.598525:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.598525:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.598527:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.598529:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.598531:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.598533:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.598534:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.598535:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.598536:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.598537:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.598538:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.598538:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.598550:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.598552:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.598553:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.598555:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.598556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.598557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.598762:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.598765:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.598765:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.598768:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.598770:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.598772:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677473.598773:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.598775:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.598776:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.598777:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677473.598780:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.598782:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677473.598785:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.598785:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677473.598786:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677473.598788:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677473.598788:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.598789:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.598789:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.2:1619677473.599363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.599368:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.599374:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677473.599374:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.599377:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.599377:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677473.599379:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.599379:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.599380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677473.599380:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.599382:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.599383:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.599383:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.599384:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.599385:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.599386:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677473.599386:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.599388:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.599390:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677473.599390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:6.0:1619677473.599391:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.599392:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677473.599393:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.599396:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:6.0:1619677473.599396:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.599396:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.599399:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677473.599399:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.0:1619677473.599400:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.599400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.0:1619677473.599401:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.599401:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677473.599402:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.599403:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677473.599403:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:1.0:1619677473.599403:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.599404:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.599404:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677473.599404:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.599405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677473.599405:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:1.0:1619677473.599405:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.599406:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.599407:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:1.0:1619677473.599407:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677473.599408:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:1.0:1619677473.599408:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.599409:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.2:1619677473.599410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:1.0:1619677473.599410:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.599410:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.599412:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.599416:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.599418:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.599419:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.599419:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:17.0:1619677473.599420:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.599421:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.599421:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.599422:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.599422:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.599422:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:17.0:1619677473.599422:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.599422:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.599422:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:20.0:1619677473.599423:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.599423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:17.0:1619677473.599424:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.599425:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.599425:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677473.599426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:1.0:1619677473.599426:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677473.599428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677473.599428:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.599428:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.599429:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.599430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677473.599430:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.599434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.599438:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.599440:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.599440:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677473.599441:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.599442:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.599442:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.599443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677473.599444:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.599444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.599446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.599524:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.599527:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.599528:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.599530:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.599533:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.599534:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.599535:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.599537:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.599539:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.599541:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.599544:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.599547:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.599550:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.599551:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.599551:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.599553:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.599554:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.599555:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.599556:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.599560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.599567:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.599569:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.599570:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.599570:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.599574:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.599576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.599862:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.599865:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.599866:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.599869:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.599871:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.599872:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.599874:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.599875:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.599876:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.599877:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.599881:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.599883:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.599885:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.599885:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.599886:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.599887:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.599888:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.599888:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.599889:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.599889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.2:1619677473.599893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.599896:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:24.0:1619677473.599898:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:18.0:1619677473.599898:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:24.0:1619677473.599899:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.599899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:24.0:1619677473.599901:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.599901:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.599902:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.599902:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.599903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000400:00000200:18.0:1619677473.599904:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.599907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.2:1619677473.599910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.599913:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.599916:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.599917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.599917:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.599918:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:8.0:1619677473.599919:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:12.0:1619677473.599946:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.599948:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.599950:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.599953:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.599953:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.599954:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677473.599956:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.599957:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.599958:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.599959:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677473.599962:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.599964:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677473.599966:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.599967:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677473.599967:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677473.599969:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677473.599969:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.599971:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677473.599972:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677473.599973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.599986:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677473.599988:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677473.599989:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.599990:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:21.0:1619677473.600004:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.600006:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.600009:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.600009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.600011:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.600013:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.600014:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.600015:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.600016:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.600017:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.600020:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.600022:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.600029:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.600030:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.600030:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.600032:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.600032:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.600033:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.600034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.600038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.600046:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.600048:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.600049:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.600051:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.600052:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.600053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677473.600073:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.600076:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.600077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.600077:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.600079:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.600082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.600228:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.600230:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.600231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.600233:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.600235:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.600236:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.600237:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.600238:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.600239:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.600240:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.600243:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.600245:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.600247:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.600248:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.600249:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.600250:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.600251:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.600251:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.600252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.600264:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.600266:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.600267:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.600269:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.600269:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.600271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.600288:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.600291:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.600292:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.600293:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.600294:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.600298:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.600803:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.600805:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.600806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.600808:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.600811:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.600812:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.600813:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.600815:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.600816:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.600817:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.600820:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.600822:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.600824:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.600825:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.600826:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.600827:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.600828:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.600829:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.600830:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.600831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677473.600842:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.600843:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.600845:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.600846:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.600846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.600846:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.600846:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.600847:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.600848:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.600851:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.600860:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.600862:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.600863:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.600863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.600864:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.600865:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.600964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.600970:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.600973:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.600974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.600976:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.600978:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.600979:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.600981:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.600982:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.600983:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.600984:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.600987:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.600989:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.600991:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.600992:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.600992:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.600994:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.600995:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.600996:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.600996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.601000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.601008:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.601010:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.601010:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.601011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.601012:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.601013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.601320:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.601323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.601323:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.601325:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.601326:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (21)++
00000400:00000200:6.0:1619677473.601327:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.601329:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.57@o2ib3
00000800:00000200:10.0:1619677473.601329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000400:00000200:6.0:1619677473.601329:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.601331:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.601333:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.57@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.601333:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.601334:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.601335:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.601336:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.601338:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677473.601339:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.601339:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.601341:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.601342:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.601343:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.601343:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.601344:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677473.601344:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.601345:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.601346:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677473.601347:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.601347:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.601348:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.601348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677473.601349:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 344 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677473.601352:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.601353:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677473.601353:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677473.601355:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677473.601355:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a061ff1000] (21)++
00000800:00000200:26.0:1619677473.601356:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a061ff1000] (22)--
00000800:00000200:26.0:1619677473.601357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (21)--
00000800:00000200:21.0:1619677473.601360:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.2:1619677473.601362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677473.601363:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.601363:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.601365:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.601366:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.601367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677473.601371:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.601374:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.601375:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.601375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.601377:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.601380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.601413:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (21)++
00000800:00000200:10.0:1619677473.601416:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.57@o2ib3
00000800:00000200:26.0:1619677473.601417:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000400:00000200:10.0:1619677473.601419:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.57@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.601421:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.601423:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677473.601424:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.601426:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.601426:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.601427:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677473.601430:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.601432:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677473.601434:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.601435:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677473.601435:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:10.0:1619677473.601436:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677473.601437:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a061ff1000] (21)++
00000800:00000200:10.0:1619677473.601438:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a061ff1000] (22)++
00000800:00000200:10.0:1619677473.601440:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a061ff1000] (23)--
00000800:00000200:10.0:1619677473.601444:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (23)--
00000800:00000200:10.0:1619677473.601446:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)++
00000800:00000200:10.0:1619677473.601447:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.57@o2ib3
00000800:00000200:26.0:1619677473.601448:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (23)++
00000800:00000200:26.0:1619677473.601449:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a061ff1000] (24)--
00000800:00000200:13.0:1619677473.601449:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:10.0:1619677473.601449:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.57@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.601450:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (23)--
00000800:00000200:26.0:1619677473.601451:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000400:00000200:10.0:1619677473.601451:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.601452:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.601452:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.601452:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:13.0:1619677473.601454:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.601454:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.601455:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677473.601456:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:10.0:1619677473.601456:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.601457:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677473.601459:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:10.0:1619677473.601461:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.601462:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.601465:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.601466:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.601466:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.601468:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.601468:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a061ff1000] (21)++
00000800:00000200:10.0:1619677473.601469:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a061ff1000] (22)--
00000800:00000200:10.0:1619677473.601469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (21)--
00000800:00000200:22.0:1619677473.601481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.601484:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.601485:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.601485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.601487:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (21)++
00000400:00000200:22.0:1619677473.601487:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.601489:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.57@o2ib3
00000800:00000200:10.0:1619677473.601490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000800:00000200:22.0:1619677473.601491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677473.601493:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.57@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.601495:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.601497:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.601498:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.601499:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.601500:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.601501:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.601504:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.601505:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.601509:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.601510:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.601510:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.601513:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.601513:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a061ff1000] (22)++
00000800:00000200:10.0:1619677473.601513:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)++
00000800:00000200:26.0:1619677473.601514:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a061ff1000] (24)--
00000800:00000200:26.0:1619677473.601515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (23)--
00000800:00000200:10.0:1619677473.601515:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.57@o2ib3
00000800:00000200:26.0:1619677473.601516:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000400:00000200:10.0:1619677473.601517:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.57@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.601518:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677473.601518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:10.0:1619677473.601520:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:10.0:1619677473.601521:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.601522:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.601523:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.601523:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.601526:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:10.0:1619677473.601526:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.601527:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.601528:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677473.601529:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677473.601530:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.601530:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:10.0:1619677473.601530:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677473.601531:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.601532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.601532:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677473.601532:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a061ff1000] (21)++
00000800:00000200:10.0:1619677473.601533:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a061ff1000] (22)--
00000800:00000200:10.0:1619677473.601533:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (21)--
00000800:00000200:20.0:1619677473.601534:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.601537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.601544:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.601546:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:26.0:1619677473.601547:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (21)++
00000400:00000200:24.0:1619677473.601547:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.601547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677473.601549:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.57@o2ib3
00000800:00000200:10.0:1619677473.601549:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)++
00000400:00000200:24.0:1619677473.601550:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.601551:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.57@o2ib3
00000400:00000200:26.0:1619677473.601552:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.57@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.601552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677473.601554:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.601554:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.57@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.601555:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677473.601557:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.601558:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.601559:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677473.601559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:26.0:1619677473.601560:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677473.601561:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.601563:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.601563:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677473.601564:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.601565:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:10.0:1619677473.601565:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.601565:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:10.0:1619677473.601566:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.601567:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:8.0:1619677473.601567:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.601568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.601568:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.601569:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.601569:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:10.0:1619677473.601570:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.601571:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677473.601573:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.601574:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:26.0:1619677473.601574:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:10.0:1619677473.601574:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.601574:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677473.601575:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.601577:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677473.601577:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a061ff1000] (23)++
00000800:00000200:10.0:1619677473.601577:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.601577:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a061ff1000] (23)++
00000800:00000200:26.0:1619677473.601579:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a061ff1000] (25)++
00000800:00000200:26.0:1619677473.601580:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a061ff1000] (26)--
00000800:00000200:10.0:1619677473.601580:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a061ff1000] (26)++
00000800:00000200:26.0:1619677473.601581:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (26)--
00000800:00000200:10.0:1619677473.601581:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a061ff1000] (26)--
00000800:00000200:9.2:1619677473.601582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.601583:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (24)--
00000800:00000200:10.0:1619677473.601583:0:19940:0:(o2iblnd_cb.c:929:kiblnd_post_tx_locked()) 10.51.2.57@o2ib3(1): redundant or enough NOOP
00000800:00000200:10.0:1619677473.601583:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a061ff1000] (24)--
00000800:00000200:10.0:1619677473.601584:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000800:00000200:26.0:1619677473.601585:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.601586:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.601586:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.601588:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.601589:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.601590:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)++
00000800:00000200:12.0:1619677473.601590:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:10.0:1619677473.601590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:26.0:1619677473.601593:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a061ff1000] (23)--
00000800:00000200:26.0:1619677473.601593:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000800:00000200:12.0:1619677473.601593:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.601594:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.601595:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (21)++
00000400:00000200:12.0:1619677473.601596:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.601597:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677473.601598:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.57@o2ib3
00000800:00000200:12.0:1619677473.601598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:10.0:1619677473.601599:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000400:00000200:26.0:1619677473.601601:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.57@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677473.601601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:26.0:1619677473.601603:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.601605:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.601606:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.601608:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.601608:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:26.0:1619677473.601609:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.601609:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677473.601609:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:26.0:1619677473.601610:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.601610:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677473.601611:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.601612:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:26.0:1619677473.601613:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.601615:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.601616:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.601617:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.601617:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.601619:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.601619:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a061ff1000] (21)++
00000800:00000200:26.0:1619677473.601620:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a061ff1000] (22)--
00000800:00000200:26.0:1619677473.601620:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (21)--
00000800:00000200:9.2:1619677473.601624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.601630:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (21)++
00000800:00000200:26.0:1619677473.601632:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.601632:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.601632:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.57@o2ib3
00000400:00000200:26.0:1619677473.601633:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.601634:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.57@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.601635:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.601636:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.601636:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.601637:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000400:00000200:10.0:1619677473.601637:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677473.601638:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:10.0:1619677473.601638:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.601640:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.601641:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.601642:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.601644:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.601647:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.601650:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)++
00000800:00000200:10.0:1619677473.601650:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.601651:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.57@o2ib3
00000800:00000200:10.0:1619677473.601651:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.601651:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:26.0:1619677473.601653:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.57@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.601653:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:26.0:1619677473.601654:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.601654:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a061ff1000] (23)++
00000800:00000200:10.0:1619677473.601655:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a061ff1000] (24)--
00000800:00000200:10.0:1619677473.601655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (23)--
00000400:00000200:26.0:1619677473.601656:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677473.601656:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000400:00000200:26.0:1619677473.601658:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.601659:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.601660:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.601661:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677473.601664:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.601665:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677473.601666:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677473.601667:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.601668:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.601670:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677473.601670:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.601670:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)++
00000800:00000200:26.0:1619677473.601671:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677473.601671:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677473.601671:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.601672:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677473.601672:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.57@o2ib3
00000800:00000200:26.0:1619677473.601673:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677473.601674:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a061ff1000] (23)++
00000400:00000200:10.0:1619677473.601674:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.57@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.601675:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a061ff1000] (24)--
00000800:00000200:26.0:1619677473.601675:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (23)--
00000400:00000200:10.0:1619677473.601676:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.601677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000400:00000200:10.0:1619677473.601678:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677473.601679:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.601679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677473.601681:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.601682:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.601682:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677473.601685:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677473.601687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677473.601687:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677473.601689:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.601689:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677473.601690:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.601690:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:10.0:1619677473.601691:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677473.601692:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a061ff1000] (21)++
00000800:00000200:10.0:1619677473.601692:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a061ff1000] (22)++
00000800:00000200:2.0:1619677473.601692:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.601693:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.601694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677473.601694:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a061ff1000] (23)--
00000800:00000200:10.0:1619677473.601694:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000800:00000200:18.0:1619677473.601696:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677473.601696:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677473.601699:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.601701:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.601701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.601702:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:18.0:1619677473.601703:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677473.601703:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.601704:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000400:00000200:18.0:1619677473.601705:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677473.601705:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.601705:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.601706:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.601707:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677473.601707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677473.601708:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)++
00000400:00000200:18.0:1619677473.601708:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.601709:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.601710:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a061ff1000] (23)--
00000800:00000200:26.0:1619677473.601711:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (22)--
00000800:00000200:26.0:1619677473.601712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a061ff1000] (21)--
00000400:00000200:18.0:1619677473.601714:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.601715:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.601718:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.601719:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.601720:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.601722:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.601722:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.601724:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.601725:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.601725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.601731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677473.601734:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677473.601736:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677473.601737:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.601738:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.601738:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:12.0:1619677473.601739:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.601739:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.601740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.601741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.601742:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.601767:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.601770:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.601770:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.601771:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.601772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.601773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677473.601823:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.601826:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.601827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.601829:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.601832:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.601833:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.601834:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.601836:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.601837:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.601838:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.601841:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.601843:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677473.601843:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.601845:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.601846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677473.601846:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.601846:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.601847:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677473.601848:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.601848:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.601849:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.601850:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:10.0:1619677473.601851:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.601851:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:10.0:1619677473.601852:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677473.601852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:10.0:1619677473.601853:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.601855:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.601856:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.601856:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.601859:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.601861:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.601862:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677473.601863:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:10.0:1619677473.601864:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.601865:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677473.601865:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:10.0:1619677473.601865:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.601865:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.601865:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.601866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.601866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:10.0:1619677473.601867:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.601867:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.601868:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.601869:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677473.601869:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.601870:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.601870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677473.601872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.601882:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.601883:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.601885:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677473.601885:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.601885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.601886:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677473.601886:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.601886:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.601888:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.601890:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.602144:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.602147:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.602148:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.602148:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.602151:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.602153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.602198:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.602200:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.602201:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.602204:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.602207:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.602208:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.602210:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.602211:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.602212:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.602213:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.602217:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.602218:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.602221:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.602222:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.602222:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.602224:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.602225:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.602225:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.602226:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.602238:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.602240:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.602241:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.602242:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.602243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.602245:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.602487:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.602490:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.602492:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.602493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.602494:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.602496:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.602497:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.602498:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.602499:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.602500:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.602504:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.602506:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.602508:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.602508:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.602509:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.602511:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.602511:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.602513:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.602514:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.602514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.602525:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677473.602525:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.602527:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.602528:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.602528:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.602528:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.602529:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.602529:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.602531:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.602535:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.602537:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.602540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.602540:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.602541:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.602542:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.602542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.602563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.602571:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.602573:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.602574:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.602577:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.602580:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.602582:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677473.602583:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.602585:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.602586:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.602587:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677473.602591:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.602592:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677473.602596:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.602597:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677473.602597:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677473.602599:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677473.602600:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.602601:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.602602:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:22.0:1619677473.602613:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.602615:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.602617:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.602617:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.602619:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.602621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.603047:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.603049:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.603050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.603051:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.603053:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.603055:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.603056:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.603057:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.603058:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.603059:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.603061:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.603062:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.603063:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.603064:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.603065:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677473.603066:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677473.603067:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.603067:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.603067:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.603069:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:24.0:1619677473.603070:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.603070:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677473.603071:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677473.603071:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:24.0:1619677473.603072:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96508566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.603072:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.603073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677473.603074:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.603075:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.603076:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.603078:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.603080:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.603083:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.603083:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:24.0:1619677473.603084:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677473.603084:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:24.0:1619677473.603085:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.603085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.603085:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:24.0:1619677473.603086:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.603086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677473.603087:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677473.603087:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:24.0:1619677473.603088:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.603088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:21.0:1619677473.603088:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.603089:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.603090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.603092:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.603099:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.603102:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.603103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.603103:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.603106:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.603108:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.603975:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.603977:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.603978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.603979:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.603981:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.603983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.604090:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.604093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.604093:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.604095:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.604097:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.604099:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.604100:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.604101:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.604102:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.604103:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.604106:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.604108:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.604111:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.604112:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.604112:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.604114:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.604115:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.604115:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.604116:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.604128:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.604130:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:5.2:1619677473.604130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:20.0:1619677473.604131:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.604133:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.604134:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.604135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.604136:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.604139:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.604140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.604142:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.604144:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.604146:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:21.0:1619677473.604147:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:1.0:1619677473.604147:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.604148:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.604149:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.604150:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.604151:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000400:00000200:1.0:1619677473.604151:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677473.604152:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.604153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.604154:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677473.604154:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.604156:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.604158:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.604159:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.604160:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.604161:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.604162:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.604162:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.604163:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.604175:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.604177:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.604178:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.604178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.604180:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.604182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.604466:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.604469:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.604470:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.604471:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.604474:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.604476:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.604477:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.604478:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.604479:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.604480:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.604483:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.604485:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.604488:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.604489:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.604490:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.604491:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.604492:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.604492:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.604493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.604498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.604507:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.604509:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.604511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.604511:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.604513:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.604517:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.604765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.604775:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.604780:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.604781:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.604784:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.604788:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.604790:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.604792:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.604795:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.604796:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.604798:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.604803:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.604805:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.604808:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.604809:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.604810:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.604813:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.604813:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.604815:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.604815:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.604818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.604824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.604827:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.604828:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.604828:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.604830:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.604832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.604961:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.604964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.604964:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.604966:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.604968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.604970:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.604971:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.604972:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.604973:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.604974:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.604978:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.604980:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.604982:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.604983:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.604984:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.604985:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.604986:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.604986:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.604987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.604999:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.605001:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.605002:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.605004:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.605004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.605006:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.605107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677473.605116:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.605120:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.605122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.605125:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.605129:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.605131:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.605132:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.605135:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.605136:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.605138:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.605143:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.605146:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.605149:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.605150:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.605151:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.605153:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.605154:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.605155:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.605156:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.605159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.605162:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.605165:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.605166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.605166:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.605167:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.605169:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:8.2:1619677473.605316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.605324:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.605327:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.605328:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.605329:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.605330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.605333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.605756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.605762:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.605764:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.605766:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.605767:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.605769:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.605770:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.605771:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.605773:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.605774:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.605775:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.605778:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.605780:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.605782:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.605783:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.605784:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.605785:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.605786:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.605787:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.605788:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.605789:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.605792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677473.605794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.605798:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.605800:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677473.605800:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677473.605801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.605802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677473.605802:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.605803:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.605804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.605805:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.605808:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677473.605820:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.605823:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.605823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677473.605823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:22.0:1619677473.605825:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.605827:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.605829:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677473.605830:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:22.0:1619677473.605830:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.605831:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.605832:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.605834:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:22.0:1619677473.605834:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.605834:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:22.0:1619677473.605838:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.605839:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.605840:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.605841:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.605843:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:22.0:1619677473.605843:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.605844:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.605844:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.605845:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:26.0:1619677473.605846:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.605846:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677473.605847:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.605847:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677473.605848:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677473.605848:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.605848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:26.0:1619677473.605851:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.605853:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.605856:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.605857:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.605857:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.605859:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.605859:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.605860:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677473.605860:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677473.605861:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677473.605863:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.605864:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.605865:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.605865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.605868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.605869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.605870:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.605872:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.605872:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.605874:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.605875:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.605877:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677473.605877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.605885:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.605888:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.605889:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.605890:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.605893:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.605894:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.605896:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.605897:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.605898:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.605899:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.605902:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.605903:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.605905:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.605905:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.605906:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.605907:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.605908:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.605908:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.605909:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.605913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.605919:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.605921:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.605922:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.605922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.605924:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.605926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.606122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.606127:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.606129:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.606131:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.606132:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.606134:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.606135:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.606137:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.606138:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.606139:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.606140:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.606143:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.606145:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.606147:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.606148:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.606149:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.606150:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.606151:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.606152:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.606152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.606164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.606166:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.606167:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.606167:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.606169:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.606172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.606182:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.606185:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.606186:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.606186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.606187:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.606188:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.606395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.606402:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.606404:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.606405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.606407:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.606410:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.606411:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.606413:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.606414:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.606415:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.606416:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.606420:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.606421:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.606425:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.606426:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.606426:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.606428:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.606429:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.606430:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.606430:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:4.2:1619677473.606656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.606664:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.606667:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677473.606667:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.606668:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.606669:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.606670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.606670:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:26.0:1619677473.606671:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.606671:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.606673:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677473.606674:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.606675:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.606676:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.606677:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.606677:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677473.606678:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.606681:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.606683:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.606686:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.606687:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.606687:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.606689:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.606689:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.606690:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.606691:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.606702:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.606705:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.606706:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.606706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.606708:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.606711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.606723:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.606725:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.606726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.606726:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.606727:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.606728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677473.606873:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.606875:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.606876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.606878:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.606879:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:1.0:1619677473.606880:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.606882:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.606883:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:1.0:1619677473.606883:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.606884:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.606886:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.606886:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.606887:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.606888:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677473.606889:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.606891:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.606893:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677473.606894:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.606895:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.606896:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677473.606897:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677473.606897:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.606897:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677473.606898:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.606899:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677473.606899:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.606899:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.606900:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.606901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:22.0:1619677473.606902:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.606904:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.606907:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.606908:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.606909:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.606911:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.606912:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.606913:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.606913:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.606914:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.606915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.606915:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.606916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.606916:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.606917:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.606920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.606920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.0:1619677473.606925:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.606926:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.606928:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677473.606928:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677473.606928:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:22.0:1619677473.606929:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.606929:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.606930:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.606931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.606931:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.606931:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.606932:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:20.0:1619677473.606934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677473.606934:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.606937:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.606938:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.606940:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.606941:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.606942:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.606943:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.606947:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.606948:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.606951:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.606952:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.606953:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.606954:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.606955:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.606956:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.606957:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.606958:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:4.2:1619677473.606963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:12.0:1619677473.606968:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677473.606970:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:12.0:1619677473.606970:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.606971:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.606972:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.606973:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.606973:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.606974:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677473.606974:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.606975:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.607766:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.607769:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.607770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.607772:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.607774:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.607775:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.607777:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.607778:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.607779:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.607780:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.607784:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.607785:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.607788:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.607789:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.607789:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.607791:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.607791:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.607792:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.607792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.607805:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.607807:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.607811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677473.607812:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.607814:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.607816:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:21.0:1619677473.607818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677473.607819:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.607819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:21.0:1619677473.607820:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:17.0:1619677473.607821:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.607822:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.607823:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.607823:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.607824:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (22)--
00000400:00000200:17.0:1619677473.607824:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.607825:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:17.0:1619677473.607826:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677473.607827:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.607828:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.607830:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.607831:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.607834:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.607836:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.607838:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.607839:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.607840:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.607841:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.607842:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.607842:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.607843:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677473.607855:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.607857:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.607858:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.607858:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.607861:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.607863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.607922:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.607924:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.607925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.607926:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.607928:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.607931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.607943:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.607945:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.607946:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.607946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.607947:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.607948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.608151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.608159:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.608161:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.608163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.608164:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.608167:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.608168:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.608170:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.608171:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.608172:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.608173:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.608176:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.608177:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.608180:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.608181:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.608181:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.608183:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.608184:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.608185:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.608186:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.608187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.608193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.608196:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.608199:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.608199:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.608200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.608200:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.608202:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.608202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.608203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677473.608203:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.608206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.608616:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.608618:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.608619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.608621:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.608623:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.608624:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.608626:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.608627:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.608628:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.608629:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.608632:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.608633:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.608636:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.608636:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.608637:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.608639:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.608639:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.608640:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.608640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.608654:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.608656:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.608657:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.608658:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.608660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.608661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.608885:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919860fab000] (21)++
00000800:00000200:6.0:1619677473.608888:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.2@o2ib3
00000800:00000200:22.0:1619677473.608889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919860fab000] (22)--
00000400:00000200:6.0:1619677473.608891:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.608893:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.608894:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.608895:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.608896:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.608897:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.608898:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.608901:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.608902:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 464 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.608904:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.608905:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.608905:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.608907:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.608907:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919860fab000] (21)++
00000800:00000200:6.0:1619677473.608908:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919860fab000] (22)--
00000800:00000200:6.0:1619677473.608908:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919860fab000] (21)--
00000800:00000200:21.0:1619677473.608920:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.608922:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.608923:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.608925:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.608926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.608927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.609207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677473.609216:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.609219:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.609220:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.609222:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.609224:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.609225:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.609226:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.609227:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.609228:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.609229:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.609232:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.609234:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.609236:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.609236:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:10.0:1619677473.609237:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:10.0:1619677473.609238:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:10.0:1619677473.609239:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.609239:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.609240:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.609245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.609249:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.609251:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.609252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.609252:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.609254:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.609256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:5.2:1619677473.609336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.609342:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.609345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.609345:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.609348:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.609350:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.609351:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.609353:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.609354:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.609355:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.609356:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.609359:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.609360:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.609363:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.609363:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.609364:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.609366:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.609366:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.609367:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.609367:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.609380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.609383:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.609384:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.609384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.609386:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.609388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.609391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.609395:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.609398:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.609400:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.609401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.609402:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.609404:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.609405:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.609407:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.609408:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.609409:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.609412:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.609414:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.609417:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.609418:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.609418:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.609420:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.609421:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.609422:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.609422:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.609426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.609433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.609436:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.609436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.609437:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.609439:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.609442:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.609631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677473.609639:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.609641:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677473.609642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677473.609644:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677473.609646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:8.0:1619677473.609647:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.609648:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.609649:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.609651:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.609652:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.609653:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:8.0:1619677473.609653:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677473.609656:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:8.0:1619677473.609656:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.609657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.0:1619677473.609658:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677473.609659:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.609660:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.609661:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.609661:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:26.0:1619677473.609662:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677473.609662:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:26.0:1619677473.609663:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.609663:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:26.0:1619677473.609664:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.609664:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000400:00000200:26.0:1619677473.609665:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.609665:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.609665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:26.0:1619677473.609666:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.609668:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.609670:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:9.2:1619677473.609670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.609671:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.609672:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.609673:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.609674:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.609674:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.609675:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.609675:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:2.0:1619677473.609677:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:9.2:1619677473.609679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677473.609680:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.609681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.609681:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.609683:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.609685:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:2.0:1619677473.609687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677473.609688:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.609688:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.609689:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.609691:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.609693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677473.609737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.609744:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.609747:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.609747:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.609749:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.609752:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.609753:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.609755:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.609756:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.609757:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.609758:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.609761:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.609762:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.609765:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.609766:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.609766:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.609768:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.609768:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.609769:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.609770:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.609774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.609782:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.609784:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.609785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.609786:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.609788:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.609791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.609846:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.609849:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.609850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.609852:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.609854:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.609855:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.609857:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.609858:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.609859:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.609860:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.609863:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.609865:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.609867:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.609868:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.609868:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.609870:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.609870:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.609871:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.609871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.609884:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.609887:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.609888:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.609890:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.609890:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.609892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.609924:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.609927:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.609927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.609928:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.609930:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.609933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.610064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.610072:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.610075:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.610075:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.610077:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.610079:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.610080:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.610081:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.610082:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.610083:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.610084:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.610087:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.610088:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.610091:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.610092:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.610092:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.610094:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.610094:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.610095:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.610095:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.610099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.610106:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.610108:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.610109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.610109:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.610111:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.610113:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.610118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.610123:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.610126:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.610127:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.610127:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.610127:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.610128:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.610714:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.610717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.610717:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.610719:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.610721:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.610723:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.610724:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.610725:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.610726:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.610727:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.610730:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.610732:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.610734:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.610735:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.610736:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.610737:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.610738:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.610739:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.610740:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.610740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.610751:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.610752:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.610753:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.610753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.610754:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.610754:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.610755:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.610757:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.610758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.610759:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.610773:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.610775:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.610776:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.610777:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.610777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.610779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.610848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:4.2:1619677473.610854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677473.610855:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.610858:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677473.610859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677473.610861:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.610862:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677473.610863:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677473.610864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677473.610864:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.610865:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677473.610865:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.610866:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.610867:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.610868:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.610869:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.610869:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.610870:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.610870:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677473.610871:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.610873:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.610873:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.610873:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.610874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677473.610874:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.610875:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.610876:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.610876:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.610878:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.610879:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.610880:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.610880:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677473.610881:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677473.610881:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.610882:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.610882:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:10.0:1619677473.610883:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.610883:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000400:00000200:2.0:1619677473.610883:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.0:1619677473.610884:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:2.0:1619677473.610884:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.610885:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.610886:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.610887:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.610887:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:2.0:1619677473.610887:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677473.610888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:2.0:1619677473.610888:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677473.610889:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.610889:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.610891:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.610892:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.610892:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677473.610892:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.610894:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.610897:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.610898:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:4.2:1619677473.610898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677473.610898:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677473.610899:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.610899:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (23)++
00000800:00000200:2.0:1619677473.610899:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (23)++
00000400:00000200:18.0:1619677473.610901:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.610902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677473.610902:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (24)--
00000800:00000200:2.0:1619677473.610902:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.610903:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.610904:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677473.610905:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677473.610905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:6.0:1619677473.610905:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:2.0:1619677473.610905:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677473.610906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.610906:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.610907:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:9.2:1619677473.610907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.610908:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.610909:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.610909:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.610912:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677473.610915:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.610918:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.610918:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000800:00000200:18.0:1619677473.610919:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (24)++
00000800:00000200:2.0:1619677473.610919:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000800:00000200:18.0:1619677473.610920:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (25)--
00000400:00000200:2.0:1619677473.610920:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.610921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.610922:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.610922:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.610924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.611416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.611424:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.611426:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.611427:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.611429:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.611432:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.611434:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.611435:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.611436:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.611437:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.611439:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.611442:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.611443:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.611446:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.611447:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.611447:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.611449:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.611449:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.611450:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.611451:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.611464:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.611467:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.611468:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.611468:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.611470:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.611473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.611590:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.611593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.611593:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.611597:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.611599:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.611600:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.611602:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.611604:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.611605:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.611606:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.611612:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.611614:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.611617:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.611618:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.611619:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.611621:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.611622:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.611622:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.611623:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.611634:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.611636:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.611637:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.611639:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677473.611639:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.611640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.611641:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677473.611641:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:22.0:1619677473.611642:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.611642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.611644:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.611647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.611648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.611655:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.611658:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.611658:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:6.0:1619677473.611659:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677473.611660:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.611662:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.611663:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.611663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.611663:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:26.0:1619677473.611664:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:6.0:1619677473.611664:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.611665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677473.611666:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.611668:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677473.611668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677473.611669:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.611670:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.611673:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.611675:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677473.611675:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.611678:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:26.0:1619677473.611679:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.611679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677473.611680:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.611680:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.611682:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:24.0:1619677473.611682:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.611683:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.611684:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.611685:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.611685:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:24.0:1619677473.611685:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.611687:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.611688:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96508517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.611688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:24.0:1619677473.611689:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.611690:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.611691:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677473.611691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:24.0:1619677473.611694:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.611696:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.611697:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.611698:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:24.0:1619677473.611699:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.611699:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.611700:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.611700:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:10.0:1619677473.611700:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:18.0:1619677473.611701:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.611701:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677473.611701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:26.0:1619677473.611702:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:24.0:1619677473.611702:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.611702:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.611703:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.611703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677473.611704:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.611707:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677473.611708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.611716:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.611719:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.611720:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.611720:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.611723:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.611727:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.612640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.612642:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.612643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.612645:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.612647:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.612649:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.612650:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.612651:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.612653:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.612653:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.612657:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.612659:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.612661:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.612662:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.612662:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.612664:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.612665:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.612665:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.612666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.612679:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.612681:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.612682:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.612684:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.612685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.612686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.612999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.613007:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.613010:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.613011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.613013:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.613016:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.613017:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.613019:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.613020:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.613021:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.613022:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.613025:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.613026:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.613030:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.613030:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.613031:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.613033:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.613033:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.613034:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.613034:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.613046:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.613049:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.613050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.613050:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.613053:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.613056:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.613328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677473.613337:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.613340:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.613344:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.613345:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.613347:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.613349:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677473.613351:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.613354:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.613355:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.613357:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677473.613362:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.613365:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677473.613369:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.613370:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677473.613371:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677473.613374:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677473.613375:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.613376:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.613377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.613380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.613389:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.613392:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.613393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.613393:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.613395:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.613399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677473.613493:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.613496:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.613497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.613498:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.613501:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.613502:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.613503:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.613505:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.613506:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.613507:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.613510:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.613512:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.613514:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.613515:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.613515:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.613517:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.613517:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.613518:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.613518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.613531:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.613533:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.613534:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.613535:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.613536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.613537:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.613693:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.613696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.613696:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.613697:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.613699:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.613702:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.613793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.613801:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.613804:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.613805:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.613807:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.613809:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.613811:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677473.613812:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.613814:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.613815:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.613815:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677473.613819:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.613820:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677473.613822:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.613823:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677473.613823:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677473.613825:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677473.613825:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.613826:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.613826:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:5.2:1619677473.614043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.614049:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.614052:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.614053:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.614054:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.614056:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.614058:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.614059:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.614060:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.614061:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.614062:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.614066:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.614068:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.614070:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.614071:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.614072:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.614074:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.614074:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.614075:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.614075:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.614087:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.614090:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.614091:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.614091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.614093:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.614095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.614102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:9.2:1619677473.614142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.614146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:20.2:1619677473.614148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.614153:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.614155:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.614155:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:18.0:1619677473.614157:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.614158:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:18.0:1619677473.614159:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.614160:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677473.614163:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.614163:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.614164:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677473.614165:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.614165:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.614166:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.614168:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.614169:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.614170:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.614173:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.614175:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.614178:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.614179:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.614179:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.614181:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.614182:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.614183:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.614183:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.614196:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.614199:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.614201:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.614203:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.614203:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.614207:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.614345:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.614348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.614348:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.614350:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.614352:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.614354:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.614355:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.614357:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.614358:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.614359:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.614362:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.614364:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.614366:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.614367:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.614368:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.614369:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.614370:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.614371:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.614372:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.614372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.614382:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.614384:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677473.614384:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.614385:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.614386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.614386:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.614387:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.614388:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.614390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.614390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.614400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.614402:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.614403:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.614404:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.614405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.614406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.614938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:9.2:1619677473.614978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.614982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:18.0:1619677473.614987:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.614990:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.614990:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.614991:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.614993:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.614997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.615150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.615158:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.615161:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.615162:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.615163:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.615166:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.615167:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.615169:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.615170:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.615171:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.615172:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.615175:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.615176:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.615179:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.615180:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.615181:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.615182:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.615183:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.615183:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.615184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.615196:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.615199:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.615200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.615200:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.615202:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.615205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.615277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.615282:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.615285:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.615288:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.615288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.615290:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.615292:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677473.615299:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.615301:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.615303:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.615304:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677473.615308:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.615310:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.615312:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.615313:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.615314:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:9.2:1619677473.615315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677473.615316:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.615316:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.615317:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.615317:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.615322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677473.615324:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.615327:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.615328:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.615328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.615330:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.615331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:24.0:1619677473.615332:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.615333:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.615334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.615336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.615336:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.615337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.615341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:22.0:1619677473.615349:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677473.615350:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677473.615351:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.0:1619677473.615352:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677473.615352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.615353:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000400:00000200:22.0:1619677473.615354:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.615354:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.615355:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:22.0:1619677473.615356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.615356:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:22.0:1619677473.615358:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.615359:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.615361:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.615362:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.615363:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.615365:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.615367:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.615369:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.615370:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.615371:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.615372:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.615373:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.615374:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.615374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.615386:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.615388:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.615389:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.615390:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.615391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.615392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.615479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677473.615486:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.615489:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.615490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677473.615492:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.615494:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.615496:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.615497:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.615499:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.615500:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.615501:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.615504:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.615506:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.615508:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.615508:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.615509:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.615511:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.615511:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.615512:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.615512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677473.615516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677473.615523:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.615524:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.615526:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:6.0:1619677473.615526:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:24.0:1619677473.615527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.615527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677473.615527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:6.0:1619677473.615528:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.615529:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.615531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677473.615531:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.615534:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.615542:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.615545:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.615546:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.615546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.615548:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.615548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.615760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.615767:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.615770:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.615771:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.615773:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.615776:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.615777:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.615778:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.615780:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.615781:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.615781:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.615785:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.615787:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.615790:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.615790:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.615791:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.615793:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.615793:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.615794:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.615794:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.615806:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.615809:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.615810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.615810:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.615812:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.615815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.615825:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.615827:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.615828:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.615828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.615829:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.615830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.615908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.615915:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.615918:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.615919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.615920:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.615923:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.615925:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.615926:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.615927:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.615928:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.615929:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.615932:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.615933:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.615936:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.615937:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.615937:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.615939:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.615939:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.615940:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.615941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.615952:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.615954:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.615955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.615956:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.615957:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.615960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.616205:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.616208:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.616209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.616210:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.616212:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.616214:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.616215:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.616216:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.616217:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.616219:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.616222:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.616224:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.616227:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.616228:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.616228:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.616230:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.616230:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.616231:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.616232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.616244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.616246:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.616247:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.616249:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.616251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.616252:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.617073:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.617076:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.617077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.617078:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.617081:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.617082:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.617083:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.617084:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.617085:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.617086:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.617089:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.617091:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.617093:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.617094:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.617094:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.617096:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.617097:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.617097:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.617098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.617111:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.617113:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.617114:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.617115:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.617117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.617117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.617138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.617146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.617149:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.617150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.617151:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.617153:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.617155:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.617156:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.617158:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.617159:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.617160:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.617163:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.617165:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.617168:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.617169:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.617169:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.617171:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.617171:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.617173:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.617174:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.617174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.617180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677473.617185:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.617187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677473.617187:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.617188:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.617188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.617189:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.617190:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677473.617190:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.617190:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.617193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.617220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.617227:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.617230:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.617231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.617233:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.617236:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.617238:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.617239:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.617240:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.617241:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.617242:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.617245:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.617246:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.617249:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.617249:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.617250:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.617251:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.617252:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.617253:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.617253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.617265:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.617267:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.617268:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.617270:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.617272:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.617273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.617690:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.617693:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.617694:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.617694:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.617696:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.617699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.617743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677473.617751:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.617753:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677473.617754:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677473.617756:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.617759:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.617760:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.617761:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.617763:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.617764:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.617764:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.617768:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.617769:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.617772:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.617773:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.617773:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.617775:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.617775:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.617776:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.617776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677473.617788:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.617791:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.617792:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.617792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.617794:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.617798:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.617805:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.617808:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.617809:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.617809:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.617810:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.617811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.617961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.617968:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.617970:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.617971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.617973:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.617976:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.617977:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.617978:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.617980:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.617980:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.617981:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.617984:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.617986:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.617988:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.617989:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.617989:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.617991:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.617992:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.617993:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.617993:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.617997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.618006:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.618009:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.618010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.618010:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.618012:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.618016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.618017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.618025:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.618027:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.618028:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.618028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.618029:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.618030:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.618130:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.618133:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.618133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.618135:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.618137:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.618139:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.618140:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.618142:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.618143:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.618144:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.618147:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.618148:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.618151:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.618152:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.618152:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.618154:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.618154:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:4.2:1619677473.618154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:22.0:1619677473.618155:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.618156:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.618157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.618161:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.618163:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.618164:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.618166:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.618167:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.618168:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:26.0:1619677473.618169:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.618169:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.618170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677473.618171:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:21.0:1619677473.618171:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.618171:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.618171:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:26.0:1619677473.618172:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.618173:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.618173:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.618174:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.618174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677473.618175:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677473.618175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677473.618179:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.618181:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677473.618184:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.618184:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677473.618185:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677473.618186:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.618187:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677473.618187:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.618187:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.618188:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.0:1619677473.618189:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.618190:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.618190:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.618191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.618192:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.618293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.618299:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.618302:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.618303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.618306:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.618308:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.618310:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.618311:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.618313:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.618314:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.618315:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.618318:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.618320:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.618322:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.618323:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.618324:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.618325:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.618326:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.618327:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.618327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.618332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.618340:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.618342:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.618342:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.618343:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.618344:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.618346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.618823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.618829:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.618832:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.618833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.618834:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.618837:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.618839:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.618840:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.618841:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.618842:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.618843:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.618847:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.618848:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.618850:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.618851:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.618851:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.618853:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.618853:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.618854:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.618855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.618860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.618868:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.618871:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.618872:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.618872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.618874:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.618877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.618881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.618890:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.618891:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.618891:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.618892:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.618893:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.618894:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.618894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.618895:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.618895:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677473.618896:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.618897:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.618899:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677473.618900:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.618901:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.618902:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.618903:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.618906:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.618908:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.618911:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.618912:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.618912:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.618914:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.618915:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.618915:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.618916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677473.618928:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.618930:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.618932:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.618932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.618934:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.618936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.619015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.619017:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.619018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.619020:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.619022:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.619023:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.619024:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.619025:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.619027:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.619027:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.619031:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.619033:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.619035:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.619036:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.619036:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.619038:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.619038:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.619039:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.619039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.619051:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.619054:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.619054:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.619056:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.619057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.619058:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.619293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.619302:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.619305:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.619305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.619308:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.619311:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.619312:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.619313:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.619315:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.619316:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.619317:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.619320:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.619322:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.619324:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.619325:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.619326:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.619327:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.619328:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.619328:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.619329:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.619333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.619342:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.619345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.619345:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.619346:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.619348:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.619351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677473.619360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.619366:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.619368:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.619369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.619371:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.619372:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.619374:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.619375:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.619377:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.619378:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.619379:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.619382:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.619384:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.619386:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.619387:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.619387:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.619389:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.619390:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.619391:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.619392:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.619392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.619397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.619403:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.619404:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.619404:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.619405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.619406:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.619407:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.619408:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.619408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.619411:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.619413:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.619440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.619442:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.619443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.619443:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.619446:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.619448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.619859:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.619862:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.619863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.619865:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.619866:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.619868:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.619869:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.619870:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.619871:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.619872:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.619875:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.619877:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.619879:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.619880:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.619881:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.619882:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.619883:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.619883:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.619884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.619895:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.619897:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.619898:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.619900:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.619901:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.619902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.620013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.620021:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.620024:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.620025:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.620027:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.620029:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.620031:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.620032:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.620033:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.620034:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.620035:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.620039:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.620041:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.620044:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.620044:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.620045:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.620047:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.620047:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.620048:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.620048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.620060:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.620063:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.620064:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.620064:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.620067:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.620070:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.620532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.620539:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.620542:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.620543:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.620545:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.620547:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.620549:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.620550:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.620551:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.620552:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.620553:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.620557:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.620558:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.620561:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.620562:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.620562:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.620564:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.620564:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.620566:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.620567:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.620567:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.620573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677473.620578:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.620581:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677473.620581:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677473.620582:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.620582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.620583:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.620584:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.620584:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.620584:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.620587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.620712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:9.2:1619677473.620749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.620756:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.620759:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.620760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.620760:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.620762:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.620765:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.620795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.620802:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.620804:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.620805:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.620807:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.620810:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.620811:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.620812:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.620814:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.620814:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.620815:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.620818:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.620820:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.620823:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.620823:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.620824:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.620826:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.620826:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.620827:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.620828:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.620829:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.620834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.620839:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.620841:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.620841:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.620842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.620843:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677473.620843:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.620844:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.620845:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.620845:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.620847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.620861:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.620864:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.620864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.620865:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.620866:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.620866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.620895:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.620897:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.620898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.620899:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.620901:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.620903:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.620904:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.620905:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.620906:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.620907:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.620909:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.620911:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.620913:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.620914:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.620915:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.620916:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.620917:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.620917:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.620918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.620930:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.620932:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.620933:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.620934:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.620935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.620937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.621422:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.621425:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.621426:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.621426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.621429:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.621432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.621595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.621600:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.621603:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.621605:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.621605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.621607:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.621609:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677473.621611:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.621612:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.621613:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.621614:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677473.621618:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.621619:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677473.621622:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.621623:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677473.621624:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677473.621625:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677473.621626:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.621627:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677473.621629:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677473.621629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:9.2:1619677473.621631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677473.621634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.621637:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:10.0:1619677473.621638:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:20.0:1619677473.621639:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677473.621640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.621641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:10.0:1619677473.621641:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.621642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.621642:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.621644:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.621647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677473.621783:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.621786:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.621787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.621789:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.621791:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.621792:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.621793:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.621795:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.621796:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.621797:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.621800:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.621801:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.621804:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.621805:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.621805:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.621807:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.621807:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.621808:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.621809:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.621810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.621820:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.621821:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677473.621821:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.621822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.621823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.621823:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.621824:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.621825:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.621826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.621828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.621840:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.621843:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.621844:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.621844:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.621845:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.621846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.621970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.621977:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.621979:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.621980:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.621980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.621982:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.621984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.622045:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b98600] (21)++
00000800:00000200:22.0:1619677473.622048:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b98600] (22)--
00000800:00000200:6.0:1619677473.622048:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.10@o2ib3
00000400:00000200:6.0:1619677473.622051:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.622054:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.622055:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:6.0:1619677473.622057:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.622058:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.622059:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.622060:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:6.0:1619677473.622063:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.622065:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:6.0:1619677473.622068:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.622069:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:6.0:1619677473.622069:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:6.0:1619677473.622071:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:6.0:1619677473.622072:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b98600] (21)++
00000800:00000200:6.0:1619677473.622073:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b98600] (22)--
00000800:00000200:6.0:1619677473.622073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b98600] (21)--
00000800:00000200:24.2:1619677473.622077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.622081:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.622084:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.622085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (23)--
00000400:00000200:8.0:1619677473.622085:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.622087:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.622090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.622651:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.622654:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.622655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.622656:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.622659:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.622660:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.622661:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.622663:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.622664:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.622664:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.622667:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.622668:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.622670:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.622671:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.622671:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.622673:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.622674:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.622674:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.622675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.622688:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.622690:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.622691:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.622693:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.622694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.622695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.622777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.622784:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.622787:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.622788:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.622788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.622790:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.622792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677473.623440:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.623442:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.623443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.623445:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677473.623446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:17.0:1619677473.623448:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.623449:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.623451:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.623452:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.623453:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:17.0:1619677473.623453:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.623454:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677473.623456:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:17.0:1619677473.623457:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.623457:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.623457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677473.623459:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:26.0:1619677473.623460:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.623462:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.623462:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.623463:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.623463:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:26.0:1619677473.623464:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:8.0:1619677473.623464:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:26.0:1619677473.623465:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96508462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.623465:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:26.0:1619677473.623466:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677473.623466:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.623466:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.0:1619677473.623466:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:26.0:1619677473.623467:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677473.623467:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:8.0:1619677473.623467:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.623468:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677473.623468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:26.0:1619677473.623472:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677473.623472:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677473.623473:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:8.0:1619677473.623476:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.623477:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.623477:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:9.2:1619677473.623477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.623478:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677473.623479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.623480:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.623480:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677473.623480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:26.0:1619677473.623481:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.0:1619677473.623481:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.623482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677473.623482:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:24.0:1619677473.623483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:20.0:1619677473.623483:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.623483:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.623484:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677473.623485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.623485:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.623487:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.623490:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677473.623491:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677473.623491:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.623491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.623493:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.623494:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.623495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.623495:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.623496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.623499:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677473.623500:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.623502:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.623503:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.623504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.623504:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.623505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677473.623684:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.623687:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.623687:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.623690:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.623692:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.623693:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.623694:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.623695:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.623697:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.623697:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.623701:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.623703:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.623705:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.623706:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.623706:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.623708:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.623708:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.623709:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.623709:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.623721:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.623724:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.623724:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.623726:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.623727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.623728:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.623923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.623930:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.623933:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.623934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.623935:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.623938:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.623940:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.623941:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.623942:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.623943:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.623944:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.623947:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.623949:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.623951:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.623952:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.623953:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.623955:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.623955:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.623956:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.623957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677473.623962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677473.623969:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677473.623971:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677473.623972:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.623973:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.623973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677473.623975:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677473.623976:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.623977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677473.623979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:18.0:1619677473.623980:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.623984:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.623986:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.623988:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.623991:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.623992:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.623994:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.623999:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.624002:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.624006:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.624008:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.624009:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.624011:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.624012:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.624014:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.624016:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.624017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.624021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.624029:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.624032:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.624033:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.624035:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:5.2:1619677473.624063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.624069:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.624072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.624072:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.624075:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.624077:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.624078:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.624080:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.624081:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.624082:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.624083:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.624086:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.624088:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.624090:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.624091:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.624092:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.624093:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.624094:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.624094:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.624095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.624107:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.624109:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.624110:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.624110:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.624113:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.624115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.624269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677473.624276:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677473.624278:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677473.624279:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677473.624281:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.624283:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.624285:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677473.624286:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.624287:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.624289:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.624289:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677473.624292:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.624297:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.624299:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.624299:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.624300:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.624302:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.624302:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677473.624303:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677473.624304:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677473.624305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:8.2:1619677473.624308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677473.624312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.624317:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.624319:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677473.624319:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.624319:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.624320:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677473.624321:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677473.624321:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.624322:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.624322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677473.624323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.2:1619677473.624399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.624406:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.624408:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.624409:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.624410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.624410:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.624410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677473.624491:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.624494:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.624494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.624495:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.624497:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.624500:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.624579:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.624581:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.624582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.624584:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.624586:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.624587:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.624588:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.624590:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.624591:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.624591:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.624594:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.624596:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.624599:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.624599:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.624600:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.624602:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.624602:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.624603:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.624603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.624616:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.624618:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.624619:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.624620:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.624621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.624622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.624982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.624989:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.624992:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.624993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.624995:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.624998:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.624999:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.625000:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.625002:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.625003:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.625004:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.625007:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.625009:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.625012:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.625013:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.625014:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.625015:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.625016:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.625017:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.625018:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.625019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677473.625021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677473.625024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677473.625030:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.625031:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677473.625032:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677473.625033:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677473.625033:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.625034:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677473.625034:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.625035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677473.625036:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.625038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677473.625076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.625084:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.625087:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.625088:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.625090:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.625092:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.625094:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.625095:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.625096:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.625097:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.625098:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.625101:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.625103:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.625105:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.625106:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.625107:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.625108:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.625109:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.625110:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.625110:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.625122:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.625125:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.625126:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.625126:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.625128:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.625131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.625143:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.625146:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.625147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.625147:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.625148:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.625148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.625338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.625345:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.625348:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.625349:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.625351:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.625353:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.625355:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.625356:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.625357:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.625359:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.625359:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.625362:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.625364:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.625366:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.625367:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.625367:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.625369:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.625370:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.625371:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.625371:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.625384:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.625387:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.625388:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.625388:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.625390:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.625395:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.625396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.625398:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.625399:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.625400:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.625400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.625401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.625455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.625457:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.625458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.2:1619677473.625459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:6.0:1619677473.625459:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.625462:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.625463:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.625464:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.625465:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677473.625466:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677473.625466:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.625467:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.625469:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.625469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.625472:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.625472:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.625474:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677473.625475:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.625476:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:6.0:1619677473.625477:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677473.625478:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.625478:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.625478:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677473.625479:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.625480:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.625480:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.625480:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:18.0:1619677473.625482:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677473.625482:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.625483:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.625483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:18.0:1619677473.625485:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.625487:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.625488:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.625489:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.625490:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.625491:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.625492:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.625492:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:22.0:1619677473.625494:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:18.0:1619677473.625494:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.625494:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.0:1619677473.625495:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.625496:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.625497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.625497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:9.2:1619677473.625497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677473.625498:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.625499:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.625500:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.2:1619677473.625500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:21.0:1619677473.625503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.625503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677473.625506:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.625507:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677473.625508:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.625508:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.625509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.625509:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.625510:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.625511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677473.625511:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.625512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677473.625517:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.625520:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.625521:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.625521:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.625522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.625523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.625562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677473.625569:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.625572:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.625573:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.625575:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.625578:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.625579:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.625581:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.625582:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.625583:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.625584:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.625588:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.625589:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 984 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.625593:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.625594:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.625594:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.625596:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.625597:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.625598:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.625599:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.625599:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:4.2:1619677473.625605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:21.0:1619677473.625610:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677473.625612:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:21.0:1619677473.625612:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.625613:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.625614:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000400:00000200:21.0:1619677473.625614:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.625615:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:20.0:1619677473.625615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.625616:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.625618:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.625811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.625819:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.625822:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.625823:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.625824:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.625827:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.625828:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.625829:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.625830:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.625831:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.625832:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.625835:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.625836:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.625839:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.625839:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.625840:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.625841:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.625842:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.625843:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.625843:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.625855:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.625858:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.625859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.625859:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.625861:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.625864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.626497:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.626500:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.626501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.626503:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.626504:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.626506:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677473.626507:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.626508:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.626510:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.626510:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677473.626513:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.626515:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677473.626518:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.626519:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677473.626520:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677473.626521:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:1.0:1619677473.626522:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677473.626522:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:22.0:1619677473.626523:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:1.0:1619677473.626523:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:6.0:1619677473.626524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:1.0:1619677473.626524:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.626525:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:22.0:1619677473.626526:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.626528:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.626528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:22.0:1619677473.626529:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.626530:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.626532:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.626533:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.626534:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.626536:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.626536:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:17.0:1619677473.626537:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:22.0:1619677473.626538:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.626538:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.626539:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.626539:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:22.0:1619677473.626540:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.626540:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:10.0:1619677473.626540:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.626541:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677473.626541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677473.626542:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:10.0:1619677473.626542:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.626544:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.626544:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.626545:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677473.626545:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677473.626546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:9.2:1619677473.626548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.626555:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.626557:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.626558:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:20.0:1619677473.626558:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677473.626558:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.626559:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.626560:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.0:1619677473.626561:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.626561:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.626563:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.626564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.626566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.626674:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.626677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.626677:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.626678:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.626680:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.626682:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.627385:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.627388:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.627388:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.627390:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.627392:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.627393:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.627394:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.627396:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.627397:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.627398:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.627400:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.627402:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.627404:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.627405:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.627406:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.627407:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.627408:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.627409:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.627409:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.627422:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.627424:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.627425:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.627427:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.627428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.627428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:1.0:1619677473.627483:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.627486:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.627486:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677473.627489:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.627491:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.627493:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677473.627494:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.627496:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.627497:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677473.627498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:1.0:1619677473.627498:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677473.627501:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677473.627501:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.627503:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677473.627504:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:1.0:1619677473.627505:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.627506:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:21.0:1619677473.627507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.627507:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.627507:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.627508:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.627509:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:20.0:1619677473.627510:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.627510:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.627510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:20.0:1619677473.627512:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677473.627513:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.627514:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.627515:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677473.627515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:20.0:1619677473.627517:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.627521:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.627522:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677473.627523:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:20.0:1619677473.627525:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.627525:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:20.0:1619677473.627526:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:17.0:1619677473.627526:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.627526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.0:1619677473.627527:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:17.0:1619677473.627528:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.627529:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677473.627529:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677473.627530:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.627531:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.627531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677473.627707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.627713:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.627716:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.627717:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.627717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.627719:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.627721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.628241:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.628244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.628244:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.628247:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.628249:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.628250:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.628251:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.628252:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.628253:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.628254:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.628257:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.628259:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.628261:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.628262:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.628263:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.628264:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.628265:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.628265:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.628266:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.628278:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.628280:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.628281:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.628283:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.628283:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.628285:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.628299:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.628302:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.628303:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.628304:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.628304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.628305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.628373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.628376:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.628379:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.628381:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.628382:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.628383:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.628385:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.628386:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.628388:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.628389:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.628390:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.628393:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.628395:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.628398:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.628399:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.628399:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.628401:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.628401:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.628402:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.628403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.628408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.628415:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.628417:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.628418:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.628418:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.628421:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.628423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.628427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.628435:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.2:1619677473.628436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.628437:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.628439:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.628440:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.628440:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.628442:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.628442:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000400:00000200:24.0:1619677473.628443:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.628444:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.628444:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.628445:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677473.628445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:24.0:1619677473.628446:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.628448:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.628449:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.628450:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.628452:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:24.0:1619677473.628453:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.628455:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.628455:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.628456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:24.0:1619677473.628458:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.628459:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:22.0:1619677473.628459:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.628460:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.628461:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.628461:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.628462:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.628462:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.628463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677473.628463:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677473.628465:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.628466:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.628467:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.628468:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677473.628473:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.628475:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677473.628476:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.628477:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.628478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.628479:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.628480:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:6.0:1619677473.628480:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.628481:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677473.628483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.628484:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677473.628484:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.628484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.628486:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677473.628487:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677473.628488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.628498:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677473.628499:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677473.628500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.628501:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:22.0:1619677473.628614:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.628617:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.628617:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.628618:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.628620:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.628623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.628631:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.628634:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.628635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.628635:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.628636:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.628637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.628868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.628875:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.628879:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.628879:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.628881:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.628883:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.628885:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.628886:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.628887:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.628888:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.628889:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.628892:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.628894:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.628897:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.628897:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.628898:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.628900:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.628900:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.628901:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.628901:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.628906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.628915:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.628918:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.628919:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.628919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.628922:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.628925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.628926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677473.628934:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.628936:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.628937:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.628937:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.628938:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.628939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.629227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.629233:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.629235:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.629236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.629236:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.629238:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.629240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.629320:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.629323:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.629324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.629326:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.629329:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.629330:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:5.2:1619677473.629331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677473.629332:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.629333:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.629334:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.629335:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677473.629337:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677473.629338:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.629340:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677473.629340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.629340:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677473.629342:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.629343:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:1.0:1619677473.629343:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.629344:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:1.0:1619677473.629345:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.629346:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.629346:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:1.0:1619677473.629346:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677473.629347:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:1.0:1619677473.629347:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.629348:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.629349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:1.0:1619677473.629349:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.629350:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.629351:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.629354:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.629356:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.629358:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:1.0:1619677473.629359:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.629359:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677473.629360:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:1.0:1619677473.629360:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677473.629361:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.629361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.629362:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:1.0:1619677473.629362:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.629362:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.0:1619677473.629363:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:1.0:1619677473.629363:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.629363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:21.0:1619677473.629364:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.629366:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.629367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.629368:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.629375:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.629377:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.629378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.629378:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.629380:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.629383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.629411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.629419:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.629421:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.629422:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.629424:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.629427:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.629429:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.629430:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.629431:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.629432:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.629433:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.629437:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.629438:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.629442:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.629442:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.629443:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.629444:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.629445:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.629446:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.629446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.629451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.629459:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.629462:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.629463:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.629463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.629466:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.629469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.629496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677473.629504:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677473.629507:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677473.629508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677473.629510:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677473.629513:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.629514:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.629516:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.629517:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.629518:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.629518:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.629521:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.629523:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.629525:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.629526:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.629526:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.629528:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.629528:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677473.629529:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677473.629530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677473.629534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.629542:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.629544:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.629545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.629546:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.629548:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.629550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.629553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.629561:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.629563:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.629564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.629564:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.629565:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.629566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677473.629891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.629899:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.629902:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.629903:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.629905:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.629907:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.629909:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.629910:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.629911:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.629912:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.629913:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.629916:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.629917:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.629920:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.629921:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.629921:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.629923:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.629924:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.629924:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.629925:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.629937:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.629940:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.629941:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.629941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.629943:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.629946:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.630133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.630140:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.630143:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.630144:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.630146:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.630148:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.630150:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.630151:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.630152:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.630153:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.630154:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.630157:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.630158:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.630161:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.630162:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.630162:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.630164:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.630164:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.630165:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.630165:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.630178:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.630180:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.630181:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.630181:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.630183:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.630186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.630212:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.630215:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.630216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.630218:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.630219:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.630221:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.630222:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.630223:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.630224:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.630225:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.630228:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.630230:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.630232:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.630233:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.630234:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.630235:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.630236:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.630236:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.630237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.630249:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.630251:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.630252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.630254:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.630255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.630256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.630467:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.630470:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.630471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.630471:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.630473:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.630476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.630865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.630873:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.630876:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.630877:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.630879:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.630882:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.630883:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.630884:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.630885:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.630886:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.630887:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.630890:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.630892:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.630894:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.630895:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.630896:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.630897:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.630898:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.630899:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.630900:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.630901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.630906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.630911:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.630913:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.630913:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.630914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.630915:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.630915:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.630916:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.630917:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.630917:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.630919:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.631124:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.631127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.631127:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.631129:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.631131:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.631132:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.631133:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.631134:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.631136:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.631137:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.631139:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.631141:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.631143:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.631144:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.631145:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.631146:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.631147:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.631147:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.631148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.631160:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.631162:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.631163:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.631165:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.631165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.631167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.631208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.631214:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.631217:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.631219:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.631220:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.631223:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.631224:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.631226:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.631227:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.631228:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.631229:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.631232:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.631234:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.631236:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.631237:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.631238:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.631239:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.631240:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.631241:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.631241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.631246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.631255:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.631257:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.631258:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.631258:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.631260:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.631263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677473.631773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.631814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.631823:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.631826:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.631827:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.631827:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.631829:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.631833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.632164:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.632167:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.632168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.632170:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.632172:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.632174:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.632176:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.632177:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.632178:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.632178:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:22.0:1619677473.632179:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.632180:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.632181:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.632183:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.632183:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.632185:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.632186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.632187:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.632187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.632188:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.632189:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.632191:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.632192:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.632193:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.632193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677473.632195:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.632196:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.632197:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.632197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.632198:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.632199:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.632204:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.632206:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.632207:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.632209:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.632210:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.632211:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.632222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.632225:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.632225:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.632226:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.632227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.632228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.632275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.632282:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.632285:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.632286:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.632286:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.632287:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.632287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.632668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.632675:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.632678:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.632678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.632680:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.632683:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.632684:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677473.632685:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.632687:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.632688:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.632689:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677473.632692:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.632693:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677473.632696:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.632697:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677473.632697:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677473.632699:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677473.632700:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.632701:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.632701:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677473.632706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.632714:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.632716:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.632717:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.632719:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.632719:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.632721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677473.633095:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.633098:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.633099:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:20.2:1619677473.633100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677473.633101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:1.0:1619677473.633101:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.633103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677473.633103:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.633105:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677473.633106:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677473.633106:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.633107:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.633108:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:1.0:1619677473.633108:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.633109:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.633110:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.633111:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.633112:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.633112:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.633114:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.633115:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.633116:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677473.633117:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.633118:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.633118:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.633119:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.633121:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.633121:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.633122:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.633123:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677473.633124:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.633124:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.633124:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:21.0:1619677473.633125:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.633126:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.633126:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:21.0:1619677473.633127:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.633127:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.633127:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:21.0:1619677473.633128:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.633129:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.633132:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.633133:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.633133:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.633134:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.633135:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.633136:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.633136:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.633137:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.633138:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.633138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.633138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677473.633138:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.633139:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.633140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677473.633140:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677473.633142:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677473.633142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:6.0:1619677473.633142:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.633143:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.633143:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.633144:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677473.633144:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677473.633145:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.0:1619677473.633145:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677473.633146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.633147:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677473.633147:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.633148:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.633148:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.633148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:17.0:1619677473.633149:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:22.0:1619677473.633150:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.633150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677473.633151:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:22.0:1619677473.633152:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677473.633152:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.633153:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.633153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:22.0:1619677473.633154:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.633155:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.633156:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:22.0:1619677473.633156:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.633157:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677473.633157:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677473.633159:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.633160:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.633160:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.633160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.633163:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.633163:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.633165:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.633166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.633166:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.633166:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.633168:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.633169:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.633170:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.633171:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.633172:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677473.633177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.633182:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.633182:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.633184:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.633184:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.633184:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.633185:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:20.0:1619677473.633185:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.633185:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.633186:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.633186:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.633186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.633188:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.633188:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.0:1619677473.633189:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.634008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677473.634014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:8.0:1619677473.634016:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.634018:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.634019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677473.634020:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:8.0:1619677473.634021:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.634023:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.634023:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.634024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:8.0:1619677473.634025:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677473.634026:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.634027:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.634028:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.634029:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.634029:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.634031:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677473.634031:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677473.634033:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.634034:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.634036:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.634037:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677473.634037:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677473.634040:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677473.634041:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.634043:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.634043:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:8.0:1619677473.634044:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.634044:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.634046:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.634046:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.634047:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.634047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677473.634047:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.634047:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:22.0:1619677473.634048:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677473.634048:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677473.634050:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:22.0:1619677473.634051:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:2.0:1619677473.634051:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.634051:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:6.0:1619677473.634052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:2.0:1619677473.634052:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:22.0:1619677473.634054:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.634056:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.634056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:22.0:1619677473.634058:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.634060:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.634062:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.634063:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.634063:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.634064:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:22.0:1619677473.634064:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.634064:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.634065:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.634067:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.634067:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:6.0:1619677473.634067:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677473.634067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:26.0:1619677473.634068:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.634069:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.634070:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.634071:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.634071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.634072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.634073:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:1.0:1619677473.634073:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.634074:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.634075:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.634075:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677473.634076:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677473.634077:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677473.634077:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.634078:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.634078:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.634079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:1.0:1619677473.634079:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.634081:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.634083:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.634084:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.634085:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.634086:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.634087:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677473.634091:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:1.0:1619677473.634091:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.634093:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:1.0:1619677473.634093:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:21.0:1619677473.634094:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.634095:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.634096:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.634096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:1.0:1619677473.634096:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.634097:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677473.634098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.634099:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.634099:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.634100:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.634101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.634112:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.634115:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.634116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.634116:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.634118:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.634120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.634153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.634159:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.634161:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.634162:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.634162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.634164:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.634167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.634266:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.634269:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.634270:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.634270:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.634272:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.634275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.634410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.634417:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.634420:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.634421:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.634422:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.634425:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.634427:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.634428:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.634430:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.634430:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.634431:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.634434:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.634436:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.634439:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.634439:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.634440:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.634442:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.634442:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.634443:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.634443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.634456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.634459:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.634460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.634460:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.634462:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.634465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.634475:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.634478:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.634478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.634479:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.634480:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.634480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.634499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.634506:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.634509:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.634510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.634512:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.634515:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.634516:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.634517:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.634519:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.634520:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.634520:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.634524:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.634525:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.634527:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.634528:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.634529:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.634530:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.634531:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.634532:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.634533:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.634534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677473.634540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677473.634545:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677473.634546:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.634548:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:12.0:1619677473.634549:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677473.634549:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.634550:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:12.0:1619677473.634550:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.634553:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.634555:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.634556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.634565:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.634568:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.634570:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.634571:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.634573:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.634574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677473.634742:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.634745:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.634746:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.634748:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.634750:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.634752:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.634753:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.634754:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.634755:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.634756:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.634759:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.634761:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.634763:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.634764:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.634765:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.634766:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.634767:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.634768:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.634768:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.634773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.634782:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:4.2:1619677473.634783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677473.634784:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.634786:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.634787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.634788:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.634791:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677473.634791:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.634794:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.634795:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.634797:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.634799:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.634801:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.634802:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.634804:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.634805:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.634806:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.634809:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.634811:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.634814:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.634815:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.634815:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.634817:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.634817:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.634818:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.634819:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.634830:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.634833:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.634834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.634834:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.634836:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.634839:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.634854:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.634857:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.634858:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.634858:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.634859:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.634860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.634868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.634875:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.634878:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.634879:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.634881:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.634885:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.634886:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.634887:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.634889:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.634890:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.634891:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.634894:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.634896:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.634898:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.634899:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.634899:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.634901:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.634902:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.634903:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.634903:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.634918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.634925:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.634927:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.634928:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.634929:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.634930:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.634931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.635128:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.635131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.635131:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.635133:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.635135:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.635137:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.635138:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.635139:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.635140:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.635141:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.635144:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.635146:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.635148:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.635149:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.635149:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.635151:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.635152:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.635152:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.635153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.635164:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.635167:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.635167:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.635169:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.635170:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.635171:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.635809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.635817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.635820:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.635821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.635822:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.635826:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.635827:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.635828:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.635830:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.635831:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.635832:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.635835:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.635837:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.635839:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.635840:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.635841:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.635843:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.635843:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.635845:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.635846:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.635846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.635853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.635860:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.2:1619677473.635860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.635861:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.635862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.635863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677473.635869:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.635872:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.635873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.635874:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.635876:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.635879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.635942:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.635945:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.635946:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.635946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.635949:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.635952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.636005:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.636007:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.636008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.636010:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.636012:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.636013:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.636014:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.636015:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.636016:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.636017:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.636020:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.636022:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.636024:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.636025:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.636026:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.636027:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.636027:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.636028:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.636028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.636041:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.636043:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.636044:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.636045:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.636046:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.636048:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.636049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677473.636057:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.636060:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.636061:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:21.0:1619677473.636062:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:26.0:1619677473.636063:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.636065:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.636066:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.636066:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.636066:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000400:00000200:26.0:1619677473.636067:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:21.0:1619677473.636067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677473.636068:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.636068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677473.636070:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.636071:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.636072:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677473.636075:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.636077:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677473.636080:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.636081:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677473.636082:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.636084:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677473.636084:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.636085:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.636085:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.636100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.636109:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.636112:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.636113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.636113:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.636115:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.636118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.636491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.636499:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.636502:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.636503:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.636504:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.636508:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.636509:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.636510:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.636511:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.636512:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.636513:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.2:1619677473.636514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:10.0:1619677473.636516:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.636517:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.636519:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.636520:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677473.636520:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.636520:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.636521:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.636522:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.636522:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677473.636523:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:10.0:1619677473.636523:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677473.636524:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.636526:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.636527:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.636528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677473.636529:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.636530:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.636532:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.636533:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.636534:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677473.636535:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:18.0:1619677473.636537:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.636537:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.636538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.636538:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.636539:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677473.636541:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.636543:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.636543:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677473.636544:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.636545:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:9.2:1619677473.636545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677473.636547:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.636547:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.636548:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.636549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677473.636551:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.636554:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.636555:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.636555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.636556:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.636557:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:13.0:1619677473.636560:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.636563:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.636563:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.636565:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.636567:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.636568:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677473.636866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:22.0:1619677473.636868:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.636871:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.636872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.636873:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:22.0:1619677473.636874:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.636876:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:22.0:1619677473.636876:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.636876:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:22.0:1619677473.636878:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.636880:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.636880:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.636881:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.636883:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.636883:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.636884:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677473.636884:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.636887:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.636888:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.636888:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.636889:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.636890:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677473.636890:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.636893:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.636894:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.636894:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.636894:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.636896:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677473.636896:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.636897:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677473.636898:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.636898:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:26.0:1619677473.636899:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:22.0:1619677473.636899:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:26.0:1619677473.636900:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677473.636900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.636901:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.636901:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.636902:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.636904:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.636904:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677473.636906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:4.2:1619677473.636909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677473.636910:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.636912:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.636912:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.636913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.636914:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.636914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.636915:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.636915:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:21.0:1619677473.636916:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.636916:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:20.0:1619677473.636917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677473.636917:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.636918:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:21.0:1619677473.636919:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677473.636919:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.636920:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.636921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.636921:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.636922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.636922:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:1.0:1619677473.637114:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.637116:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677473.637117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677473.637120:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.637122:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.637123:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677473.637125:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.637126:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.637127:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.637128:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677473.637132:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.637135:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677473.637138:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.637138:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677473.637139:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677473.637141:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677473.637141:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.637142:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.637143:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677473.637147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.637154:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.637157:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.637158:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.637159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.637160:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.637163:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677473.637950:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.637953:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.637954:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.637956:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.637958:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.637959:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.637961:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.637962:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.637963:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.637964:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.637968:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.637969:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.637972:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.637973:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.637973:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.637975:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.637976:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.637976:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.637977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.637990:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.637992:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.637993:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.637995:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.637996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.637997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.638051:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.638053:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.638054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.638055:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.638057:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.638060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.638274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677473.638282:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.638285:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.638287:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.638289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.638289:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.638291:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:12.0:1619677473.638292:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.638298:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.638300:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.638300:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:12.0:1619677473.638304:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.638305:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677473.638307:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.638308:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:12.0:1619677473.638309:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:12.0:1619677473.638310:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:12.0:1619677473.638311:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.638312:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.638312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.638316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.638324:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.638326:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.638327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.638327:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.638330:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.638332:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677473.638439:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.638442:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.638442:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.638444:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.638447:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.638448:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.638449:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.638450:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.638451:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.638452:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.638455:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.638457:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.638459:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.638460:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.638461:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.638462:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.638463:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.638464:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677473.638465:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.638466:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.638469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.638477:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677473.638479:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677473.638479:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.638479:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.638480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677473.638481:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.638482:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.638484:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.638485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.638486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.638641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.638644:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.638647:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677473.638649:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.638649:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.638651:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.638653:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.638654:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.638656:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.638656:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.638658:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.638661:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.638663:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.638665:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.638666:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.638667:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.638668:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.638669:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.638670:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.638670:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.638674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.638682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.638685:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.638686:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.638686:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.638688:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.638690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677473.638837:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.638840:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.638841:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.638842:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.638844:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.638846:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.638847:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.638848:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.638849:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.638850:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.638854:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.638856:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.638858:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.638859:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.638859:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.638861:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.638861:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.638862:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.638862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.638874:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.638876:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.638877:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.638879:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.638880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.638881:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.638979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.638986:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.638989:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.638990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.638992:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.638994:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.638996:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.638997:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.638998:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.638999:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.639000:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.639004:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.639006:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.639009:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.639009:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.639010:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.639012:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.639012:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.639013:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.639013:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.639025:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.639028:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.639029:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.639029:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.639031:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.639034:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.639189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.639197:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.639200:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.639201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.639203:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.639205:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.639206:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.639207:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.639208:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.639209:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.639210:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.639213:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.639215:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.639218:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.639218:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.639219:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.639221:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.639221:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.639222:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.639223:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.639224:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.639229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677473.639234:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.639236:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.639237:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.639237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.639238:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.639238:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.639239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.639240:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.639240:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.639243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.639406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.639412:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.639414:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.639415:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.639417:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.639419:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.639421:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.639422:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.639423:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.639424:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.639426:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.639429:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.639431:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.639433:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.639434:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.639434:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.639436:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.639436:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.639438:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.639439:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.639439:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.639445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.639450:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.639450:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.639452:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.639453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677473.639453:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.639454:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677473.639454:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.639455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.639456:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.639459:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.639613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.639621:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.639624:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.639625:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.639627:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.639629:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.639631:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.639632:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.639634:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.639635:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.639636:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.639639:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.639641:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.639644:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.639645:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.639645:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.639647:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.639647:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.639648:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.639648:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.639660:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.639663:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.639664:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.639664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.639666:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.639669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.639681:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.639684:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.639685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.639685:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.639686:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.639687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.639733:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.639735:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.639736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.639738:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.639740:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.639741:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.639743:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.639744:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.639745:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.639746:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.639749:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.639750:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.639753:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.639754:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.639754:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.639756:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.639756:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.639757:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.639757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.639770:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.639772:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.639773:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.639775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.639775:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.639777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.639790:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.639792:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.639793:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.639794:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.639794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.639795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.639941:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.639944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.639944:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.639945:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.639947:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.639949:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.640530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677473.640538:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.640541:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.640541:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.640543:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.640546:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.640548:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677473.640549:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.640550:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.640551:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.640552:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.640554:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.640556:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677473.640558:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e98a78900] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.640559:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:26.0:1619677473.640560:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:26.0:1619677473.640561:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:26.0:1619677473.640562:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.640562:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.640563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677473.640568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:8.0:1619677473.640572:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.640574:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7ca00] (23)--
00000800:00000200:24.0:1619677473.640575:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000400:00000200:8.0:1619677473.640575:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.640577:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.640579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.2:1619677473.640586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:24.0:1619677473.640589:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)++
00000800:00000200:24.0:1619677473.640591:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.640592:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)++
00000800:00000200:8.0:1619677473.640592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.640593:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7ca00] (22)--
00000800:00000200:24.0:1619677473.640593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:22.0:1619677473.640779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.640782:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.640782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.640784:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.640786:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.640788:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.640789:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.640790:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.640791:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.640792:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.640796:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.640797:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.640800:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.640801:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.640801:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.640803:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.640803:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.640805:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.640806:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.640806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.640816:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.640818:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.640818:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.640819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.640819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.640821:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.640821:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.640823:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.640824:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.640825:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.641020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:9.2:1619677473.641025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677473.641028:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.641030:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:8.0:1619677473.641031:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:2.0:1619677473.641031:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.641033:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677473.641033:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.641034:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.641034:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.641036:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.641037:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:18.0:1619677473.641038:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.641039:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.641039:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:18.0:1619677473.641040:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.641041:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.641042:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.641046:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.641048:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.641051:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.641052:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.641052:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.641054:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.641055:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.641056:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.641057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:4.2:1619677473.641190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.641193:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.641196:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.641198:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.641199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.641201:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.641202:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.641204:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.641205:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.641206:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.641207:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.641210:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.641211:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.641214:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.641215:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.641215:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.641217:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.641217:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.641218:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.641219:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.641230:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.641233:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.641234:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.641234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.641236:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.641238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.641244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.641252:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.641255:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.641256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.641258:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.641260:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.641262:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.641263:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.641264:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.641265:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.641266:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.641269:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.641271:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.641273:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.641274:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.641274:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.641276:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.641277:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.641278:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.641279:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.641279:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.641281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.641285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677473.641287:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.641290:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.641290:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.641291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.641293:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677473.641293:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.641297:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.641299:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677473.641299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.641300:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677473.641561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.641567:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.641570:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.641571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.641573:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677473.641576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:2.0:1619677473.641576:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.641577:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.641579:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.641580:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.641581:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.641582:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677473.641583:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:2.0:1619677473.641585:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.641586:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.641586:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:2.0:1619677473.641587:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677473.641589:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.641589:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.641590:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.641590:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677473.641592:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.641592:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:10.0:1619677473.641593:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:2.0:1619677473.641593:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.641593:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.641594:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677473.641595:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.641596:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.641597:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.641598:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677473.641599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677473.641601:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.641603:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677473.641606:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:10.0:1619677473.641606:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.641607:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.641607:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677473.641608:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.641609:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.641609:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.641609:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677473.641609:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677473.641610:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:18.0:1619677473.641611:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.641611:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677473.641613:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677473.641615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.641624:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:9.2:1619677473.641624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677473.641626:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.641627:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.641629:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.641629:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677473.641631:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677473.641632:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:2.0:1619677473.641634:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.641635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.641635:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.641635:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.641636:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.641659:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.641661:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.641662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.641664:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.641666:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.641667:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.641669:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.641670:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.641671:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.641672:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.641675:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.641677:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.641679:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.641680:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.641681:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.641682:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.641683:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.641684:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.641684:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.641696:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.641698:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.641699:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.641701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.641701:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.641703:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.641831:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.641834:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.641835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.641837:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.641839:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.641841:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.641842:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.641843:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.641844:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.641845:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.641848:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.641850:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.641852:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.641853:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.641853:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.641855:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.641855:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.641856:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.641856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.641861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.641868:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.641870:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.641871:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.641873:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.641874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.641875:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677473.641933:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.641935:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.641936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.641937:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.641939:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.641941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.642095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677473.642103:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.642105:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.642106:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677473.642108:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.642111:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.642112:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.642113:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.642115:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.642116:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.642118:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.642120:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.642122:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.642124:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.642124:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.642125:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.642126:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.642127:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.642128:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.642128:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677473.642132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.642138:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.642140:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.642140:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.642141:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.642142:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.642144:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.642558:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.642561:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.642562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.642563:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.642566:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.642567:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.642568:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.642570:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.642571:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.642572:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.642575:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.642577:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.642579:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.642580:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.642581:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.642582:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.642583:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.642583:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.642584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.642596:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.642598:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.642599:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.642600:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.642601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.642602:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.643240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.643248:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.643251:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.643252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.643253:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.643255:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.643257:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:4.2:1619677473.643257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:8.0:1619677473.643258:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.643259:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.643260:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.643261:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677473.643262:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.643264:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:8.0:1619677473.643264:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.643265:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:21.0:1619677473.643267:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.643267:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.0:1619677473.643268:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.643269:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.643269:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.643270:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:21.0:1619677473.643271:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:8.0:1619677473.643271:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:21.0:1619677473.643272:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.643272:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.643273:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.643273:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:21.0:1619677473.643274:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.643275:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.643276:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:9.2:1619677473.643277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:21.0:1619677473.643280:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.643281:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.643284:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.643285:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.643286:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.643286:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677473.643288:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.643288:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.643289:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677473.643289:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677473.643290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677473.643290:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.643290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.643292:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.643300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.643595:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.643597:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.643598:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.643601:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.643607:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.643608:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.643610:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.643611:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.643612:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.643614:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.643617:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.643618:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.643621:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.643621:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.643622:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.643624:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.643624:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.643625:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.643626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677473.643637:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.643639:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.643640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.643640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.643641:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:6.0:1619677473.643642:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.643643:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.643644:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.643645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:20.0:1619677473.643646:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.643647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.643648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.643658:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.643660:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.643661:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.643662:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.643662:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.643663:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.643664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.643665:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.643666:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.643666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.643667:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.643668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.643982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.643989:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.643992:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.643993:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.643995:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.643998:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.643999:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.644000:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.644002:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.644003:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.644004:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.644007:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.644008:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.644011:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.644012:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.644013:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.644014:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.644015:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.644015:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.644016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.644027:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.644030:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.644031:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.644031:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.644033:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.644037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.644038:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.644040:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.644040:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677473.644042:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.644044:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.644045:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677473.644046:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.644048:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.644049:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.644050:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677473.644053:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.644055:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.644057:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.644058:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677473.644059:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677473.644061:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677473.644061:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.644061:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.644062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677473.644067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.644075:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.644077:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.644078:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.644079:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.644081:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.644083:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677473.644150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.644156:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.644159:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.644160:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.644162:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.644164:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.644166:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.644167:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.644168:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.644169:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.644170:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.644173:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.644175:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.644177:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.644178:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.644179:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.644180:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.644181:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.644181:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.644182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.644194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.644196:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.644197:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.644197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.644199:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.644201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.644212:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.644215:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.644216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.644216:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.644217:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.644218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.644343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.644350:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.644352:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.644354:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.644355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.644357:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.644358:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.644360:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677473.644360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:21.0:1619677473.644361:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.644362:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.644363:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.644366:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.644367:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677473.644368:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677473.644370:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:21.0:1619677473.644371:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.644371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677473.644372:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.644372:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:24.0:1619677473.644373:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.644374:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.644375:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.644375:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677473.644376:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.644376:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:24.0:1619677473.644378:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.644380:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96508351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.644381:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677473.644381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:24.0:1619677473.644383:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.644384:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.644388:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677473.644388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677473.644389:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.644389:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:24.0:1619677473.644392:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.644392:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:24.0:1619677473.644393:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.644393:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.644393:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.644394:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:1.0:1619677473.644395:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.644396:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677473.644396:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677473.644397:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.644397:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.644398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677473.644398:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677473.644399:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.644400:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.644402:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677473.644402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:10.0:1619677473.644405:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.644406:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.644407:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.644408:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677473.644409:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:10.0:1619677473.644410:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.644410:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.644412:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.644412:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.644413:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.644415:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.644415:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.644417:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.644418:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677473.644420:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.644420:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.644421:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.644423:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.644423:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.644424:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.644425:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.644438:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.644440:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.644441:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.644442:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.644444:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.644446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.644456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.644459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.644459:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.644461:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.644464:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.644465:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.644466:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.644467:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.644468:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.644469:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.644472:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.644474:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.644477:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.644477:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.644478:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.644480:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.644480:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.644481:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.644482:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.644483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.644493:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.644493:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.644494:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.644495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.644496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.644496:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.644496:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.644498:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.644499:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.644500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.645317:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.645319:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.645320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.645322:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.645324:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.645325:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.645326:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.645328:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.645328:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.645329:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.645332:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.645333:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.645335:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.645336:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.645337:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.645338:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.645339:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.645339:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.645340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.645353:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.645355:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.645356:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.645357:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.645359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.645360:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.645451:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.645454:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.645455:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.645455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.645458:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.645460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.645702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677473.645710:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.645713:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.645715:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.645717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.645718:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.645719:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677473.645720:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.645721:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.645722:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.645723:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677473.645726:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.645728:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677473.645731:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.645731:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677473.645732:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677473.645734:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677473.645734:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.645735:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.645736:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.645740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.645749:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.645752:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.645753:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.645753:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.645755:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.645758:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677473.646097:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.646100:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.646100:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677473.646103:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.646105:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.646106:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677473.646108:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.646109:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.646110:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.646111:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677473.646114:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.646115:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677473.646118:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.646119:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677473.646120:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.646121:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677473.646122:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.646123:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.646123:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.646128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.646134:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.646137:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.646138:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.646139:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.646141:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.646144:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.646245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.646252:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.646255:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.646256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.646258:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.646261:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.646263:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.646264:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.646265:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.646266:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.646267:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.646271:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.646272:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.646275:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.646276:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.646277:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.646278:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.646279:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.646279:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.646280:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.646285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.646299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:2.0:1619677473.646300:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:20.0:1619677473.646303:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677473.646303:0:13335:0:(o2iblnd_cb.c:2075:kiblnd_close_conn_locked()) closing conn to 10.0.10.54@o2ib7
00000800:00000200:2.0:1619677473.646303:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:8.0:1619677473.646304:0:13335:0:(o2iblnd.c:419:kiblnd_unlink_peer_locked()) peer_ni[ffff919e98a78900] -> 10.0.10.54@o2ib7 (2)--
00000400:00000200:2.0:1619677473.646305:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.646306:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:18.0:1619677473.646306:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.646307:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:8.0:1619677473.646307:0:13335:0:(o2iblnd_cb.c:3321:kiblnd_cm_callback()) conn[ffff919e43e7ca00] (20)--
00000400:00000200:2.0:1619677473.646307:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:20.0:1619677473.646308:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.646309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:18.0:1619677473.646309:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677473.646310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:20.0:1619677473.646311:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.646311:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.646311:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:20.0:1619677473.646312:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.646314:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.646314:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.646315:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.646316:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.646317:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677473.646318:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677473.646318:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.646321:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677473.646321:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.646322:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.646323:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677473.646323:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677473.646325:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.646326:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.646327:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:18.0:1619677473.646328:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.646329:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.646330:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.646330:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:18.0:1619677473.646330:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677473.646331:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:18.0:1619677473.646333:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.646334:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677473.646335:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677473.646337:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677473.646337:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.646339:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677473.646340:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677473.646341:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677473.646342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677473.646344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.646348:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.646349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.646349:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.646350:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:6.0:1619677473.646351:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.646351:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:10.0:1619677473.646353:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.646354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:6.0:1619677473.646354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677473.646354:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000400:00000200:10.0:1619677473.646355:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.646355:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677473.646355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:6.0:1619677473.646356:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:2.0:1619677473.646356:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:22.0:1619677473.646357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677473.646358:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.646359:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.646360:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:6.0:1619677473.646361:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.646362:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.646364:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.646365:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.646366:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.646367:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.646370:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.646372:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.646375:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.646376:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.646376:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.646378:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.646378:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.646379:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.646380:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.646392:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.646394:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.646395:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.646396:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.646397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.646399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.646501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.646509:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.646512:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.646513:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.646516:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.646518:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.646519:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.646521:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.646522:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.646523:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.646524:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.646528:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.646530:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.646533:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.646534:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.646534:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.646536:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.646536:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.646537:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.646538:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.646542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.646549:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.646552:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.646553:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.646554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.646555:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.646557:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.646563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.646569:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.646572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.646572:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.646572:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.646573:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.646574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.2:1619677473.646867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (19)++
00000800:00000200:24.0:1619677473.646869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (20)++
00000800:00000200:8.0:1619677473.646873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (21)--
00000800:00000200:24.0:1619677473.646878:0:19939:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff919e43e7ca00] (20)--
00000800:00000200:24.0:1619677473.646879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (19)--
00000800:00000200:23.0:1619677473.646939:0:19929:0:(o2iblnd_cb.c:3568:kiblnd_connd()) conn[ffff919e43e7ca00] (18)--
00000800:00000200:9.2:1619677473.646972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.646978:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.646980:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.646981:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.646981:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.646983:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.646986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:22.0:1619677473.647225:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.647227:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.647228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.647230:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.647233:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.647234:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.647235:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.647237:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.647238:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.647239:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.647242:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.647243:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.647246:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.647247:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.647247:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.647249:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.647249:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.647250:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.647250:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.647262:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.647264:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.647265:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.647267:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.647268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.647270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.647271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.647279:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677473.647282:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677473.647282:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.647283:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677473.647284:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.647285:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677473.647285:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.647286:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.647287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.647288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677473.647288:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.647289:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.647291:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.647292:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.647299:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.647300:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.647303:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.647305:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.647308:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.647309:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.647309:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.647311:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.647311:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.647313:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.647314:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.647315:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.647321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.647328:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.647330:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.647331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.647332:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.647336:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.647339:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.647340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.647340:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.647342:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.647345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.647359:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.647362:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.647363:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.647363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.647364:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.647365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.647591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.647599:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.647602:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.647603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.647605:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.647608:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.647609:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.647611:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.647612:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.647613:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.647614:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.647618:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.647619:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.647622:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.647622:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.647623:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.647625:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.647625:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.647626:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.647627:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.647628:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.647631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.647633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677473.647638:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677473.647639:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.647641:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:17.0:1619677473.647642:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677473.647642:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677473.647643:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.647643:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677473.647644:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.647645:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.647648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.648125:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.648128:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.648129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.648131:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.648133:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.648134:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.648136:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.648137:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.648138:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.648139:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.648142:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.648143:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.648146:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.648146:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.648147:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.648149:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.648149:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.648150:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.648151:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.648152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.648162:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.648164:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.648164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.648164:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.648165:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.648166:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.648167:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.648169:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.648169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.648171:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.648476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.648484:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.648487:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.648488:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.648490:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.648492:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.648494:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.648495:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.648496:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.648497:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.648498:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.648502:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.648503:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.648506:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.648507:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.648507:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.648509:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.648510:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.648510:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.648511:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.648524:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.648527:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.648528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.648528:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.648530:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.648533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.648644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.648651:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.648653:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.648654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.648654:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.648656:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.648657:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.648658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.648659:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.648660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.648661:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.648663:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.648667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.649182:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:5.2:1619677473.649184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677473.649185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.649185:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.649188:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.649189:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.649190:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677473.649191:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.649192:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.649193:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677473.649193:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677473.649193:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:6.0:1619677473.649194:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.649196:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677473.649196:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.649199:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677473.649199:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677473.649200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677473.649200:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677473.649201:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:1.0:1619677473.649202:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.649203:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.649204:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.649205:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.649205:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:1.0:1619677473.649205:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.649206:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.649207:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.649207:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:1.0:1619677473.649209:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.649210:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.649210:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:6.0:1619677473.649211:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.649211:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.649212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:1.0:1619677473.649212:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.649214:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.649215:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.649215:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:10.0:1619677473.649216:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.649216:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677473.649217:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677473.649218:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.649218:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677473.649219:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96508326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.649219:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.649219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:20.0:1619677473.649220:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:10.0:1619677473.649220:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.649221:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.649222:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.649223:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.649224:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.649226:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.649226:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.649226:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.649229:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.649229:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.649231:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.649232:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.649233:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.649233:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.649234:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.649235:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.649235:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.649235:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.649235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.649236:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.649237:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.649237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.649239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.649248:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.649250:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.649251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.649251:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.649253:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.649256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.649587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677473.649594:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.649597:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677473.649598:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677473.649600:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.649603:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.649605:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677473.649606:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.649607:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.649608:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.649609:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677473.649612:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.649613:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.649616:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.649617:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677473.649618:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.649619:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.649620:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.649620:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677473.649621:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677473.649637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.649645:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.649648:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.649648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.649649:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.649651:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.649654:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.649656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.649663:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.649666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.649666:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.649667:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.649668:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.649668:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.649823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.649831:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.649834:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.649835:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.649837:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.649839:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.649841:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.649842:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.649843:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.649844:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.649845:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.649848:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.649850:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.649853:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.649853:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.649854:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.649856:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.649856:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.649857:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.649857:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.649863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.649872:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.649874:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.649875:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.649875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.649877:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.649880:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.649978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.649983:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.649986:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.649987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.649989:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.649991:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.649992:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.649993:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.649995:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.649996:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.649997:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.649999:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.650001:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.650003:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.650004:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.650004:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.650006:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.650006:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.650007:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.650007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.650012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.650019:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.650020:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.650021:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.650022:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.650022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.650023:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.650034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:22.0:1619677473.650039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.650041:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:2.0:1619677473.650041:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677473.650042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.650044:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.650044:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.650045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.650045:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.650045:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.650046:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:22.0:1619677473.650047:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.650048:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.650050:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.650051:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.650052:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.650053:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.650056:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.650057:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.650060:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.650060:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.650061:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.650063:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.650063:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.650064:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.650065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.650076:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.650079:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.650079:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.650081:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.650082:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.650083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.650221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.650224:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.650226:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.650228:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.650229:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.650231:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.650232:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.650234:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.650235:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.650236:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.650237:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.650240:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.650242:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.650244:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.650245:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.650246:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.650248:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.650248:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.650249:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.650251:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.650251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677473.650256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.650261:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.650263:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.650264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.650265:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.650268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.650275:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.650278:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.650279:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.650279:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.650281:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.650283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.650969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:6.0:1619677473.650971:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.650974:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.650975:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.650975:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:6.0:1619677473.650976:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.650977:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.650979:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:6.0:1619677473.650979:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.650980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:6.0:1619677473.650980:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.650981:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.650982:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.650983:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.650984:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:6.0:1619677473.650984:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.650985:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677473.650986:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.650987:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.650988:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.650989:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.650995:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:6.0:1619677473.650995:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.650997:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677473.650998:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.651001:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.651001:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677473.651001:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.651002:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677473.651002:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.651003:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.651004:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677473.651005:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.651006:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677473.651006:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677473.651007:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.651007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.651007:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.651008:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.651019:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.651021:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:9.2:1619677473.651021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677473.651022:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.651024:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.651024:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.651026:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.651028:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.651031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.651031:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.651032:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.651034:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.651037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.651040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:9.2:1619677473.651041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.651043:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.651044:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.651044:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.651045:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.651046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677473.651047:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.651050:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.651050:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.651051:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.651052:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.651052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.651142:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.651145:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.651146:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.651146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.651148:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.651151:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.651763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677473.651767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.651772:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.651774:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.651775:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.651777:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.651777:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.651779:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.651780:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.651781:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.651782:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.651783:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.651785:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.651786:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.651787:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.651788:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.651790:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.651790:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.651791:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.651792:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.651792:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.651793:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.651796:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.651796:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.651797:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.651798:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.651798:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.651800:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.651800:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.651801:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.651801:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.651802:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.651802:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.651802:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.651803:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677473.651804:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.651804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.651805:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.651806:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.651808:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.651808:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.651810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:12.0:1619677473.651813:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:4.2:1619677473.651813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677473.651815:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.651816:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.651817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:12.0:1619677473.651818:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.651819:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.651820:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:13.0:1619677473.651820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:8.0:1619677473.651820:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.651821:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:12.0:1619677473.651822:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:8.0:1619677473.651822:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.651823:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.651824:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677473.651824:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.651824:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.652012:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.652015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.652015:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.652018:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.652020:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.652021:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.652022:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.652024:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.652025:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.652026:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.652029:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.652030:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.652033:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.652034:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.652034:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.652036:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.652036:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.652038:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.652039:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.652039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.652049:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.652051:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.652051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.652052:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.652053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.652053:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.652054:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.652056:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.652056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.652058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.652195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.652203:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.652206:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.652206:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.652209:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.652211:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.652212:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677473.652214:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.652215:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.652216:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.652217:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677473.652220:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.652222:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677473.652224:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.652225:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677473.652225:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677473.652227:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677473.652227:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.652228:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.652228:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:4.2:1619677473.652372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.652379:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.652382:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.652383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.652384:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.652386:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.652388:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.652389:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.652390:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.652391:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677473.652391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000400:00000200:24.0:1619677473.652392:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.652395:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.652397:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.652398:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:24.0:1619677473.652400:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.652400:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.652401:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:24.0:1619677473.652401:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.652402:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.652403:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.652403:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.652403:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.652404:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.652404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677473.652405:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.652407:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.652408:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.652409:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.652410:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.652411:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.652414:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.652415:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.652416:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.652418:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.652419:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.652419:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677473.652419:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.652420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.652420:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.652421:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.652421:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.652422:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.652422:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000400:00000200:6.0:1619677473.652422:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.652425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.652427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677473.652435:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677473.652436:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:22.0:1619677473.652437:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.652439:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677473.652439:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:6.0:1619677473.652439:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.652440:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.652440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:18.0:1619677473.652440:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.652440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.652442:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.652446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.652449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677473.652456:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.652459:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.652460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.652460:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.652461:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.652462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677473.652840:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.652843:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.652844:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.652844:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.652846:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.652849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.652920:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.652922:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.652923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.652925:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.652927:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.652928:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.652929:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.652930:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.652931:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.652932:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.652935:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.652937:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.652939:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.652940:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.652941:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.652942:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.652943:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.652944:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.652944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.652956:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.652959:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.652960:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.652961:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.652962:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.652964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.653095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.653102:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.653104:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.653105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.653105:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.653107:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.653109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.653324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.653332:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.653334:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.653335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.653337:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.653340:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.653341:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.653342:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.653343:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.653344:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.653345:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.653348:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.653350:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.653353:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.653353:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.653354:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.653355:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.653356:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.653357:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.653357:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.653371:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.653373:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.653374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.653374:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.653377:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.653379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.653781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.653788:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.653791:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.653791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677473.653794:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:18.0:1619677473.653794:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.653796:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.653797:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:18.0:1619677473.653798:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:6.0:1619677473.653798:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.653800:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.653800:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.653801:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.653802:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.653803:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.653803:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677473.653804:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.653805:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.653808:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.653808:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677473.653809:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.653810:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.653810:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:22.0:1619677473.653814:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.653814:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.653815:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677473.653816:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.653816:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.653818:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.653818:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677473.653819:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.653819:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677473.653819:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677473.653820:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677473.653820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677473.653822:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.653822:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.653823:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.653824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.653835:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.653837:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.653838:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.653840:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.653841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.653842:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.653975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.653982:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.653985:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.653986:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.653986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.653988:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.653990:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.653995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.654001:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.654004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.654004:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.654004:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.654005:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.654006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.654551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.654557:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.654560:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.654561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.654563:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.654565:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.654567:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.654568:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.654569:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.654570:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.654571:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.654574:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.654575:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.654578:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.654579:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.654579:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.654581:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.654581:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.654582:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.654582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.654596:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.654598:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.654599:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.654600:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.654602:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.654604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.654617:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.654620:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.654620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.654621:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.654622:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.654622:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.654713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.654721:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.654724:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.654724:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.654727:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.654729:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.654731:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.654732:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.654733:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.654734:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.654735:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.654738:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.654740:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.654742:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.654743:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.654744:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.654745:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.654746:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.654747:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.654747:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.654759:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.654761:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.654762:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.654763:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.654765:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.654768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.654817:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.654819:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.654820:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.654821:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.654823:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.654825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.654840:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.654843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.654843:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.654845:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.654847:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.654848:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.654849:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.654851:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.654852:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.654853:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.654856:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.654858:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.654860:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.654861:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.654862:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.654864:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.654864:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.654865:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.654865:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.654877:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.654879:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.654880:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.654882:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.654883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.654884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.654900:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.654903:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.654904:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.654904:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.654905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.654906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.655002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677473.655009:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.655012:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.655013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.655015:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.655018:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.655019:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.655020:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.655021:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.655023:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.655024:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.655027:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.655029:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.655033:0:19940:0:(o2iblnd_cb.c:1635:kiblnd_launch_tx()) peer_ni[ffff919e58c4f100] -> 10.0.10.54@o2ib7 (1)++
00000800:00000200:10.0:1619677473.655038:0:19940:0:(o2iblnd_cb.c:1440:kiblnd_connect_peer()) peer_ni[ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2)++
00000800:00000200:4.2:1619677473.655673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.655680:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.655683:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.655684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.655685:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.655688:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.655689:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.655690:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96508297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.655692:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.655692:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.655693:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.655696:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.655698:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.655700:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.655701:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.655702:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.655703:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.655704:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.655705:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.655705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.655718:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.655720:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.655722:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.655722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.655724:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.655726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.655735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.655738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.655738:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.655739:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.655740:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.655741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.655916:0:19940:0:(o2iblnd_cb.c:1392:kiblnd_resolve_addr()) bound to port 1023
00000800:00000200:10.0:1619677473.655918:0:19940:0:(o2iblnd_cb.c:1642:kiblnd_launch_tx()) peer_ni[ffff919e58c4f100] -> 10.0.10.54@o2ib7 (3)--
00000800:00000200:10.0:1619677473.655919:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.655920:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.655922:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.655922:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:7.0:1619677473.655924:0:13229:0:(o2iblnd_cb.c:3191:kiblnd_cm_callback()) 10.0.10.54@o2ib7 Addr resolved: 0
00000800:00000200:4.2:1619677473.655928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:7.0:1619677473.655930:0:13229:0:(o2iblnd_cb.c:3208:kiblnd_cm_callback()) 10.0.10.54@o2ib7: connection bound to ib1:10.0.10.237:mlx5_1
00000800:00000200:26.0:1619677473.655935:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.655937:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.655938:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.655939:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:7.0:1619677473.656078:0:13229:0:(o2iblnd_cb.c:3232:kiblnd_cm_callback()) 10.0.10.54@o2ib7 Route resolved: 0
00000800:00000200:7.0:1619677473.656093:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 0: ffff919e4116c000 0x12c116c000(0x12c116c000)
00000800:00000200:7.0:1619677473.656093:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 1: ffff919e4116d000 0x12c116d000(0x12c116d000)
00000800:00000200:7.0:1619677473.656094:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 2: ffff919e4116e000 0x12c116e000(0x12c116e000)
00000800:00000200:7.0:1619677473.656095:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 3: ffff919e4118e000 0x12c118e000(0x12c118e000)
00000800:00000200:7.0:1619677473.656095:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 4: ffff919e4118f000 0x12c118f000(0x12c118f000)
00000800:00000200:7.0:1619677473.656096:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 5: ffff919e41190000 0x12c1190000(0x12c1190000)
00000800:00000200:7.0:1619677473.656097:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 6: ffff919e41191000 0x12c1191000(0x12c1191000)
00000800:00000200:7.0:1619677473.656097:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 7: ffff919e41192000 0x12c1192000(0x12c1192000)
00000800:00000200:7.0:1619677473.656098:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 8: ffff919e41193000 0x12c1193000(0x12c1193000)
00000800:00000200:7.0:1619677473.656098:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 9: ffff919e41194000 0x12c1194000(0x12c1194000)
00000800:00000200:7.0:1619677473.656099:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 10: ffff919e41195000 0x12c1195000(0x12c1195000)
00000800:00000200:7.0:1619677473.656100:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 11: ffff919e41196000 0x12c1196000(0x12c1196000)
00000800:00000200:7.0:1619677473.656100:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 12: ffff919e41197000 0x12c1197000(0x12c1197000)
00000800:00000200:7.0:1619677473.656101:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 13: ffff919e41198000 0x12c1198000(0x12c1198000)
00000800:00000200:7.0:1619677473.656101:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 14: ffff919e41199000 0x12c1199000(0x12c1199000)
00000800:00000200:7.0:1619677473.656102:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 15: ffff919e4119a000 0x12c119a000(0x12c119a000)
00000800:00000200:7.0:1619677473.656102:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 16: ffff919e4119b000 0x12c119b000(0x12c119b000)
00000800:00000200:7.0:1619677473.656103:0:13229:0:(o2iblnd.c:1350:kiblnd_map_rx_descs()) rx 17: ffff919e4119c000 0x12c119c000(0x12c119c000)
00000800:00000200:22.0:1619677473.656560:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.656563:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.656563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.656564:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.656566:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.656569:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.656637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.656643:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.656645:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.656647:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.656648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.656650:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.656651:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.656653:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.656654:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.656655:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.656656:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.656659:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.656661:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.656663:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.656664:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.656665:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.656666:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.656667:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.656668:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.656668:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.656674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.656682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.656685:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.656686:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.656686:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.656688:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.656690:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.657004:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.657007:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.657008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.657010:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677473.657011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:6.0:1619677473.657012:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.657013:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.657015:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.657016:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.657017:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.657018:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.657020:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677473.657021:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.657023:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:6.0:1619677473.657023:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.657024:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.657026:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.657026:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.657027:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:10.0:1619677473.657028:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.657028:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.657029:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677473.657030:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:6.0:1619677473.657030:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677473.657031:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96508292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.657031:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.657032:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:10.0:1619677473.657033:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.657033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677473.657034:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.657035:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.657038:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.657040:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.657042:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.657043:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.657043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677473.657043:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.657044:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.657045:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.657045:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.657045:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677473.657045:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.657046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.657046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.657046:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.657046:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.657047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677473.657048:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.657049:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.657050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.657051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.657058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.657061:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.657062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.657062:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.657064:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.657066:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.657204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.657210:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.657213:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.657214:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.657215:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.657218:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.657220:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.657221:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96508290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.657222:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.657223:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.657224:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.657227:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.657229:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.657232:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.657232:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.657233:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.657235:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.657235:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.657242:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.657243:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.657244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:7.0:1619677473.657245:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657245:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657246:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657246:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657247:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657247:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657247:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657248:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657248:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657248:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657249:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657249:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:4.2:1619677473.657249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:7.0:1619677473.657250:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657250:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657250:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657251:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657251:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657252:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657252:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657252:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657253:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657253:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657253:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657254:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657255:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657255:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657255:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:8.0:1619677473.657256:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:7.0:1619677473.657256:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657256:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657256:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657257:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657257:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:8.0:1619677473.657258:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:7.0:1619677473.657258:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:7.0:1619677473.657258:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:7.0:1619677473.657258:0:13229:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (19)++
00000800:00000200:8.0:1619677473.657259:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:7.0:1619677473.657259:0:13229:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (20)--
00000800:00000200:6.0:1619677473.657259:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.657260:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.657262:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.657263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.657264:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.657266:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.657269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.657432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.657438:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.657440:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.657442:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.657443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.657445:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.657447:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.657448:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.657449:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.657450:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.657451:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.657455:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.657456:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.657459:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.657460:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.657460:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.657462:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.657462:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.657463:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:21.0:1619677473.657464:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677473.657465:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677473.657469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677473.657472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.657478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.657480:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.657480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:21.0:1619677473.657481:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.657481:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677473.657481:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.657482:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.657483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.657484:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677473.657485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.657489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.657497:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.657499:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.657500:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.657500:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677473.657501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.657502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677473.657519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677473.657528:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.657530:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.657533:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.657535:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.657536:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.657537:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677473.657538:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.657540:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.657541:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.657542:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677473.657546:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.657547:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677473.657550:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.657551:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677473.657552:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677473.657553:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677473.657554:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.657555:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677473.657556:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677473.657557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677473.657560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677473.657563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677473.657567:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.657570:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:12.0:1619677473.657570:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:26.0:1619677473.657571:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.657571:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.657573:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.657573:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677473.657573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.657574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:26.0:1619677473.657576:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:22.0:1619677473.657904:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.657907:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.657907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.657909:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.657911:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.657913:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.657914:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.657915:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.657916:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.657918:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.657921:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.657923:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.657925:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.657926:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.657927:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.657928:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.657929:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.657930:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.657930:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.657942:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.657944:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.657945:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.657946:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.657948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.657949:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.657993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677473.658000:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.658003:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.658004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.658005:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.658008:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.658009:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.658010:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.658012:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.658013:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.658014:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.658017:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.658018:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.658021:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 0
00000800:00000200:10.0:1619677473.658022:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 0
00000800:00000200:10.0:1619677473.658023:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.658024:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.658024:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:4.2:1619677473.658061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.658068:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.658070:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.658071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.658072:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.658075:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.658076:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.658077:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96508285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.658078:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.658079:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.658080:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.658083:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.658084:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.658087:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.658088:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.658088:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.658090:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.658090:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.658091:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.658093:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.658093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.658099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677473.658104:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.658106:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.658106:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.658107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.658108:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677473.658108:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.658109:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.658110:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.658110:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.658112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.658405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.658412:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.658414:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.658415:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.658415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.658417:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.658419:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.658508:0:13335:0:(o2iblnd_cb.c:3299:kiblnd_cm_callback()) ESTABLISHED(active): 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.658509:0:13335:0:(o2iblnd_cb.c:2312:kiblnd_connreq_done()) 10.0.10.54@o2ib7: active(1), version(12), status(0)
00000800:00000200:8.0:1619677473.658511:0:13335:0:(o2iblnd_cb.c:2339:kiblnd_connreq_done()) conn[ffff919e42d9c400] (19)++
00000800:00000200:8.0:1619677473.658512:0:13335:0:(o2iblnd_cb.c:2382:kiblnd_connreq_done()) conn[ffff919e42d9c400] (20)++
00000800:00000200:8.0:1619677473.658513:0:13335:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677473.658515:0:13335:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677473.658519:0:13335:0:(o2iblnd_cb.c:2402:kiblnd_connreq_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.2:1619677473.658526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677473.658530:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (23)++
00000800:00000200:8.0:1619677473.658531:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (24)--
00000400:00000200:8.0:1619677473.658532:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.658533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (23)++
00000800:00000200:24.0:1619677473.658533:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (24)--
00000400:00000200:24.0:1619677473.658534:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.658534:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.658536:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.658537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:8.0:1619677473.658537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (23)--
00000800:00000200:8.0:1619677473.658538:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:24.2:1619677473.658576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677473.658579:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677473.658582:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.658582:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677473.658584:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.54@o2ib7 : PUT - routed
00000800:00000200:24.0:1619677473.658586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677473.658733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.658739:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.658741:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.658743:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.658744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.658745:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.658746:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.658747:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.658749:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.658750:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.658751:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.658753:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.658755:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.658758:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.658758:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.658759:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.658760:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.658761:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.658762:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.658763:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.658764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677473.658769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.658773:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677473.658774:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.658775:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.658776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.658777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.658778:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.658779:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.658779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.658780:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.658783:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.658957:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.658959:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.658960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.658962:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.658964:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.658965:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.658966:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.658968:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.658968:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.658970:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.658973:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.658974:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.658977:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.658977:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:4.2:1619677473.658977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:6.0:1619677473.658978:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.658980:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.658980:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.658981:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.658981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677473.658985:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.658989:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.658990:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.658991:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.658993:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:26.0:1619677473.658994:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.658996:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:20.0:1619677473.658996:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.658996:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.658997:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.658998:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.658998:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.658999:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.658999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677473.659000:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677473.659000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677473.659004:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.659006:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.659009:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.659010:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.659010:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.659012:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.659012:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.659014:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.659015:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.659016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.659018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.659021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677473.659027:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.659029:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.659030:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.659031:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677473.659031:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:10.0:1619677473.659032:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677473.659032:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.659033:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.659035:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.659039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677473.659156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.659162:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.659165:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.659165:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.659167:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.659170:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.659171:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.659172:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.659174:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.659175:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.659176:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.659180:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.659182:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.659185:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.659186:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.659186:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.659188:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.659189:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.659190:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.659190:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.659213:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.659215:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.659216:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.659218:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:4.2:1619677473.659219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:13.0:1619677473.659220:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.659221:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.659226:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.659229:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.659230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.659232:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.659235:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.659236:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677473.659237:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.659239:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.659240:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.659241:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677473.659244:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.659245:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677473.659247:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.659248:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677473.659249:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.659250:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.659251:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.659251:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.659252:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.659256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.659262:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.659263:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.659264:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.659265:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.659266:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.659267:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:5.2:1619677473.659267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.659273:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677473.659277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677473.659277:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.659278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.659280:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.659282:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.659283:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:1.0:1619677473.659284:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.659285:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.659286:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677473.659287:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.659287:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:1.0:1619677473.659287:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.659288:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.659288:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:1.0:1619677473.659288:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.659289:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.659292:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.659301:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.659304:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.659304:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.659305:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.659307:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.659307:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.659309:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.659310:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.659310:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.659316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.659322:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.659322:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.659324:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.659324:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.659324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.659325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.659326:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.659326:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.659328:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.659330:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.659519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.659526:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.659529:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.659530:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.659532:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.659535:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.659536:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.659537:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.659539:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.659540:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.659541:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.659544:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.659546:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.659548:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.659548:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.659549:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.659551:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.659551:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.659552:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.659553:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.659554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.659557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.659560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.659566:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.659566:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.659566:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677473.659567:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.659567:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.659568:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.659569:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.659569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.659570:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.659570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.659678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.659685:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.659688:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.659689:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.659690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.659691:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.659694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.659828:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.659831:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.659832:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.659834:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.659836:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.659837:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.659838:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.659840:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.659841:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.659842:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.659845:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.659847:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.659849:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.659850:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.659851:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.659852:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.659853:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.659854:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.659854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.659866:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.659868:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.659869:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.659871:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.659872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.659874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.659885:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.659888:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.659889:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.659889:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.659890:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.659891:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.660328:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.660331:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.660332:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.660332:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.660334:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.660337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.660344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.660347:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.660348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.660348:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.660349:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.660350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.660391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.660399:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.660402:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.660403:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.660405:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.660406:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.660408:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.660409:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.660410:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.660411:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.660412:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.660415:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.660416:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.660419:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.660420:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.660421:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.660422:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.660423:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.660424:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.660424:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.660447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.660449:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.660450:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.660450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.660453:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.660456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.660703:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.660706:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.660707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.660709:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.660711:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.660712:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.660714:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.660715:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.660716:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.660717:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.660720:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.660721:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.660724:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.660725:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.660725:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.660727:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.660727:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.660729:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.660730:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.660730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.660740:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.660741:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.660742:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.660743:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.660743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.660743:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.660744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.660745:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.660746:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.660747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.660765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.660772:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.660775:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.660776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.660778:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.660781:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.660782:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.660784:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.660785:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.660786:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.660787:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.660791:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.660792:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.660795:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.660796:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.660796:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.660798:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.660799:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.660800:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.660800:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.660822:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.660824:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.660825:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.660826:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.660828:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.660830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.661732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.661735:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.661736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.661738:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.661740:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.661741:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.661742:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.661744:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.661745:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.661745:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.661748:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.661750:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.661752:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.661753:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.661754:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.661755:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.661756:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.661757:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.661757:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.661769:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.661771:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.661772:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.661774:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.661775:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.661776:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.662206:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.662209:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.662210:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.662210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.662212:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.662215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.662361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.662369:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.662372:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.662373:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.662374:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.662377:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.662378:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.662380:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.662381:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.662382:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.662383:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.662385:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.662387:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.662389:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.662390:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.662390:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.662392:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.662392:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.662393:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.662394:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.662395:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.662398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.662401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677473.662405:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.662407:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.662408:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.662408:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.662408:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.662410:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677473.662410:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.662411:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.662412:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.662412:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.662418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.662424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.662427:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.662428:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.662428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.662429:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.662429:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.662596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.662599:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.662601:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.662603:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.662605:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.662607:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:21.0:1619677473.662608:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.662608:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.662609:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.662610:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.662610:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:20.0:1619677473.662611:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677473.662613:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.662614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.662615:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.662616:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.662617:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:6.0:1619677473.662618:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.662620:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677473.662620:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.0:1619677473.662621:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:6.0:1619677473.662621:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.662622:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.662623:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:6.0:1619677473.662623:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.662623:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.662624:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677473.662624:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.662625:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.662625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677473.662628:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677473.662630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677473.662630:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.662632:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.662633:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.662633:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.662635:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.662635:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.662636:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.662636:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:1.0:1619677473.662638:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.662641:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.662642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.662642:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.662644:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.662646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677473.662649:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.662651:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.662652:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.662654:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.662655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.662656:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.662859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.662866:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.662869:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.662870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.662872:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.662874:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.662876:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.662877:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.662879:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.662880:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.662881:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.662884:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.662886:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.662889:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.662890:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.662890:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.662892:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.662892:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.662893:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.662894:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.662906:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.662909:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.662909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.662910:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.662912:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.662914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.663398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.663405:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.663408:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.663409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.663411:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.663413:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.663415:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.663416:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.663417:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.663418:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.663419:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.663423:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.663424:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.663427:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.663428:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.663428:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.663430:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.663431:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.663431:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.663431:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.663437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.663446:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.663449:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.663449:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.663450:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.663452:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.663455:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.663474:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.663477:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.663478:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.663479:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.663482:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.663483:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.663484:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.663486:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.663487:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.663488:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.663491:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.663493:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.663495:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.663496:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.663497:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.663498:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.663499:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.663499:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.663500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.663512:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.663514:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.663515:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.663517:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.663518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.663519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.663531:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.663534:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.663535:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.663535:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.663536:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.663537:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.663545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677473.663550:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.663552:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.663554:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.663555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.663557:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.663559:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.663560:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.663561:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.663562:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.663563:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.663567:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.663569:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.663572:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.663573:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.663573:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.663576:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.663576:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.663577:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677473.663578:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677473.663579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:4.2:1619677473.663584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.663588:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677473.663590:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677473.663591:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.663592:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677473.663768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.663772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.663775:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.663778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.663778:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.663780:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.663782:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.663784:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.663785:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.663786:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.663788:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.663791:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.663793:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.663797:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.663797:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.663798:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.663800:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.663801:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.663802:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.663803:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.663806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.663813:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.663815:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.663816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.663817:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.663819:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.663822:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.663971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.663978:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.663980:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.663981:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.663983:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.663985:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.663987:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.663988:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.663989:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.663990:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.663991:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.663995:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.663996:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.663999:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.664000:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.664000:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.664002:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.664003:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.664003:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.664004:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.664016:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.664018:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.664019:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.664019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.664021:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.664024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.664034:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.664037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.664037:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.664038:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.664039:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.664039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.664246:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.664248:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.664249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.664250:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.664252:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.664254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.664265:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.664267:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.664268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.664269:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.664269:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.664270:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.664300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677473.664309:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.664311:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.664314:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.664316:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.664316:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.664318:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677473.664319:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.664321:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.664322:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.664322:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677473.664326:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.664328:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677473.664331:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.664332:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677473.664333:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677473.664334:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677473.664335:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.664336:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.664336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677473.664341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:9.2:1619677473.664342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.664348:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677473.664349:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677473.664351:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.664352:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.664352:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.664352:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.664353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.664353:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.664354:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.664356:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.664357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677473.664360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.664361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.664369:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.664371:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.664372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.664373:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.664374:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.664374:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.664520:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.664522:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.664523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.664525:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.664527:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.664529:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.664530:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.664532:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.664532:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.664534:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.664537:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.664539:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.664541:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.664542:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.664543:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.664545:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.664545:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.664547:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.664548:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.664548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.664558:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.664561:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.664561:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.664561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.664562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.664563:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.664563:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.664565:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.664566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.664568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.664846:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.664849:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.664850:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.664852:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.664854:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.664855:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.664856:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.664858:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.664859:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.664860:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.664864:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.664866:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.664868:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.664869:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.664869:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.664871:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.664872:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.664873:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677473.664874:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.664874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:22.0:1619677473.664885:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.664888:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.664889:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.664889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.664891:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.664893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677473.665032:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677473.665034:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677473.665035:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.665036:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:4.2:1619677473.665174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.665182:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.665185:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.665186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.665188:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.665190:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.665192:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.665193:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677473.665194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:10.0:1619677473.665194:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.665195:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.665196:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.665199:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.665200:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677473.665201:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.665203:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.665203:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.665204:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677473.665205:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.665205:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:2.0:1619677473.665205:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677473.665206:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.665207:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.665208:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.665208:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677473.665208:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.665210:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.665211:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.665213:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.665213:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.665215:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.665218:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:12.0:1619677473.665219:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677473.665219:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677473.665221:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.665222:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.665222:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.665222:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.665223:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:12.0:1619677473.665224:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.665225:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:12.0:1619677473.665226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.665226:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.665226:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677473.665227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677473.665227:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.665231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:12.0:1619677473.665239:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677473.665240:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.665241:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.665242:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:12.0:1619677473.665242:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677473.665243:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.665243:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:17.0:1619677473.665244:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.665244:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.665244:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.665245:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.665246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:1.0:1619677473.665246:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.665248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.665248:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.665249:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677473.665250:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.665250:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.665252:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.665253:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.665254:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.665255:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.665258:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.665260:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.665262:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.665263:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.665264:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.665266:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.665266:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.665267:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.665267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.665272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.665280:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.665283:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.665284:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.665286:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.665287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.665288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677473.665393:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.665396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.665396:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.665398:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.665400:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.665402:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.665403:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.665404:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.665405:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.665406:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.665410:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.665412:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.665415:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.665415:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.665416:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.665418:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.665418:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.665418:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.665419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.665433:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.665435:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.665436:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.665438:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:4.2:1619677473.665439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677473.665440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.665442:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.665447:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.665450:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.665451:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.665453:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.665455:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.665457:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.665458:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.665459:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.665460:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.665461:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.665465:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.665467:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.665470:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.665471:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.665471:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.665473:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.665473:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.665474:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.665474:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.665486:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.665489:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.665490:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.665490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.665493:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.665496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.665581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:8.2:1619677473.665622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.665630:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.665633:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.665634:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.665634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.665636:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.665640:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.665641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.665648:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.665650:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.665651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.665652:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.665653:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.665653:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677473.665731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.665738:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.665740:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.665741:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.665741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.665743:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.665745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.666038:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.666041:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.666042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.666042:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.666045:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.666047:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.666228:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.666230:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.666231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.666233:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.666235:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.666236:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.666237:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.666239:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.666240:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.666241:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.666243:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.666245:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.666247:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.666248:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.666249:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.666250:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.666251:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.666252:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.666252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.666264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.666266:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.666267:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.666269:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.666270:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.666271:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.666610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.666616:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.666619:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.666620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.666622:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.666625:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.666626:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.666627:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.666629:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.666630:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.666631:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.666634:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.666636:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.666638:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.666639:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.666640:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.666642:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.666642:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.666643:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.666644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.666649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.666656:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.666658:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.666659:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.666659:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.666661:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.666662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.666905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.666911:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.666913:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.666916:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.666916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.666918:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.666920:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.666922:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.666923:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.666924:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.666925:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.666928:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.666929:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.666932:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.666933:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.666934:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.666936:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.666936:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.666937:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.666937:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.666942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.666949:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.666951:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.666952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.666953:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.666955:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.666957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.667153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.667161:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.667163:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.667164:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.667166:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.667169:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.667171:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.667172:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.667173:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.667174:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.667175:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.667177:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.667179:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.667182:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.667183:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.667183:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.667185:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.667186:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.667186:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.667186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.667199:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.667202:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.667203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.667203:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.667206:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.667208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.667252:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.667255:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.667256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.667258:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.667259:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.667261:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.667262:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.667263:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.667264:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.667265:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.667269:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.667270:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.667273:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.667274:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.667274:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.667276:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.667276:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.667277:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.667277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.667289:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.667291:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.667292:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.667295:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.667298:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.667300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.667309:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.667311:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.667312:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.667313:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.667314:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.667315:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.667516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.667523:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.667526:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.667527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.667529:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.667532:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.667534:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.667535:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.667537:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.667538:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.667539:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.667542:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.667544:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.667546:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.667547:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.667548:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.667550:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.667550:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.667551:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.667552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.667563:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.667566:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.667567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.667567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.667570:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.667573:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.667583:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.667585:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.667586:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.667586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.667587:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.667588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.667709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.667716:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.667719:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.667720:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.667722:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.667725:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.667726:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.667727:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.667729:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.667730:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.667731:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.667734:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.667735:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.667738:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.667738:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.667739:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.667741:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.667741:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.667742:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.667742:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.667746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.667752:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.667755:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.667756:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.667756:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.667758:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.667760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.668134:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.668137:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.668138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.668140:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.668142:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.668143:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.668144:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.668147:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.668148:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.668149:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.668152:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.668153:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.668156:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.668157:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.668158:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.668159:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.668160:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.668160:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.668161:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.668162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.668172:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.668174:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.668174:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.668174:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.668175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.668176:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.668177:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.668179:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.668179:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.668181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.668347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.668350:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.668351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.668351:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.668353:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.668356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.668821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.668824:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.668826:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.668828:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.668829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.668832:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.668833:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.668835:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.668836:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.668837:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.668838:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.668841:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.668843:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.668846:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.668846:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.668847:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.668849:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.668849:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.668850:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.668851:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.668856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.668864:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.668867:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.668868:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.668868:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.668870:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.668873:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677473.669018:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.669021:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.669021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.669024:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.669026:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.669027:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.669028:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.669030:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.669031:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.669032:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.669035:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.669036:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.669039:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.669039:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.669040:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.669042:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.669042:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.669042:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.669043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.669055:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.669057:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.669058:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.669059:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.669060:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.669061:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.669237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.669243:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.669246:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.669247:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.669249:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.669251:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.669252:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.669254:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.669255:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.669256:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.669257:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.669260:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.669262:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.669264:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.669265:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.669266:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.669267:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.669268:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.669268:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.669269:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.669280:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.669283:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.669284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.669284:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.669286:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.669288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.670069:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.670072:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.670072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.670073:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.670075:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.670078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.670089:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.670092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.670092:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.670094:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.670096:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.670097:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.670098:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.670100:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.670101:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.670102:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.670105:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.670107:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.670109:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.670110:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.670111:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.670112:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.670113:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.670113:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.670114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.670126:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.670129:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.670129:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.670131:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.670132:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.670133:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.670286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.670294:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.670299:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.670302:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.670304:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.670306:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.670308:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.670309:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.670310:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.670311:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.670312:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.670316:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.670317:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.670320:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.670321:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.670321:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.670323:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.670324:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.670324:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.670324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.670337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.670340:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.670341:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.670341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.670343:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.670346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.670851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.670858:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.670861:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.670862:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.670864:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.670866:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.670868:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.670869:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.670871:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.670872:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.670873:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.670876:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.670877:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.670880:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.670881:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.670882:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.670884:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.670884:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.670886:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.670887:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.670887:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.670892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.670898:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.670900:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.670901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.670902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677473.670965:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.670967:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.670968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.670970:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.670972:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.670973:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.670975:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.670976:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.670977:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.670978:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.670981:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.670983:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.670986:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.670986:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.670987:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.670988:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.670989:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.670989:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.670990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.671002:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.671004:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.671005:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.671007:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.671008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.671009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.671029:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.671032:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.671033:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.671034:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.671034:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.671035:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.671172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.671176:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.671178:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.671180:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.671182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.671182:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.671184:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.671185:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.671186:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.671187:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.671188:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.671191:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.671193:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.671195:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.671196:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.671197:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.671198:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.671199:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.671199:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.671200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.671211:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.671214:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.671215:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.671215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.671217:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.671219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.671230:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.671233:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.671234:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.671234:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.671235:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.671235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.671845:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.671848:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.671848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.671850:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.671852:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.671854:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.671855:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.671856:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.671857:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.671858:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.671861:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.671863:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.671865:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.671866:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.671866:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.671868:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.671869:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.671870:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.671871:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.671871:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.671882:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.671884:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.671884:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.671885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.671886:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.671886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.671887:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.671889:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.671890:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.671891:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.671972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.671980:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.671983:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.671984:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.671986:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.671988:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.671990:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.671991:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.671993:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.671994:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.671995:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.671997:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.671999:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.672002:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.672003:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.672004:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.672005:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.672006:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.672006:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.672007:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.672018:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.672021:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.672022:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.672022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.672025:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.672027:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.672134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677473.672178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677473.672180:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.672183:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.672184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.672184:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.672186:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.672186:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.672188:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:6.0:1619677473.672189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677473.672190:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.672190:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.672192:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.672195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677473.672245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677473.672254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.672257:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677473.672258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677473.672259:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.672263:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.672264:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677473.672266:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.672267:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.672268:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.672269:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677473.672272:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.672273:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677473.672276:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.672276:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:24.0:1619677473.672277:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:24.0:1619677473.672278:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:24.0:1619677473.672279:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.672280:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677473.672281:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677473.672282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:4.2:1619677473.672287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000800:00000200:13.0:1619677473.672292:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:8.0:1619677473.672294:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677473.672299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (23)--
00000800:00000200:13.0:1619677473.672299:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.672300:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.672300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:13.0:1619677473.672301:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.672301:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677473.672302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:13.0:1619677473.672304:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:4.2:1619677473.672404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.672412:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.672414:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.672415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.672417:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.672419:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.672421:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.672422:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.672423:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.672424:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.672425:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.672428:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.672429:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.672432:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.672433:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.672433:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.672435:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.672435:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.672436:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.672437:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.672438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.672443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677473.672449:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.672450:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677473.672452:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677473.672452:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677473.672453:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.672453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677473.672454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.672454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677473.672456:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.672458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.672911:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.672914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.672914:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.672916:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.672918:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.672920:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.672921:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.672922:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.672923:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.672924:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.672927:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.672929:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.672932:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.672932:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.672933:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.672935:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.672935:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.672936:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.672936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.672948:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.672950:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.672951:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.672953:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.672954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.672955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.672966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:4.2:1619677473.672970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.672975:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.672977:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.672979:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.672980:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.672981:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.672982:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.672984:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.672985:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.672986:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.672987:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.672989:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.672990:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.672991:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.672992:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.672993:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.672995:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.672995:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.672996:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.672997:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.672997:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 968 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:26.0:1619677473.673000:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.673000:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.673001:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677473.673002:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.673002:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677473.673004:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.673005:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.673005:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.673005:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677473.673005:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677473.673007:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.673007:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.673007:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.673008:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.673008:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:26.0:1619677473.673009:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (23)++
00000800:00000200:26.0:1619677473.673010:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.2:1619677473.673010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677473.673010:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.673011:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.673013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.673017:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:4.2:1619677473.673017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677473.673019:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677473.673020:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:8.0:1619677473.673020:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.673021:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.673021:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.673022:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.673023:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.673024:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677473.673025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.673025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677473.673026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.673026:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:8.0:1619677473.673026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:10.0:1619677473.673027:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.673028:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.673034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.673040:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.673042:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.673043:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.673043:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.673044:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.673044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.673379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.673385:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.673388:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.673389:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.673391:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.673394:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.673395:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.673396:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.673398:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.673399:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.673400:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.673403:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.673404:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.673406:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.673407:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.673407:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.673409:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.673409:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.673410:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.673411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.673416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.673423:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.673425:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.673426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.673426:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.673428:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.673429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.673437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.673444:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.673446:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.673447:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.673447:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.673447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.673448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.673466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.673474:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.673477:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.673477:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.673479:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.673482:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.673484:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677473.673485:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.673486:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.673487:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.673488:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677473.673491:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.673493:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677473.673496:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.673497:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677473.673497:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677473.673499:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677473.673500:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.673500:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.673501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.673506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.673515:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.673518:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.673519:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.673519:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.673521:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.673524:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677473.673743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.673750:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.673753:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.673753:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.673755:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.673758:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.673759:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.673760:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.673761:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.673762:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.673763:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.673766:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.673767:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.673769:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.673770:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.673770:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.673772:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.673772:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.673773:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.673773:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.673785:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.673788:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.673788:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.673790:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.673792:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.673793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677473.673820:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.673823:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.673823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.673826:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.673828:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.673829:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.673830:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.673832:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.673833:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.673834:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.673837:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.673839:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.673841:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.673842:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.673843:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.673844:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.673845:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.673845:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.673846:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.673858:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.673860:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.673861:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.673863:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.673864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.673865:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.674005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.674011:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.674014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.674014:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.674016:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.674019:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.674020:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.674021:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.674023:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.674024:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.674025:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.674029:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.674030:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.674033:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.674034:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.674034:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.674036:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.674036:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.674037:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.674037:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.674049:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.674052:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.674053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.674053:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.674055:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.674057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.674065:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.674068:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.674068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.674069:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.674070:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.674070:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.674149:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.674151:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.674152:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.674152:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.674155:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.674157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.674702:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.674705:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.674705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.674707:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.674710:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.674711:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.674712:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.674713:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.674714:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.674715:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.674718:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.674720:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.674722:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.674723:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.674723:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.674725:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.674725:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.674726:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.674727:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.674739:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.674741:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.674742:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.674744:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.674745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.674746:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.674757:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.674760:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.674761:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.674761:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.674762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.674763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.674812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.674817:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.674819:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.674822:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.674822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.674824:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.674826:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.674827:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.674828:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.674829:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.674830:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.674834:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.674835:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.674838:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.674838:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.674839:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.674841:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.674841:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.674843:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.674844:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.674844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.674848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677473.674849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.674852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.674854:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.674855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.674856:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677473.674856:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.674858:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.674859:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.674860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.674861:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.674863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.674965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.674968:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.674971:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.674973:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.674974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.674975:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.674976:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.674978:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.674979:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.674980:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.674981:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.674984:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.674986:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.674988:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.674989:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.674989:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.674991:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.674991:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.674992:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.674993:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.674994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677473.675000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677473.675004:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.675006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677473.675006:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.675007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.675007:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.675008:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.675009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:6.0:1619677473.675009:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.0:1619677473.675010:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.675011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.675021:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.675024:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.675024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.675025:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.675026:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.675026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.675710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.675718:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.675721:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.675722:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.675724:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.675726:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.675727:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.675729:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.675730:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.675731:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.675732:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.675735:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.675736:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.675739:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.675740:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.675741:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.675742:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.675743:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.675743:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.675744:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.675757:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.675760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.675760:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.675761:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.675763:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.675766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.675836:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.675838:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.675839:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.675839:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.675842:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.675844:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.675920:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.675923:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.675923:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.675926:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.675927:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.675929:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.675930:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.675931:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.675932:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.675933:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.675936:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.675938:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.675941:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.675941:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.675942:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.675944:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.675944:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.675945:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.675946:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.675947:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.675957:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.675959:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.675959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.675959:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.675960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.675961:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.675962:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.675964:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.675965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.675966:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.676288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.676295:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.676300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.676303:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.676303:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.676305:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.676307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.676775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.676783:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.676786:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.676787:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.676789:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.676792:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.676793:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.676794:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.676795:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.676796:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.676796:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677473.676797:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.676799:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.676800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677473.676801:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.676803:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:6.0:1619677473.676804:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.676805:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.676806:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.676807:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677473.676807:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.676808:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.676808:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677473.676809:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677473.676809:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.676810:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677473.676810:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.676811:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677473.676811:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.676812:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677473.676812:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.676815:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.676817:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:4.2:1619677473.676817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677473.676820:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.676821:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.676822:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.676824:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.676824:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.676824:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.676825:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.676825:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:22.0:1619677473.676826:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.676826:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.676826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:10.0:1619677473.676827:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.676827:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677473.676828:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.676828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.676829:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.676831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.676837:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.676840:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.676840:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.676842:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.676843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.676845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.677353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.677360:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.677363:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.677365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.677366:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.677369:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.677370:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.677371:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.677373:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.677374:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.677375:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.677378:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.677380:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.677383:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.677384:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.677384:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.677386:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.677387:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.677388:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.677388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.677401:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.677404:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.677405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.677405:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.677407:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.677410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.677621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.677627:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.677630:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.677631:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.677633:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.677636:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.677637:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.677638:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.677640:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.677641:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.677642:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.677645:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.677647:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.677650:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.677650:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.677651:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.677653:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.677654:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.677654:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.677655:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.677666:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.677669:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.677670:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.677672:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.677674:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:13.0:1619677473.677674:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.677674:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677473.677677:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.677678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.677680:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.677682:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.677683:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.677685:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.677686:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.677687:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.677688:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.677691:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.677693:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.677695:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.677696:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.677697:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.677698:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.677699:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.677700:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.677700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.677712:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.677715:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.677715:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.677717:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.677718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.677719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.677793:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.677795:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.677796:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.677797:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.677799:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.677802:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.677811:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.677814:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.677815:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.677815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.677816:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.677817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.678227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.678235:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.678238:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.678239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.678241:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.678244:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.678246:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677473.678247:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.678248:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.678249:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.678250:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677473.678250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677473.678254:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.678256:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.678257:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.678258:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.678259:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.678260:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677473.678260:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677473.678261:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.678262:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677473.678263:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000400:00000200:10.0:1619677473.678263:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.678264:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.678265:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.678267:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.678268:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.678269:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.678270:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.678271:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.678272:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.678272:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.678273:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.678274:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.678277:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.678277:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:4.2:1619677473.678277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.678278:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.678279:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.678280:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.678281:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.678282:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677473.678283:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.678284:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.678285:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.678285:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.678285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.678291:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.678297:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.678297:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.678299:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.678300:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.678303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677473.678330:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.678333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.678333:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.678336:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.678338:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.678339:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.678341:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.678342:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.678343:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.678344:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.678347:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.678349:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.678351:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.678352:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.678353:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.678354:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.678355:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.678355:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.678356:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.678361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.678370:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.678372:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.678373:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.678374:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.678376:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.678377:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677473.678724:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.678727:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.678727:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.678730:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.678732:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.678733:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.678734:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.678735:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.678737:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.678738:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.678741:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.678742:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.678745:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.678746:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.678746:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.678748:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.678749:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.678749:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.678750:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.678762:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.678765:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.678765:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.678767:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.678768:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.678769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.678781:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.678784:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.678785:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.678786:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.678786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.678787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.678832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.678838:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.678841:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.678842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.678843:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.678846:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.678847:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.678849:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.678850:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.678851:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.678852:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.678855:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.678857:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.678860:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.678861:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.678861:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.678863:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.678863:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.678865:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.678866:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.678866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.678871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.678876:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.678877:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.678879:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.678879:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677473.678880:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.678880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.678880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.678881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.678882:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.678885:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677473.679084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.679092:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.679094:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.679095:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.679097:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.679098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.679101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.679110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.679118:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.679120:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.679121:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.679122:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.679123:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.679123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677473.679206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.679211:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.679213:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.679215:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.679216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.679217:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.679219:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.679220:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.679222:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.679223:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.679224:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.679227:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.679229:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.679232:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.679233:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.679233:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.679235:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.679236:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.679236:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.679237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.679284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.679290:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.679294:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.679297:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.679299:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.679301:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.679304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.679607:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.679610:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.679611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.679613:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.679615:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.679616:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.679617:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.679619:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.679620:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.679621:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.679623:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.679625:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.679627:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.679628:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.679629:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.679630:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.679631:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.679632:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.679633:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.679634:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.679644:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.679645:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677473.679645:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.679646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.679647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.679647:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.679648:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.679649:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.679651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.679652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.679785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.679792:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.679794:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.679795:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.679795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.679797:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.679799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.679818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.679824:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.679826:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.679827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.679827:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.679828:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.679829:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.680477:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.680480:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.680481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.680483:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.680485:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.680486:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.680488:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.680489:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.680490:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.680491:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.680495:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.680496:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.680499:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.680499:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.680500:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.680502:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.680502:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.680503:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.680503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.680517:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.680519:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.680520:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.680522:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.680523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.680524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.680587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.680594:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.680598:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.680599:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.680600:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.680602:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.680604:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.680605:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.680606:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.680607:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.680608:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.680612:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.680613:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.680616:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.680617:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.680617:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.680619:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.680619:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.680620:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.680620:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.680633:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.680635:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.680636:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.680637:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.680639:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.680641:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.680922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677473.680929:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.680931:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.680932:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.680934:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.680936:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.680938:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677473.680939:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.680940:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.680941:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.680942:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677473.680945:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.680946:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677473.680949:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.680950:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677473.680950:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677473.680952:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677473.680953:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.680953:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.680954:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:9.2:1619677473.680958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.680970:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.680974:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.680976:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.680976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.680979:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.680983:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.681010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.681017:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.681020:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.681020:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.681022:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.681025:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.681026:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.681028:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.681029:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.681030:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.681031:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.681034:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.681035:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.681038:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.681039:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.681039:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.681041:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.681041:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.681042:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.681042:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.681047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.681056:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.681060:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.681062:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.681062:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.681065:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.681070:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677473.681181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.681184:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.681187:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.681189:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.681190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.681192:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.681193:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.681195:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.681196:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.681197:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.681198:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.681201:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.681203:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.681205:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.681206:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.681206:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.681208:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.681209:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.681209:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.681210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.681216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.681224:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.681227:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.681228:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.681228:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.681230:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.681232:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.681508:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.681510:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.681511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.681513:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.681515:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.681516:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.681518:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.681519:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.681520:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.681521:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.681524:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.681526:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.681528:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.681529:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.681530:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.681531:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.681532:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.681533:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.681533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.681546:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.681549:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.681549:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.681551:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.681552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.681553:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.681622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.681630:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.681633:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.681634:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.681636:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.681639:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.681641:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.681642:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.681643:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.681644:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.681645:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.681649:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.681650:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.681653:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.681654:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.681654:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.681656:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.681656:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.681657:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.681657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.681670:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.681672:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.681673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.681674:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.681676:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.681678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.681919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.681926:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.681929:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.681930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.681933:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.681936:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.681937:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.681938:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.681940:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.681942:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.681943:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.681947:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.681948:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.681950:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.681951:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:9.2:1619677473.681951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677473.681952:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.681953:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.681954:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.681955:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.681956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677473.681959:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:9.2:1619677473.681959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.681961:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.681962:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.681962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.681964:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.681966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677473.681966:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.681969:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.681969:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.681970:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.681971:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.681973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677473.681998:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.682001:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.682001:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677473.682003:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.682005:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.682007:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.682008:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.682009:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.682010:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.682012:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.682015:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.682017:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.682019:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.682020:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.682021:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.682022:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.682023:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.682024:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.682025:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.682025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.682029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.682036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677473.682037:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677473.682037:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.682038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677473.682039:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677473.682039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677473.682040:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.682042:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.682043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.682044:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677473.682363:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.682366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.682366:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.682368:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.682370:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.682372:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.682373:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.682374:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.682375:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.682377:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.682380:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.682382:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.682384:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.682385:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.682385:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.682387:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.682387:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.682388:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.682388:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.682402:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.682404:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.682405:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.682406:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.682407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.682408:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.682422:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.682424:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.682425:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.682426:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.682426:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.682427:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.682524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.682531:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.682534:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.682534:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.682537:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.682539:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.682541:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.682542:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.682544:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.682545:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.682546:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.682549:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.682551:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.682554:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.682555:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.682555:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.682557:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.682558:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.682559:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.682559:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.682570:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.682573:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.682574:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.682574:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.682577:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.682580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.2:1619677473.683000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677473.683003:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677473.683005:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.683006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677473.683008:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.54@o2ib7 : PUT - routed
00000800:00000200:24.0:1619677473.683010:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677473.683022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.683029:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.683031:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.683032:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.683034:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.683037:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.683038:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677473.683040:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.683041:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.683042:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.683043:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677473.683046:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.683047:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.683051:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.683052:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677473.683052:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.683054:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.683054:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.683055:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.683056:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.683061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.683069:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.683071:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.683072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.683073:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.683075:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.683078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.683079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.683086:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.683088:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.683089:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.683089:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.683090:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.683091:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677473.683153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.683159:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.683161:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.683162:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.683162:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.683164:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.683166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.683213:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.683215:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.683216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.683218:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.683220:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.683222:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.683223:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.683224:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.683225:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.683227:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.683230:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.683232:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.683234:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.683235:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.683235:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.683237:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.683237:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.683239:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.683240:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.683240:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.683250:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.683250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.683252:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.683253:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.683253:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.683253:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.683254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.683255:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.683257:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.683258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.683419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.683426:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.683429:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.683430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.683432:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.683434:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.683435:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.683437:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.683438:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.683439:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.683440:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.683443:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.683444:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.683447:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.683448:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.683448:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.683450:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.683450:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.683451:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.683452:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.683457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.683465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.683467:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.683468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.683468:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.683470:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.683473:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677473.683544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.683552:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.683555:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.683556:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.683558:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.683561:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.683562:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.683564:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.683565:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.683566:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.683567:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.683570:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.683571:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.683574:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.683574:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.683575:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.683576:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.683577:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.683578:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.683579:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.683580:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.683583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.683585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677473.683589:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.683591:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677473.683592:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677473.683592:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.683592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.683594:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:24.0:1619677473.683594:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.683595:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.683596:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677473.683596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.683599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:9.2:1619677473.683602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.683607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.683608:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.683609:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.683609:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.683610:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.683610:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.683611:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.683611:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.683612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.683613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.683613:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.683614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.684057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.684062:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.684065:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.684066:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.684067:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.684070:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.684071:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.684072:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.684073:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.684074:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.684075:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.684078:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.684080:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.684082:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.684083:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.684084:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.684086:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.684086:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.684087:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.684087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.684100:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.684102:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.684103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.684103:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.684105:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.684108:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.684118:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.684120:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.684121:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.684121:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.684122:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.684123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.684231:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.684233:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.684234:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.684235:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.684237:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.684238:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.684239:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.684241:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.684242:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.684243:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.684245:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.684247:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.684249:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.684250:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.684251:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.684253:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.684253:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.684253:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.684254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.684266:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.684268:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.684269:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.684271:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.684272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.684273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.684423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677473.684427:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.684429:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.684431:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.684432:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.684434:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.684436:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677473.684437:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.684438:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.684439:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.684440:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.684443:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.684445:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.684448:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.684449:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677473.684450:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677473.684452:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677473.684452:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.684453:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.684454:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.2:1619677473.685119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:22.0:1619677473.685120:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.685123:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.685124:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:2.0:1619677473.685125:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:22.0:1619677473.685126:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.685128:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.685128:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:22.0:1619677473.685129:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:18.0:1619677473.685129:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:22.0:1619677473.685131:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.685131:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.685133:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.685134:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.685134:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.685135:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677473.685136:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677473.685137:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.685138:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677473.685139:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.685140:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.685141:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:2.0:1619677473.685141:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677473.685144:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.685145:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:2.0:1619677473.685145:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.685146:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.685147:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.685148:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677473.685148:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677473.685149:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.685149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:2.0:1619677473.685151:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.685152:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677473.685152:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677473.685154:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677473.685155:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.685156:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677473.685157:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677473.685158:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677473.685160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.0:1619677473.685161:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.685163:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.2:1619677473.685163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:21.0:1619677473.685164:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.685165:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.685167:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:21.0:1619677473.685168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.685169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677473.685169:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:10.0:1619677473.685170:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.685171:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677473.685171:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:10.0:1619677473.685171:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.685172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.685173:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:10.0:1619677473.685174:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.685177:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677473.685438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.685446:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.685449:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.685450:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.685451:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.685453:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.685455:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.685456:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.685457:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.685458:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.685459:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.685462:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.685463:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.685466:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.685467:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.685468:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.685469:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.685470:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.685471:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.685471:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.685484:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.685487:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.685488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.685488:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.685490:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.685494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.685548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.685556:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.685559:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.685560:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.685560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.685562:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.685564:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.685899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.685905:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.685907:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.685908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.685908:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.685910:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.685912:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.685924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.685931:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.685934:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.685935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.685936:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.685939:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.685940:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.685942:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.685943:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.685944:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.685944:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.685947:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.685949:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.685952:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.685953:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.685953:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.685955:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.685955:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.685956:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.685957:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.685969:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.685971:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.685972:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.685972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.685975:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.685978:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.686015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.686018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.686018:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.686020:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.686022:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.686023:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.686024:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.686026:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.686027:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.686028:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.686031:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.686033:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.686035:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.686036:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.686036:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.686038:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.686039:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.686039:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.686039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.686052:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.686054:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.686055:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.686056:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.686057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.686058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.686069:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.686072:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.686073:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.686073:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.686074:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.686075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.686752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.686759:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.686762:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.686763:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.686765:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.686768:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.686769:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.686771:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.686772:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.686773:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.686774:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.686778:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.686779:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.686782:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.686783:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.686784:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.686786:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.686786:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.686787:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.686789:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.686789:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677473.686795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677473.686800:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677473.686802:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:13.0:1619677473.686803:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.686804:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.686804:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:13.0:1619677473.686805:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.686805:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.686806:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.686807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.686809:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677473.687036:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.687039:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.687040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.687042:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.687044:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.687045:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.687046:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677473.687046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677473.687048:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.687048:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677473.687049:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.687050:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.687051:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:22.0:1619677473.687054:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.687054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677473.687056:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:20.0:1619677473.687057:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.687058:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.687059:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:20.0:1619677473.687059:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.687060:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.687061:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.687061:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:22.0:1619677473.687062:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:20.0:1619677473.687062:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.687063:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:20.0:1619677473.687063:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.687064:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677473.687064:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.687065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.687065:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.687068:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.687070:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.687072:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.687073:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.687074:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.687075:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.687076:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677473.687076:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.687077:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.687077:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.687078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677473.687078:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.687079:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.687079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677473.687080:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.687080:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.687080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.687081:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.687082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.687083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.687091:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.687093:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.687094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.687094:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.687097:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.687099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677473.687308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.687315:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.687317:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.687318:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.687318:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.687320:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.687322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.687327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.687334:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.687337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.687337:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.687339:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.687341:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.687343:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.687344:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.687345:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.687346:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.687347:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.687351:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.687352:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.687355:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.687356:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.687356:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.687358:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.687359:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.687359:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.687360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.687372:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.687375:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.687376:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.687376:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.687378:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.687382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.687392:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.687394:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.687395:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.687395:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.687396:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.687397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.687442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.687448:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.687450:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.687451:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.687451:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.687453:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.687455:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.687463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.687469:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.687471:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.687472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.687472:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.687473:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.687474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.687804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:24.0:1619677473.687812:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677473.687815:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.3@o2ib3
00000800:00000200:8.0:1619677473.687815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:24.0:1619677473.687817:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.687820:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.687821:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677473.687822:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.687824:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.687825:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.687825:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.687828:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.687830:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.687832:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.687833:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677473.687834:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677473.687835:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677473.687836:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677473.687837:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000800:00000200:24.0:1619677473.687837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:22.0:1619677473.687900:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.687903:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.687903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.687905:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.687908:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.687909:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.687910:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.687912:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.687913:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.687914:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.687917:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.687919:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.687922:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.687922:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.687923:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.687925:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.687925:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.687926:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.687927:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.687938:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.687940:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.687941:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.687943:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.687944:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.687946:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.688403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.688410:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.688413:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.688413:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.688416:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.688418:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.688420:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.688421:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.688423:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.688424:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.688425:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.688428:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.688430:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.688433:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.688434:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.688434:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.688436:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.688437:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.688438:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.688438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:6.0:1619677473.688735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.688737:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.688738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.688740:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.688742:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.688744:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.688745:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.688746:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.688747:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.688748:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.688752:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.688753:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.688756:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.688757:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.688758:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.688759:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.688760:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.688761:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.688761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.688773:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.688775:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.688776:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.688778:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.688779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.688780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.688811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677473.688817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677473.688819:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.688821:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.688822:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677473.688823:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677473.688824:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.688826:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:26.0:1619677473.688827:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.688827:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677473.688828:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677473.688829:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.688830:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.688831:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.688832:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.688832:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.688833:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677473.688834:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677473.688835:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.688836:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.688838:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677473.688838:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677473.688839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677473.688839:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677473.688841:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:2.0:1619677473.688842:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.688843:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.688844:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677473.688844:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677473.688845:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677473.688845:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677473.688846:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677473.688846:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.688847:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.688847:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677473.688847:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677473.688848:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677473.688848:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677473.688848:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.688849:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677473.688849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677473.688850:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677473.688851:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.688851:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:9.2:1619677473.688852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677473.688852:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:17.0:1619677473.688853:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.688853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677473.688854:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.688856:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.688856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:17.0:1619677473.688857:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677473.688858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000400:00000200:17.0:1619677473.688858:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.688859:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677473.688859:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.688861:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:17.0:1619677473.688862:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.688862:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.688863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677473.688864:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.688865:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:17.0:1619677473.688865:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677473.688865:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.688865:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677473.688866:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.688866:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:17.0:1619677473.688867:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.688867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.688868:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.688868:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:8.0:1619677473.688868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677473.688868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677473.688869:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677473.688869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677473.688870:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:18.0:1619677473.688871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677473.688871:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.688872:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.688872:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.688885:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.688888:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.688889:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.688889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.688891:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.688893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.689536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.689543:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.689546:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.689547:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.689547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.689549:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.689550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.689703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c78ba0400] (20)++
00000800:00000200:8.0:1619677473.689710:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba0400] (21)++
00000800:00000200:8.0:1619677473.689713:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.20@o2ib3
00000800:00000200:24.0:1619677473.689714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba0400] (22)--
00000400:00000200:8.0:1619677473.689716:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.1.20@o2ib3 : GET - for me
00000400:00000200:8.0:1619677473.689720:0:19938:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.1.20@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:8.0:1619677473.689722:0:19938:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.1.20@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:8.0:1619677473.689723:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba0400] (21)++
00000800:00000200:8.0:1619677473.689725:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c78ba0400] (22)++
00000800:00000200:8.0:1619677473.689726:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba0400] (23)--
00000400:00000200:8.0:1619677473.689728:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.1.20@o2ib3 local destination
00000400:00000200:8.0:1619677473.689731:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.1.20@o2ib3(10.51.1.20@o2ib3:10.51.1.20@o2ib3) : REPLY try# 0
00000800:00000200:4.2:1619677473.689732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c78ba0400] (22)++
00000800:00000200:8.0:1619677473.689733:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.1.20@o2ib3
00000800:00000200:8.0:1619677473.689735:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919fedea5280] -> 10.51.1.20@o2ib3 (2) version: 12
00000800:00000200:8.0:1619677473.689736:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919c78ba0400] (23)++
00000800:00000200:8.0:1619677473.689737:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c78ba0400] (24)++
00000800:00000200:8.0:1619677473.689739:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919c78ba0400] (25)--
00000800:00000200:8.0:1619677473.689739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba0400] (24)--
00000800:00000200:8.0:1619677473.689740:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba0400] (23)++
00000800:00000200:8.0:1619677473.689741:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c78ba0400] (24)--
00000800:00000200:8.0:1619677473.689742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba0400] (23)--
00000800:00000200:8.0:1619677473.689743:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba0400] (22)++
00000800:00000200:8.0:1619677473.689745:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c78ba0400] (23)--
00000400:00000200:8.0:1619677473.689745:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.689747:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba0400] (22)--
00000400:00000200:8.0:1619677473.689747:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.1.20@o2ib3: REPLY: OK
00000800:00000200:8.0:1619677473.689748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba0400] (21)--
00000800:00000200:6.0:1619677473.689781:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.689784:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.689785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.689787:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.689789:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.689790:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.689792:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.689793:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.689794:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.689795:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.689798:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.689800:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.689802:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.689803:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.689804:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.689805:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.689806:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.689807:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.689807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.689819:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.689822:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.689822:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.689824:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.689825:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.689826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.689838:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.689841:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.689842:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.689842:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.689843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.689844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.689979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.689986:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.689990:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.689990:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.689993:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.689995:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.689996:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.689998:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.689999:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.690000:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.690001:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.690005:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.690006:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.690010:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.690010:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.690011:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.690013:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.690013:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.690013:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.690014:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.690019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.690028:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.690031:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.690031:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.690032:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.690034:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.690037:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.690247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.690254:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.690257:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.690258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.690260:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.690262:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.690264:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.690265:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.690266:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.690267:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.690268:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.690271:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.690273:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.690276:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.690276:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.690277:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.690279:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.690279:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.690279:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.690280:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.690293:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:9.2:1619677473.690298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.690299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.690303:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.690304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.690305:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:22.0:1619677473.690306:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.690308:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.690309:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.690309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.690309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.690311:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.690314:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.690315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.690321:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.690323:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.690324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.690324:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.690325:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.690326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677473.690562:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.690565:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677473.690566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677473.690568:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.690570:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.690571:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677473.690573:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.690574:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.690575:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.690576:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677473.690579:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.690581:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677473.690583:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.690584:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677473.690585:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677473.690586:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677473.690587:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.690587:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677473.690588:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677473.690593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.690600:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.690603:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.690604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.690604:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.690606:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.690609:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.690612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.690619:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.690621:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.690622:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.690623:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.690624:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.690624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677473.690633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.690637:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.690639:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677473.690641:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.690642:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.690644:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.690645:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.690647:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.690648:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.690649:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.690650:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.690652:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.690654:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.690656:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.690657:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.690658:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.690658:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677473.690659:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.690660:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:6.0:1619677473.690660:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.690661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.690661:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.690661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:6.0:1619677473.690663:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.690665:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.690666:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.690668:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.690670:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.690671:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.690672:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.690674:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.690676:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.690676:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.690676:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.690678:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.690678:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.690681:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.690681:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.690682:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.690682:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.690683:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.690686:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.690686:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.690688:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.690689:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.690689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.690700:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.690701:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.690702:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.690702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.690703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.690704:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.690704:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.690706:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.690707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.690709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.690770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.690778:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.690780:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.690781:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.690783:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.690786:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.690787:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.690789:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.690790:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.690791:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.690792:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.690795:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.690797:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.690799:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.690800:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.690801:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.690803:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.690803:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.690804:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.690804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.690816:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.690819:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.690820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.690821:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.690823:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.690826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.690958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.690965:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.690968:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.690969:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.690970:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.690973:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.690975:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677473.690976:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.690977:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.690978:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.690979:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677473.690982:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.690984:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.690986:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.690987:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677473.690987:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.690989:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677473.690990:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.690990:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.690991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.690996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.691003:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.691006:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.691007:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.691007:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.691010:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.691013:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677473.691325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.691332:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.691334:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.691335:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.691335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.691337:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.691339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.691536:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.691539:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.691539:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.691541:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.691544:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.691545:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.691546:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.691547:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.691548:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.691549:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.691553:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.691554:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.691557:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.691558:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.691558:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.691560:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.691561:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.691562:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.691562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.691575:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.691577:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.691578:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.691579:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.691581:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.691582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677473.692214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.692217:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.692218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.692220:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.692222:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.692224:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677473.692225:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.692226:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.692227:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.692228:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677473.692232:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.692233:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677473.692236:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.692237:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677473.692238:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677473.692240:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.692240:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.692241:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.692242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.692246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.692253:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.692256:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.692257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.692257:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.692260:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.692262:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677473.692291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.692299:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.692302:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.692303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.692304:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.692306:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.692308:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.692309:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.692311:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.692312:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.692313:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.692316:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.692317:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.692320:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.692321:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.692322:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.692323:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.692324:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.692325:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.692326:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.692327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.692333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.692339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.692340:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.692341:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677473.692342:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677473.692342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.692343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677473.692343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.692344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677473.692345:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.692348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.692585:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.692587:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.692588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.692590:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.692592:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.692593:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.692594:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.692596:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.692596:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.692597:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.692600:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.692602:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.692604:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.692605:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.692606:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.692607:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.692608:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.692609:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.692609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.692622:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.692624:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.692625:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.692626:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.692627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.692629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.692634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.692642:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.692644:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.692645:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.692647:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.692650:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.692651:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.692653:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.692654:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.692655:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.692656:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.692659:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.692660:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.692664:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.692665:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.692665:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.692667:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.692668:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.692668:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.692669:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.692674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.692682:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.692684:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.692685:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.692687:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.692688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.692690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.693300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.693306:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.693308:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.693310:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.693311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.693312:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.693314:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.693315:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.693316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:21.0:1619677473.693321:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.693322:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.693323:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:21.0:1619677473.693324:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677473.693325:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.693326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.693326:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.693327:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.693329:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.693330:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677473.693331:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677473.693332:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.693333:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.693334:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.693336:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.693336:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.693337:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.693338:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.693342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.693350:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.693353:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.693354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.693354:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.693356:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.693358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.693359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.693367:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.693369:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.693370:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.693370:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.693371:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.693372:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.693425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677473.693432:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.693435:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.693436:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.693438:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.693441:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.693442:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677473.693443:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.693444:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.693445:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.693446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677473.693446:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.693448:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.693449:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.693450:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:6.0:1619677473.693451:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.693453:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.693454:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.693454:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.693456:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.693457:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:6.0:1619677473.693457:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.693459:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.693460:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.693460:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677473.693461:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000400:00000200:6.0:1619677473.693462:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.693463:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677473.693463:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:6.0:1619677473.693464:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.693465:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.693466:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.693467:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:6.0:1619677473.693467:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.693467:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.2:1619677473.693468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:6.0:1619677473.693468:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.693470:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.693470:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677473.693471:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:6.0:1619677473.693471:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.693472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:4.2:1619677473.693472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:24.0:1619677473.693474:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677473.693475:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.693475:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677473.693478:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.693479:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:24.0:1619677473.693480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:26.0:1619677473.693481:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.693482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.693483:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:20.0:1619677473.693483:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.693485:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.693486:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.693488:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.693489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.693490:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.693506:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.693508:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.693509:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.693509:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.693510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.693511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.693549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.693556:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.693559:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.693560:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.693562:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.693564:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.693566:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.693567:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.693568:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.693569:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.693570:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.693574:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.693575:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.693578:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.693578:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.693579:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.693580:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.693581:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.693582:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.693583:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.693583:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.693586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.693589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677473.693592:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:5.2:1619677473.693593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677473.693594:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.693595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.693595:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.693597:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.693597:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.693598:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677473.693599:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.693599:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.693599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.693601:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677473.693601:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.693602:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.693604:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.693606:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.693608:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.693609:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.693610:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.693611:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.693613:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.693615:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.693617:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.693620:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.693620:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.693621:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.693623:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.693623:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.693624:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.693625:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.693637:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.693639:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.693640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.693641:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.693643:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.693645:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.693657:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.693660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.693660:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.693661:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.693662:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.693662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.693786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.693789:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.693792:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.693794:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.693795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.693796:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.693798:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.693799:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.693801:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.693802:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.693803:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.693806:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.693808:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.693810:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.693811:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.693812:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.693813:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.693814:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.693815:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.693815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.693826:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.693829:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.693830:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.693830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.693832:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.693834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.693845:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.693847:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.693848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.693849:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.693850:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.693850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677473.693930:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.693933:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.693934:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.693936:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.693938:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.693940:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.693941:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.693942:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.693943:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.693944:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.693947:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.693949:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.693951:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.693952:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.693953:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.693954:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.693955:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.693956:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.693956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.693960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.693968:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.693971:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.693971:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.693974:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.693974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.693980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.693987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.2:1619677473.693993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.693997:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.693999:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.694000:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.694001:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.694001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.694002:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.694003:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677473.694004:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.694004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677473.694005:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.694006:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.694009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.694305:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.694308:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.694308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.694310:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.694313:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.694314:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.694315:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.694316:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.694318:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.694319:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.694322:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.694324:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.694327:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.694327:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.694328:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.694330:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.694330:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.694332:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.694333:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.694333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.694344:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.694344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.694346:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.694346:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.694347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.694347:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.694347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.694349:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.694349:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.694351:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.695080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.695089:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.695091:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.695092:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.695094:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.695096:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.695098:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.695099:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.695100:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.695102:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.695103:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.695106:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.695107:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.695111:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.695111:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.695112:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.695113:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.695114:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.695115:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.695115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.695115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.695122:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.695125:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.695125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.695126:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.695127:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.695127:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.695129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.695130:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.695131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.695131:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.695134:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.695137:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.695316:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.695319:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.695320:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.695322:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.695324:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.695325:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.695326:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.695327:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.695328:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.695329:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.695333:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.695335:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.695337:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.695338:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.695339:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.695340:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.695341:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.695342:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.695342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.695356:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.695358:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.695359:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.695360:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.695361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.695362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.695440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.695446:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.695449:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.695450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.695452:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.695454:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.695456:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.695457:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.695458:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.695459:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.695460:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.695463:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.695465:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.695468:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.695469:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.695470:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.695471:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.695472:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.695473:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.695474:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.695475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.695480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.695486:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.695488:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.695489:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.695489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.695512:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.695515:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.695516:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.695517:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.695519:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.695520:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.695535:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.695537:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.695538:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.695539:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.695540:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.695541:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.695590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677473.695593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677473.695597:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.695599:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:26.0:1619677473.695600:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677473.695600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.695600:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.695602:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.695603:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.695604:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.695605:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.695605:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677473.695608:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.695610:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:9.2:1619677473.695610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:26.0:1619677473.695611:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.695612:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.695613:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.695614:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677473.695616:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:26.0:1619677473.695617:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.695618:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677473.695619:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.695620:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.695620:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677473.695621:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.695621:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677473.695622:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:24.0:1619677473.695622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.695623:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.695624:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.695625:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.695626:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.695627:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.695628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.695633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677473.695638:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.695640:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.695641:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.695641:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.695642:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.695643:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.695644:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677473.695644:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677473.695644:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.695648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.696226:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.696229:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.696230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.696232:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.696234:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.696235:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.696236:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.696237:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.696238:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.696239:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.696242:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.696243:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.696246:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.696247:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.696247:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.696249:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.696250:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.696251:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.696251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.696262:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.696265:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.696265:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.696267:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.696268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.696269:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.696829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.696836:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.696838:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.696839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.696841:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.696844:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.696845:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.696847:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.696848:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.696849:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.696850:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.696853:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.696855:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.696857:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.696857:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.696858:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.696859:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.696860:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.696861:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.696861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.696866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.696873:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.696875:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.696876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.696876:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.696877:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.696879:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.696890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.696897:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.696900:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.696900:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.696901:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.696901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.696901:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677473.697107:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.697110:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.697111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.697113:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.697115:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.697116:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.697118:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.697119:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.697120:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.697121:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.697124:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.697125:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.697128:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.697129:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.697129:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.697131:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.697131:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.697132:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.697133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.697145:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.697147:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.697148:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.697150:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.697150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.697152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.697165:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.697168:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.697169:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.697169:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.697170:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.697171:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.697392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.697395:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.697397:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.697400:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.697401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.697402:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.697403:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.697405:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.697406:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.697407:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.697408:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.697411:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.697413:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.697415:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.697416:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.697417:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.697419:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.697419:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.697420:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.697420:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.697434:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.697436:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.697437:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.697437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.697439:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.697441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.697562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677473.697568:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.697571:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677473.697572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677473.697574:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.697577:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.697578:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677473.697579:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.697581:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.697582:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.697583:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677473.697586:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.697588:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677473.697591:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.697592:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677473.697593:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677473.697595:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677473.697595:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.697596:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677473.697596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677473.697601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.697608:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.697611:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.697612:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.697612:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.697615:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.697618:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.697628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.697635:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.697638:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.697639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.697639:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.697641:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.697643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.698154:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.698157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.698157:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.698160:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.698162:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.698163:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.698164:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.698166:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.698167:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.698168:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.698171:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.698172:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.698175:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.698176:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.698176:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.698178:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.698179:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.698180:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.698181:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.698181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.698192:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.698194:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.698194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.698194:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.698195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.698196:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.698197:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.698198:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.698199:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.698201:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.698269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.698276:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.698279:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.698280:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.698282:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.698284:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.698286:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.698287:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.698288:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.698289:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.698290:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.698298:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.698300:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.698303:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.698304:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.698305:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.698306:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.698307:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.698308:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.698308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.698321:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.698324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.698324:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.698325:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.698327:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.698330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.698483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.698490:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.698493:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.698494:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.698496:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.698498:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.698500:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.698501:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.698503:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.698504:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.698505:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.698508:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.698509:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.698512:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.698513:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.698513:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.698515:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.698516:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.698516:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.698517:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.698532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.698540:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.698543:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.698544:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.698545:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.698547:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.698550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677473.699017:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.699020:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.699021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.699023:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.699025:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.699027:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.699028:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.699029:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.699030:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.699031:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.699034:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.699036:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.699039:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.699039:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.699040:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.699042:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.699042:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.699043:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.699044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.699055:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.699058:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.699058:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.699060:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.699061:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.699063:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.699344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.699352:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.699354:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.699355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.699355:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.699357:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.699359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.699739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.699747:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.699749:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.699750:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.699752:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.699755:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.699756:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.699757:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.699758:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.699759:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.699760:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.699763:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.699765:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.699767:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.699768:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.699769:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.699770:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.699771:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.699772:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.699772:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.699785:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.699787:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.699788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.699789:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.699791:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.699794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.699804:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.699807:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.699807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.699808:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.699809:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.699809:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.699880:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.699882:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.699883:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.699885:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.699887:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.699888:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.699890:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.699891:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.699892:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.699893:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.699896:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.699898:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.699900:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.699901:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.699902:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.699903:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.699904:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.699905:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.699905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.699917:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.699919:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.699920:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.699921:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.699922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.699924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677473.700466:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677473.700469:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.4.65@o2ib3
00000800:00000200:1.0:1619677473.700470:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:17.0:1619677473.700473:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.700475:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.700477:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677473.700478:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.700480:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.700481:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.700482:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677473.700485:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.700487:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677473.700489:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.700490:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677473.700491:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677473.700493:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677473.700493:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677473.700494:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:17.0:1619677473.700494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:20.0:1619677473.700508:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.700510:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.700511:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.700512:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.700515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.700516:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.700519:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677473.700522:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:17.0:1619677473.700523:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:1.0:1619677473.700524:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.700527:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.700528:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:1.0:1619677473.700528:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677473.700529:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.700530:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.700531:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000400:00000200:1.0:1619677473.700531:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.700532:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.700532:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:1.0:1619677473.700532:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.700533:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677473.700534:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677473.700536:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.700537:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677473.700539:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.700540:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677473.700540:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677473.700542:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677473.700543:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677473.700543:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:1.0:1619677473.700544:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:9.2:1619677473.700548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.700556:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.700559:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.700560:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.700560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.700562:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.700565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677473.700598:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677473.700601:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:1.0:1619677473.700601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:17.0:1619677473.700603:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.700605:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.700607:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677473.700608:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.700610:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.700611:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.700611:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677473.700615:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.700617:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677473.700620:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.700620:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677473.700621:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677473.700623:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677473.700623:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677473.700624:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:17.0:1619677473.700625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:9.2:1619677473.700628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.700636:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.700638:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.700639:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.700639:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.700641:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.700644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.700646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.700648:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.700650:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.700650:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.700650:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.700652:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.700654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.700699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677473.700706:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:26.0:1619677473.700707:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677473.700708:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:17.0:1619677473.700709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:26.0:1619677473.700710:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.700710:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677473.700712:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.700713:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.700714:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.700716:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.700716:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677473.700718:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677473.700718:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.700720:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.700720:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.700721:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.700722:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.700723:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.700723:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677473.700725:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677473.700727:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.700730:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.700730:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.700732:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.700733:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677473.700733:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)++
00000800:00000200:1.0:1619677473.700734:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.700735:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:1.0:1619677473.700735:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677473.700736:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.700737:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677473.700737:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677473.700738:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677473.700738:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.700738:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (23)++
00000800:00000200:26.0:1619677473.700739:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.700740:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.700740:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677473.700740:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.700740:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb3800] (24)++
00000800:00000200:26.0:1619677473.700741:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677473.700741:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (25)--
00000400:00000200:17.0:1619677473.700742:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677473.700742:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (24)--
00000800:00000200:8.2:1619677473.700743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:17.0:1619677473.700744:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.700744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677473.700744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (23)--
00000800:00000200:1.0:1619677473.700745:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:17.0:1619677473.700746:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677473.700746:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.700747:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.700747:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.700748:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677473.700749:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.700751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677473.700751:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.700752:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.700752:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677473.700753:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677473.700753:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (23)++
00000800:00000200:24.0:1619677473.700754:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677473.700754:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.700754:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.700754:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb3800] (24)--
00000800:00000200:1.0:1619677473.700755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (23)--
00000400:00000200:8.0:1619677473.700756:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.700756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:17.0:1619677473.700757:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.700758:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:17.0:1619677473.700759:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.700759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677473.700761:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677473.700761:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677473.700762:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:17.0:1619677473.700763:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:9.2:1619677473.700764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.2:1619677473.700765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:9.2:1619677473.700767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.700771:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.700774:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677473.700774:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.700775:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.700775:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677473.700775:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.700776:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677473.700776:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.700777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677473.700777:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.700778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.700778:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:1.0:1619677473.700778:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677473.700779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.700779:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.700779:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:2.0:1619677473.700781:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.700783:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677473.700785:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677473.700788:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:1.0:1619677473.700788:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:17.0:1619677473.700790:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.700792:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.700794:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677473.700795:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.700796:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.700797:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.700798:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.700801:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.700802:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.700804:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.700805:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.700806:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.700807:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.700808:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:17.0:1619677473.700809:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:17.0:1619677473.700809:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:1.0:1619677473.700819:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677473.700821:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:22.0:1619677473.700822:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677473.700823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:1.0:1619677473.700823:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.700824:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.700825:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.700826:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.700826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:1.0:1619677473.700826:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677473.700827:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.700829:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.700829:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.700830:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.700831:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677473.700833:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.700835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.700835:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.700836:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.700837:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.700837:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.700839:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.700839:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677473.700840:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:1.0:1619677473.700841:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:8.2:1619677473.700845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.700852:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.700854:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.700855:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.700856:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.700857:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.700859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677473.700868:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677473.700870:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:17.0:1619677473.700871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:1.0:1619677473.700872:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.700873:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.700874:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677473.700875:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.700877:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.700877:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.700878:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677473.700881:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.700882:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677473.700884:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.700884:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:1.0:1619677473.700885:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677473.700886:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677473.700887:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677473.700887:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb3800] (22)++
00000800:00000200:1.0:1619677473.700888:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (23)--
00000800:00000200:1.0:1619677473.700889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:9.2:1619677473.700892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677473.700899:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)++
00000800:00000200:18.0:1619677473.700900:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:17.0:1619677473.700901:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb3800] (23)--
00000800:00000200:17.0:1619677473.700901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:17.0:1619677473.700902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:18.0:1619677473.700903:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.700904:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.700904:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677473.700904:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.700906:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.700907:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.700908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677473.700908:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:6.0:1619677473.700910:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.700912:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.700913:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.700914:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.700916:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.700917:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.700918:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.700920:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.700922:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.700925:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.700925:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.700926:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.700928:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.700928:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.700929:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.700930:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:1.0:1619677473.700933:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677473.700936:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:17.0:1619677473.700937:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:1.0:1619677473.700939:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.700941:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:1.0:1619677473.700941:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.700942:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:21.0:1619677473.700943:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.700944:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.700944:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.700945:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.700946:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.700946:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.700947:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:1.0:1619677473.700947:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.700948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677473.700951:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.700953:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677473.700955:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.700956:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677473.700957:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677473.700959:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:1.0:1619677473.700959:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:1.0:1619677473.700960:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:1.0:1619677473.700960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000800:00000200:20.2:1619677473.700977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677473.700983:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.700985:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.700986:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.700988:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.700991:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.700992:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677473.700994:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.700995:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.700996:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.700997:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677473.701000:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.701002:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677473.701005:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:2.0:1619677473.701006:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.701006:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677473.701007:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.65@o2ib3
00000800:00000200:2.0:1619677473.701007:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677473.701008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (22)--
00000800:00000200:2.0:1619677473.701009:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:17.0:1619677473.701010:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.65@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.701010:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.701010:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.701011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:17.0:1619677473.701013:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.701014:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677473.701015:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.701015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677473.701017:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.701018:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.701018:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677473.701021:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.701021:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677473.701023:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677473.701023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:24.0:1619677473.701024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.701024:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.701025:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.701026:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677473.701027:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.701028:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:8.0:1619677473.701028:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.701029:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:17.0:1619677473.701029:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:20.0:1619677473.701031:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677473.701031:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677473.701031:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb3800] (21)++
00000800:00000200:8.0:1619677473.701031:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677473.701032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677473.701032:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb3800] (22)--
00000400:00000200:20.0:1619677473.701033:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.701033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb3800] (21)--
00000400:00000200:20.0:1619677473.701035:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677473.701036:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.701037:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.701038:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.701039:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677473.701042:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.701044:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677473.701046:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.701046:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:20.0:1619677473.701047:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677473.701048:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677473.701049:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677473.701050:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:12.0:1619677473.701050:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:20.0:1619677473.701051:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.701051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:12.0:1619677473.701051:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.701054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677473.701054:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.701055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677473.701057:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:18.0:1619677473.701063:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.701066:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.701067:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.701067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.701070:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.701072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.701180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.701185:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.701188:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.701189:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.701191:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.701193:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.701195:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.701196:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.701197:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.701198:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.701199:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.701202:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.701204:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.701207:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.701208:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.701209:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.701211:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.701211:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.701212:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.701212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677473.701621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.701628:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.701631:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.701631:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.701632:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.701633:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.701635:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.701637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.701643:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.701645:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.701646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.701646:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.701647:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.701649:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.701656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.701661:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.701664:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.701665:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.701665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.701666:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.701666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.701771:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.701774:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.701774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.701776:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.701779:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.701780:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.701781:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.701783:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.701784:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.701785:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.701788:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.701789:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.701792:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.701792:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.701793:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.701795:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.701795:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.701797:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.701798:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.701798:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.701808:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.701810:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.701810:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.701811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.701812:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.701812:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.701813:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.701815:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.701816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.701817:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.702551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.702559:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.702562:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.702563:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.702564:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.702566:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.702568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.702682:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.702684:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.702685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.702687:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.702690:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.702691:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.702692:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.702694:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.702694:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.702695:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.702698:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.702700:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.702703:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.702703:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.702704:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.702706:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.702706:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.702707:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.702708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.702719:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.702722:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.702722:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.702724:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.702725:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.702726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.702970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.702976:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.702979:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.702980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.702982:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.702985:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.702986:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.702987:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.702989:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.702990:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.702991:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.702994:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.702996:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.702998:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.702999:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.702999:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.703001:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.703002:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.703003:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.703004:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.703004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.703009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.703014:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.703015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.703016:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.703017:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677473.703017:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.703018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.703019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.703019:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.703020:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.703023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.703149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.703156:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.703159:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.703160:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.703162:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.703164:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.703166:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.703167:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.703169:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.703169:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.703171:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.703173:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.703175:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.703178:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.703179:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.703179:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.703181:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.703182:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.703182:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.703183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.703194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.703197:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.703198:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.703198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.703200:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.703203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.703293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.703301:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.703303:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.703304:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.703304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.703306:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.703308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.703726:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.703729:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.703730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.703732:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.703734:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.703735:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.703737:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.703738:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.703739:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.703740:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.703743:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.703745:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.703748:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.703749:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.703749:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.703751:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.703751:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.703752:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.703753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.703764:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.703767:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.703767:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.703769:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.703770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.703772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.703783:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.703786:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.703787:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.703787:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.703788:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.703789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.703889:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.703892:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.703892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.703894:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.703897:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.703898:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677473.703900:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.703901:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.703902:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.703903:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677473.703906:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.703908:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:4.2:1619677473.703908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677473.703911:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.703912:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677473.703912:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677473.703914:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677473.703915:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.703916:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.703916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:10.0:1619677473.703916:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.703919:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.703920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.703922:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.703925:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.703926:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.703927:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.703928:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.703929:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.703930:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.703934:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.703936:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.703939:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.703939:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.703940:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.703942:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.703942:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.703943:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.703943:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.703955:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.703958:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.703959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.703959:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.703961:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.703965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.703969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.703976:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.703979:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.703980:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.703982:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.703984:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.703985:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.703987:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.703988:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.703989:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.703989:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.703992:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.703993:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.703996:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.703997:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.703997:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.703998:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.703999:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.704000:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.704000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.704012:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.704014:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.704015:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.704016:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.704018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.704019:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677473.704224:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.704227:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.704228:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.704230:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.704233:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.704234:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677473.704235:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.704237:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.704238:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.704239:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677473.704242:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.704244:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677473.704247:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.704248:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677473.704248:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677473.704250:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677473.704250:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.704251:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.704251:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.704256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.704262:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.704265:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.704266:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.704266:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.704268:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.704271:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:22.0:1619677473.704560:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.704563:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.704563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.704565:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.704569:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.704570:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.704571:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.704573:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.704574:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.704575:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.704578:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.704580:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.704582:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.704583:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.704584:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.704585:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.704586:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.704587:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.704588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.704600:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.704603:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.704603:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.704605:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.704606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.704607:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.704978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.704984:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.704987:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.704988:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.704990:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.704993:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.704994:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.704996:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.704997:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.704998:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.704999:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.705002:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.705004:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.705006:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.705007:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.705007:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.705009:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.705010:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.705011:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.705011:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.705015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.705022:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.705025:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.705025:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.705026:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.705027:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.705028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.705036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.705044:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.705046:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.705047:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.705047:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.705047:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.705048:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.705268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.705276:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.705278:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.705279:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.705281:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.705283:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.705285:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.705286:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.705287:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.705288:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.705289:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.705292:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.705299:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.705303:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.705303:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.705304:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.705306:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.705306:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.705307:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.705309:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.705309:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.705315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.705321:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.705323:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.705324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.705324:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.705325:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.705325:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677473.705325:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.705327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.705327:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.705330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677473.705330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.705337:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.705340:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.705341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.705341:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.705343:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.705345:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.705431:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.705434:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.705435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.705437:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.705439:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.705441:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.705442:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.705443:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.705444:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.705445:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.705449:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.705451:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.705453:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.705454:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.705454:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.705456:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.705457:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.705458:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.705459:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.705460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.705469:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.705470:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.705471:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.705472:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.705472:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.705472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.705473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.705474:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.705475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.705476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.705902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.705908:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.705911:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.705911:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.705911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.705913:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.705915:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.705968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.705974:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.705977:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.705978:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.705980:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.705982:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.705984:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.705985:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.705987:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.705988:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.705989:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.705993:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.705995:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.705998:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.705998:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.705999:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.706001:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.706001:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.706002:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.706003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:4.2:1619677473.706079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.706086:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.706089:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.706090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.706092:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.706094:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.706096:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.706097:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.706098:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.706099:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.706100:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.706103:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.706105:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.706107:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.706108:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.706108:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.706110:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.706110:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.706111:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.706111:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.706115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.706122:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.706124:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.706125:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.706125:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.706127:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.706129:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.706133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.706139:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.706141:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.706142:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.706142:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.706143:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.706143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.706212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.706220:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.706222:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.706223:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.706225:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.706227:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.706229:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.706230:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.706231:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.706232:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.706233:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.706236:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.706237:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.706240:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.706241:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.706241:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.706243:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.706243:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.706244:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.706246:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.706246:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677473.706249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677473.706252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.0:1619677473.706258:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677473.706259:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:21.0:1619677473.706260:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677473.706261:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:21.0:1619677473.706261:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.706262:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.706263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:21.0:1619677473.706263:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.706264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.706266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.706270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.706278:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.706280:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.706281:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.706282:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.706283:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.706284:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677473.706359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677473.706367:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.706370:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.706371:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.706372:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.706375:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.706377:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.706378:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.706379:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.706380:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.706381:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.706384:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.706386:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.706388:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.706389:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.706390:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.706391:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.706392:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.706393:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.706393:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.706406:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.706408:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.706409:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.706411:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.706412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.706414:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.706496:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.706499:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.706500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.706502:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.706504:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.706506:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.706507:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.706508:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.706509:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.706510:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.706514:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.706515:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.706518:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.706519:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.706519:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.706521:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.706521:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.706522:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.706523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.706534:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.706536:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.706537:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.706539:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.706540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.706541:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.706558:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.706560:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.706561:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.706562:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.706563:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.706564:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.706720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.706728:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.706730:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.706731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.706731:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.706733:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.706735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.706943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.706949:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.706952:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.706952:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.706953:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.706954:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.706956:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.707340:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.707343:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.707344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.707346:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.707348:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.707349:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.707351:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.707352:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.707353:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.707354:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.707356:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.707358:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.707361:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.707361:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.707362:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.707364:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.707364:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.707365:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.707366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.707378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.707381:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.707381:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.707383:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.707384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.707385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.707600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677473.707607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.707611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.707611:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677473.707613:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.707615:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.707617:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.707618:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.707620:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.707620:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.707621:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.707624:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.707626:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.707629:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.707630:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.707630:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.707632:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.707633:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.707635:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677473.707636:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677473.707636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:4.2:1619677473.707642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:22.0:1619677473.707646:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.707649:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:22.0:1619677473.707649:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.707650:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.707650:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.707651:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677473.707652:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:22.0:1619677473.707652:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.707652:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677473.707659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.707661:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.707663:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.707664:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.707664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.707665:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.707666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.707745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.707751:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.707754:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.707755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.707757:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.707760:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.707761:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.707763:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.707764:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.707765:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.707766:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.707769:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.707771:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.707774:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.707774:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.707775:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.707777:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.707777:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.707778:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.707778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.707791:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.707794:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.707795:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.707795:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.707797:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.707799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.707803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.707810:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677473.707812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677473.707813:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677473.707814:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677473.707814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677473.707815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:24.0:1619677473.707816:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.707816:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.707817:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.707817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:24.0:1619677473.707819:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.707820:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.707821:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.707823:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.707824:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.707825:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.707827:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.707829:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.707832:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.707832:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.707833:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.707835:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.707835:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.707836:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.707837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.707848:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.707850:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.707852:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.707852:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.707854:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.707857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.707995:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.707997:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677473.707998:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677473.708000:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.708002:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.708003:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677473.708004:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.708006:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.708007:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.708008:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677473.708011:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.708012:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677473.708015:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.708016:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677473.708016:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677473.708018:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677473.708019:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.708020:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677473.708021:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677473.708021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:8.2:1619677473.708025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677473.708032:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677473.708033:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:1.0:1619677473.708033:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:6.0:1619677473.708034:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.708035:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:1.0:1619677473.708035:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.708036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.708036:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.708038:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.708041:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677473.708143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.708146:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.708149:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677473.708152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.708152:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.708155:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.708157:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.708159:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.708161:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.708162:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.708163:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677473.708167:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:20.0:1619677473.708169:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.708170:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.708171:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.708172:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677473.708173:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.708175:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677473.708176:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.708176:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:22.0:1619677473.708177:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.0:1619677473.708177:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.708178:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:22.0:1619677473.708179:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96508038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.708179:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677473.708180:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.708181:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.708181:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:22.0:1619677473.708182:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.708182:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.708183:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.708184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:22.0:1619677473.708185:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.708187:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.2:1619677473.708188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677473.708190:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.708190:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.708191:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677473.708192:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677473.708193:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.708193:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.708194:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.708195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:17.0:1619677473.708195:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.708196:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.708196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.708197:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.708199:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:17.0:1619677473.708199:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.708200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677473.708201:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677473.708202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677473.708203:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.708205:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.708206:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.708208:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.708209:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.708210:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.708616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.708624:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.708627:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.708628:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.708630:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.708633:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.708634:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.708636:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.708637:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.708638:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.708639:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.708643:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.708644:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.708647:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.708648:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.708649:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.708650:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.708651:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.708651:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.708652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.708666:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.708669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.708669:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.708670:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.708672:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.708675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677473.709065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.709072:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.709075:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.709076:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.709076:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.709078:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.709080:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.709090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.709096:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.709099:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.709099:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.709099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.709100:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.709101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.709176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677473.709183:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.709186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:22.0:1619677473.709186:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.0:1619677473.709186:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677473.709188:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677473.709189:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:8.0:1619677473.709190:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.709190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.709192:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.709192:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.709193:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.709194:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.709194:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.709195:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677473.709196:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.709197:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.709197:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677473.709198:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.709199:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.709200:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.709205:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.709206:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677473.709208:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677473.709209:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.709211:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.709211:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.709212:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677473.709212:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.709213:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677473.709214:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677473.709214:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.709215:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.709216:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:8.0:1619677473.709216:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.709216:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:22.0:1619677473.709217:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.709217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677473.709217:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.709218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:20.0:1619677473.709227:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.709229:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.709230:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.709230:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.709231:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677473.709232:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.709232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.709233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.709233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677473.709234:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.709234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.709237:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.709238:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.709238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.709240:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.709243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.710029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.710031:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.710032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.710034:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.710036:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.710037:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.710038:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.710039:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.710040:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.710041:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.710045:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.710047:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.710049:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677473.710049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677473.710050:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.710050:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.710052:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.710053:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.710053:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.710054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:10.0:1619677473.710056:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.710059:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.710060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.710062:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.710064:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.710066:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677473.710067:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:10.0:1619677473.710067:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.710068:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.710069:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:10.0:1619677473.710069:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.710070:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.710070:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.710072:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.710072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677473.710075:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.710077:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.710077:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.710080:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.710081:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.710081:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.710083:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.710084:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.710084:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.710085:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677473.710088:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.710090:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.710091:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.710092:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.710092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.710094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.710097:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.710100:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.710101:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.710101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.710103:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.710107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.710782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.710789:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.710792:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.710793:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.710795:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.710797:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.710799:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.710800:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.710801:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.710802:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.710803:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.710806:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.710808:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.710810:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.710811:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.710812:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.710813:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.710814:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.710815:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.710816:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.710817:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677473.710820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677473.710823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677473.710829:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.710831:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677473.710831:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:21.0:1619677473.710832:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.710833:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.710833:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677473.710834:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.710834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.710835:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677473.710837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.710875:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.710878:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.710879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.710881:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.710882:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.710884:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.710885:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.710887:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.710888:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.710889:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.710892:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.710894:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.710896:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.710897:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.710898:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.710899:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.710900:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.710901:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.710901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.710914:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.710916:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.710917:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.710919:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.710920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.710921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.710958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.710967:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.710971:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.710973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.710977:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.710980:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.710982:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.710984:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.710987:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.710988:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.710990:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.710995:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.710998:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.711001:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.711002:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.711003:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.711005:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.711006:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.711007:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.711008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.711011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.711020:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.711024:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.711025:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.711026:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.711028:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.711030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.2:1619677473.711050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677473.711053:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677473.711056:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.711057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677473.711060:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.6.11@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.54@o2ib7 : PUT - routed
00000800:00000200:24.0:1619677473.711062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677473.711156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.711164:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.711166:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.711167:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.711169:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.711172:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.711173:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.711175:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96508027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.711176:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.711177:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.711177:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.711181:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.711182:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677473.711182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677473.711185:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.711186:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.711186:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.711188:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.711188:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677473.711188:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:26.0:1619677473.711189:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.711189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.711190:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.711191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.711191:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.711193:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.711195:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.711202:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.711205:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.711206:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.711206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.711208:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.711212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.711220:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.711222:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.711223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.711224:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.711225:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.711225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.711535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.711542:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.711545:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.711545:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.711547:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.711550:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.711551:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.711552:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.711553:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.711554:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.711555:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.711558:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.711560:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.711562:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.711563:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.711564:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.711565:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.711566:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.711566:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.711567:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677473.711580:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.711582:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.711583:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.711583:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.711586:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.711589:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.711916:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.711918:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.711919:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.711921:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.711923:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.711924:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.711925:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.711927:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.711928:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.711929:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.711932:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.711933:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.711936:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.711937:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.711937:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.711939:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.711940:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.711940:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.711941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.711953:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.711955:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.711956:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.711957:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.711958:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.711960:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.712504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.712510:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.712513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.712513:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.712515:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.712518:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.712519:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.712520:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.712522:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.712523:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.712524:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.712527:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.712528:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.712531:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.712532:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.712532:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.712534:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.712534:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.712535:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.712536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.712550:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.712552:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.712553:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.712553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.712555:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.712558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.712766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.712774:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.712775:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677473.712777:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677473.712778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677473.712778:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677473.712778:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:24.0:1619677473.712780:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.712781:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.712783:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.712783:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.712785:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677473.712785:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677473.712787:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.712787:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.712789:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.712790:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.712791:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.712792:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.712793:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677473.712793:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.712799:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.712800:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.712801:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.712803:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.712804:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.712806:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.712806:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.712806:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677473.712807:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.712808:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.712808:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677473.712809:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.712809:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677473.712810:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.712810:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:24.0:1619677473.712811:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.712811:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.712811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677473.712812:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.712813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.712819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677473.712823:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.712824:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.712825:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.712826:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677473.712826:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.712827:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677473.712827:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:20.0:1619677473.712828:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.712828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677473.712829:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.712829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677473.712829:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.712830:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677473.712831:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677473.712831:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.712831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677473.712832:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.712832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.712834:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.712837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.712918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.712925:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.712928:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.712928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.712930:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.712933:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.712934:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.712935:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.712937:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.712938:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.712939:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.712942:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.712943:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.712946:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.712947:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.712948:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.712949:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.712950:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.712950:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.712951:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.712963:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.712966:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.712967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.712968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.712970:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.712973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.713233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3b200] (20)++
00000800:00000200:21.0:1619677473.713239:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3b200] (21)++
00000800:00000200:21.0:1619677473.713241:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.36@o2ib3
00000400:00000200:21.0:1619677473.713244:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.36@o2ib3 : GET - for me
00000800:00000200:20.0:1619677473.713244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3b200] (22)--
00000400:00000200:21.0:1619677473.713247:0:19935:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.36@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:21.0:1619677473.713249:0:19935:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.36@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:21.0:1619677473.713250:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3b200] (21)++
00000800:00000200:21.0:1619677473.713251:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3b200] (22)--
00000400:00000200:21.0:1619677473.713253:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.36@o2ib3 local destination
00000400:00000200:21.0:1619677473.713257:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.36@o2ib3(10.51.5.36@o2ib3:10.51.5.36@o2ib3) : REPLY try# 0
00000800:00000200:21.0:1619677473.713259:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.36@o2ib3
00000800:00000200:21.0:1619677473.713261:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9197e5a60900] -> 10.51.5.36@o2ib3 (2) version: 12
00000800:00000200:21.0:1619677473.713261:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197e8d3b200] (21)++
00000800:00000200:21.0:1619677473.713262:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e8d3b200] (22)++
00000800:00000200:21.0:1619677473.713263:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197e8d3b200] (23)--
00000800:00000200:21.0:1619677473.713264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3b200] (22)--
00000800:00000200:4.2:1619677473.713269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3b200] (21)++
00000800:00000200:20.0:1619677473.713273:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3b200] (22)++
00000800:00000200:20.0:1619677473.713275:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e8d3b200] (23)--
00000400:00000200:20.0:1619677473.713276:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.713277:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.36@o2ib3: REPLY: OK
00000800:00000200:21.0:1619677473.713278:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3b200] (22)--
00000800:00000200:20.0:1619677473.713280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3b200] (21)--
00000800:00000200:9.2:1619677473.713369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.713376:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.713378:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.713379:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.713379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.713381:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.713383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677473.713391:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.713394:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677473.713394:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677473.713397:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.713399:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.713401:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677473.713402:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.713403:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.713404:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.713405:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677473.713408:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.713410:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677473.713413:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.713414:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677473.713415:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677473.713416:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677473.713417:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.713418:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677473.713418:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677473.713430:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.713433:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.713434:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.713435:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.713438:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.713439:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677473.713447:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677473.713450:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677473.713451:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.713451:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677473.713452:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.713453:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:22.0:1619677473.713670:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.713673:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.713674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.713676:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.713678:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.713680:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.713681:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.713682:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.713683:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.713684:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.713688:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.713690:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.713692:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.713693:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.713693:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.713695:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.713696:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.713697:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.713697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.713709:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.713711:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.713712:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.713714:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.713714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.713716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.713727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.0:1619677473.713731:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677473.713733:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677473.713734:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.713735:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.713735:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.713736:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677473.713736:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.713736:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677473.713738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:18.0:1619677473.713739:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.713741:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.713743:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.713744:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.713746:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.713746:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.713748:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.713751:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.713753:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.713756:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.713757:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.713758:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.713759:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.713760:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.713761:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.713762:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.713774:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.713777:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.713778:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.713781:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.713781:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.713784:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677473.714374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.714380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.714383:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.714385:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.714385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.714387:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.714389:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.714390:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.714392:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.714393:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.714394:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.714397:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.714399:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.714401:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.714402:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.714403:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.714404:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.714405:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.714405:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.714406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.714411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.714420:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.714422:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.714423:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.714423:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.714425:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.714428:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.714431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.714438:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.714441:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.714441:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.714442:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.714442:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.714443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.714748:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.714751:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.714752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.714754:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.714756:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.714757:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.714759:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.714760:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.714761:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.714762:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.714766:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.714767:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.714770:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.714771:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.714771:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.714773:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.714774:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.714775:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.714775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.714787:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.714789:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.714790:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.714792:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.714793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.714794:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.714852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.714860:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.714863:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.714864:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.714866:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.714869:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.714871:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.714872:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.714873:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.714874:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.714875:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.714878:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.714879:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.714882:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.714883:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.714884:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.714885:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.714886:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.714887:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.714888:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.714889:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.714894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.714899:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.714902:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.714902:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.714903:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.714903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.714904:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.714905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.714906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.714906:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.714909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.715092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.715099:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.715101:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.715102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.715102:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.715104:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.715106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.715587:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.715590:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.715591:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.715593:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.715595:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.715596:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.715598:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.715599:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.715600:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.715601:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.715605:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.715607:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.715609:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.715610:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.715610:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.715612:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.715613:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.715614:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.715614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.715627:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.715630:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.715630:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.715632:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.715633:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.715634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.715759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677473.715767:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.715769:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677473.715770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677473.715772:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.715775:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.715776:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677473.715778:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96508008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.715779:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.715780:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.715781:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.715784:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.715786:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.715788:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.715789:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677473.715790:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677473.715791:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677473.715792:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.715793:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677473.715794:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677473.715795:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:4.2:1619677473.715800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.715807:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677473.715809:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677473.715810:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.715811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:4.2:1619677473.715828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.715835:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.715838:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.715839:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.715840:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.715843:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.715845:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.715846:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.715847:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.715848:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.715849:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.715852:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.715853:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.715856:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.715857:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.715857:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.715859:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.715859:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.715860:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.715860:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.715865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.715874:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.715876:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.715877:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.715878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.715880:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.715883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677473.716269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.716277:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.716279:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.716280:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.716282:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.716285:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.716286:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.716288:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96508006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.716289:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.716290:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.716291:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:4.2:1619677473.716296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:26.0:1619677473.716298:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.716300:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677473.716300:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:26.0:1619677473.716302:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.716302:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:26.0:1619677473.716303:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.716303:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.716304:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.716305:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.716305:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677473.716305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.716305:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.716306:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677473.716307:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.716309:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.716311:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96508005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.716311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:20.0:1619677473.716312:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.716313:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.716314:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.716317:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:20.0:1619677473.716317:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.716319:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.716320:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.716321:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.716321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677473.716322:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677473.716323:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.716323:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.716323:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.716325:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.716326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.716326:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.716327:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.716328:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.716328:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677473.716333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677473.716339:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.716339:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.716341:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:22.0:1619677473.716342:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677473.716342:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:22.0:1619677473.716343:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.716343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.716343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.716345:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.716347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.716357:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.716359:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.716360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.716360:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.716361:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.716362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.716488:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.716490:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.716491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.716493:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.716495:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.716496:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.716497:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.716498:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.716499:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.716500:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.716504:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.716505:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.716508:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.716509:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.716509:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.716511:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.716512:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.716513:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.716514:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.716514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.716525:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.716526:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.716526:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.716527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.716528:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.716528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.716529:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.716531:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.716531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.716533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.717144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.717150:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.717153:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.717154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.717154:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.717156:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.717157:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.717167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.717173:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.717176:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.717177:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.717177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:5.2:1619677473.717177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677473.717178:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.717178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.717183:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.717186:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.717187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.717189:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.717191:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.717192:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.717194:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96508001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.717195:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.717196:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.717197:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.717199:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.717201:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.717203:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.717204:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.717205:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.717206:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.717207:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.717208:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.717208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.717221:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.717223:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.717224:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.717224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.717227:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.717229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.717241:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.717243:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.717244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.717244:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.717245:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.717246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.717504:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.717506:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.717507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677473.717507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677473.717509:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.717511:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.717513:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.717514:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.717515:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.717515:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677473.717516:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.717517:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677473.717518:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.717519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677473.717521:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.717522:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.717524:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677473.717525:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.717527:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.717527:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:8.0:1619677473.717527:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677473.717528:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677473.717528:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96507998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.717530:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.717530:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677473.717530:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.717531:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:8.0:1619677473.717531:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.717532:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:8.0:1619677473.717532:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.717536:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.717538:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.717540:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.717541:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.717542:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.717544:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.0:1619677473.717544:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.717544:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.717545:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677473.717546:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677473.717546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677473.717547:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.717549:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.717550:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.717551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.717557:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.717560:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.717561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.717561:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.717563:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.717566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.717568:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.717571:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.717572:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.717573:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.717573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.717575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.717780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.717788:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.717791:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.717792:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.717794:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.717796:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.717797:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.717799:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.717800:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.717801:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.717802:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.717805:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.717807:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.717809:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.717810:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.717811:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.717813:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.717813:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.717814:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.717815:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.717816:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.717821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677473.717826:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.717829:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.717829:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.717830:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.717830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.717831:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.717832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.717832:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.717833:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.717836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.718357:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.718360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.718360:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.718362:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.718364:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.718365:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.718367:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.718368:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.718369:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.718370:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.718374:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.718375:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.718378:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.718378:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.718379:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.718381:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.718381:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.718382:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.718383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.718396:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.718398:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.718399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.718400:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.718401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.718403:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.718455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.718463:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.718466:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.718467:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.718469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.718469:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.718471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.718767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.718774:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.718776:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.718777:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.718777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.718779:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.718782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.718963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907b396800] (20)++
00000800:00000200:18.0:1619677473.718972:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b396800] (21)++
00000800:00000200:18.0:1619677473.718977:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.3@o2ib3
00000800:00000200:2.0:1619677473.718978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b396800] (22)--
00000400:00000200:18.0:1619677473.718982:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.6.3@o2ib3 : GET - for me
00000400:00000200:18.0:1619677473.718986:0:19933:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.6.3@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:18.0:1619677473.718990:0:19933:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.6.3@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:18.0:1619677473.718992:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907b396800] (21)++
00000800:00000200:18.0:1619677473.718994:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907b396800] (22)--
00000400:00000200:18.0:1619677473.718997:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.6.3@o2ib3 local destination
00000800:00000200:18.0:1619677473.719001:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919064fef680] -> 10.51.6.3@o2ib3 (2) version: 12
00000800:00000200:18.0:1619677473.719003:0:19933:0:(o2iblnd.c:1251:kiblnd_query()) peer_ni 10.51.6.3@o2ib3 ffff919064fef680, alive 0 secs ago
00000400:00000200:18.0:1619677473.719009:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.6.3@o2ib3(10.51.6.3@o2ib3:10.51.6.3@o2ib3) : REPLY try# 0
00000800:00000200:18.0:1619677473.719012:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.6.3@o2ib3
00000800:00000200:18.0:1619677473.719015:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919064fef680] -> 10.51.6.3@o2ib3 (2) version: 12
00000800:00000200:18.0:1619677473.719017:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91907b396800] (21)++
00000800:00000200:18.0:1619677473.719018:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907b396800] (22)++
00000800:00000200:18.0:1619677473.719020:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91907b396800] (23)--
00000800:00000200:18.0:1619677473.719021:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b396800] (22)--
00000800:00000200:5.2:1619677473.719025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907b396800] (21)++
00000800:00000200:2.0:1619677473.719032:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b396800] (22)++
00000800:00000200:2.0:1619677473.719035:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907b396800] (23)--
00000400:00000200:2.0:1619677473.719036:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.719038:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b396800] (22)--
00000400:00000200:2.0:1619677473.719040:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.6.3@o2ib3: REPLY: OK
00000800:00000200:2.0:1619677473.719042:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b396800] (21)--
00000800:00000200:20.2:1619677473.719077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.719084:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.719088:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.719090:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.719092:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.719096:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.719098:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.719100:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.719102:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.719104:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.719106:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.719110:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.719113:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.719116:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.719117:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.719118:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.719120:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.719121:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.719122:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.719124:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.719124:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.719126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.719129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677473.719135:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.719137:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677473.719137:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:17.0:1619677473.719138:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.719138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.719140:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:17.0:1619677473.719141:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.719141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677473.719143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677473.719143:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677473.719208:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.719211:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.719212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.719214:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.719216:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.719218:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.719219:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.719220:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.719221:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.719222:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.719226:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.719228:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.719230:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.719231:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.719232:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.719233:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.719234:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.719235:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.719235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.719247:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.719249:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.719250:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.719252:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.719252:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.719254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.719360:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.719363:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:20.2:1619677473.719364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677473.719364:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677473.719366:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.719368:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.719369:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677473.719371:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.719372:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677473.719373:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:1.0:1619677473.719373:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.719374:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677473.719377:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.719378:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:1.0:1619677473.719378:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.719380:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:18.0:1619677473.719382:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.719382:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.719383:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.719384:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:18.0:1619677473.719386:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.719386:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.719386:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.719387:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000400:00000200:18.0:1619677473.719388:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677473.719388:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677473.719389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:18.0:1619677473.719390:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677473.719392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:18.0:1619677473.719393:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.719394:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.719396:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.719401:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.719401:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.719402:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.719402:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000400:00000200:17.0:1619677473.719403:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.719403:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677473.719403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677473.719404:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677473.719404:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.719405:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.719405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677473.719407:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.719407:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677473.719408:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.719409:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.719412:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.719413:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.719414:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677473.719416:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.719417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:4.2:1619677473.719418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:9.2:1619677473.719419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.719422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677473.719427:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.719429:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677473.719430:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.719431:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:21.0:1619677473.719432:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.719432:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.719432:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.719433:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677473.719433:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.719434:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.719435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677473.719436:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:18.0:1619677473.719436:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.719437:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.719438:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677473.719438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677473.719439:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.719440:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677473.719440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:20.0:1619677473.719441:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.719445:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.719447:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.719449:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.719450:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677473.719451:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677473.719453:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677473.719454:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.719454:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.719455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677473.719467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.719475:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.719478:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.719479:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.719481:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.719483:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.719484:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.719486:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.719487:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.719488:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.719489:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.719493:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.719494:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.719496:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.719497:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.719497:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.719499:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.719499:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.719499:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.719500:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.719504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.719510:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.719512:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.719513:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.719514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.719515:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.719516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.719528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.719534:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.719537:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.719537:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.719537:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.719538:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.719539:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677473.719649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677473.719659:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.719663:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.719667:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.719669:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.719671:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.719673:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677473.719675:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.719676:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.719678:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.719679:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677473.719683:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.719685:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:4.2:1619677473.719685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677473.719689:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.719689:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:12.0:1619677473.719690:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:12.0:1619677473.719691:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:12.0:1619677473.719692:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.719693:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.719693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:10.0:1619677473.719693:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.719695:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.719696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677473.719697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:10.0:1619677473.719698:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.719700:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.719701:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.719703:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.719704:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.719705:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.719705:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:10.0:1619677473.719706:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.719708:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.719709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:10.0:1619677473.719709:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677473.719709:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.719711:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.719712:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.719714:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.719714:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677473.719715:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.719716:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.719717:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.719718:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.719718:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.719719:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.719732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.719735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.719735:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.719736:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.719738:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.719742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.720038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.720045:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.720048:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.720049:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.720050:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.720051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.720054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.720215:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.720217:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.720218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.720220:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.720222:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.720224:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.720225:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.720226:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.720227:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.720228:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.720232:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.720233:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.720236:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.720237:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.720237:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.720239:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.720240:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.720241:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.720242:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.720242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.720252:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.720253:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.720255:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.720255:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.720256:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.720256:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.720256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.720258:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.720259:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.720260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.720662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.720670:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.720673:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.720673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.720676:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.720678:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.720680:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.720681:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.720682:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.720683:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.720684:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.720688:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.720690:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.720693:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.720694:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.720695:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.720696:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.720697:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.720697:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.720698:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.720703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.720711:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.720713:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.720714:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.720716:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.720717:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.720719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.720721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.720729:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.720731:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.720732:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.720733:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.720733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.720734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677473.720793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.720799:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.720801:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.720802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.720802:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.720804:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.720806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.720852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.720858:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.720860:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.720862:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.720863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.720865:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.720867:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.720868:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.720870:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.720871:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.720872:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.720876:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.720877:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.720880:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.720880:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.720881:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.720883:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.720884:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.720884:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.720885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.720889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.720898:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.720900:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.720901:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.720901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.720903:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.720905:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.721044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.721051:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.721054:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.721055:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.721057:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.721059:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.721061:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.721062:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.721063:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.721064:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.721065:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:6.0:1619677473.721066:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677473.721068:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.721068:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.721069:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677473.721070:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:6.0:1619677473.721071:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.721072:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.721073:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.721073:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:6.0:1619677473.721073:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.721075:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.721075:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677473.721075:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677473.721076:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677473.721076:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.721077:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.721077:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.721078:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.721079:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:9.2:1619677473.721080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:6.0:1619677473.721082:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.721084:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.721086:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677473.721087:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.721087:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.721088:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677473.721089:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.721090:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.721090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.721090:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.721090:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.721091:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677473.721092:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.721092:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677473.721094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.721104:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.721106:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.721107:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.721108:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.721110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.721111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.721119:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.721122:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.721123:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.721123:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.721124:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.721125:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.721187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.721194:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.721197:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.721198:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.721200:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.721202:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.721203:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.721205:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.721206:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.721207:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.721208:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.721212:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.721213:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.721216:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.721216:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.721217:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.721219:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.721219:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.721220:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.721220:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.721232:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.721234:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.721235:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.721236:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.721239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.721240:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.721250:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.721253:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.721254:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.721254:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.721255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.721256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677473.721907:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.721909:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.721910:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.721912:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.721914:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.721916:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.721917:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.721918:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.721919:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.721920:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.721924:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.721925:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.721928:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.721928:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.721929:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.721931:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.721931:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.721932:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.721933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.721945:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.721947:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.721948:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.721949:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.721950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.721951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.721961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677473.721965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:1.0:1619677473.721967:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677473.721969:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:1.0:1619677473.721969:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.721970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677473.721971:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:1.0:1619677473.721973:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.721974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.721974:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.721975:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.721977:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.721977:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677473.721978:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.721978:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.721980:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.721980:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.721981:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.721982:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677473.721983:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.721984:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.721986:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.721986:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.721989:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677473.721991:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.721992:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.721993:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:20.0:1619677473.721994:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.721994:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.721995:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677473.721996:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.721996:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677473.721997:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.721997:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677473.721998:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.721998:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.721999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677473.722000:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.722000:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.722001:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.722002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:5.2:1619677473.722004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.722009:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.722011:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677473.722011:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.722012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.722013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.722014:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.722014:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.722015:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.722015:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.722016:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.722017:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.722018:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.722020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.722020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.722021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.722021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.722441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.722448:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.722451:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.722452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.722452:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.722454:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.722456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.722612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.722620:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.722623:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.722624:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.722626:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.722628:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.722630:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.722631:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.722632:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.722633:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.722634:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.722638:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.722639:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.722642:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.722643:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.722643:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.722645:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.722646:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.722646:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.722647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.722659:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.722662:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.722663:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.722663:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.722665:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.722668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.722942:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.722944:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.722945:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.722947:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.722950:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.722951:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.722953:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.722954:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.722955:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.722956:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.722960:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.722961:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.722964:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.722965:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.722965:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.722967:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.722968:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.722969:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.722969:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.722981:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.722983:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.722984:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.722985:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.722986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.722987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.723010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.723017:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.723020:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.723021:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.723023:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.723025:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.723027:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.723028:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.723030:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.723031:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.723032:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.723035:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.723037:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.723040:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.723041:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.723041:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.723043:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.723044:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.723044:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.723045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.723065:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.723068:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.723069:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.723069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.723071:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.723075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.723083:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.723085:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.723086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.723087:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.723088:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.723088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.723723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.723730:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.723732:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.723733:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.723735:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.723738:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.723739:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.723740:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.723742:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.723743:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.723744:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.723747:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.723749:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.723751:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.723752:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.723753:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.723754:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.723755:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.723756:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.723756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677473.723761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.723770:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.723772:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.723773:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.723773:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.723775:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.723778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.723791:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.723793:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.723794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.723796:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.723798:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.723800:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.723801:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.723802:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.723803:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.723804:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.723808:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.723809:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.723812:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.723812:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.723813:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.723815:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.723815:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.723817:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.723818:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.723818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.723828:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.723829:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.723830:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.723831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.723832:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.723832:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.723832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.723834:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.723835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.723836:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.723871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.723877:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.723879:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.723880:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.723882:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.723884:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.723886:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.723887:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.723888:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.723889:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.723890:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.723894:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.723895:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.723898:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.723899:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.723899:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.723901:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.723901:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.723902:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.723902:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.723906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.723915:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.723917:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.723918:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.723918:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.723920:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.723922:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.724449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.724457:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.724459:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677473.724459:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:24.0:1619677473.724460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.724460:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.724462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.724462:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:8.0:1619677473.724463:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.724464:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.724465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:1.0:1619677473.724467:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.724468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:1.0:1619677473.724468:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677473.724469:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.724472:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.724473:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.724474:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:1.0:1619677473.724474:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677473.724478:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.724478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:1.0:1619677473.724478:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.724479:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677473.724479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677473.724480:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677473.724480:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677473.724481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.724483:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.724484:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677473.724485:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677473.724487:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677473.724487:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677473.724487:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.724488:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.724488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:26.0:1619677473.724490:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.724491:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.724493:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.724495:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.724497:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.724498:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.724499:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.724500:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.724501:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677473.724501:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:26.0:1619677473.724505:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.724505:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.724506:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.724507:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:12.0:1619677473.724508:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.724510:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.724510:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:13.0:1619677473.724510:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677473.724511:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:12.0:1619677473.724512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:26.0:1619677473.724513:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.724513:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.724514:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.724516:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.724516:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.724522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.724527:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.724529:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.724530:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.724531:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.724531:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677473.724531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.724532:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.724533:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.724533:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.724537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.724636:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.724639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.724639:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.724641:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.724643:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.724644:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.724646:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.724647:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.724649:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.724650:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.724653:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.724655:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.724658:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.724659:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.724659:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.724661:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.724661:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.724662:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.724663:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.724675:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.724677:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.724678:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.724680:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.724680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.724682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.724687:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.724688:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677473.724692:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.724695:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.724696:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.724697:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.724698:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:4.2:1619677473.724698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.0:1619677473.724700:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.724701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.724705:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.724708:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:8.0:1619677473.724708:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677473.724709:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.0:1619677473.724710:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:26.0:1619677473.724711:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.724711:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.724711:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.724713:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.724714:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.724715:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677473.724715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:26.0:1619677473.724716:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.724718:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.724719:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.724720:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.724723:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677473.724723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.724724:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.724725:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.724726:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.724727:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.724728:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.724728:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:24.0:1619677473.724728:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.724729:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.724730:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.724730:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:24.0:1619677473.724731:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.724732:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.724732:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.724732:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.724733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.724733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:4.2:1619677473.724736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.724742:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.724743:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.724743:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.724745:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.724746:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.724746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.724747:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.724747:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677473.724748:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.724748:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677473.725675:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.725678:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.725679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.725681:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.725683:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.725685:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.725686:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.725687:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.725688:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.725689:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.725693:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.725694:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.725697:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.725698:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.725698:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.725700:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.725701:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.725702:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.725702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.725715:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.725717:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.725718:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.725720:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.725722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.725722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677473.726208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.726240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.726248:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.726250:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.726251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.726251:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.726253:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.726255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.726272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.726280:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.726282:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.726283:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.726285:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.726287:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.726288:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.726289:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.726290:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.726291:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.726292:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.726299:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.726301:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.726303:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.726304:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.726304:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.726306:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.726306:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.726307:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.726307:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.726312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.726319:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.726321:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.726322:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.726322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.726323:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.726325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.726356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.726362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.726365:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.726366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.726368:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.726371:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.726372:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.726373:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.726375:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.726375:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.726376:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.726379:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.726381:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.726382:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.726383:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.726384:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.726385:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.726385:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.726386:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.726387:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.726391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.726398:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.726400:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.726401:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.726401:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.726402:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.726403:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.726507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.726511:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.726514:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.726516:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.726517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.726518:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.726519:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.726521:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.726522:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.726523:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.726524:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.726527:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.726528:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.726531:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.726532:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.726532:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.726534:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.726535:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.726536:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.726536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.726540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.726546:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.726549:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:6.0:1619677473.726549:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677473.726550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.726550:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.726551:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.726552:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:8.0:1619677473.726552:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:6.0:1619677473.726554:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.726555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677473.726556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.726558:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.726559:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.726560:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.726561:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.726562:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.726565:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.726567:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.726569:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.726570:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.726571:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.726572:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.726573:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.726574:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.726575:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.726590:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.726592:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.726593:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.726594:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:21.0:1619677473.726595:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.726596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677473.726597:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677473.726597:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:21.0:1619677473.726598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:17.0:1619677473.726600:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.726602:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.726604:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677473.726605:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.726606:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.726607:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.726608:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677473.726611:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.726613:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677473.726616:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.726617:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677473.726618:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677473.726619:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677473.726620:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.726621:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677473.726621:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677473.726633:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.726636:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.726637:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.726639:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.726640:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.726642:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:31.0:1619677473.726671:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.726673:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.726676:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.726678:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.726679:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.726681:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.726682:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.726683:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.726684:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.726687:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.726689:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.726691:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.726692:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.726692:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.726694:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.726694:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.726695:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.726696:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.726713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.727145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.727153:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.727155:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.727156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.727158:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.727160:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.727161:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.727163:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.727164:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.727165:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.727165:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677473.727167:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:10.0:1619677473.727168:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.727170:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.727171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:10.0:1619677473.727171:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677473.727173:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.727174:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.727175:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.727176:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:1.0:1619677473.727176:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.727177:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.727178:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.727178:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000400:00000200:1.0:1619677473.727178:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677473.727179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:1.0:1619677473.727179:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.727180:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.727181:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.727182:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677473.727184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:1.0:1619677473.727185:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.727187:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.727190:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.727191:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.727191:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.727192:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:1.0:1619677473.727193:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.727193:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:20.0:1619677473.727194:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:1.0:1619677473.727194:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:20.0:1619677473.727195:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.727195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:20.0:1619677473.727197:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.727198:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.727200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.727207:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.727210:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.727211:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.727211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.727213:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.727216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.727385:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.727388:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.727389:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.727391:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.727393:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.727394:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.727395:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.727396:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.727398:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.727399:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.727402:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.727404:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.727407:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.727407:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.727408:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.727410:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.727410:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.727411:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.727412:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.727413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.727424:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.727425:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.727425:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.727426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.727427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.727427:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.727428:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.727430:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.727431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.727432:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.727493:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.727522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.727523:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.727524:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.727526:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.727529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.728009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.728017:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.728019:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.728020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.728022:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.728025:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.728026:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.728027:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.728028:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.728029:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.728030:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.728033:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.728035:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.728038:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.728038:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.728039:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.728041:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.728041:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.728042:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.728042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.728054:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.728057:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.728058:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.728058:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.728061:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.728064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.728223:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.728226:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.728228:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.728230:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.728231:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.728233:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.728234:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.728235:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.728236:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.728239:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.728240:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.728243:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.728243:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.728244:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.728246:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.728246:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.728247:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.728248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.728252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.728258:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.728261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.728261:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.728261:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.728264:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.728266:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.728332:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.728352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.728359:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.728361:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.728362:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.728362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.728364:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.728366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.728423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:22.0:1619677473.728425:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.728428:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.728428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.728430:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:22.0:1619677473.728430:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.728433:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:22.0:1619677473.728433:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.728434:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677473.728434:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:22.0:1619677473.728435:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.728437:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.728437:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.728439:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.728440:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.728440:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.728442:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.728443:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.728444:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.728445:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.728447:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.728447:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.728448:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677473.728449:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.728450:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.728451:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:26.0:1619677473.728452:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.728453:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.728453:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677473.728454:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677473.728454:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.728455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677473.728457:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.728458:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.728458:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.728460:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.728461:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.728461:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.728462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.728466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.728469:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.728471:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.728472:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.728474:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.728475:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.728476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.728477:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.728477:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.728477:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677473.728479:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.728479:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.728480:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.728481:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.728481:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.728483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.728484:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677473.728485:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.729263:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.729266:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.729267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.729269:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.729271:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.729272:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.729273:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.729275:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.729276:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.729277:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.729280:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.729282:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.729284:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.729285:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.729286:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.729287:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.729288:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.729289:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.729289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.729302:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.729305:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.729305:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.729307:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.729308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.729309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.729385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.729393:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.729396:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.729397:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.729399:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.729402:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.729403:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.729404:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.729405:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.729406:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.729407:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.729411:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.729412:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.729415:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.729416:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.729416:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.729418:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.729419:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.729419:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.729420:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.729434:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.729436:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.729437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.729438:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.729440:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.729446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.729447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.729449:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.729450:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.729450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.729451:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.729452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.730118:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.730121:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.730122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.730123:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.730125:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.730127:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.730128:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.730129:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.730130:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.730131:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.730134:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.730136:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.730139:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.730139:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.730140:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.730142:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.730142:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.730143:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.730143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:4.2:1619677473.730145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677473.730152:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:20.0:1619677473.730155:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.0:1619677473.730155:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677473.730156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:20.0:1619677473.730157:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.730158:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.730158:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.730160:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.730161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677473.730161:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.730162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677473.730162:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.730164:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.730165:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.730166:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.730167:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.730171:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.730173:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.730176:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.730177:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.730178:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.730179:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.730180:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677473.730181:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677473.730182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:22.0:1619677473.730193:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.730196:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.730197:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.730197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.730199:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.730202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.730222:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.730225:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.730227:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.730229:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.730230:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.730232:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.730233:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.730234:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.730235:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.730238:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.730239:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.730241:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.730242:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.730243:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.730244:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.730245:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.730246:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.730246:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.730250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.730257:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.730259:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.730260:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.730260:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.730262:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.730264:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.730270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.730291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.730298:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.730301:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.730302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.730302:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.730304:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.730305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.730309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.730315:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.730318:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.730318:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.730319:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.730320:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.730320:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.731119:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.731122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.731122:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.731124:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.731127:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.731128:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.731129:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.731130:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.731131:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.731133:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.731136:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.731138:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.731140:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.731141:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.731141:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.731143:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.731144:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.731145:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.731146:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.731146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.731157:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.731158:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.731159:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.731160:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.731161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.731161:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.731161:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.731163:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.731164:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.731165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.731472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.731478:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.731481:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.731482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.731484:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.731486:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.731487:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.731488:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.731490:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.731491:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.731492:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.731495:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.731497:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.731500:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.731501:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.731501:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.731503:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.731504:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.731505:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.731505:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.731517:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.731520:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.731521:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.731523:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.731525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.731525:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677473.731632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.731639:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.731642:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.731643:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.731645:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.731647:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.731649:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.731650:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.731651:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.731652:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.731653:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.731656:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.731658:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.731660:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.731661:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.731662:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.731664:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.731664:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.731665:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.731665:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.731669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.731677:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.731679:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.731680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.731680:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.731682:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.731684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.731688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.731694:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.731698:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.731698:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.731699:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.731699:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.731700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.731733:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.731737:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.731741:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.731744:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.731746:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.731748:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.731750:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.731752:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.731753:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.731758:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.731761:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:4.2:1619677473.731761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677473.731764:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.731764:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677473.731765:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.731766:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677473.731767:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677473.731768:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.731769:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677473.731769:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677473.731770:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677473.731770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677473.731771:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677473.731772:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.731773:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677473.731775:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.731776:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.731777:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.731778:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677473.731781:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.731783:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677473.731786:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.731786:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677473.731787:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677473.731788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677473.731789:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677473.731790:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.731791:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677473.731792:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677473.731793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677473.731795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.731798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677473.731802:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677473.731803:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677473.731805:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:2.0:1619677473.731805:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677473.731806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677473.731806:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677473.731807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:2.0:1619677473.731807:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.731809:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.731812:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.731816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.731824:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.731827:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.731828:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.731828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.731829:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.731829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.731982:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.731985:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.731985:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.731987:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.731990:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.731991:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.731992:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677473.731993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000400:00000200:6.0:1619677473.731994:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.731995:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.731996:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.731998:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.732000:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677473.732002:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:6.0:1619677473.732002:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.732003:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.732004:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.732005:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677473.732006:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:6.0:1619677473.732006:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.732007:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.732007:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:31.0:1619677473.732009:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.732012:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.732014:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.732016:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.732019:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.732019:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:31.0:1619677473.732020:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.732021:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:31.0:1619677473.732022:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.732022:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.732024:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.732024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.732027:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677473.732028:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.732032:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.732035:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.732037:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.732037:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677473.732038:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.732040:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.732040:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677473.732041:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:20.0:1619677473.732041:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677473.732042:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:20.0:1619677473.732042:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.732042:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677473.732043:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:21.0:1619677473.732043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.732046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.732052:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.732055:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.732056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.732056:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.732058:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.732060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.732293:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.732327:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.732329:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.732330:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.732332:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.732335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.732338:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677473.732812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.732815:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.732816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.732818:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.732820:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.732822:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.732823:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.732824:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.732825:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.732826:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.732830:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.732831:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.732834:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.732834:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.732835:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.732837:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.732837:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.732838:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.732839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.732850:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.732853:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.732853:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.732855:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.732856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.732857:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.732992:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.732995:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.732997:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.732998:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.733000:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.733001:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.733002:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.733003:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.733004:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.733007:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.733009:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.733011:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.733012:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.733013:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.733014:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.733015:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.733016:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.733016:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.733033:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.733059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.733064:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.733067:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.733068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.733069:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.733073:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.733074:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.733075:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.733077:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.733078:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.733079:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.733082:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.733084:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.733085:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.733086:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.733087:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.733089:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.733089:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.733090:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.733091:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.733095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.733102:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.733104:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.733105:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.733105:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.733106:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.733108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.733118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.733125:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.733128:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.733129:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.733131:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.733134:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.733135:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.733137:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.733138:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.733139:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.733140:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.733143:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.733145:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.733148:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.733149:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.733149:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.733151:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.733152:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.733153:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.733154:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.733154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.733160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.733165:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.733167:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.733168:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.733169:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.733169:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:6.0:1619677473.733169:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.733170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.733171:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677473.733172:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.733175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.733450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.733458:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.733461:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.733462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.733464:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.733466:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.733468:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677473.733469:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.733470:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.733471:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.733472:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677473.733475:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.733477:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677473.733479:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.733479:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677473.733480:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677473.733482:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677473.733482:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.733482:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.733483:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677473.733504:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677473.733510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.733512:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677473.733513:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677473.733514:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.733514:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677473.733515:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.733516:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677473.733543:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.733545:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.733546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.733547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.733571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677473.733572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:24.0:1619677473.733578:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:26.0:1619677473.733580:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:24.0:1619677473.733580:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.733581:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.733581:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677473.733582:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:24.0:1619677473.733583:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.733583:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.733585:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.733585:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677473.733587:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.733588:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.733589:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.733590:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.733591:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.733592:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.733595:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.733596:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.733599:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.733600:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.733600:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.733602:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.733603:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.733604:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.733605:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.733606:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.733608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.733611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677473.733616:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.733618:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677473.733619:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.733620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.733620:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.733621:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.733621:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677473.733622:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.733623:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677473.733626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.733631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.733638:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.733641:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.733641:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.733642:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.733643:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.733643:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677473.733823:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.733825:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.733826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.733828:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.733830:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.733832:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.733833:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.733834:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.733835:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.733836:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.733839:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.733840:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.733843:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.733843:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.733844:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.733846:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.733846:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.733847:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.733848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.733860:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.733862:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.733863:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.733864:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.733865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.733867:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.733895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.733902:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.733905:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.733906:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.733908:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.733911:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.733912:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.733914:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.733915:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.733916:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.733917:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.733920:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.733921:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.733924:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.733925:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.733926:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.733927:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.733928:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.733928:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.733929:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.733952:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.733955:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.733956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.733956:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.733959:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.733962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.733990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.733996:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.733999:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.734000:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.734002:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.734004:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.734006:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.734007:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.734009:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.734010:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.734011:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.734014:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.734016:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.734018:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.734019:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.734020:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.734021:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.734022:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.734022:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.734023:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.734046:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.734048:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.734049:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.734049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.734051:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.734053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.734279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.734286:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.734289:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.734290:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.734292:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.734299:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.734300:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.734302:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.734303:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.734304:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.734305:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.734309:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.734310:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.734313:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.734314:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.734314:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.734316:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.734317:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.734318:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.734319:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.734319:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.734325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.734332:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.734334:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.734335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.734336:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.734341:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.734344:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.734345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.734345:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.734347:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.734351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.734467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.734471:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.734473:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.734475:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.734476:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.734478:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.734479:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.734480:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.734482:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.734483:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.734484:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.734487:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.734488:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.734490:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.734491:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.734492:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.734494:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.734494:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.734495:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.734496:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.734507:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.734509:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.734510:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.734510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.734512:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.734515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.734523:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.734525:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.734526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.734526:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.734528:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.734528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.734683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.734685:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.734686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.734688:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.734690:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.734691:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.734692:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.734693:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.734694:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.734695:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.734698:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.734699:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.734702:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.734703:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.734703:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.734705:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.734705:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.734707:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.734708:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.734708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.734719:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.734720:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.734720:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.734721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.734722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.734723:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.734723:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.734725:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.734726:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.734727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.735447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.735450:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.735450:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.735450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.735451:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.735452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.735544:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.735546:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.735547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.735548:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.735550:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.735551:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.735552:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.735554:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.735555:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.735556:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.735558:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.735560:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.735562:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.735562:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.735563:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.735564:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.735565:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.735565:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.735566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.735579:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.735581:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.735582:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.735584:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.735585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.735586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.735597:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.735600:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.735601:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.735601:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.735602:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.735603:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.735670:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.735673:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.735675:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.735678:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.735679:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.735681:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.735682:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.735683:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.735684:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.735687:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.735689:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.735691:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.735692:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.735693:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.735694:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.735695:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.735696:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.735696:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.735713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.735777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.735796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.735803:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.735806:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.735807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.735807:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.735809:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.735811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.736258:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.736261:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.736263:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.736265:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.736266:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.736267:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.736269:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.736270:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.736270:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.736274:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.736275:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.736277:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.736278:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.736279:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.736280:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.736281:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.736281:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.736282:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.736300:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677473.736387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.736395:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.736398:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.736399:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.736400:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.736403:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.736404:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.736406:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.736407:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.736408:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.736409:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.736412:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.736414:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.736416:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.736417:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.736417:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.736419:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.736419:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.736420:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.736420:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.736425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.736431:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.736434:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.736435:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.736435:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.736436:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.736438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.736549:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.736552:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.736553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.736555:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.736557:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.736558:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.736560:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.736561:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.736562:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.736563:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.736566:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.736567:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.736570:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.736570:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.736571:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.736573:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.736573:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.736574:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.736575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.736586:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.736588:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.736589:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.736591:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.736591:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.736593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.736624:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.736627:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.736629:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.736631:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.736633:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.736634:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.736635:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.736636:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.736637:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.736640:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.736642:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.736644:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.736645:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.736645:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.736647:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.736647:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.736649:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.736650:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.736651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.736665:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.736665:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.736666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.736668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.736677:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.736680:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.736680:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.736681:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.736681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.736682:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.736727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.736733:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.736736:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.736737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677473.736737:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:2.0:1619677473.736739:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.736740:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.736741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:2.0:1619677473.736741:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.736743:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677473.736744:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.736745:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.736746:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.736747:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.736747:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.736749:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677473.736749:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.736751:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.736753:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677473.736753:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.736754:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.736755:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677473.736756:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:1.0:1619677473.736758:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.736760:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.736760:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677473.736761:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.736761:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.736763:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:1.0:1619677473.736763:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.736764:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677473.736764:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.736764:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677473.736765:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.736765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677473.736766:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.736766:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.736767:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.736768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.736772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:31.0:1619677473.736776:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.736779:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.736780:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.736780:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:31.0:1619677473.736782:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.736782:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.736783:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.736785:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:20.0:1619677473.736785:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.736786:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.736788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677473.736963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677473.736970:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.736973:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677473.736973:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.736976:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.106@o2ib7 : GET - for me
00000400:00000200:26.0:1619677473.736978:0:19941:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.106@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:26.0:1619677473.736980:0:19941:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.106@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:26.0:1619677473.736982:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.736983:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.736984:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677473.736988:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : REPLY try# 0
00000800:00000200:26.0:1619677473.736990:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677473.736991:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.736992:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.736992:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.736994:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.736994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:9.2:1619677473.736999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.737005:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.737007:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.737008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.737008:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.737009:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: REPLY: OK
00000800:00000200:10.0:1619677473.737011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:31.0:1619677473.737173:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.737193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.737199:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.737202:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.737203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.737203:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.737204:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.737207:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.737439:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.737442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.737442:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.737444:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.737447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.737448:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.737449:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.737451:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.737452:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.737453:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.737456:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.737458:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.737461:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.737461:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.737462:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.737464:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.737464:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.737465:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.737466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.737478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.737481:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.737481:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.737483:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:4.2:1619677473.737484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.737486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.737487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.737488:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.737490:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.737492:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.737493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.737494:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.737495:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.737496:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.737498:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.737499:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.737500:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.737503:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.737505:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.737508:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.737508:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.737509:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.737511:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.737512:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.737512:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.737514:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.737514:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677473.737517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677473.737519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677473.737523:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.737523:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.737525:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:24.0:1619677473.737526:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677473.737526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.737527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677473.737527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.737528:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.737530:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.737533:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.737540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.737547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.737549:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.737550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.737550:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.737551:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.737551:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.737581:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.737584:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.737586:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.737588:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.737589:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.737590:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.737592:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.737593:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.737594:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.737596:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.737598:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.737600:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.737601:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.737602:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.737604:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.737604:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.737605:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.737605:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.737609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.737615:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.737617:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.737618:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.737619:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.737620:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.737622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.737925:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.737928:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.737929:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.737931:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.737934:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.737935:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677473.737936:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.737938:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.737939:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.737940:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677473.737943:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.737945:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677473.737948:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.737949:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677473.737950:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677473.737952:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677473.737952:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.737954:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.737954:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:31.0:1619677473.737971:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:4.2:1619677473.738208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.738216:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.738219:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.738219:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.738221:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.738224:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.738225:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.738227:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.738228:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.738229:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.738230:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.738234:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.738236:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.738239:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.738239:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.738240:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.738242:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.738242:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.738243:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.738244:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.738255:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.738258:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.738259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.738259:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.738261:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.738264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.738285:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.738288:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.738289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.738291:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.738299:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.738300:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.738301:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.738303:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.738304:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.738305:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.738308:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.738310:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.738312:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.738313:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.738313:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.738315:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.738316:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.738317:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.738318:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.738319:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.738329:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.738330:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.738331:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.738332:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.738332:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.738332:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.738333:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.738334:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.738335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.738336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.738449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.738454:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.738456:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.738458:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.738459:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.738461:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.738463:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.738464:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.738466:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.738466:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.738468:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.738471:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.738473:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.738475:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.738476:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.738477:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.738478:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.738479:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.738479:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.738480:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.738485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.738494:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.738497:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.738498:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.738498:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.738500:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.738503:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.738506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.738513:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.738516:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.738516:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.738517:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.738518:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.738518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.738555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.738563:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.738566:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.738567:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.738569:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.738572:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.738573:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.738575:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.738576:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.738577:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.738578:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.738581:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.738583:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.738586:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.738587:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.738588:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.738589:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.738590:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.738590:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.738591:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.738595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.738603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.738606:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.738606:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.738608:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.738609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.738612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677473.738857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.738862:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.738865:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.738867:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.738867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.738869:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.738870:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.738872:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.738873:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.738874:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.738875:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.738878:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.738880:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.738882:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.738883:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.738884:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.738886:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.738886:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.738887:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.738887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.738891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.738898:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.738901:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.738902:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.738902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.738905:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.738907:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.739122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.739130:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.739133:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.739134:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.739136:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.739138:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.739140:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.739141:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.739142:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.739143:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.739144:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.739148:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.739150:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.739153:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.739154:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.739154:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.739156:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.739156:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.739157:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.739157:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.739170:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.739173:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.739173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.739174:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.739176:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.739179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.739323:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.739326:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.739327:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.739328:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.739330:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.739332:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.739332:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.739334:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.739335:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.739336:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.739339:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.739340:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:4.2:1619677473.739340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:6.0:1619677473.739343:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.739344:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.739344:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.739346:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.739347:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.739347:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.739348:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:6.0:1619677473.739348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677473.739351:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.739352:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.739354:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.739356:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.739358:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.739359:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.739360:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.739360:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:26.0:1619677473.739362:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.739362:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.739362:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.739363:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677473.739365:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.739366:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677473.739367:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.739367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.739369:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.739372:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.739373:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.739373:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.739375:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.739375:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.739376:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.739377:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677473.739378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.739380:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.739381:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.739382:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.739383:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.739384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:13.0:1619677473.739397:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.739399:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.739400:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.739402:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.739404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.739406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677473.739714:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.739717:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.739719:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.739722:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.739723:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.739724:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.739726:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.739727:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.739728:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.739731:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.739732:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.739735:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.739735:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.739736:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.739738:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.739738:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.739739:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.739740:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.739743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.739751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.739753:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.739754:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.739754:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.739756:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.739758:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.740154:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.740157:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.740157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.740159:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.740162:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.740163:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.740164:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.740166:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.740167:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.740168:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.740171:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.740172:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.740175:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.740176:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.740176:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.740178:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.740179:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.740180:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.740180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.740192:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.740194:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.740195:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.740197:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.740197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.740199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.740499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.740528:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.740531:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.740531:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.740533:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.740535:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677473.740998:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.741001:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.741002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.741003:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.741006:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.741007:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.741008:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.741010:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.741011:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.741012:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.741015:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.741016:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.741019:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.741019:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.741020:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.741022:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.741022:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.741023:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.741024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.741035:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.741038:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.741038:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.741040:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.741041:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.741042:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.741049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677473.741054:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.741057:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.741059:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.741059:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.741062:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.741063:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677473.741064:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.741066:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.741067:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.741068:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677473.741070:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.741072:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677473.741075:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.741075:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677473.741076:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677473.741078:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677473.741078:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.741079:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.741079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677473.741084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.741093:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.741098:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.741099:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.741100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.741102:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.741106:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.741438:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.741440:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.741442:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.741444:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.741445:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.741447:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.741448:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.741449:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.741450:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.741455:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.741457:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.741469:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.741470:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.741471:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.741472:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.741473:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.741475:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (23)++
00000800:00000200:31.0:1619677473.741476:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (24)--
00000800:00000200:31.0:1619677473.741477:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.741479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.741487:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:31.0:1619677473.741488:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.741489:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.741490:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.741490:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:31.0:1619677473.741491:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.741491:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.741491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.741493:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:24.0:1619677473.741493:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.741495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677473.741495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.741496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.741503:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.741505:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.741506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.741506:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.741507:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.741508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.741536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.741543:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.741548:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.741549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.741552:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.741556:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.741558:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.741560:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.741563:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.741564:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.741566:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.741571:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.741574:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.741578:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.741579:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.741580:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.741582:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.741583:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.741585:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.741587:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.741587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.741588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.741593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677473.741597:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.741599:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677473.741600:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:17.0:1619677473.741600:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.741601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.741602:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.741604:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.741605:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677473.741605:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677473.741607:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.741608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.741616:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.741618:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.741619:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.741619:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.741620:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.741621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677473.741647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.741655:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.741658:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.741659:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.741661:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.741663:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.741665:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.741666:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.741668:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.741669:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.741670:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.741673:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.741675:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.741677:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.741678:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.741678:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.741680:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.741680:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.741681:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.741681:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.741686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.741692:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.741694:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.741695:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.741695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.741697:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.741699:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.741836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f391e00] (20)++
00000800:00000200:26.0:1619677473.741844:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f391e00] (21)++
00000800:00000200:26.0:1619677473.741847:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.1.59@o2ib3
00000800:00000200:10.0:1619677473.741847:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f391e00] (22)--
00000400:00000200:26.0:1619677473.741849:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.59@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.741851:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.741853:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.741854:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.741855:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.741856:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.741857:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.741860:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.59@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.741861:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 352 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.741864:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.741865:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677473.741865:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.741867:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.741867:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f391e00] (21)++
00000800:00000200:26.0:1619677473.741868:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f391e00] (22)--
00000800:00000200:26.0:1619677473.741868:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f391e00] (21)--
00000800:00000200:20.0:1619677473.741881:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.741883:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.741884:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.741885:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.741887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.741888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.742025:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677473.742035:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.742038:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.742039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.742041:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.742043:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.742044:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.742046:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.742046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677473.742047:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.742048:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.742049:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.742052:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.742052:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677473.742053:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677473.742055:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.742056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.742056:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677473.742056:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.742057:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.742057:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677473.742058:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677473.742058:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.742059:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.742059:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677473.742060:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677473.742060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.742061:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677473.742061:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:31.0:1619677473.742062:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677473.742062:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.742063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:9.2:1619677473.742066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.742072:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.742073:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:24.0:1619677473.742075:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:21.0:1619677473.742075:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.742075:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:24.0:1619677473.742076:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.742076:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.742076:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677473.742077:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677473.742077:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.742077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677473.742078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:21.0:1619677473.742078:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.742079:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.742080:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.742082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.742091:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.742094:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.742095:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.742095:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.742096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.742097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.742241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.742247:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.742249:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.742251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.742252:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.742255:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.742257:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.742258:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.742259:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.742260:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.742261:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.742264:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.742266:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.742268:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.742269:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.742270:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.742272:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.742272:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.742273:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.742274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.742278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.742285:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.742287:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.742288:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.742288:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.742289:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.742291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.742496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677473.742504:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677473.742507:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677473.742509:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.742512:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.742513:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677473.742515:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.742516:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.742517:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.742518:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677473.742521:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.742522:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677473.742525:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.742526:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:31.0:1619677473.742526:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677473.742528:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:31.0:1619677473.742529:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677473.742530:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:31.0:1619677473.742532:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:31.0:1619677473.742532:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:20.2:1619677473.742538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:12.0:1619677473.742542:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.742545:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.742545:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.742547:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.742549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.742550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:22.0:1619677473.742924:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.742926:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.742927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.742929:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.742931:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.742933:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.742934:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.742935:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.742936:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.742937:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.742941:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.742942:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.742945:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.742946:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.742946:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.742948:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.742949:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.742950:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.742950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.742961:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.742964:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.742964:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.742966:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.742967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.742968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.743377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.743385:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.743387:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.743388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.743390:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.743393:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.743394:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.743395:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.743396:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.743397:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.743398:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.743402:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.743404:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.743406:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.743407:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.743408:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.743410:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.743410:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.743411:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.743411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.743425:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.743427:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.743428:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.743428:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.743431:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.743433:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.743466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.743487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.743493:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.743496:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.743497:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.743497:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.743499:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.743501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.743779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.743782:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.743783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.743784:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.743787:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.743788:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.743789:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.743790:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.743791:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.743792:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.743795:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.743797:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.743799:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.743800:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.743800:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.743802:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.743803:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.743804:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.743804:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.743816:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.743818:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.743819:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.743820:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.743822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.743823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.743866:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.743869:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.743871:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.743873:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.743875:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.743876:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.743877:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.743878:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.743879:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.743883:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.743885:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.743887:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.743888:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.743889:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.743890:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.743891:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.743892:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.743892:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.743896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.743900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677473.743902:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.743905:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.743906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.743906:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.743907:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677473.743908:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.743910:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677473.743910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.743911:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.743913:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.743915:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.743917:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.743918:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.743919:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.743920:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.743921:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.743924:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.743926:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.743929:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.743930:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.743930:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.743932:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.743932:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.743933:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.743935:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.743935:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.743941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.743946:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.743948:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.743948:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.743949:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.743950:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677473.743950:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.743951:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.743952:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.743952:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.743955:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.743965:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.743967:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.743968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.743969:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.743970:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.743970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.744705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.744711:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.744714:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.744714:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.744716:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.744719:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.744720:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.744721:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.744723:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.744724:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.744725:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.744728:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.744730:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.744732:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.744733:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.744733:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.744735:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.744736:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.744736:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.744737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.744751:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.744753:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.744754:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.744754:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.744756:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.744759:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.744805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.744813:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.744815:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.744816:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.744818:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.744821:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.744822:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.744824:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.744825:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.744826:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.744827:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.744830:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.744832:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.744835:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.744835:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.744836:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.744838:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.744838:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.744839:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.744839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.744851:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.744854:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.744855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.744856:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.744858:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.744861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.744884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.744888:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.744890:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.744893:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.744894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.744896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.744898:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.744900:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.744901:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.744902:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.744904:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.744907:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.744909:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.744912:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.744913:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.744914:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.744916:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.744917:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.744918:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.744919:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.744920:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.744923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677473.744925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.744931:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.744933:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.744934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677473.744934:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677473.744935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677473.744939:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.744941:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.744942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.744945:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.744948:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677473.745331:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.745354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.745360:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.745363:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.745364:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.745364:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.745366:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.745368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.745477:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.745480:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677473.745481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677473.745483:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.745486:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.745487:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677473.745489:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.745490:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.745492:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.745492:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677473.745496:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.745498:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677473.745501:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.745502:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677473.745503:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677473.745505:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677473.745505:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.745506:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.745507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:31.0:1619677473.745518:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.745520:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.745521:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.745523:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.745526:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677473.745664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677473.745671:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.745674:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677473.745675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677473.745677:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.745680:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.745681:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677473.745682:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.745684:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.745685:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.745686:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.745689:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.745691:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677473.745693:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677473.745695:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.745696:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677473.745696:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.745698:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.745699:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677473.745699:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677473.745700:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:21.0:1619677473.745713:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.745715:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.745716:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.745718:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.745718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.745721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.745723:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.745725:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.745726:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.745728:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.745730:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.746004:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.746007:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.746008:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.746010:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.746013:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.746015:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.746017:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.746018:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.746020:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.746021:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.746024:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.746026:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.746028:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.746029:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.746030:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.746032:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.746032:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.746033:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.746034:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.746045:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.746047:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.746048:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.746050:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.746051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.746052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.746055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.746062:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.746065:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.746065:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:21.0:1619677473.746068:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:10.0:1619677473.746068:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.746070:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.746071:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.746071:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.0:1619677473.746072:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.746072:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.746073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677473.746073:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.746074:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677473.746074:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.746075:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.746076:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.746079:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.746081:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.746084:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.746085:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.746086:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.746088:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.746088:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.746089:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.746089:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.746096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.746105:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.746107:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.746108:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.746109:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.746110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.746112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.746122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.746131:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.746133:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.746134:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.746135:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.746136:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.746137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:31.0:1619677473.746200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.746229:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.746231:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.746232:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.746233:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.746235:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.746248:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.746250:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.746251:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.746252:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.746252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.746253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:17.0:1619677473.746336:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.746338:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677473.746339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677473.746341:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.746343:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.746344:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677473.746346:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.746347:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.746348:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.746349:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677473.746352:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.746353:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677473.746356:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.746356:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677473.746357:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.746359:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677473.746359:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677473.746360:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.746360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.746365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.746373:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.746375:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.746376:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.746378:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.746379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.746380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677473.746404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.746412:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.746415:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.746416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.746418:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.746420:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.746422:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.746423:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.746425:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.746425:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.746426:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.746430:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.746432:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.746434:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.746435:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.746435:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.746437:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.746438:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.746438:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.746439:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.746443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.746449:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.746452:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.746453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.746453:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.746455:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.746457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.746855:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.746858:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.746859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.746860:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.746863:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.746864:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.746865:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.746867:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.746868:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.746869:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.746872:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.746874:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.746876:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.746877:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.746878:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.746879:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.746880:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.746882:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.746883:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.746883:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.746894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.746895:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.746896:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.746896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.746897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.746897:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.746898:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.746900:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.746901:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.746902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.747370:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.747373:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.747375:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.747377:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.747378:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.747379:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.747381:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.747383:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.747384:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.747388:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.747391:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.747398:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.747399:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.747399:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.747401:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.747402:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.747403:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.747405:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.747405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.747407:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (23)--
00000800:00000200:9.2:1619677473.747407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:9.2:1619677473.747410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.747416:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.747417:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.747417:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.747418:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:31.0:1619677473.747419:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677473.747419:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.747419:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.747421:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.747421:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677473.747421:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.747421:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.747422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677473.747424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.747424:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.747425:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.747425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.747776:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.747779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.747779:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.747782:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.747784:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.747785:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.747786:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.747788:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.747789:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.747790:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.747793:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.747795:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.747797:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.747798:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.747799:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.747801:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.747801:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.747802:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.747803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.747814:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.747817:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.747817:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.747819:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.747820:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.747821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.748270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677473.748275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.0:1619677473.748277:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.748280:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.748281:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677473.748281:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:8.0:1619677473.748282:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.748283:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.748284:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:8.0:1619677473.748285:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.748286:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.748287:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.748288:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.748288:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.748290:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.748290:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.748292:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.748292:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.748299:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677473.748300:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.748302:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.748303:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.748306:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677473.748307:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.748308:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.748310:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.748312:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.748312:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.748313:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677473.748313:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.748314:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.748315:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.748316:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677473.748316:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.748316:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677473.748317:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677473.748317:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:8.0:1619677473.748318:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.748318:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677473.748318:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.748319:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.748323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.748326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677473.748331:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677473.748332:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677473.748334:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:18.0:1619677473.748334:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.748334:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677473.748335:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.748335:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:2.0:1619677473.748335:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:22.0:1619677473.748336:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.748336:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.748337:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.748337:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:22.0:1619677473.748338:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.748338:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677473.748339:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.748342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677473.748345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.748352:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.748355:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.748356:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.748356:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.748357:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.748358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.748677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.748685:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.748688:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.748689:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.748691:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.748693:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.748694:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.748696:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.748697:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.748698:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.748699:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.748702:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.748704:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.748707:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.748708:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.748708:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.748710:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.748710:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.748711:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.748711:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.748724:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.748727:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.748728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.748728:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.748731:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.748734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.748787:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.748790:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.748790:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.748793:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.748795:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.748796:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.748797:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.748798:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.748800:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.748801:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.748804:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.748805:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.748808:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.748809:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.748809:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.748811:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.748812:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.748812:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.748813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.748824:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.748827:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.748827:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.748829:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.748830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.748831:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.749079:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.749081:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.749083:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.749086:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.749087:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.749088:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.749089:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.749090:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.749091:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.749094:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.749096:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.749098:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.749099:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.749100:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.749101:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.749102:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.749103:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.749105:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.749105:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.749107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.749113:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.749116:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.749117:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.749117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.749119:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.749121:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.749128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.749134:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.749136:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.749137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.749137:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.749138:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.749139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.749373:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.749393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.749400:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.749402:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.749403:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.749403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.749404:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.749407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.749409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.749415:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.749417:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.749418:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.749418:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.749419:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.749420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.749424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.749431:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.749434:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.749434:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.749436:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.749438:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.749440:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.749441:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.749442:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.749444:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.749444:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.749448:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.749449:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.749452:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.749453:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.749454:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.749456:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.749456:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.749457:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.749458:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.749469:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.749472:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.749473:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.749474:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.749476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.749477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677473.749563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677473.749567:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.749570:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.749572:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.749573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.749574:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.749575:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.749576:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.749577:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.749578:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.749579:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.749582:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.749584:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.749586:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.749587:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.749588:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.749589:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.749590:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.749590:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.749591:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.749595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.749603:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.749605:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.749606:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.749608:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677473.749609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.749610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677473.749650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.749656:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.749659:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.749659:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.749662:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.749664:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.749666:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.749667:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.749668:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.749669:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.749670:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.749673:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.749675:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.749677:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.749678:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.749679:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.749681:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:1.0:1619677473.749681:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.749681:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.749682:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.749682:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.749684:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.749685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.749687:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.749689:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.749691:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.749693:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677473.749693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677473.749695:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:6.0:1619677473.749695:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.749696:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.749697:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.749697:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.749697:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.749699:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.749701:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677473.749701:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.749702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.749703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.749703:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.749704:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.749705:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677473.749706:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.749707:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:26.0:1619677473.749708:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.749708:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:26.0:1619677473.749710:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.749711:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677473.749712:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677473.749712:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677473.749713:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.749713:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.749713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:26.0:1619677473.749714:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.749715:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.749715:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:26.0:1619677473.749716:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.749717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.749717:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.749718:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000400:00000200:26.0:1619677473.749719:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.749719:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.749720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677473.749721:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.749724:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.749724:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677473.749725:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.749726:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677473.749726:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.749727:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.749728:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.749728:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.749729:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:20.0:1619677473.749729:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.749730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.749731:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.749731:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677473.749731:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.749737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677473.749741:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.749744:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677473.749744:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677473.749745:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.749745:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.749745:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.749747:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.749747:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.749747:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.749748:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.0:1619677473.749748:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.749749:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.749749:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.749750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.749750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.749751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.750323:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.750326:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.750329:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.750331:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.750332:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.750333:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.750335:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.750336:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.750336:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.750340:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.750341:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.750343:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.750343:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.750344:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.750345:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.750346:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.750347:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.750348:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.750365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677473.750447:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.750450:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677473.750451:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677473.750453:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.750457:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.750458:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677473.750459:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.750461:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.750462:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.750463:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677473.750466:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.750467:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.750470:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.750471:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677473.750471:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677473.750473:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677473.750474:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.750475:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677473.750475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:6.0:1619677473.750488:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.750490:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.750491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.750492:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.750494:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.750496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.750513:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.750516:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.750516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.750518:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.750520:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.750521:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.750522:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.750523:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.750525:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.750526:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.750529:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.750531:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.750533:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.750534:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.750535:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.750537:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.750537:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.750538:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.750539:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.750540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.750551:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.750552:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.750553:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.750553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.750554:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.750554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.750555:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.750557:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.750558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.750559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.750854:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.750856:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.750859:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.750861:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.750862:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.750864:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.750865:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.750866:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.750867:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.750870:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677473.750871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677473.750872:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.750874:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.750874:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677473.750875:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.750876:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.750877:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677473.750877:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677473.750878:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677473.750878:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.750879:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.750879:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677473.750879:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.750881:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.750882:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.750883:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.750884:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.750885:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.750886:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.750890:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.750891:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.750895:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.750896:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677473.750896:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.750896:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.750898:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.750898:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.750899:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.750900:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.750905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.750913:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.750916:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.750917:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.750917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.750919:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.750921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677473.751044:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.751047:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.751049:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.751051:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.751053:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.751054:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.751055:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.751057:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.751058:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.751061:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.751063:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.751065:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.751066:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.751066:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.751068:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.751068:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.751070:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.751070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.751074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.751080:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.751082:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.751083:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.751083:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.751085:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.751087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.751243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.751251:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.751253:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.751254:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.751256:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.751259:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.751260:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.751261:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.751262:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.751263:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.751264:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.751267:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.751269:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.751272:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.751273:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.751273:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.751275:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.751275:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.751276:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.751276:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.751281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.751290:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.751292:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.751293:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.751297:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677473.751300:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.751303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677473.751413:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.751415:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677473.751416:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677473.751418:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.751421:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.751422:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677473.751423:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.751425:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.751426:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.751427:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677473.751430:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.751431:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677473.751434:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.751434:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677473.751435:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677473.751437:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677473.751437:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.751438:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.751438:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677473.751444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.751452:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.751454:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.751455:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.751455:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.751458:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.751460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677473.751555:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.751558:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.751559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.751561:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.751563:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.751564:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.751566:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.751567:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.751568:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.751569:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.751572:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.751574:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.751577:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.751577:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.751578:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.751580:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.751580:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.751581:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.751582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.751593:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.751595:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.751596:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.751598:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.751599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.751600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.751610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.751618:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.751620:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.751621:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.751623:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.751626:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.751627:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.751629:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.751630:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.751631:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.751632:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.751635:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.751637:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.751640:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.751640:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.751641:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.751643:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.751643:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.751644:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.751644:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.751648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.751655:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.751657:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.751658:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.751658:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.751660:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.751662:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677473.752261:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b4c00] (21)++
00000800:00000200:17.0:1619677473.752264:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.68@o2ib3
00000800:00000200:1.0:1619677473.752264:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4c00] (22)--
00000400:00000200:17.0:1619677473.752267:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.68@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.752269:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.752271:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677473.752272:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.752273:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.752274:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.752275:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677473.752279:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.68@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.752281:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677473.752284:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.752284:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677473.752285:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677473.752287:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677473.752288:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b4c00] (21)++
00000800:00000200:17.0:1619677473.752288:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b4c00] (22)--
00000800:00000200:17.0:1619677473.752289:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b4c00] (21)--
00000800:00000200:21.0:1619677473.752303:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.752305:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.752306:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.752308:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.752308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.752310:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.752437:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.752439:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.752440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.752442:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.752444:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.752446:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.752447:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.752449:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.752450:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.752451:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.752454:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.752455:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.752458:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.752458:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.752459:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.752461:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.752462:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.752462:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.752463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.752475:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.752477:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.752478:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.752480:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.752481:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.752482:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.752493:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.752496:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.752496:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.752497:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.752498:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.752499:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.752691:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.752714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.752721:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.752723:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.752724:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.752724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.752726:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.752728:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.752786:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.752789:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.752789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.752791:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.752794:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.752796:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677473.752797:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.752798:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.752799:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.752800:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677473.752803:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.752805:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677473.752808:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.752809:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677473.752810:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677473.752812:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677473.752812:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.752814:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677473.752815:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677473.752815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:13.0:1619677473.752825:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:6.0:1619677473.752826:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:13.0:1619677473.752828:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:6.0:1619677473.752828:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677473.752829:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:13.0:1619677473.752829:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.752829:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:13.0:1619677473.752831:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.752833:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.752834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.752847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.752850:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.752851:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.752852:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.752852:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.752854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677473.753027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.753033:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.753036:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.753038:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.753038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.753040:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.753042:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.753043:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.753045:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.753046:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.753047:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.753050:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.753052:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.753054:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.753055:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.753055:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.753057:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.753058:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.753058:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.753059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.753071:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.753073:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.753074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.753075:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.753077:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.753079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.753090:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.753092:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.753093:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.753093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.753094:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.753095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.753267:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.753269:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.753270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.753272:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.753274:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.753276:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.753277:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.753278:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.753279:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.753280:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.753283:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.753284:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.753287:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.753287:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.753288:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.753290:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.753290:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.753291:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.753291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.753305:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.753308:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.753308:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.753310:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.753311:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.753312:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.753661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.753669:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.753671:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.753672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.753674:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.753677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.753679:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.753680:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.753681:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.753682:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.753683:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.753687:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.753688:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.753691:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.753692:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.753693:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.753694:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.753695:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.753696:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.753697:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.753698:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677473.753702:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677473.753704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677473.753705:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.753707:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.753709:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.753710:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:22.0:1619677473.753710:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677473.753711:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.753711:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677473.753712:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:31.0:1619677473.753713:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.753713:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677473.753713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677473.753714:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.753714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677473.753714:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.753716:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.753716:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677473.753716:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677473.753719:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.753721:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677473.753722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.753723:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.753724:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.753725:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.753726:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.753727:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.753729:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.753730:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.753731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.753732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.753739:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.753741:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.753742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.753742:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.753744:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.753746:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.753908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.753914:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.753916:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.753918:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.753919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.753921:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.753923:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677473.753924:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.753925:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.753926:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.753927:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677473.753931:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.753933:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.753935:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.753936:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677473.753937:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.753938:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.753939:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.753939:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677473.753940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:20.0:1619677473.753951:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.753953:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.753954:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.753956:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.753957:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.753958:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.754003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.754011:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.754014:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.754015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.754017:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.754019:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.754020:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.754021:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.754022:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.754023:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.754024:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.754027:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.754029:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.754032:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.754032:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.754033:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.754035:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.754035:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.754036:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.754038:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.754038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.754044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.754049:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.754051:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.754051:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.754052:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.754053:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677473.754053:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.754054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.754055:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.754055:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.754058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.754059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.754066:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.754069:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.754070:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.754071:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.754074:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.754076:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.754077:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.754078:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.754079:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.754080:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.754083:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.754084:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.754087:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.754087:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.754088:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.754089:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.754090:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.754091:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.754091:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.754103:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.754105:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.754106:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.754107:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.754109:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.754112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.754294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.754301:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.754304:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.754305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.754307:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.754310:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.754311:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.754312:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.754314:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.754315:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.754315:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.754319:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.754321:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.754323:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.754324:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:6.0:1619677473.754325:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677473.754325:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.754326:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.754327:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677473.754328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.754328:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:2.0:1619677473.754328:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.754329:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677473.754330:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.754332:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677473.754334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:6.0:1619677473.754334:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.754335:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.754337:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.754338:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.754339:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.754342:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.754342:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677473.754344:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677473.754345:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.754346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677473.754346:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677473.754346:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.754347:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.754348:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:1.0:1619677473.754348:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.754349:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.754350:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:1.0:1619677473.754351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.754352:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.754353:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.754353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.754364:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.754366:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.754366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.754366:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.754367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.754368:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.754369:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.754370:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.754372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.754373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.754411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.754417:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.754419:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.754420:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.754422:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.754424:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.754426:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.754427:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.754428:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.754429:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.754431:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.754434:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.754436:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.754438:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.754439:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.754440:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.754441:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.754442:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.754443:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.754444:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.754444:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.754450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.754455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.754455:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.754457:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.754457:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677473.754458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.754458:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.754458:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.754460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677473.754461:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.754463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.754496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.754518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.754524:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.754527:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.754527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.754528:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.754530:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.754531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.755022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.755029:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.755032:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.755033:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.755035:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.755037:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.755039:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677473.755040:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.755041:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.755042:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.755043:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677473.755046:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.755047:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677473.755049:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.755050:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677473.755050:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677473.755052:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677473.755052:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.755053:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.755054:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.755055:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:4.2:1619677473.755061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677473.755064:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.755067:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:10.0:1619677473.755067:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:31.0:1619677473.755068:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.755069:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.755069:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.755070:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.755071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677473.755072:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.755193:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.755196:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.755196:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677473.755198:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677473.755200:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.755202:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.755204:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.755206:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.755208:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.755208:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.755209:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.755214:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.755217:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.755220:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.755221:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.755221:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.755223:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.755224:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.755225:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.755225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677473.755234:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.755237:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.755237:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.755239:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.755239:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677473.755241:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.0:1619677473.755241:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.755242:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.755244:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.755245:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.755246:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.755257:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.755260:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.755260:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.755261:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.755262:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.755263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.755736:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.755739:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.755741:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.755743:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.755744:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.755745:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.755747:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.755748:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.755749:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.755753:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.755754:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.755756:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.755757:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.755757:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.755759:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.755760:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.755761:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.755761:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.755778:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677473.755808:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.755811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.755811:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677473.755814:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.755816:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.755817:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677473.755819:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.755820:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.755821:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.755822:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677473.755826:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.755828:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677473.755831:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.755832:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677473.755833:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677473.755835:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677473.755835:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.755836:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.755837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:31.0:1619677473.755847:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.755850:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.755851:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.755852:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.755855:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677473.755968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.755975:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.755977:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.755978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.755980:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.755983:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.755984:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677473.755985:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.755987:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.755988:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.755989:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677473.755992:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.755994:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677473.755996:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.755997:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:26.0:1619677473.755998:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677473.756000:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677473.756000:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.756001:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.756001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:13.0:1619677473.756013:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.756016:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.756017:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.756019:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.756020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.756022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677473.756033:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:22.0:1619677473.756036:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:12.0:1619677473.756037:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677473.756038:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:22.0:1619677473.756039:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:12.0:1619677473.756039:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:6.0:1619677473.756039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:12.0:1619677473.756040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.756041:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:22.0:1619677473.756042:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.756045:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.756047:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.756048:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.756049:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.756050:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.756052:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.756055:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.756056:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.756059:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.756060:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.756060:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.756062:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.756062:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.756063:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.756064:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.756075:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.756078:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.756078:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.756080:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.756081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.756082:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.756160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.756166:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.756169:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.756170:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.756172:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.756174:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.756175:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.756177:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.756178:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.756179:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.756180:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.756183:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.756185:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.756186:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.756187:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.756188:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.756189:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.756190:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.756191:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.756191:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.756196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.756203:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.756205:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.756206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.756206:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.756207:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.756209:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.756221:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.756224:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.756227:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.756229:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.756230:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.756231:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.756233:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.756234:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.756235:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.756238:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.756239:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.756242:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.756243:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.756243:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.756245:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.756246:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.756247:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.756249:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.756249:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.756250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.756258:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.756260:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.756261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.756261:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.756263:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.756265:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.756270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.756276:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.756278:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.756279:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.756279:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.756280:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.756281:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.756336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.756344:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.756346:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.756347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.756349:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.756352:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.756353:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.756354:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.756355:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.756356:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.756357:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.756360:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.756362:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.756364:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.756365:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.756366:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.756367:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.756368:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.756368:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.756369:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.756373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.756380:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.756382:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.756383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.756383:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.756385:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.756387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.756429:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.756432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.756432:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677473.756435:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.756437:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.756439:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677473.756440:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.756441:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.756442:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.756443:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677473.756447:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.756449:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.756451:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.756452:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677473.756453:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677473.756454:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677473.756455:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.756456:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.756456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677473.756460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.756468:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.756471:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.756471:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.756472:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.756474:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.756476:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677473.756496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.756503:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.756506:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.756507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.756509:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.756511:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.756512:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.756514:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.756515:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.756516:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.756517:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.756521:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.756522:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.756525:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.756526:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.756526:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.756528:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.756528:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.756530:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.756531:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.756531:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677473.756537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677473.756542:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677473.756544:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677473.756545:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677473.756545:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:12.0:1619677473.756546:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.756546:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.756547:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:12.0:1619677473.756549:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.756550:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.756552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677473.756831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.756836:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.756839:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.756841:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.756842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.756844:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.756845:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.756847:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.756848:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.756849:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.756850:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.756854:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.756855:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.756858:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.756858:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.756859:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.756860:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.756861:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.756862:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.756862:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.756867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.756875:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.756877:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.756878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.756879:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.756881:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.756883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.756919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.756923:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.756926:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.756928:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.756929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.756930:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.756931:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.756933:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.756934:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.756935:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.756936:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.756938:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.756940:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.756942:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.756943:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.756944:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.756946:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.756946:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.756947:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.756948:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.756959:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.756962:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.756963:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.756963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.756965:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.756967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.757078:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.757080:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.757081:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.757083:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.757085:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.757086:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.757087:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.757089:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.757090:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.757091:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.757094:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.757095:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.757098:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.757098:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.757099:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.757101:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.757102:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.757102:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.757103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.757114:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.757117:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.757117:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.757119:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.757120:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.757122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.757362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677473.757370:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.757373:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677473.757374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677473.757376:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.757379:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.757381:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677473.757382:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.757383:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.757384:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.757385:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677473.757389:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.757391:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677473.757393:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.757394:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677473.757394:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677473.757396:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677473.757396:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.757397:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677473.757398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677473.757402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.757409:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.757411:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.757412:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.757412:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.757414:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:26.0:1619677473.757417:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:1.0:1619677473.757627:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.757629:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.757630:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.757632:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.757634:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.757636:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677473.757637:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.757638:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.757640:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.757641:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677473.757644:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.757646:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677473.757649:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.757650:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677473.757651:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677473.757653:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677473.757653:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.757654:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677473.757655:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.757656:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677473.757659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.757665:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:17.0:1619677473.757667:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:10.0:1619677473.757667:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.757668:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.757668:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677473.757669:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:10.0:1619677473.757669:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.757670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:10.0:1619677473.757671:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.757674:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677473.757928:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.757931:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.757932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.757934:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.757936:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.757937:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.757939:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.757940:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.757941:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.757942:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.757945:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.757947:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.757949:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.757950:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.757951:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.757952:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.757953:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.757954:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.757955:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.757956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.757966:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.757967:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.757968:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677473.757969:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.757969:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.757970:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.757970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.757972:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.757973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.757974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.758379:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.758403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.758410:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.758413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.758413:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.758414:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.758415:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.758417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.758498:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.758501:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.758503:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.758505:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.758507:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677473.758508:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.758509:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.758510:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.758511:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677473.758514:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.758515:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677473.758518:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.758519:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677473.758520:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677473.758522:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677473.758522:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.758523:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.758523:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677473.758528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.758536:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.758538:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.758539:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.758541:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.758541:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.758544:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677473.758618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.758625:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.758628:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.758629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.758631:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.758634:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.758635:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.758636:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.758638:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.758639:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.758640:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.758643:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.758644:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.758647:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.758648:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.758649:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.758651:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.758651:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.758652:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.758652:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.758665:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.758668:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.758669:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.758669:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.758672:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.758675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.758684:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.758686:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.758687:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.758687:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.758689:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.758689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.758777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677473.758783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:26.0:1619677473.758785:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.758788:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.758789:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677473.758789:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677473.758790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677473.758791:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.758792:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:26.0:1619677473.758793:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.758793:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:18.0:1619677473.758793:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677473.758794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677473.758796:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.758797:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.758797:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.758797:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.758799:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.758799:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.758799:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.758800:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.758801:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677473.758801:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677473.758801:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677473.758803:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.758803:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.758805:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.758805:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.758806:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.758806:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.758807:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677473.758808:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677473.758813:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677473.758814:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.758815:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.758816:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677473.758817:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.758818:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.758819:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.758820:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.758820:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677473.758821:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.758821:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677473.758821:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677473.758822:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677473.758822:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677473.758822:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677473.758823:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677473.758823:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677473.758824:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677473.758824:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677473.758825:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.758825:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.758825:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677473.758825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677473.758826:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.758826:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.758826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677473.758827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677473.758828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677473.758837:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:1.0:1619677473.758837:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:20.0:1619677473.758840:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.758840:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.758840:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:6.0:1619677473.758841:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677473.758841:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.758842:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.0:1619677473.758843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.758843:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.758844:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:1.0:1619677473.758844:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.758845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.758845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:6.0:1619677473.758845:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.758846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:6.0:1619677473.758847:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.758851:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.758859:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.758861:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.758862:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.758863:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.758863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.758864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.758899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.758906:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.758909:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.758909:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.758912:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.758915:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.758916:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.758918:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.758919:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.758920:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.758921:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.758925:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.758927:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.758930:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.758931:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.758931:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.758933:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.758933:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.758934:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.758935:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.758947:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.758949:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.758950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.758951:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.758953:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.758956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.758966:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.758968:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.758969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.758969:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.758970:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.758971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.759185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.759190:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.759193:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.759194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.759196:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.759198:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.759200:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.759201:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.759203:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.759204:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.759205:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.759208:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.759210:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.759213:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.759213:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.759214:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.759216:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.759216:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.759217:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.759217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.759229:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.759232:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.759233:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.759233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.759235:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.759237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677473.759635:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.759638:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.759639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.759640:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.759643:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.759645:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.759646:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.759648:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.759649:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.759650:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.759653:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.759654:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.759656:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.759657:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.759657:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.759659:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.759659:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.759661:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677473.759662:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677473.759662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677473.759666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677473.759673:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677473.759675:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677473.759675:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677473.759675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677473.759676:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677473.759678:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.759678:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.759680:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.759681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.759682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.759686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.759694:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.759696:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.759697:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.759698:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.759699:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.759700:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677473.759792:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.759822:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.759824:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.759825:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.759826:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.759829:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677473.759830:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677473.759831:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.759832:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.759834:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.759836:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.759838:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.759839:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.759841:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.759842:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.759843:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.759846:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.759847:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.759850:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.759851:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.759852:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.759853:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.759854:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.759855:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.759855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.759867:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.759869:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.759870:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.759872:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.759872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.759874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.760094:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.760097:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.760100:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.760101:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.760103:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.760104:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.760105:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.760106:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.760107:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.760110:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.760112:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.760114:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.760115:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.760116:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.760117:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.760118:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.760119:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.760119:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.760123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.760129:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.760132:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.760132:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.760133:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.760135:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.760137:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.760138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.760141:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.760144:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.760146:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.760147:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.760148:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.760150:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.760151:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.760153:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.760154:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.760155:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.760158:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.760159:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.760162:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.760162:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.760163:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.760165:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.760165:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.760167:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.760168:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.760168:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677473.760173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677473.760179:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.760179:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677473.760181:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677473.760182:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.760182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:22.0:1619677473.760183:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.760183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.760183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.760185:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.760187:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.760381:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.760410:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.760413:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.760413:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.760415:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.760416:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677473.760674:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.760676:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.760677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.760679:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.760681:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.760682:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.760684:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.760685:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.760686:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.760687:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.760690:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.760692:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.760694:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.760695:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.760696:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.760697:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.760698:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.760699:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.760699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.760711:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.760713:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.760714:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.760716:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.760716:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.760718:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.760960:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.760962:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.760964:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.760966:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.760967:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.760968:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.760969:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.760970:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.760971:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.760974:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.760976:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.760978:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.760978:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.760979:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.760980:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.760981:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.760982:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.760983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.761000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.761511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.761514:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.761514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.761516:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.761519:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.761520:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.761521:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.761523:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.761524:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.761524:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.761527:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.761529:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.761531:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.761532:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.761533:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677473.761534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677473.761535:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.761535:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.761537:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.761538:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.761538:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.761542:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.761545:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.761546:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.761547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:26.0:1619677473.761548:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.761548:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:26.0:1619677473.761550:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.761550:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677473.761551:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.761551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677473.761552:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:6.0:1619677473.761552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:26.0:1619677473.761553:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.761553:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:26.0:1619677473.761554:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.761554:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:20.0:1619677473.761554:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.761555:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:26.0:1619677473.761556:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.761556:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.761557:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:21.0:1619677473.761558:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.761559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.761560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:21.0:1619677473.761561:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.761561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:26.0:1619677473.761562:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677473.761562:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:26.0:1619677473.761564:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:21.0:1619677473.761564:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.761565:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.761566:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.761567:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.761567:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677473.761568:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.761568:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.761570:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.761570:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.761571:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:21.0:1619677473.761571:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.761572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.0:1619677473.761573:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.761575:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677473.761575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677473.761576:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.761576:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.761578:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.761578:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.761579:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.761580:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677473.761583:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.2:1619677473.761584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677473.761585:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.761587:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.761588:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.761590:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.761591:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.761592:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.761594:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:9.2:1619677473.761594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:20.0:1619677473.761595:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.761597:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677473.761598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.761599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677473.761600:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.761603:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.761603:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.761604:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.761605:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.761606:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.762353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.762361:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.762363:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.762364:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.762366:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.762368:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.762369:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677473.762371:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.762372:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.762373:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.762374:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677473.762377:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.762379:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.762381:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.762382:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.762382:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.762383:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.762384:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.762385:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.762385:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.762390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.762396:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.762398:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.762398:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.762399:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.762400:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.762402:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677473.762477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.762483:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.762485:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.762487:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.762488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.762490:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.762491:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677473.762492:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.762493:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.762494:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.762495:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677473.762498:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.762500:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.762502:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.762503:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.762503:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.762505:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.762506:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.762506:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.762507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.762512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677473.762520:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677473.762520:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.762522:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:22.0:1619677473.762523:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:20.0:1619677473.762523:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.762524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.762525:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.762526:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.762526:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.762527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:22.0:1619677473.762528:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.762529:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.762531:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.762532:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.762533:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.762534:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.762537:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.762539:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.762541:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.762542:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.762543:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.762544:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.762545:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.762545:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.762546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.762557:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.762559:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.762560:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.762562:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.762562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.762564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.762575:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.762577:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.762578:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.762579:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.762580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.762581:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.763121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9196bf7b5200] (20)++
00000800:00000200:20.0:1619677473.763125:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9196bf7b5200] (21)++
00000800:00000200:20.0:1619677473.763128:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.5@o2ib3
00000400:00000200:20.0:1619677473.763130:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.5@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.763131:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9196bf7b5200] (22)--
00000400:00000200:20.0:1619677473.763132:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.763134:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:20.0:1619677473.763136:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.763137:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.763138:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.763139:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:20.0:1619677473.763142:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.5@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.763144:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:20.0:1619677473.763147:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.763148:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:20.0:1619677473.763148:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:20.0:1619677473.763151:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:20.0:1619677473.763151:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9196bf7b5200] (21)++
00000800:00000200:20.0:1619677473.763153:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9196bf7b5200] (22)++
00000800:00000200:20.0:1619677473.763154:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9196bf7b5200] (23)--
00000800:00000200:20.0:1619677473.763155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9196bf7b5200] (22)--
00000800:00000200:24.2:1619677473.763158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:4.2:1619677473.763160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9196bf7b5200] (21)++
00000800:00000200:24.0:1619677473.763161:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677473.763164:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677473.763165:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.763165:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:21.0:1619677473.763166:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9196bf7b5200] (22)++
00000400:00000200:24.0:1619677473.763167:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.763168:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9196bf7b5200] (23)--
00000800:00000200:21.0:1619677473.763169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9196bf7b5200] (22)--
00000800:00000200:21.0:1619677473.763169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9196bf7b5200] (21)--
00000800:00000200:24.0:1619677473.763170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:24.2:1619677473.763252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:8.0:1619677473.763255:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677473.763258:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677473.763259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677473.763261:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.15.5@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.54@o2ib7 : PUT - routed
00000800:00000200:8.0:1619677473.763262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:20.2:1619677473.763368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.763376:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677473.763377:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677473.763379:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677473.763380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.763380:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:21.0:1619677473.763381:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.763381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677473.763383:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.763384:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.763385:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.763386:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677473.763387:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677473.763388:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.763389:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.763390:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.763391:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.763392:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.763392:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.763393:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677473.763394:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677473.763398:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.763399:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.763401:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677473.763402:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.763403:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.763404:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677473.763404:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.763405:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677473.763405:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677473.763406:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677473.763406:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.763407:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.763407:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.763408:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677473.763408:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.763408:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.763409:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.763410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.2:1619677473.763413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.763422:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:1.0:1619677473.763422:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677473.763425:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:1.0:1619677473.763425:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:20.0:1619677473.763426:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.763426:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.763426:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.763428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.763428:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.763429:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.763431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.763432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.763433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.763441:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.763443:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.763444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.763445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.763445:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.763446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677473.763457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:4.2:1619677473.763470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.763478:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.763480:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.763481:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.763483:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.763484:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.763486:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.763487:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.763488:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.763489:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.763490:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677473.763491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:10.0:1619677473.763494:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.763495:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.763497:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:10.0:1619677473.763498:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.763499:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:10.0:1619677473.763499:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.763499:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:24.0:1619677473.763500:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.763500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677473.763501:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.763501:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000400:00000200:24.0:1619677473.763502:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.763502:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.763503:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:24.0:1619677473.763504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.763515:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.763518:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.763519:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.763519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.763522:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.763525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.763584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.763590:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.763593:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.763594:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.763596:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.763599:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.763600:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.763601:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.763603:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.763604:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.763604:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.763608:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.763609:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.763612:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.763613:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.763613:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.763615:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.763616:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.763616:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.763617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.763621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.763629:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.763632:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.763633:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.763634:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.763636:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.763639:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.763687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.763694:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.763697:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.763698:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.763700:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.763703:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.763704:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.763705:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.763706:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.763707:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.763708:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.763711:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.763712:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.763715:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.763716:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.763717:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.763718:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.763719:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.763719:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.763721:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.763721:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.763727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.763733:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.763734:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.763736:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677473.763737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.763737:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677473.763737:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.763738:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.763738:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.763740:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.763743:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.763752:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.763755:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.763756:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.763756:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.763757:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.763757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.763790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.763797:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.763800:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.763801:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.763802:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677473.763802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:10.0:1619677473.763806:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.763808:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.763809:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.763810:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.763811:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.763811:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.763812:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.763813:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.763815:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.763816:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.763818:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.763818:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.763820:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.763821:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.763821:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.763822:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:26.0:1619677473.763823:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.763823:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:26.0:1619677473.763824:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.763825:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.763825:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.763826:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.763827:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.763827:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677473.763828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.763829:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.763830:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.763833:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.763834:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.763834:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.763836:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.763836:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.763837:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.763837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:6.0:1619677473.763839:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.763841:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.763842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:9.2:1619677473.763842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:6.0:1619677473.763842:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.763845:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.763847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677473.763850:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.763852:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.763853:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.763854:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.763856:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.763859:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677473.763928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.763934:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.763937:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.763937:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.763939:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.763942:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.763943:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.763945:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.763946:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.763947:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.763948:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.763951:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.763953:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.763955:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.763956:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.763956:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.763958:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.763959:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.763959:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.763960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.763972:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.763975:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.763976:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.763976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.763978:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.763980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.763993:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.763996:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.763998:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.764001:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.764002:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.764004:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.764005:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.764006:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.764007:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.764011:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.764013:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.764015:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.764015:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.764016:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.764018:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.764018:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.764019:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677473.764019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:31.0:1619677473.764020:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677473.764025:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.764027:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.764029:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.764030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.764032:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.764033:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.764035:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.764036:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.764038:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677473.764039:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677473.764039:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.764042:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.764044:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.764047:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.764048:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.764048:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.764050:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.764051:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.764052:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.764053:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.764056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.764064:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.764066:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.764067:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.764067:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.764069:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.764072:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.764076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.764082:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.764084:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.764085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.764085:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.764086:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.764087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.764226:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.764229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.764229:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.764232:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.764234:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.764235:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.764236:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.764238:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.764239:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.764240:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.764243:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.764245:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.764247:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.764248:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.764249:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.764251:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.764251:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.764252:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.764253:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.764264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.764266:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.764267:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.764269:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.764270:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.764271:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.764685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.764693:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.764696:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.764697:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.764699:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.764701:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.764703:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.764704:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.764705:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.764706:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.764707:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.764710:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.764712:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.764715:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.764716:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:4.2:1619677473.764716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677473.764717:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.764718:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.764719:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.764720:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.764721:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.764721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:24.0:1619677473.764723:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.764726:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.764727:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.764728:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677473.764729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000400:00000200:24.0:1619677473.764731:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.764732:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.764733:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96507767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.764734:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.764734:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:24.0:1619677473.764735:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.764736:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:24.0:1619677473.764736:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.764737:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.764738:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:22.0:1619677473.764738:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.764738:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.764739:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:24.0:1619677473.764739:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.764740:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:22.0:1619677473.764740:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.764741:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.764743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677473.764744:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.764744:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.764745:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.764746:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.764747:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.764748:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.764748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.764753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.764761:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.764764:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.764764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.764765:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.764767:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.764770:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.764891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.764896:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.764899:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.764900:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.764902:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.764904:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.764905:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.764906:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.764907:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.764908:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.764909:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.764912:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.764914:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.764916:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.764916:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.764917:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.764919:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.764919:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.764920:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.764921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.764925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.764931:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.764933:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.764933:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.764935:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.764935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.764936:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.764942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.764949:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.764951:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.764952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.764953:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.764953:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.764954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.765104:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.765107:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.765109:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.765111:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.765113:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.765114:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.765115:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.765116:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.765117:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.765120:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.765122:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.765124:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.765125:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.765125:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.765127:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.765128:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.765129:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.765131:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.765131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.765145:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.765146:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.765146:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.765148:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677473.765260:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.765262:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.765263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.765265:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.765267:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.765268:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.765270:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.765271:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.765272:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.765273:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.765277:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.765279:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.765281:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.765282:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.765282:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.765284:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.765285:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.765286:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.765287:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.765288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.765299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.765300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.765301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.765301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.765302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.765302:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.765303:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.765305:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.765306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.765307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.765313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677473.765323:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.765327:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.765331:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.765331:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.765335:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.765337:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677473.765339:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.765342:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.765343:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.765345:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:13.0:1619677473.765351:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.765354:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677473.765357:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.765358:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:13.0:1619677473.765359:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:13.0:1619677473.765361:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:13.0:1619677473.765362:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.765363:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.765364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.765367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.765374:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.765377:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.765378:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.765378:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.765380:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.765383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.765386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.765391:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.765394:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.765394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.765395:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.765396:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.765396:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677473.765403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677473.765410:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.765413:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677473.765414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677473.765415:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.765419:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.765420:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677473.765422:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.765423:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.765424:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.765425:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677473.765428:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.765429:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677473.765432:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.765433:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677473.765434:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677473.765435:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677473.765436:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.765436:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.765437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677473.765441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.765450:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.765452:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.765453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.765454:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.765456:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.765459:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677473.765521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.765524:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.765526:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.765528:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.765530:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.765531:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.765533:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.765534:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.765534:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.765538:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.765539:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.765542:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.765543:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.765543:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.765545:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.765545:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.765547:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.765548:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.765549:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.765551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.765557:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.765559:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.765560:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.765560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.765562:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.765565:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.765627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.765631:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.765634:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.765636:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.765637:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.765638:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.765640:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.765641:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.765643:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.765644:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.765645:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.765648:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.765650:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.765652:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.765653:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.765654:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.765655:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.765656:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.765657:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.765658:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.765669:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.765672:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.765673:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.765673:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.765675:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.765677:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.765925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.765932:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.765935:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.765936:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.765938:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.765941:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.765942:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.765944:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.765945:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.765946:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.765947:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.765950:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.765952:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.765955:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.765956:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.765956:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.765958:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.765959:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.765960:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.765961:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.765962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.765964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.765967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677473.765973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.765975:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.765976:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.765977:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.765977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677473.765977:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.765977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.765979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677473.765980:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.765983:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677473.766060:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.766063:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.766063:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677473.766065:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.766068:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.766069:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677473.766070:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.766071:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.766073:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.766074:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677473.766077:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.766078:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.766080:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.766081:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.766082:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.766083:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.766083:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.766085:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677473.766086:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677473.766086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677473.766089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.766098:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.766098:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677473.766099:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.766099:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.766099:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677473.766100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677473.766101:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.766101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.766102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677473.766102:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:22.0:1619677473.766123:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.766126:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.766127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.766129:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.766131:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.766132:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.766134:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.766135:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.766136:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.766137:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.766140:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.766142:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.766144:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.766145:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.766146:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.766147:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.766148:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.766149:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.766149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.766166:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.766168:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.766169:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.766171:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.766171:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.766173:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.766175:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677473.766183:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.766185:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.766186:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.766187:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.766188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.766189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.766198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.766205:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.766208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.766208:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.766209:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.766210:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.766213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.766279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.766287:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.766290:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.766291:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.766292:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.766300:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.766302:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.766303:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.766304:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.766305:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.766306:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.766308:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.766310:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.766312:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.766313:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.766313:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.766315:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.766315:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.766316:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.766316:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.766321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.766328:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.766330:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.766331:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.766331:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.766332:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.766335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.766420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.766428:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.766431:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.766432:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.766433:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.766435:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.766437:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677473.766438:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.766439:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.766440:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.766440:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677473.766443:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.766446:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677473.766449:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.766449:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677473.766450:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.766451:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677473.766452:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.766453:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.766453:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.766458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.766466:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.766469:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.766470:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.766470:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.766472:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.766476:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.766984:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.766987:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.766988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.766990:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.766992:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.766993:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.766995:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.766996:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.766997:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.766998:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677473.766999:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.767002:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:6.0:1619677473.767002:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.767004:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:31.0:1619677473.767005:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.767006:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677473.767007:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.767007:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.767008:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677473.767009:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677473.767009:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677473.767010:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.767010:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.767011:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677473.767012:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.767012:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.767012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:31.0:1619677473.767014:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677473.767017:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.767019:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677473.767022:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.767023:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677473.767023:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.767024:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677473.767025:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677473.767026:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677473.767026:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677473.767027:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.767027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:20.0:1619677473.767027:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.767029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.767029:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.767031:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677473.767031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.767040:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.767042:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.767043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.767044:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.767046:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.767048:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677473.767305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.767311:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.767314:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.767314:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.767316:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.767318:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.767320:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.767321:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.767323:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.767324:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.767325:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.767328:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.767330:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.767333:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.767334:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.767335:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.767337:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.767337:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.767338:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.767339:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.767340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.767346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677473.767351:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677473.767352:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677473.767354:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:2.0:1619677473.767354:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.767355:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677473.767355:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.767355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:12.0:1619677473.767356:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.767358:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.767359:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677473.767557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.767563:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.767565:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.767568:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.767568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.767570:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.767572:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.767573:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.767575:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.767576:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.767576:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.767580:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.767582:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.767584:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.767585:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.767586:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.767587:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.767588:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.767589:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.767590:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.767591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.767594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677473.767597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.767604:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.767604:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.767605:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.767605:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677473.767606:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.767606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.767607:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.767608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.767609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677473.767610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677473.767647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.767669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.767676:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.767678:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.767679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.767679:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.767681:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.767683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.767988:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.767990:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.767991:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.767993:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.767995:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.767997:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.767998:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.767999:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.768000:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.768001:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.768004:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.768006:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.768008:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.768009:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.768009:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.768011:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.768012:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.768013:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.768013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.768024:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.768026:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.768027:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.768028:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.768030:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.768031:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.768145:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.768148:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.768150:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.768152:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.768153:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.768155:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.768156:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.768157:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.768158:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.768162:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.768163:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.768166:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.768166:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.768167:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.768169:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.768169:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.768170:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.768171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.768174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.768181:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.768183:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.768184:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.768184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.768186:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.768188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.768518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.768525:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.768528:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.768529:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.768531:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.768534:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.768536:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.768537:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.768538:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.768539:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.768540:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.768543:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.768545:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.768548:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.768548:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.768549:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.768551:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.768551:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.768552:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.768552:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.768566:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.768569:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.768569:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.768570:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.768572:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:4.2:1619677473.768574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677473.768575:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677473.768581:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.768584:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.768585:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.768587:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.768589:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.768591:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.768592:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.768593:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.768594:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.768595:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.768598:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.768599:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.768602:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.768603:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.768603:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.768605:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.768605:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.768606:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.768607:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.768608:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.768614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677473.768618:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.768620:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.768621:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.768622:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.768622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.768623:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.768624:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.768624:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.768624:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677473.768627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.768630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.768637:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.768639:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.768640:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.768641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677473.768642:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.768643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:17.0:1619677473.768643:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.768643:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.768644:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000400:00000200:17.0:1619677473.768645:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.768645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:17.0:1619677473.768646:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.768648:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.768649:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.768650:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.768651:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.768654:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.768656:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.768659:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.768660:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.768660:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.768662:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.768662:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.768663:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.768664:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.768675:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.768678:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.768679:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.768679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.768681:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.768684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.768700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677473.768726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.768735:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.768737:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.768738:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.768740:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.768740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.768743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.768748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.768756:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.768758:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.768759:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.768760:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.768760:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.768761:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.768839:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.768842:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.768843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.768845:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.768847:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.768848:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.768850:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.768851:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.768852:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.768853:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.768856:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.768858:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.768861:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.768861:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.768862:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.768864:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.768865:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.768866:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.768867:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.768867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.768877:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.768878:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.768879:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.768880:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.768880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.768880:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.768881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.768881:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.768883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.768883:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.768894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677473.768902:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677473.768905:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677473.768907:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.768910:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.768912:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677473.768913:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.768914:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.768915:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.768916:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677473.768920:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.768922:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677473.768924:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.768925:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677473.768925:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677473.768927:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.768927:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677473.768928:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677473.768929:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:31.0:1619677473.768945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:4.2:1619677473.769160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.769167:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.769170:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.769170:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.769173:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.769175:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.769177:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.769178:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.769179:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.769180:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.769181:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.769184:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.769185:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.769188:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.769189:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.769189:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.769191:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.769191:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.769192:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.769192:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.769204:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.769207:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.769207:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.769209:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.769211:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.769212:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677473.769480:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.769483:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.769485:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.769487:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.769488:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.769490:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.769491:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.769492:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.769493:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.769495:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.769497:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.769499:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.769500:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.769500:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.769502:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.769502:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.769503:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.769504:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.769508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.769515:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.769517:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.769518:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.769518:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.769521:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.769522:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.769643:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.769671:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.769673:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.769673:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.769675:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.769677:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.769679:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677473.769701:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.769704:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.769705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.2:1619677473.769705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:6.0:1619677473.769706:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.769709:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.769710:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.0:1619677473.769711:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677473.769711:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.769713:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677473.769713:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.769714:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.769715:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677473.769716:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.769716:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677473.769719:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677473.769721:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.769721:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:21.0:1619677473.769722:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677473.769723:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.769724:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.769724:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:21.0:1619677473.769725:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.769725:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677473.769726:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.769727:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677473.769727:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.769727:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.769728:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.769729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.769730:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.769732:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.769734:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.769735:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.769736:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.769738:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.769738:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.769739:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.769740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677473.769740:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.769743:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.769744:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677473.769744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:20.0:1619677473.769746:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.769747:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.769748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.769749:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.769751:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.769752:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.769754:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.769755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.769756:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677473.769757:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.769759:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.769760:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.769761:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.769761:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.769763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.770024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.770030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.770034:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.770034:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.770036:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.770039:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.770040:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.770042:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.770043:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.770044:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.770045:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.770048:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.770050:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.770053:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.770054:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.770054:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.770056:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.770056:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.770057:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.770057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.770070:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.770072:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.770073:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.770073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.770076:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.770079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.770089:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.770091:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.770092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.770092:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.770093:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.770094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.770149:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.770151:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.770152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.770154:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.770156:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.770158:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677473.770159:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.770161:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.770162:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.770163:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677473.770166:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.770168:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677473.770171:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.770172:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677473.770173:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677473.770174:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677473.770175:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.770175:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.770176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:31.0:1619677473.770193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.770521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.770524:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.770526:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.770528:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.770529:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.770530:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.770532:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.770533:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.770534:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.770537:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.770539:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.770541:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.770542:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.770542:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.770544:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.770545:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.770546:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.770548:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.770548:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.770551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.770558:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.770560:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.770561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.770561:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.770563:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.770565:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.770569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.770575:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.770577:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.770578:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.770578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.770579:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.770580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.770710:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.770712:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.770713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.770715:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.770718:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.770720:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.770721:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.770722:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.770724:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.770725:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.770728:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.770730:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.770732:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.770733:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.770734:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.770736:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.770736:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.770737:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.770738:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.770751:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.770754:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.770754:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.770756:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.770758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.770758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.771046:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.771073:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.771076:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.771077:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.771078:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.771080:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.771093:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.771096:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.771097:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.771097:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.771098:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.771099:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.771335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.771343:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.771346:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.771347:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.771348:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.771351:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.771352:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.771353:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.771355:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.771356:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.771357:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.771360:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.771361:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.771363:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.771364:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.771364:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.771365:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.771366:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.771367:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.771367:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.771372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.771378:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.771379:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.771380:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.771381:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.771382:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.771383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677473.771443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.771450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.771453:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.771454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.771456:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.771459:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.771460:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.771462:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.771463:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.771464:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.771466:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.771469:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.771471:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.771472:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.771473:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.771474:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.771476:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.771476:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.771477:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.771478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.771482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.771489:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.771491:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.771492:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.771492:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.771494:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.771495:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.771505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.771512:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.771515:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.771516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.771517:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.771519:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.771521:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.771522:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.771523:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.771524:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.771525:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.771528:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.771530:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.771532:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.771533:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.771534:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.771535:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.771536:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.771536:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.771537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.771541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.771547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.771550:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.771550:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.771551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.771552:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.771554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.771560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.771565:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.771568:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.771568:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.771568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.771569:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.771570:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.771576:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.771579:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677473.771580:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677473.771581:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.771583:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.771583:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.771583:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677473.771585:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677473.771586:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.771586:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.771588:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.771589:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.771589:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677473.771589:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.771590:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.771592:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.771593:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.771595:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.771595:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.771596:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677473.771598:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:22.0:1619677473.771600:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.771602:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.771602:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677473.771603:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677473.771603:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677473.771605:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677473.771605:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.771606:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.771606:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677473.771606:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.771606:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677473.771607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.771608:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.771609:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.771610:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.771610:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:9.2:1619677473.771611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.771619:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677473.771622:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677473.771622:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.771623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.771623:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.771625:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.771626:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.771626:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:21.0:1619677473.771628:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.771628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677473.771630:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677473.771631:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.771660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.771667:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.771670:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.771671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.771672:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.771673:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.771673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.771807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.771813:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.771815:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.771816:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.771818:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.771820:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.771821:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.771822:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.771824:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.771825:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677473.771826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:2.0:1619677473.771826:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.771830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677473.771830:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.771832:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:2.0:1619677473.771832:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.771835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.771835:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677473.771835:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.771836:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:20.0:1619677473.771837:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.771837:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:20.0:1619677473.771838:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:2.0:1619677473.771839:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.771839:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677473.771840:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677473.771840:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:20.0:1619677473.771841:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677473.771841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:20.0:1619677473.771842:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.771843:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677473.771847:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.771849:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677473.771852:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.771852:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:20.0:1619677473.771853:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (24)++
00000800:00000200:20.0:1619677473.771856:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (25)--
00000800:00000200:20.0:1619677473.771857:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677473.771858:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (24)--
00000800:00000200:20.0:1619677473.771858:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677473.771859:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677473.771860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:4.2:1619677473.771865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677473.771868:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.771870:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677473.771872:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677473.771873:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.771874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677473.771884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677473.771888:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677473.771889:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.771889:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677473.771890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.771892:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677473.771912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677473.771919:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.771922:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677473.771923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677473.771925:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.771927:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.771929:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677473.771930:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.771931:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.771933:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.771933:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677473.771937:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.771939:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677473.771942:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.771943:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677473.771943:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677473.771945:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677473.771946:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.771946:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.771947:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677473.771951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.771960:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.771963:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.771963:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.771964:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.771966:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.771969:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.771972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.771979:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.771982:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.771983:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.771983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.771984:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.771985:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.772049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.772056:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.772058:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.772059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.772061:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.772064:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.772065:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.772066:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.772068:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.772069:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.772070:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.772073:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.772074:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.772077:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.772078:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.772078:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.772080:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.772081:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.772082:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.772082:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.772086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.772094:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.772096:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.772098:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.772098:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.772100:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.772103:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.772447:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.772449:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.772450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.772452:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.772454:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.772456:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.772457:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.772459:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.772460:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.772461:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.772464:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.772465:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.772468:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.772469:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.772469:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.772471:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.772471:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.772473:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.772474:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.772474:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.772485:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.772486:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.772487:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.772487:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.772488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.772489:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.772489:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.772491:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.772492:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.772493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.772519:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.772522:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.772523:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.772524:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.772527:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.772528:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.772529:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.772531:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.772532:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.772533:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.772536:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.772538:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.772541:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.772541:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.772542:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.772543:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.772544:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.772545:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.772545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.772550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.772558:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.772560:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.772561:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.772563:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.772564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.772565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.772571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.772578:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.772583:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.772585:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.772586:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.772587:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.772588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677473.772596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.772619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.772626:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.772628:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.772629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.772629:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.772631:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.772634:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.772883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.772891:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.772893:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.772895:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.772896:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.772899:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.772900:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.772901:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.772902:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.772903:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.772904:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.772907:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.772909:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.772911:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.772912:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.772913:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.772914:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.772915:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.772915:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.772916:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.772929:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.772932:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.772933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.772933:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.772935:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.772938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.773255:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.773258:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.773260:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.773263:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.773264:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.773266:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.773267:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.773268:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.773269:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.773272:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.773274:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.773276:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.773277:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.773277:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.773279:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.773279:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.773280:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.773281:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.773285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.773291:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.773294:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.773299:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.773300:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.773302:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.773304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.773344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677473.773351:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677473.773352:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.773355:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.773356:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.773359:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.773361:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.773363:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.773364:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.773365:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.773366:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.773367:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.773370:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.773371:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677473.773372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.773374:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.773375:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.773375:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.773377:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.773378:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.773378:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677473.773379:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:26.0:1619677473.773380:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.773380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.773381:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.773382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.773382:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.773384:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.773386:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.773386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.773391:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.773392:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.773394:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.773395:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677473.773395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.773396:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.773396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.773397:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.773398:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.773402:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.773406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.773412:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.773415:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.773416:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.773419:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.773421:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.773422:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.773424:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.773425:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.773426:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.773427:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.773430:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.773432:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.773435:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.773435:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.773436:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.773437:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.773438:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.773439:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.773440:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.773441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.773446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677473.773451:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.773451:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.773454:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.773454:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.773454:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.773455:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677473.773455:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.773456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.773457:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.773461:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.773462:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.773463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.773463:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.773466:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.773468:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.773469:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.773470:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.773472:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.773473:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.2:1619677473.773474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7ca00] (17)++
00000400:00000200:6.0:1619677473.773474:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.773477:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (18)++
00000400:00000200:6.0:1619677473.773477:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.773479:0:19939:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff919e43e7ca00] (19)--
00000800:00000200:6.0:1619677473.773479:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.773480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (18)--
00000800:00000200:24.0:1619677473.773481:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (17)++
00000800:00000200:6.0:1619677473.773481:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.773482:0:19939:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff919e43e7ca00] (18)--
00000800:00000200:6.0:1619677473.773482:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677473.773483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (17)--
00000800:00000200:24.0:1619677473.773483:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (16)++
00000800:00000200:6.0:1619677473.773483:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677473.773485:0:19939:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff919e43e7ca00] (17)--
00000800:00000200:6.0:1619677473.773485:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.773485:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677473.773486:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (16)--
00000800:00000200:24.0:1619677473.773486:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (15)++
00000800:00000200:6.0:1619677473.773486:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677473.773487:0:19939:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff919e43e7ca00] (16)--
00000800:00000200:6.0:1619677473.773487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677473.773488:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (15)--
00000800:00000200:24.0:1619677473.773489:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (14)++
00000800:00000200:24.0:1619677473.773490:0:19939:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff919e43e7ca00] (15)--
00000800:00000200:24.0:1619677473.773490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (14)--
00000800:00000200:24.0:1619677473.773491:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (13)++
00000800:00000200:24.0:1619677473.773492:0:19939:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff919e43e7ca00] (14)--
00000800:00000200:24.0:1619677473.773492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (13)--
00000800:00000200:24.0:1619677473.773493:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (12)++
00000800:00000200:24.0:1619677473.773494:0:19939:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff919e43e7ca00] (13)--
00000800:00000200:24.0:1619677473.773494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (12)--
00000800:00000200:24.0:1619677473.773495:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (11)++
00000800:00000200:24.0:1619677473.773495:0:19939:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff919e43e7ca00] (12)--
00000800:00000200:24.0:1619677473.773496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (11)--
00000800:00000200:24.0:1619677473.773496:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7ca00] (10)++
00000800:00000200:24.0:1619677473.773497:0:19939:0:(o2iblnd_cb.c:158:kiblnd_drop_rx()) conn[ffff919e43e7ca00] (11)--
00000800:00000200:24.0:1619677473.773498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (10)--
00000800:00000200:20.0:1619677473.773498:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:24.0:1619677473.773499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7ca00] (9)--
00000800:00000200:20.0:1619677473.773500:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.773501:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.773503:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.773504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.773505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.773518:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.773521:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.773522:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.773522:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.773523:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.773524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.773788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677473.773795:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677473.773798:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677473.773800:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.773803:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.773804:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677473.773805:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.773807:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.773808:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.773809:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677473.773812:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.773814:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677473.773817:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.773818:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677473.773819:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677473.773820:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677473.773821:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677473.773822:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677473.773823:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677473.773827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.773835:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.773838:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.773839:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.773839:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.773841:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.773845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.773894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677473.773897:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:6.0:1619677473.773897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:22.0:1619677473.773900:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.773903:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.773905:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.773906:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.773908:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.773909:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.773910:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.773913:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.773915:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.773917:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.773918:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.773919:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.773921:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.773922:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677473.773923:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)--
00000800:00000200:22.0:1619677473.773924:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000800:00000200:21.0:1619677473.773935:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.773937:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.773938:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.773939:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.773940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.773942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.774001:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:6.0:1619677473.774004:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:22.0:1619677473.774005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:6.0:1619677473.774007:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.774009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.774011:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:6.0:1619677473.774012:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.774014:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.774015:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.774016:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677473.774019:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.774020:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677473.774023:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.774023:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677473.774024:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677473.774026:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677473.774026:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:6.0:1619677473.774027:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d5000] (22)++
00000800:00000200:6.0:1619677473.774028:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)--
00000800:00000200:6.0:1619677473.774029:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:9.2:1619677473.774032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.774038:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677473.774039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000800:00000200:8.0:1619677473.774040:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.774041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:22.0:1619677473.774041:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d5000] (23)--
00000800:00000200:22.0:1619677473.774042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:8.0:1619677473.774042:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.774043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000400:00000200:8.0:1619677473.774044:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.774046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.774116:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.774139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.774146:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.774148:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.774149:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.774149:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.774151:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677473.774151:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.774153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.774154:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.774154:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.774155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677473.774218:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:6.0:1619677473.774221:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:22.0:1619677473.774222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:6.0:1619677473.774223:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.774226:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.774228:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677473.774229:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.774231:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.774231:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.774232:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677473.774235:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.774237:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677473.774240:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.774241:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:6.0:1619677473.774242:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:6.0:1619677473.774244:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:6.0:1619677473.774244:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:6.0:1619677473.774245:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)--
00000800:00000200:6.0:1619677473.774246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000800:00000200:13.0:1619677473.774257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.774260:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.774261:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.774263:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.774265:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.774266:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:22.0:1619677473.774316:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677473.774318:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:6.0:1619677473.774318:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.774319:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677473.774321:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.774321:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.774325:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.774325:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.774326:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.774328:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.774329:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.774330:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.774331:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.774333:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677473.774334:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.774335:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.774336:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.774336:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.774336:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.774337:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677473.774340:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.774342:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.774343:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.774344:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.774344:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.774344:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.774345:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677473.774345:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.774346:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.774347:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.774347:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.774348:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)++
00000800:00000200:6.0:1619677473.774348:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.774349:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)--
00000800:00000200:22.0:1619677473.774349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:6.0:1619677473.774349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.774350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677473.774350:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677473.774351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:6.0:1619677473.774351:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000400:00000200:6.0:1619677473.774353:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.774355:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.774357:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677473.774358:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96507705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.774359:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.774360:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.774361:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:6.0:1619677473.774361:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.774363:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.774364:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.774364:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.774366:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677473.774366:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.774366:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:22.0:1619677473.774367:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.774368:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.774368:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.774369:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677473.774370:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.774370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.774370:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:6.0:1619677473.774370:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677473.774372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.774372:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:22.0:1619677473.774373:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000800:00000200:6.0:1619677473.774373:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)++
00000800:00000200:6.0:1619677473.774374:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)--
00000800:00000200:6.0:1619677473.774374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (23)--
00000800:00000200:22.0:1619677473.774375:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:6.0:1619677473.774375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.774376:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:22.0:1619677473.774377:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.774379:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677473.774379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:22.0:1619677473.774381:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677473.774382:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.774383:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.774384:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.774385:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677473.774387:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:22.0:1619677473.774389:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.774389:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:22.0:1619677473.774390:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:21.0:1619677473.774390:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.774391:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.774392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:22.0:1619677473.774393:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.774394:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677473.774394:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677473.774394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677473.774396:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677473.774397:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677473.774398:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d5000] (22)++
00000800:00000200:22.0:1619677473.774399:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)--
00000800:00000200:22.0:1619677473.774400:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:31.0:1619677473.774410:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:6.0:1619677473.774410:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000800:00000200:6.0:1619677473.774412:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d5000] (23)--
00000800:00000200:31.0:1619677473.774413:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:6.0:1619677473.774413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:22.0:1619677473.774414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000400:00000200:31.0:1619677473.774415:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.774416:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.774420:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677473.774426:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677473.774429:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:6.0:1619677473.774429:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000400:00000200:6.0:1619677473.774431:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.774434:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.774436:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677473.774437:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.774438:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.774439:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.774440:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677473.774443:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.774444:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677473.774447:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.774448:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677473.774448:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.774450:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.774451:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:6.0:1619677473.774451:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)--
00000800:00000200:6.0:1619677473.774452:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000800:00000200:9.2:1619677473.774456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.774462:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.774465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.774465:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.774466:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.774468:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.774470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.774475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.774480:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.774482:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:24.0:1619677473.774483:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.774484:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677473.774484:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:8.0:1619677473.774484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.774485:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.774485:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.774485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:22.0:1619677473.774487:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.774490:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.774491:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677473.774493:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.774494:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.774495:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.774496:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677473.774499:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.774500:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677473.774503:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.774504:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677473.774505:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.774507:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.774507:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000800:00000200:22.0:1619677473.774508:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)++
00000800:00000200:6.0:1619677473.774508:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:22.0:1619677473.774509:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (24)--
00000800:00000200:22.0:1619677473.774510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (23)--
00000400:00000200:6.0:1619677473.774510:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.774511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000800:00000200:22.0:1619677473.774512:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:9.2:1619677473.774512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:6.0:1619677473.774513:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.774514:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.774514:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677473.774515:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.774516:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.774517:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.774518:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677473.774519:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.774519:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:22.0:1619677473.774521:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677473.774521:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:6.0:1619677473.774521:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.774522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:22.0:1619677473.774522:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96507700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.774522:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.774522:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:22.0:1619677473.774524:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.774525:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.774526:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.774527:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:22.0:1619677473.774530:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.774532:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677473.774533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.774534:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.774534:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.774535:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677473.774535:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677473.774535:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.774536:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.774538:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.774538:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)++
00000800:00000200:6.0:1619677473.774538:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.774538:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)++
00000800:00000200:6.0:1619677473.774539:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (25)--
00000800:00000200:22.0:1619677473.774540:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d5000] (24)++
00000800:00000200:6.0:1619677473.774540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (24)--
00000800:00000200:22.0:1619677473.774541:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (24)--
00000800:00000200:6.0:1619677473.774541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (24)--
00000800:00000200:22.0:1619677473.774542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:9.2:1619677473.774543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.774550:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.774553:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.774553:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.774554:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.774554:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.774555:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000800:00000200:6.0:1619677473.774555:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:24.0:1619677473.774556:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.774556:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d5000] (23)--
00000400:00000200:6.0:1619677473.774556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.774557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:24.0:1619677473.774558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.774558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.774558:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.774559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000800:00000200:6.0:1619677473.774561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.774575:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.774577:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.774578:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.774578:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.774578:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.774579:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.774599:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:6.0:1619677473.774601:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:22.0:1619677473.774602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000400:00000200:6.0:1619677473.774603:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.774605:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.774606:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677473.774607:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.774609:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.774610:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.774611:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677473.774614:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.774615:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677473.774618:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.774619:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677473.774619:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677473.774621:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677473.774622:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)++
00000800:00000200:6.0:1619677473.774623:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)--
00000800:00000200:6.0:1619677473.774623:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:6.0:1619677473.774624:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677473.774626:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000800:00000200:6.0:1619677473.774626:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:22.0:1619677473.774627:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000800:00000200:9.2:1619677473.774627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:6.0:1619677473.774628:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.774630:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.774630:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.774631:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677473.774632:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.774634:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:6.0:1619677473.774634:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.774634:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.774635:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677473.774637:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.774637:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.774638:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:22.0:1619677473.774638:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.774638:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.774639:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.774640:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96507697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.774641:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.774641:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:22.0:1619677473.774642:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.774643:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.774644:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.774646:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677473.774648:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677473.774649:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.774649:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:22.0:1619677473.774650:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677473.774650:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:6.0:1619677473.774652:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:6.0:1619677473.774652:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)++
00000800:00000200:22.0:1619677473.774653:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.774653:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (24)--
00000800:00000200:22.0:1619677473.774654:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677473.774654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (23)--
00000800:00000200:22.0:1619677473.774655:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677473.774655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:22.0:1619677473.774657:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677473.774658:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677473.774658:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (22)--
00000800:00000200:22.0:1619677473.774658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000800:00000200:8.2:1619677473.774663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:12.0:1619677473.774665:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.774668:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.774669:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.774671:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.774672:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:13.0:1619677473.774673:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.774673:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:1.0:1619677473.774674:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.774675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.774675:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.774678:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.774680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677473.774858:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.774882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.774888:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.774890:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.774891:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.774891:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.774893:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.774895:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.775037:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)++
00000800:00000200:22.0:1619677473.775040:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:6.0:1619677473.775040:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.22@o2ib3
00000400:00000200:6.0:1619677473.775042:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.22@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.775045:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.775047:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677473.775048:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.775049:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.775050:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.775051:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677473.775054:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.775055:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677473.775058:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.775059:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:6.0:1619677473.775059:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677473.775061:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677473.775062:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d5000] (21)++
00000800:00000200:6.0:1619677473.775063:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d5000] (22)++
00000800:00000200:6.0:1619677473.775064:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d5000] (23)--
00000800:00000200:6.0:1619677473.775065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:8.2:1619677473.775067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677473.775074:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)++
00000800:00000200:20.0:1619677473.775075:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:22.0:1619677473.775076:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d5000] (23)--
00000800:00000200:22.0:1619677473.775077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (22)--
00000800:00000200:20.0:1619677473.775077:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:22.0:1619677473.775078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d5000] (21)--
00000400:00000200:20.0:1619677473.775078:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.775079:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.775081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.775082:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.775189:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.775192:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.775193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.775194:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.775197:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.775198:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.775199:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.775201:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.775202:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.775202:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.775205:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.775207:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.775209:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.775209:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.775210:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.775212:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.775212:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.775213:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.775213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.775225:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.775227:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.775228:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.775230:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.775231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.775232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.775313:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.775316:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.775318:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.775320:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.775322:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.775323:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.775325:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.775325:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.775327:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.775330:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.775332:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.775334:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.775335:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.775336:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.775337:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.775338:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.775339:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.775339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.775357:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677473.775483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.775491:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.775494:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.775495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.775496:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.775499:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.775501:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.775502:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.775503:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.775504:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.775505:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.775508:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.775510:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.775513:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.775514:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.775514:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.775516:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.775516:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.775517:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.775517:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.775531:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.775533:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.775534:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.775535:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.775537:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.775540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.775809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.775812:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.775815:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.775817:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.775818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.775819:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.775821:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.775822:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.775823:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.775824:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.775826:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.775829:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.775831:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.775833:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.775834:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.775834:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.775836:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.775837:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.775837:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.775838:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.775842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.775850:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.775853:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.775854:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.775854:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.775856:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.775858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.776208:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.776211:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.776212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.776214:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.776216:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.776217:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.776218:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.776220:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.776221:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.776222:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.776225:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.776227:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.776229:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.776230:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.776230:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.776232:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.776233:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.776234:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.776235:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.776235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.776246:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:4.2:1619677473.776246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677473.776248:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.776248:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.776249:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.776250:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.776250:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.776251:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.776252:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.776253:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677473.776253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.776255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.776256:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.776257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.776259:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.776261:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.776263:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.776264:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677473.776264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:26.0:1619677473.776265:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.776266:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.776267:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677473.776269:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677473.776270:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677473.776271:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.776271:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.776272:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677473.776273:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677473.776273:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:18.0:1619677473.776273:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:20.0:1619677473.776274:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677473.776274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677473.776275:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.776275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677473.776276:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.776277:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677473.776277:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677473.776278:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.776279:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677473.776279:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.776280:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.776280:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677473.776281:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.776282:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.776284:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677473.776284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677473.776285:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.776286:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.776289:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.776290:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677473.776291:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677473.776294:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.776298:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.776300:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.776301:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.776302:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:8.0:1619677473.776302:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.776303:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.776305:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.776305:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677473.776305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677473.776306:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.776307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.776320:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.776322:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.776323:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.776325:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.776327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.776327:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.776341:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.776343:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.776344:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.776345:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.776345:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.776346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677473.776883:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.776886:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.776888:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.776890:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.776892:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.776893:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.776894:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.776895:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.776896:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.776899:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.776900:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.776903:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.776904:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.776904:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.776906:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.776906:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.776908:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.776909:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.776910:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.776912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.776918:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.776921:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.776922:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.776922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.776924:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.776926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.776933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.776939:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.776942:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.776942:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.776943:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.776943:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.776944:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.777003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677473.777010:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.777013:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.777015:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.777017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.777017:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.777019:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677473.777020:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.777022:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.777022:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.777023:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677473.777024:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.777026:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.777027:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:12.0:1619677473.777027:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.777029:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:6.0:1619677473.777029:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.777031:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.777031:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677473.777032:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:6.0:1619677473.777032:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.777033:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:12.0:1619677473.777034:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677473.777034:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000400:00000200:6.0:1619677473.777034:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677473.777035:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000400:00000200:6.0:1619677473.777036:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677473.777037:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000400:00000200:6.0:1619677473.777037:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677473.777038:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:6.0:1619677473.777038:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.777041:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.777043:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:4.2:1619677473.777043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:6.0:1619677473.777046:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.777046:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.777047:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.777049:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.777049:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.777050:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:13.0:1619677473.777051:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:6.0:1619677473.777051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677473.777052:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677473.777053:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677473.777054:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.777055:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:21.0:1619677473.777062:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.777065:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.777065:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.777067:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.777068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.777069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.777203:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.777205:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.777206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.777208:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.777211:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.777212:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.777213:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.777215:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.777216:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.777217:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.777220:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.777222:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.777225:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.777225:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.777226:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.777228:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.777228:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.777229:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.777229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677473.777241:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.777244:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.777245:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.777245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.777247:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.777250:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.777261:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.777263:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.777264:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.777264:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.777265:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.777266:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.777591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677473.777599:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677473.777602:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677473.777603:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677473.777605:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.777607:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.777608:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.777610:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.777611:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.777612:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.777613:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.777617:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.777618:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.777621:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.777622:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.777623:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.777625:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.777625:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677473.777626:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (22)++
00000800:00000200:10.0:1619677473.777627:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:10.0:1619677473.777628:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:4.2:1619677473.777632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677473.777637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (21)++
00000800:00000200:20.0:1619677473.777641:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677473.777642:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677473.777643:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:26.0:1619677473.777644:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:20.0:1619677473.777645:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.777646:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.777646:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.777646:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000800:00000200:21.0:1619677473.777647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.777647:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000800:00000200:10.0:1619677473.777648:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:20.0:1619677473.777649:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.777649:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.777651:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.777651:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000400:00000200:26.0:1619677473.777653:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.777654:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.777655:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.777656:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.777657:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.777660:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.777662:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.777665:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.777665:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.777666:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.777668:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.777668:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.777669:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.777669:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.777682:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.777685:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.777686:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.777686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.777688:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.777692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.777759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.777766:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.777769:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.777770:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.777772:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.777774:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.777775:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.777776:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.777778:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.777779:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.777780:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.777783:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.777784:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.777786:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.777787:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.777787:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.777789:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.777789:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.777790:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.777790:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.777801:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.777804:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.777805:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.777806:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.777808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.777809:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677473.777842:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.777845:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.777847:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.777849:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.777850:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.777851:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.777853:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.777854:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.777854:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.777857:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.777859:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.777861:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.777862:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.777862:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.777864:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.777864:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.777865:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.777866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.777869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.777876:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677473.777877:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677473.777879:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677473.777879:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:24.0:1619677473.777880:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.777880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677473.777880:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:6.0:1619677473.777883:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.777884:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.777885:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.777886:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:6.0:1619677473.777887:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:9.2:1619677473.777888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:6.0:1619677473.777888:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.777889:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.777890:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.777891:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677473.777894:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:6.0:1619677473.777895:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.777896:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.777897:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.777897:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677473.777897:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677473.777898:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.777899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.777900:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.777900:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.777901:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.777903:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.777903:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.777904:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.777905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.777916:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.777918:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.777919:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.777921:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.777922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.777923:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.778141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.778147:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.778150:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.778151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.778153:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.778155:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.778157:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.778158:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.778159:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.778161:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.778161:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.778165:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.778166:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.778169:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.778169:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.778170:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.778172:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.778172:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.778173:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.778173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677473.778181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677473.778186:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.778188:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.778188:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.778189:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.778189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.778191:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:22.0:1619677473.778192:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.778192:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.778194:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.778195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:26.0:1619677473.778197:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.778198:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.778199:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.778201:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.778202:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.778203:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.778206:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.778208:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.778211:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.778212:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.778212:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.778214:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.778214:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.778215:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.778216:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.778227:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.778230:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.778231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.778231:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.778234:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.778237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.778297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.778304:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.778307:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.778308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.778310:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.778313:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.778314:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.778316:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.778317:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.778318:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.778319:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.778322:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.778324:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.778327:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.778328:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.778328:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.778330:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.778330:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.778332:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.778333:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.778333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.778337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.778339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677473.778345:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.778347:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.778347:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.778349:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.778349:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:2.0:1619677473.778349:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.778350:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.778350:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677473.778351:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.778355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.778629:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.778650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.778656:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.778658:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.778659:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.778659:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.778661:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.778663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.778778:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.778781:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.778783:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.778785:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.778786:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.778787:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.778788:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.778789:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.778790:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.778793:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.778795:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.778797:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.778798:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.778798:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.778800:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.778800:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.778801:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.778802:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.778805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.778812:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.778814:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.778815:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.778815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.778817:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.778819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.778899:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.778902:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.778902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.778904:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.778907:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.778908:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.778910:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.778911:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.778912:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.778913:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.778916:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.778917:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.778920:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.778921:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.778921:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.778923:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.778924:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.778925:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.778925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.778937:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.778939:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.778940:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.778942:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.778943:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.778944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.778957:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.778959:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.778960:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.778961:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.778961:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.778962:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.779149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677473.779163:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:20.2:1619677473.779164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.779170:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.779173:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.779174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.779177:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.779180:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.779182:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.779183:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.779185:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.779186:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.779187:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:9.2:1619677473.779189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677473.779190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677473.779190:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.779194:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677473.779195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:2.0:1619677473.779197:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.779198:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.779199:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.779199:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.779200:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:2.0:1619677473.779200:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.779202:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677473.779203:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:18.0:1619677473.779203:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:2.0:1619677473.779203:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.779204:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677473.779204:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:18.0:1619677473.779204:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:31.0:1619677473.779205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:18.0:1619677473.779205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:18.0:1619677473.779207:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.779208:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.779209:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.779209:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.779210:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.779210:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.779211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:12.0:1619677473.779216:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.779219:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.779220:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.779221:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.779223:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.779224:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677473.779227:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.779228:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.779229:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.779230:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.779232:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.779234:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677473.779781:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.779784:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.779785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.779787:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.779789:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.779790:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.779792:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.779793:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.779794:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.779795:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.779798:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.779800:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.779803:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.779803:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.779804:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.779806:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.779806:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.779808:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.779809:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.779809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.779820:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.779822:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.779822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.779823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.779824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.779825:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.779825:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.779827:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.779828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.779829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.779907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.779913:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.779915:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.779916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.779918:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.779921:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.779922:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.779924:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.779925:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.779926:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.779927:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.779930:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.779931:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.779934:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.779934:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.779935:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.779936:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.779937:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.779939:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.779940:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.779940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.779943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.779946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.779951:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.779953:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.779953:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.779953:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.779954:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.779954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.779955:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.779955:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.779956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.779957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.779963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.779970:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.779972:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.779973:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.779973:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.779973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.779974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.780164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.780172:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.780174:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.780175:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.780177:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.780180:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.780181:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.780183:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.780184:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.780185:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.780186:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.780189:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.780190:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.780193:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.780194:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.780194:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.780196:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.780197:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.780197:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.780198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.780210:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677473.780212:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677473.780212:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.780213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.780214:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.780217:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.780220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.780240:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.780243:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.780244:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.780245:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.780248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.780425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677473.780433:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.780435:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.780437:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.780439:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.780440:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:12.0:1619677473.780440:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.780442:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.780443:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.780444:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.780445:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677473.780448:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.780449:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677473.780451:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.780452:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677473.780453:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677473.780454:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677473.780455:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.780456:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.780457:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:31.0:1619677473.780468:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.780471:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.780472:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.780473:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.780475:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677473.780671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.780674:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.780675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.780677:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.780679:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.780680:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.780682:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.780683:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.780684:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.780685:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.780688:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.780689:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.780692:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.780693:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.780693:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.780695:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.780695:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.780696:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.780697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.780708:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.780710:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.780711:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.780713:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.780714:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.780715:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.780767:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.780770:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.780773:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.780774:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.780776:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.780777:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.780778:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.780779:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.780780:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.780783:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.780785:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.780787:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.780788:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.780789:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.780790:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.780791:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.780792:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.780792:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.780796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.780803:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.780805:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.780806:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.780806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.780808:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.780810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.780874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.780879:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.780882:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.780884:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.780884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.780886:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.780888:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.780889:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.780890:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.780891:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.780892:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.780895:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.780897:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.780899:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.780900:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.780901:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.780903:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.780903:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.780904:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677473.780905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677473.780916:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.780919:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.780920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.780920:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.780922:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.780925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.780996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677473.781003:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.781005:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677473.781006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677473.781008:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.781011:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.781012:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677473.781013:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.781015:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.781016:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.781016:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677473.781019:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.781021:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677473.781024:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.781025:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677473.781025:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677473.781027:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677473.781028:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.781028:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.781028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677473.781033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.781042:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.781045:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.781046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.781046:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.781048:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.781051:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.781460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.781468:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.781471:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.781472:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.781473:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.781476:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.781478:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.781479:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.781480:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.781481:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.781482:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.781486:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.781487:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.781489:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.781490:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.781490:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.781491:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.781492:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.781493:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.781493:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.781498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.781505:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.781507:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.781508:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.781508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.781510:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.781513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.781707:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.781710:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.781710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.781712:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.781714:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.781715:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.2:1619677473.781716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677473.781717:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.781718:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.781719:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.781719:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677473.781720:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.781722:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:22.0:1619677473.781724:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.781725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677473.781726:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:20.0:1619677473.781727:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.781728:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.781729:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:20.0:1619677473.781729:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.781730:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.781731:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.781731:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:22.0:1619677473.781732:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:20.0:1619677473.781732:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.781733:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.781733:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.781734:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.781734:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.781735:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.781739:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.781740:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.781743:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.781744:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.781744:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.781744:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.781745:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.781746:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.781746:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.781747:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:21.0:1619677473.781748:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.781748:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.781748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677473.781750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.781750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.2:1619677473.781752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.781761:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.781763:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.781764:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.781764:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.781766:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.781768:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.781772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.781779:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.781782:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.781782:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.781782:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.781783:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.781784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677473.782156:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.782158:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.782159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.782161:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.782163:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.782165:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677473.782166:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.782167:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.782168:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.782169:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.782172:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.782174:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.782176:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.782177:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.782178:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.782180:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.782180:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.782180:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.782181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677473.782193:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.782196:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.782197:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.782197:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.782199:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.782202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.782213:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.782216:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.782216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.782217:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.782218:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.782218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.782333:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.782355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.782362:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.782364:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.782365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.782366:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.782368:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.782370:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.782394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.782400:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.782403:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.782403:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.782406:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.782408:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.782409:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.782411:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.782412:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.782413:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.782414:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.782417:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.782419:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.782421:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.782422:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.782423:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.782424:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.782425:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.782426:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.782426:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.782438:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.782441:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.782442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.782442:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.782444:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.782446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.782464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.782472:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.782475:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.782475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.782477:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.782480:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.782481:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.782482:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.782484:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.782485:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.782486:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.782489:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.782490:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.782493:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.782494:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.782495:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.782496:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.782497:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.782498:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.782498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.782510:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.782512:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.782513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.782514:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.782516:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.782518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.782587:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.782590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.782590:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.782592:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.782594:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.782596:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.782597:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.782598:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.782599:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.782600:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.782603:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.782605:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.782607:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.782608:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.782609:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.782610:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.782611:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.782612:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.782612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.782624:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.782626:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.782627:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.782628:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.782630:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.782631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.782639:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.782642:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.782643:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.782644:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.782644:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.782646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.782870:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.782873:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.782874:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.782874:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.782875:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.782876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.782967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.782974:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.782977:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.782978:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.782980:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.782982:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.782984:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.782985:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.782986:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677473.782987:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677473.782987:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.782988:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.782990:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.782993:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.782993:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677473.782995:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.782995:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.782997:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.782998:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.782998:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677473.782999:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.782999:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:31.0:1619677473.783000:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.783000:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677473.783001:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.783001:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.783002:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.783003:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:31.0:1619677473.783004:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.783004:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.783005:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677473.783006:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.783008:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.783009:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.783009:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677473.783010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677473.783011:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.783011:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.783012:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.783013:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677473.783015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.783017:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.783018:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.783018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.783019:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.783020:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.783020:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.783021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.783021:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.783024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.783029:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677473.783088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.783094:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.783096:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.783098:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.783099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.783101:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.783102:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.783104:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.783105:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.783106:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.783107:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.783110:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.783111:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.783114:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.783115:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.783115:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.783117:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.783117:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.783118:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.783119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.783130:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.783133:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.783134:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.783134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.783136:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.783138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.783465:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.783468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.783468:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.783470:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.783472:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.783473:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.783474:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.783475:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.783476:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.783477:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.783480:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.783481:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.783484:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.783485:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.783485:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.783487:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.783488:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.783489:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.783490:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.783490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.783502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.783502:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.783504:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677473.783504:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.783505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.783505:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.783506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.783507:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.783508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.783509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.783591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677473.783596:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.783598:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.783600:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.783601:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.783603:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.783604:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.783605:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.783607:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.783608:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.783609:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.783612:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.783614:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.783616:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.783617:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.783618:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.783619:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.783620:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.783620:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.783621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.783626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.783633:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.783635:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.783636:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.783638:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677473.783639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.783640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677473.783692:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.783695:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677473.783696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677473.783698:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.783700:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.783702:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677473.783703:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.783705:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.783706:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.783707:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677473.783711:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.783712:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677473.783716:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.783717:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677473.783718:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677473.783720:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677473.783720:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.783721:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.783722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:31.0:1619677473.783733:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.783736:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.783737:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.783739:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.783742:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677473.783791:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.783819:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.783821:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.783822:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.783823:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.783825:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.2:1619677473.784085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677473.784088:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677473.784091:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.784091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677473.784094:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.54@o2ib7 : PUT - routed
00000800:00000200:24.0:1619677473.784095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677473.784179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677473.784187:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677473.784189:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677473.784190:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677473.784192:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.784195:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.784196:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.784197:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.784198:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.784199:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.784200:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.784203:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.784205:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 568 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.784208:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.784209:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.784209:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.784211:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.784212:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677473.784212:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677473.784213:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:21.0:1619677473.784225:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.784228:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.784228:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.784230:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.784230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.784233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.784259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.784266:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.784268:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.784269:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.784271:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.784273:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.784275:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.784276:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.784278:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.784279:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.784279:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.784282:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.784284:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.784287:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.784288:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677473.784288:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.784288:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677473.784290:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:8.0:1619677473.784290:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.784290:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.784291:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.784291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677473.784292:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677473.784297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:31.0:1619677473.784300:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.784301:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.784303:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.784304:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.784305:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.784306:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677473.784306:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:31.0:1619677473.784308:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.784308:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:31.0:1619677473.784310:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677473.784310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.784310:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.784312:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677473.784312:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.784313:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.784314:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.784315:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677473.784315:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.784316:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.784317:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.784319:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.784320:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.784321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.784328:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.784330:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.784331:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.784331:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.784333:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.784336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.784340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.784346:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.784348:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.784349:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.784349:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.784350:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.784351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.784488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.784494:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.784496:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.784497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.784499:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.784501:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.784503:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.784504:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.784505:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.784506:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.784508:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677473.784511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:18.0:1619677473.784511:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.784512:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677473.784513:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.784514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677473.784515:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.784516:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000400:00000200:22.0:1619677473.784517:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677473.784517:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.784519:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.784519:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:22.0:1619677473.784520:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.784520:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.784521:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677473.784522:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.784523:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.784524:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.784525:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677473.784525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:22.0:1619677473.784526:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.784529:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.784531:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.784533:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.784533:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:22.0:1619677473.784534:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.784534:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677473.784535:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:22.0:1619677473.784536:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:17.0:1619677473.784536:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.784536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:22.0:1619677473.784537:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.784538:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.784538:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:17.0:1619677473.784538:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.784541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677473.784549:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.784551:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.784552:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.784554:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.784555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.784556:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.784583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.784605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.784612:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.784614:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.784615:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.784615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.784617:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677473.784617:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.784619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.784620:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.784620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.784621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677473.785369:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.785372:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.785373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.785374:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.785377:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.785379:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.785380:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.785381:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.785382:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.785383:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.785386:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.785388:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.785390:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.785391:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.785392:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.785393:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.785394:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.785395:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.785396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.785409:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.785411:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.785412:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.785414:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.785415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.785416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.785425:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.785427:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.785428:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.785429:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.785429:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.785430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.785462:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.785464:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.785467:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.785469:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.785471:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.785472:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.785473:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.785474:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.785475:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.785479:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.785481:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.785491:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.785492:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.785493:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.785495:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.785496:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)++
00000800:00000200:31.0:1619677473.785497:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (25)--
00000800:00000200:31.0:1619677473.785498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (24)--
00000800:00000200:31.0:1619677473.785500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.785501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.785509:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.785511:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.785512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.785512:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.785514:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.785517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.785518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.785524:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.785526:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.785527:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.785527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.785528:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.785528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.785626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.785632:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.785635:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.785636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.785637:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.785640:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.785642:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.785643:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.785644:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.785645:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.785646:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.785649:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.785650:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.785653:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.785654:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.785654:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.785656:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.785657:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.785657:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.785658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.785670:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.785673:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.785674:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.785674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.785676:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.785680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.785970:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.785973:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.785975:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.785978:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.785979:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.785980:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.785981:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.785982:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.785983:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.785986:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.785988:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.785990:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.785990:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.785991:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.785992:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.785993:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.785994:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.785995:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.785998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.786005:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.786007:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.786008:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.786008:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.786010:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.786012:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.786196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.786204:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.786206:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.786207:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.786209:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.786212:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.786213:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.786214:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.786215:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.786216:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.786217:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.786220:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.786222:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.786224:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.786224:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.786225:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.786226:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.786227:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.786228:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.786228:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.786232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.786238:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.786240:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.786241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.786241:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.786243:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.786245:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.786251:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.786254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.786254:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.786256:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.786258:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.786259:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.786261:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.786262:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.786263:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.786264:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.786267:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.786268:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.786271:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.786272:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.786272:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.786274:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.786275:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.786276:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.786276:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.786287:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.786290:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.786290:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.786292:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.786293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.786301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.786407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.786414:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.786417:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.786418:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.786420:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.786422:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.786424:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.786425:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.786426:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.786427:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.786428:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.786431:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.786433:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.786435:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.786436:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.786437:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.786438:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.786439:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.786440:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.786441:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.786441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677473.786447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677473.786452:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677473.786454:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677473.786455:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.786455:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.786456:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.786457:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:12.0:1619677473.786457:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677473.786458:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.786459:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.786460:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677473.786655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:10.0:1619677473.786663:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677473.786665:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677473.786666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:10.0:1619677473.786668:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.786670:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.786672:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.786673:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.786674:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.786675:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.786676:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.786679:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.786680:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.786683:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.786684:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.786685:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.786686:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.786687:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677473.786687:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677473.786688:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677473.786693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.786702:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.786705:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.786706:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.786706:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.786708:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.786711:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.787153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.787160:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.787163:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.787164:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.787165:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.787168:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.787169:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.787170:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.787171:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.787172:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.787173:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.787176:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.787177:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.787180:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.787181:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.787181:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.787183:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.787183:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.787184:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.787186:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.787186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.787192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677473.787197:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.787199:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677473.787199:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.787200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.787201:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.787201:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.787202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677473.787202:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677473.787203:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.787206:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.787316:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.787319:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.787319:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.787321:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.787323:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.787324:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.787325:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.787327:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.787328:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.787329:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.787333:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.787334:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.787337:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.787338:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.787338:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.787340:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.787341:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.787342:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.787343:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.787343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.787354:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.787356:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.787356:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.787357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.787358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.787358:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.787359:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.787361:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.787362:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.787363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.787601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:5.2:1619677473.787606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.787612:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.787615:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.787616:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.787617:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.787618:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.787618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677473.787619:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.787621:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.787622:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.787623:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.787624:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.787625:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677473.787628:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:17.0:1619677473.787628:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.787630:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677473.787631:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.787632:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.787632:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677473.787633:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.787633:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.787633:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677473.787635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:17.0:1619677473.787635:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:31.0:1619677473.787636:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:17.0:1619677473.787636:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.787637:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.787637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677473.787648:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.787651:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.787651:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.787652:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677473.787653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677473.787653:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:30.0:1619677473.787654:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677473.787655:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.787656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.787656:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.787659:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.787661:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.787765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.787768:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.787770:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677473.787773:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.787773:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.787775:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.787776:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.787778:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.787779:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.787780:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.787781:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.787784:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.787786:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.787788:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.787789:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.787790:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.787792:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.787792:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.787793:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.787795:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.787795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.787798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677473.787800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677473.787806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:1.0:1619677473.787806:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677473.787807:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.787809:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677473.787809:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677473.787810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677473.787810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.787810:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.787811:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.787812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677473.787812:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.787813:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:20.0:1619677473.787813:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.787815:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677473.787815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:21.0:1619677473.787818:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.787819:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.787821:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.787822:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.787823:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.787824:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:4.2:1619677473.787825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:21.0:1619677473.787827:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.787829:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.787832:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.787832:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.787833:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677473.787833:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677473.787835:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.787836:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:10.0:1619677473.787836:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.787837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677473.787837:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.787838:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.787838:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:10.0:1619677473.787838:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.787841:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.787841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:10.0:1619677473.787843:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:4.2:1619677473.787843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000400:00000200:10.0:1619677473.787844:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.787845:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.787846:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.787847:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.787847:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:10.0:1619677473.787847:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.787850:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677473.787850:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:10.0:1619677473.787850:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.787851:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:10.0:1619677473.787851:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.787851:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.787852:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.787852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:24.0:1619677473.787853:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:10.0:1619677473.787854:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.787855:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.787856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.787856:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.787857:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.787858:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.787859:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.787859:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.787871:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.787874:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.787874:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.787875:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.787877:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.787881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.787893:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.787895:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.787896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.787897:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.787898:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.787898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.787935:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.787938:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.787940:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.787942:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.787943:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.787944:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.787945:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.787946:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.787947:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.787948:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.787951:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.787952:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.787955:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.787956:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.787956:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.787958:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.787958:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.787959:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.787960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.787963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.787970:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.787972:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.787973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.787973:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.787975:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.787977:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.788176:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.788179:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.788180:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.788182:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.788184:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.788185:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.788186:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.788188:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.788189:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.788190:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.788194:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.788196:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.788198:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.788199:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.788199:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.788201:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.788202:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.788203:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.788203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.788215:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.788217:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.788218:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.788219:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.788220:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.788222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.789025:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.789028:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.789029:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.789031:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.789033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.789034:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.789035:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677473.789037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:6.0:1619677473.789037:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.789037:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.789038:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.789041:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.789043:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677473.789043:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677473.789045:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.789046:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.789046:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677473.789046:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.789047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677473.789048:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.789048:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.789049:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:2.0:1619677473.789049:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.789050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:2.0:1619677473.789051:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.789053:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.789054:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.789055:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.789056:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.789057:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.789060:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.789062:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677473.789062:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.789063:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.789064:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:2.0:1619677473.789064:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:20.0:1619677473.789065:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.789065:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:20.0:1619677473.789067:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.789067:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.789067:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:21.0:1619677473.789068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677473.789068:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677473.789069:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677473.789069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.789074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677473.789081:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677473.789082:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677473.789084:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.789084:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677473.789085:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.789085:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000400:00000200:18.0:1619677473.789085:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.789085:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677473.789086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.789087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:18.0:1619677473.789087:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.789089:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.789269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.789275:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.789277:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.789280:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.789280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.789282:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.789284:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.789285:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.789286:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.789288:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.789289:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.789292:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.789297:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.789300:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.789300:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.789301:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.789303:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.789303:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.789304:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.789305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.789309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.789316:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.789319:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.789319:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.789320:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.789322:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.789324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.789326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.789332:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.789335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.789335:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.789336:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.789337:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.789337:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.789483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.789490:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.789492:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.789493:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.789495:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.789497:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.789498:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.789499:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.789501:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.789502:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.789503:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.789506:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.789508:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.789511:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.789512:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.789513:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.789514:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.789515:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.789516:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.789517:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.789518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.789523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.789528:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677473.789529:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677473.789531:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.789531:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677473.789532:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:2.0:1619677473.789532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677473.789533:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.789535:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677473.789536:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.789538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677473.789549:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677473.789552:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677473.789552:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.789553:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677473.789554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.789555:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677473.789756:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.789759:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.789761:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.789762:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.789763:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.789764:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.789765:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.789766:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.789767:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.789768:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.789772:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.789773:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.789775:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.789776:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.789777:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.789778:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.789779:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.789780:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.789782:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.789782:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.789792:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.789794:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.789794:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.789796:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.789796:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.789797:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.789797:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.789799:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.789800:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.789801:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.789886:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.789889:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.789891:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.789893:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.789893:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.789895:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.789896:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.789897:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.789898:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.789899:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.789902:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.789904:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.789906:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.789907:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.789907:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.789909:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.789909:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.789910:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.789911:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.789915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.789921:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.789923:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.789924:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.789924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.789926:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.789928:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.789965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.789968:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.789970:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:17.0:1619677473.789970:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.789972:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:21.0:1619677473.789973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.789973:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.789973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:20.0:1619677473.789976:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.789976:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.789978:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677473.789978:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.789979:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.789980:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.789981:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.789982:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.789982:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.789983:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677473.789983:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.789985:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.789986:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.789988:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.789990:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:17.0:1619677473.789990:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.789993:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.789994:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.789994:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677473.789995:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677473.789996:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.789997:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677473.789997:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677473.789997:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.789998:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.789999:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:17.0:1619677473.789999:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677473.790000:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:20.0:1619677473.790001:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:17.0:1619677473.790001:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:20.0:1619677473.790002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:17.0:1619677473.790002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.790005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.2:1619677473.790006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.790011:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:12.0:1619677473.790012:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:8.0:1619677473.790012:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677473.790014:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677473.790014:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:24.0:1619677473.790015:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677473.790015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:12.0:1619677473.790015:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:8.0:1619677473.790015:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:12.0:1619677473.790016:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.790016:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.790018:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.790019:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.790020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.790021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677473.790022:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.790062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.790064:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.790065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.790067:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.790070:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.790071:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.790072:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.790074:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.790075:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.790076:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.790080:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.790081:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.790084:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.790085:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.790086:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.790087:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677473.790087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677473.790088:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.790089:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.790089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677473.790095:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.790098:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.790099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.790101:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.790101:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.790103:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:24.0:1619677473.790104:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.790104:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.790106:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.790106:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.790106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:24.0:1619677473.790107:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96507612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.790109:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.790109:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:24.0:1619677473.790110:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.790111:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.790114:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.790116:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.790119:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.790120:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.790120:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.790122:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.790123:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.790125:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.790126:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.790127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.790132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.790136:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.790139:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.790140:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.790140:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:6.0:1619677473.790140:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.790142:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:6.0:1619677473.790142:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.790143:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.790144:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.790146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.790718:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.790721:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.790723:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.790726:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.790726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.790727:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.790729:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.790730:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.790731:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.790732:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.790734:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.790736:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.790738:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.790739:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.790739:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.790741:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.790741:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.790742:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.790743:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.790754:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.790756:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.790757:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.790758:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.790760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.790761:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.790898:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.790901:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.790902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.790904:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.790906:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.790907:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.790908:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.790910:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.790911:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.790912:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.790915:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.790917:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.790920:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.790920:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.790921:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.790923:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.790923:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.790925:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.790926:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.790926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677473.790927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.790934:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677473.790936:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677473.790937:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:6.0:1619677473.790937:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:10.0:1619677473.790938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677473.790939:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.790939:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:26.0:1619677473.790940:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.790940:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.790940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.790941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:26.0:1619677473.790942:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.790942:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.790943:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677473.790944:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677473.790944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677473.790945:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.790946:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.790947:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.790948:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.790952:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.790953:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.790956:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.790957:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.790957:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.790959:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.790959:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.790960:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.790961:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.790965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.790972:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.790974:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.790975:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.790975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.790977:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.790979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.791100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.791107:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.791109:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.791110:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.791111:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.791114:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.791115:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.791116:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.791117:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.791118:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.791119:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.791122:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.791123:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.791126:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.791127:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.791127:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.791129:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.791129:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.791130:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.791130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.791143:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.791145:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.791146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.791147:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.791149:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.791152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.791488:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.791491:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.791493:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.791495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.791495:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.791496:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.791497:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.791499:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.791500:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.791500:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.791503:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.791505:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.791507:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.791508:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.791508:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.791510:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.791510:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.791512:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.791513:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.791513:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.791516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.791523:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.791526:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.791527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.791527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.791528:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677473.791528:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.791530:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.791531:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677473.791531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.791532:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677473.791748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677473.791754:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.791756:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.791758:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.791759:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.791761:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.791762:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.791763:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.791765:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.791765:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.791766:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.791774:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677473.791775:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677473.791775:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677473.791778:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.0:1619677473.791779:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.791779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.791780:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.791780:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:22.0:1619677473.791781:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.791782:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:22.0:1619677473.791783:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.791783:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:22.0:1619677473.791784:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.0:1619677473.791784:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.791784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:22.0:1619677473.791786:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.791787:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.791788:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.791789:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.791792:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.791794:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677473.791795:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:22.0:1619677473.791796:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.791797:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:31.0:1619677473.791798:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:22.0:1619677473.791798:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677473.791799:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.791800:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.791800:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.791800:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677473.791801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677473.791801:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.791802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677473.791804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.0:1619677473.791813:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.791815:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.791816:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.791817:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.791818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.791820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.791925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.791933:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.791935:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.791936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.791939:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.791940:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.791942:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677473.791944:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.791945:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.791946:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.791947:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677473.791950:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.791952:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.791955:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.791956:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677473.791956:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677473.791958:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677473.791958:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.791959:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.791959:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:30.0:1619677473.791970:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.791973:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.791973:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.791975:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.791977:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.791977:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.792052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.792059:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.792062:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.792063:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.792065:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.792067:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.792068:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.792070:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.792071:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.792072:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.792072:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.792075:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.792077:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.792080:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.792080:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.792081:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.792082:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.792083:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.792083:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.792084:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.792096:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.792099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.792099:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.792100:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.792102:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.792105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.792355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.792361:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.792364:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.792365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.792367:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.792370:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.792371:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.792372:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.792374:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.792375:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.792375:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.792378:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.792380:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.792382:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.792383:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.792384:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.792385:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.792386:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.792387:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.792388:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.792389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.792394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677473.792400:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.792400:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.792402:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.792402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.792403:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.792403:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.792404:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.792404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.792406:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.792408:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.792418:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.792420:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.792421:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.792421:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.792422:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.792423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.792742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.792750:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.792752:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.792753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.792755:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.792758:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.792759:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.792760:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.792762:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.792763:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.792764:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.792767:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.792769:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.792772:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.792772:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.792773:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.792775:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.792775:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.792776:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.792777:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.792778:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.792783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.792788:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.792791:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.792791:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.792792:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.792792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.792793:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.792793:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.792793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000400:00000200:22.0:1619677473.792794:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.792795:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.792797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.792801:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677473.792804:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677473.792806:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.792809:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.792809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677473.792810:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677473.792811:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.792813:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.792814:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.792815:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677473.792818:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.792819:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677473.792822:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.792823:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677473.792824:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677473.792825:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677473.792826:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677473.792827:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677473.792827:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677473.792832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.792841:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.792844:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.792845:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.792847:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677473.792850:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.792852:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677473.792853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.792855:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.792856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.792858:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.792860:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.792861:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.792862:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.792864:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.792865:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.792866:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.792869:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.792871:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.792874:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.792874:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.792875:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.792877:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.792878:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.792879:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.792879:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.792890:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.792893:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.792894:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.792895:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.792896:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677473.792896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.792897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.792899:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.792901:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.792903:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.792903:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.792904:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.2:1619677473.792905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:30.0:1619677473.792906:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.792907:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.792908:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.792909:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677473.792910:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.792912:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.792912:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:30.0:1619677473.792913:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.792914:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.792914:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:18.0:1619677473.792914:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.792915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:30.0:1619677473.792916:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677473.792916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.792917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.792918:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677473.792918:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.792919:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.792919:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.792921:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.792921:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677473.792921:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.792922:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677473.792922:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677473.792923:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677473.792924:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.792925:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.792926:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.792927:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677473.792931:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.792932:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.792933:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:18.0:1619677473.792935:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.792936:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:18.0:1619677473.792936:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677473.792936:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677473.792937:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.792938:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.792938:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677473.792939:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:30.0:1619677473.792940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677473.792940:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:31.0:1619677473.792941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:18.0:1619677473.792941:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677473.792941:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677473.792946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:22.0:1619677473.792952:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677473.792952:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677473.792954:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:30.0:1619677473.792955:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677473.792955:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:2.0:1619677473.792955:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:22.0:1619677473.792956:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.792956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:6.0:1619677473.792956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.792958:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.792959:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.792959:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000400:00000200:22.0:1619677473.792959:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.792960:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.792961:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.792962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.792979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.792981:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.792982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.792982:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.792983:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.792984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.793490:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.793493:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.793495:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.793497:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.793497:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.793499:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.793500:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.793501:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.793502:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.793503:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.793506:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.793508:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.793510:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.793511:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.793512:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.793513:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.793514:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.793515:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.793516:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.793520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.793527:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.793530:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.793531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.793531:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.793533:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.793535:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.793538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.793544:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.793546:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.793547:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.793547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.793548:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.793549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.793693:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.793696:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.793696:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.793698:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.793700:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.793702:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.793703:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.793705:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.793706:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.793707:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.793710:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.793712:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.793715:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.793715:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.793716:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.793718:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.793718:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.2:1619677473.793718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677473.793719:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.793720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.793721:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.793723:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.793725:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.793727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.793728:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:20.0:1619677473.793728:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.793729:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.793730:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.793730:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.793731:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.793732:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.793733:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.793734:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.793735:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.793737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.793738:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:20.0:1619677473.793738:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.793740:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.793742:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.793743:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.793744:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.793746:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.793746:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.793747:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.793748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.793752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.793760:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.793762:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.793763:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.793763:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.793766:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.793768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.794477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677473.794484:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.794487:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.794488:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677473.794489:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.794492:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.794494:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.794495:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.794497:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.794498:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.794499:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.794501:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.794503:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.794506:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.794507:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.794507:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.794509:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.794510:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.794510:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.794511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677473.794538:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.794540:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.794541:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:6.0:1619677473.794541:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.794543:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.794543:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.794545:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.794547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.794548:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.794548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.794549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.794550:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.794551:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.794553:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.794554:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.794555:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.794558:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.794560:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.794563:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.794563:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.794564:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.794566:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.794566:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.794567:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.794568:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.794569:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.794579:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.794581:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.794581:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.794581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.794582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.794583:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.794584:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.794586:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.794587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.794588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.794692:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.794695:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.794697:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.794699:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.794699:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.794701:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.794702:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.794703:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.794704:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.794705:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.794708:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.794710:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.794712:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.794713:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.794713:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.794715:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.794715:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.794716:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.794717:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.794727:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.794729:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.794730:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.794732:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.794733:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.794734:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.794754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.794761:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.794764:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.794765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.794766:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.794769:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.794771:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.794772:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.794773:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.794774:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.794775:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.794778:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.794780:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.794783:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.794784:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.794784:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.794786:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.794787:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.794788:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.794788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.794802:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.794805:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.794806:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.794809:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.794810:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.794811:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.794812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.794812:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)++
00000800:00000200:12.0:1619677473.794813:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (24)--
00000800:00000200:12.0:1619677473.794813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.794815:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677473.794898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.794906:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.794909:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.794910:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.794912:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.794914:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.794915:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.794917:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.794918:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.794919:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.794920:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.794923:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.794924:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.794927:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.794928:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.794928:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.794930:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.794930:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.794931:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.794931:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.794942:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.794944:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.794945:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.794946:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.794948:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.794949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677473.795122:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.795124:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.795126:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.795128:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.795128:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.795129:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.795130:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.795131:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.795132:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.795133:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.795136:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.795137:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.795140:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.795140:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.795141:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.795142:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.795143:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.795144:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.795144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.795148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.795155:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.795157:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.795158:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.795158:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.795160:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.795162:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.795446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.795452:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.795455:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.795456:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.795458:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.795461:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.795462:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.795463:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.795465:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.795466:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.795467:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.795470:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.795472:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.795474:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.795475:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.795476:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.795478:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.795478:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.795479:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.795479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.795493:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.795496:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.795497:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.795497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.795499:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.795502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.795608:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.795611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.795611:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.795613:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.795615:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.795616:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.795618:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.795619:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.795620:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.795621:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.795624:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.795626:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.795628:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.795629:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.795630:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.795632:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.795632:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.795633:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.795633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.795645:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.795647:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.795648:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.795650:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.795651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.795652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.795664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.795671:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.795674:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.795675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.795677:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.795679:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.795680:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.795681:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.795683:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.795684:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.795685:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.795688:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.795689:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.795692:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.795692:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.795693:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.795695:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.795695:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.795696:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.795696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.795700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.795707:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.795710:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.795710:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.795711:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.795713:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.795715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.795878:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.795881:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.795883:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.795885:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.795886:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.795886:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.795887:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.795889:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.795889:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.795890:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.795893:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.795895:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.795897:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.795897:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.795898:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.795900:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.795900:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.795902:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.795903:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.795904:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.795905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.795911:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.795913:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.795914:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.795914:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.795916:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.795917:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677473.795918:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.795919:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.795920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.795921:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.795922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.795928:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.795930:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.795931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.795931:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.795932:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.795932:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.795949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677473.795957:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677473.795959:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677473.795961:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.795963:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.795964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677473.795965:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677473.795966:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.795968:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.795969:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.795970:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677473.795972:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.795974:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677473.795976:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.795977:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:31.0:1619677473.795978:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:31.0:1619677473.795979:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677473.795980:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677473.795981:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677473.795981:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677473.795985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.795992:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.795994:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.795995:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.795995:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.795997:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.795999:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677473.796089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.796096:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.796098:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.796099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.796101:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.796103:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.796105:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.796106:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.796107:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.796108:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.796109:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.796112:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.796114:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.796116:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.796117:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.796118:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.796119:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.796120:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.796120:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.796121:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.796133:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.796136:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.796137:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.796137:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.796139:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.796142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.796510:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.796513:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.796514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.796516:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.796517:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.796519:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.796520:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.796522:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.796523:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.796524:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.796527:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.796529:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.796531:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.796532:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.796533:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.796535:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.796535:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.796536:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.796537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.796550:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.796552:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.796553:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.796554:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.796555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.796556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.796568:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.796571:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.796572:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.796572:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.796573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.796574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.796819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.796826:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.796829:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.796830:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.796832:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.796834:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.796836:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.796837:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.796838:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.796839:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.796840:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.796843:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.796845:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.796848:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.796849:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.796849:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.796851:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.796851:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.796852:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.796853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.796865:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.796868:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.796868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.796869:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.796871:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.796874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.796888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.796895:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.796898:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.796899:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.796901:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.796904:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.796905:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.796906:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.796908:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.796909:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.796910:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.796913:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.796914:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.796917:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.796918:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.796918:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.796920:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.796920:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.796921:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.796921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.796933:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.796936:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.796937:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.796937:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.796940:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.796943:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.797015:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.797018:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.797021:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.797023:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.797023:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.797024:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.797025:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.797026:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.797027:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.797028:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.797031:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.797033:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.797035:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.797036:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.797036:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.797038:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.797038:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.797039:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.797040:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.797044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.797050:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.797052:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.797053:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.797053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.797055:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.797057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.797259:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.797261:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.797263:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.797265:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.797265:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.797266:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.797268:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.797269:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.797270:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.797271:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.797274:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.797275:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.797277:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.797278:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.797279:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.797280:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.797281:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.797281:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.797282:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.797286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.797292:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.797296:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.797299:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.797299:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.797301:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.797303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.797400:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.797404:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.797404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.797406:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.797408:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.797409:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.797411:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.797412:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.797414:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.797415:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.797418:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.797419:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.797422:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.797423:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.797423:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.797425:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.797426:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.797427:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.797427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.797440:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.797442:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.797443:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.797444:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.797445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.797447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.797559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.797567:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.797570:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.797571:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.797573:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.797575:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.797577:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.797578:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.797579:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.797580:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.797581:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.797584:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.797586:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.797589:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.797589:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.797590:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.797592:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.797592:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.797593:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.797593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677473.797594:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.797596:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.797598:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.797600:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.797601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.797601:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677473.797603:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.797604:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.797605:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.797606:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677473.797606:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677473.797608:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.797608:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.797609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.797609:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.797610:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677473.797611:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.797613:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.797614:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677473.797615:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677473.797615:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.797617:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677473.797617:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.797618:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.797619:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677473.797623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677473.797624:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.797627:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.797628:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.797628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.797629:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.797630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.797632:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.797634:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.797635:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.797637:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.797637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.797639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.798456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.798459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.798459:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.798461:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.798463:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.798465:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.798466:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.798468:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.798468:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.798469:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.798472:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.798474:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.798476:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.798477:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.798478:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.798480:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.798480:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.798481:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.798482:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.798483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.798494:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.798494:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.798496:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.798496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.798496:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.798497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.798497:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.798499:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.798500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.798501:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.798795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677473.798802:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.798805:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677473.798805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677473.798808:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.798810:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.798812:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.798813:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.798814:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.798815:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.798817:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.798819:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.798821:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.798822:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.798823:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.798824:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.798825:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.798825:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.798826:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.798826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677473.798830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.798837:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.798838:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.798839:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.798840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.798841:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.798843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.798877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.798883:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.798886:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.798887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.798889:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.798892:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.798893:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.798895:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.798896:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.798897:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.798899:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.798902:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.798903:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.798905:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.798906:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.798907:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.798908:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.798909:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.798910:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.798910:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.798915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.798922:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.798924:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.798924:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.798925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.798926:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.798927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677473.798933:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.798936:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.798937:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.798940:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.798942:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.798944:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677473.798945:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.798947:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.798948:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.798949:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677473.798953:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.798955:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.798957:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.798958:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.798958:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.798960:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.798961:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.798962:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677473.798964:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.798965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.2:1619677473.798966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.798978:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.798980:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.798980:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:17.0:1619677473.798981:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.798982:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.798983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.798985:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.798985:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.798988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.798988:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.798989:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.798990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677473.798991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.798991:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.798993:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677473.799022:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.799025:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.799027:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.799029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.799030:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.799032:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.799033:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.799034:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.799035:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.799036:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.799040:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.799042:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.799044:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.799045:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.799046:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.799047:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.799048:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.799049:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.799050:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.799051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.799053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.799059:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.799061:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.799062:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.799062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.799064:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.799065:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677473.799066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.799067:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.799068:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.799069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.799071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.799077:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.799080:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.799081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.799081:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.799081:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.799082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.799292:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.799296:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.799301:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.799303:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.799305:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.799306:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.799307:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.799309:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.799310:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.799311:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.799314:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.799316:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.799319:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.799319:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.799320:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.799322:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.799323:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.799324:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.799324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.799336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.799339:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.799339:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.799341:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.799342:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.799343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677473.799470:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677473.799475:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677473.799476:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677473.799480:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.799482:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.799484:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677473.799486:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.799489:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.799490:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.799492:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.799497:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.799500:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.799503:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.799504:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.799505:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.799508:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.799509:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677473.799510:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:17.0:1619677473.799511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000800:00000200:22.0:1619677473.799522:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.799525:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.799525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.799526:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.799528:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.799530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.799647:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.799650:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.799652:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.799654:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.799655:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.799656:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.799657:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.799659:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.799660:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.799661:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.799664:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.799666:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.799669:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.799669:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.799670:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.799671:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.799672:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.799673:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.799674:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.799701:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.799703:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.799704:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.799706:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.799707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.799708:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.800116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.800119:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.800122:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.800124:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.800125:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.800126:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.800128:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.800129:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.800130:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.800131:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.800133:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.800136:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.800138:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.800140:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.800141:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.800142:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.800144:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.800144:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.800145:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.800145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.800151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.800158:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.800161:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.800162:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.800162:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.800164:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.800166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.800170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677473.800171:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.800174:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.800174:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.800176:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.800177:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:6.0:1619677473.800178:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.800180:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677473.800180:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:17.0:1619677473.800181:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.800181:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.800182:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:6.0:1619677473.800182:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.800183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:6.0:1619677473.800183:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.800184:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.800185:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.800188:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.800190:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.800193:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.800194:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.800194:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.800196:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.800196:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.800197:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.800198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.800209:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.800212:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.800212:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.800214:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.800215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.800216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.800229:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.800232:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.800233:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.800234:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.800234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.800235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.800364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677473.800371:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.800374:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677473.800375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677473.800376:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.800378:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.800379:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.800380:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.800382:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.800383:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.800384:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.800387:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.800389:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.800390:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.800391:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.800391:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.800393:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.800393:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.800394:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677473.800395:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677473.800396:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:9.2:1619677473.800399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677473.800401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677473.800405:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:4.2:1619677473.800407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.800408:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.800408:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.800409:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677473.800410:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.800410:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:26.0:1619677473.800411:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677473.800411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.800412:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:10.0:1619677473.800412:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677473.800413:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000400:00000200:26.0:1619677473.800414:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.800414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.800415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:26.0:1619677473.800417:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.800418:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.800419:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.800420:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.800421:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.800422:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.800425:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.800426:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.800428:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.800429:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.800430:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.800431:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.800432:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.800432:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.800433:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.800437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.800443:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.800445:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.800446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.800446:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.800448:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.800450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.800660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677473.800668:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677473.800671:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677473.800673:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.800675:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.800676:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677473.800677:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677473.800678:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.800679:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.800680:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.800681:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677473.800684:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.800685:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677473.800688:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.800689:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677473.800689:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677473.800691:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677473.800691:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677473.800692:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677473.800693:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677473.800697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.800706:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.800708:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.800709:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.800710:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.800712:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.800715:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.800716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.800723:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.800726:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.800726:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.800727:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.800728:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.800728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.801002:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.801005:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.801007:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.801009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.801010:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.801011:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.801012:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.801014:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.801015:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.801015:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.801018:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.801020:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.801022:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.801023:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.801023:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.801025:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.801026:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.801027:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.801027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.801031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.801037:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.801040:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.801041:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.801041:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.801042:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.801044:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.801167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.801173:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.801176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.801176:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.801179:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.801181:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.801182:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.801183:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.801185:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.801186:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.801187:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.801190:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.801192:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.801194:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.801195:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.801195:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.801197:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:1.0:1619677473.801197:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.801197:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.801198:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.801198:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.801199:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.801200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.801202:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.801205:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.801207:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677473.801208:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677473.801208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000400:00000200:22.0:1619677473.801210:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.801212:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.801213:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:17.0:1619677473.801214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:6.0:1619677473.801214:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.801216:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:6.0:1619677473.801216:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.801216:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.801217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:22.0:1619677473.801218:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.801218:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.801220:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:17.0:1619677473.801220:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.801221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.801222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:17.0:1619677473.801223:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.801224:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:22.0:1619677473.801225:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677473.801225:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.801226:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:17.0:1619677473.801226:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.801227:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677473.801227:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.801228:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677473.801228:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.801229:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.801230:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.801230:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.801231:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677473.801232:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000400:00000200:17.0:1619677473.801232:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.801233:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.801234:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:17.0:1619677473.801234:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:6.0:1619677473.801234:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677473.801235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:6.0:1619677473.801236:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.801237:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.801238:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:17.0:1619677473.801238:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677473.801238:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.801239:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96507556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.801240:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677473.801240:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677473.801241:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:8.0:1619677473.801241:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:6.0:1619677473.801241:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677473.801242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000400:00000200:6.0:1619677473.801242:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.801244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.801244:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:8.0:1619677473.801244:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:6.0:1619677473.801244:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.801245:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000400:00000200:8.0:1619677473.801245:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.801246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677473.801246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677473.801247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:8.0:1619677473.801248:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.801249:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.801251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.801252:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.801254:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:6.0:1619677473.801254:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.801255:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677473.801256:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:6.0:1619677473.801256:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677473.801257:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.801257:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677473.801257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677473.801258:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:18.0:1619677473.801259:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.801259:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.801260:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:18.0:1619677473.801262:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677473.801271:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.801273:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.801274:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.801276:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.801277:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.801279:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.801329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677473.801334:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.801336:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.801339:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.801339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.801342:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.801343:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.801345:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.801346:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.801347:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.801348:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.801351:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.801354:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.801357:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.801358:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.801359:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.801360:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.801361:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.801362:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.801362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677473.801374:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.801377:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677473.801378:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.801380:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.801382:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.801383:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677473.801439:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.801442:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.801444:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.801446:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.801446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677473.801446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677473.801447:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.801448:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.801450:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.801451:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.801451:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.801453:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677473.801454:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.801455:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677473.801456:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.801456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.801458:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.801459:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.801459:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.801460:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:24.0:1619677473.801460:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.801462:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.801462:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677473.801462:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.801463:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96507553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677473.801464:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677473.801464:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677473.801465:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:24.0:1619677473.801465:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677473.801466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677473.801466:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677473.801467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:24.0:1619677473.801470:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.801471:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.801474:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.801474:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.801475:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.801475:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.801476:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:8.0:1619677473.801476:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.801477:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.801478:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.801479:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677473.801479:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.801479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.801480:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.801480:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.801481:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:30.0:1619677473.801482:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.801482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677473.801482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677473.801483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.801484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:4.2:1619677473.801487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.801493:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.801494:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.801494:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.801495:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677473.801496:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677473.801496:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.801496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.801497:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.801497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.801498:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677473.801498:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.801498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677473.801499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677473.801500:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.801500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.801502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.802080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.802087:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.802089:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.802090:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.802092:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.802094:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.802096:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.802097:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.802098:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.802099:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.802100:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.802103:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.802104:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677473.802105:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677473.802107:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.802108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:2.0:1619677473.802109:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677473.802110:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.802110:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.802110:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.802112:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:6.0:1619677473.802113:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.802113:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677473.802114:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677473.802114:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677473.802115:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677473.802115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:6.0:1619677473.802117:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.802118:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.802119:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.802122:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.802124:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677473.802125:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:6.0:1619677473.802126:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.802127:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677473.802128:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:6.0:1619677473.802128:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677473.802129:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.802129:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.802130:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677473.802131:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.802131:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:31.0:1619677473.802132:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677473.802132:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:30.0:1619677473.802133:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:6.0:1619677473.802133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.802143:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.802143:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.802145:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677473.802145:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.802146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.802146:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.802147:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.802148:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.802149:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.802150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.802176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.802184:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.802187:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.802187:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.802189:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.802192:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.802193:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.2:1619677473.802194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000400:00000200:10.0:1619677473.802194:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.802196:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.802197:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.802198:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677473.802199:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.802201:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:10.0:1619677473.802201:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.802202:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:21.0:1619677473.802204:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.802204:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:10.0:1619677473.802205:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.802206:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.802206:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.802207:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:21.0:1619677473.802208:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677473.802209:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.802209:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.802209:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:21.0:1619677473.802210:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.802210:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.802210:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:21.0:1619677473.802211:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.802212:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677473.802216:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.802218:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 600 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.802220:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.802221:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677473.802221:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.802223:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:21.0:1619677473.802223:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.802223:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.802224:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:22.0:1619677473.802225:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:21.0:1619677473.802225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677473.802226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.802227:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.802229:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.802231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677473.802236:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.802238:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.802239:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.802240:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.802241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.802243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.802243:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.802246:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.802247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.802247:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.802248:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.802249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.802327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.802334:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.802337:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.802338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.802340:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.802342:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.802343:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.802345:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.802346:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.802347:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.802348:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.802351:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.802352:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.802355:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.802356:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.802357:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.802358:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.802359:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.802360:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.802361:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.802362:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.802368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.802373:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.802375:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.802375:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677473.802377:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.802377:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677473.802377:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.802378:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.802379:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.802379:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.802383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.802795:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.802798:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.802800:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.802802:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.802802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.802803:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.802804:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.802806:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.802806:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.802807:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.802811:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.802812:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.802814:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.802815:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.802815:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.802817:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.802817:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.802818:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.802819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.802829:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.802831:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.802832:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.802833:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.802835:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.802836:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.802955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:6.0:1619677473.802958:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.802961:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:26.0:1619677473.802962:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:22.0:1619677473.802962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.802964:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:6.0:1619677473.802964:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.802965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:6.0:1619677473.802966:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.802968:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.802968:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.802969:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.802970:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.802971:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.802973:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677473.802973:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.802974:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.802975:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.802977:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.802979:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.802979:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.802980:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.802982:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.802985:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.802986:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.802987:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.802987:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677473.802988:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.802989:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.802990:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677473.802991:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.802991:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677473.802991:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.802992:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677473.802992:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677473.802994:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.802994:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.802995:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.802996:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.803000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677473.803003:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.803005:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.803006:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.803008:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.803008:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:20.0:1619677473.803009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.803010:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677473.803011:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.803011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.803012:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.803014:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.803018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.0:1619677473.803021:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.803024:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.803025:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.803025:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.803026:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.803027:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.804039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.804042:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.804043:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.804045:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.804047:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.804048:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.804050:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.804051:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.804052:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.804053:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.804057:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.804058:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.804061:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.804062:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.804062:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.804064:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.804065:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.804066:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.804066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.804079:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.804081:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.804082:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.804083:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.804085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.804086:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.804811:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.804814:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.804816:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.804818:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.804818:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.804819:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.804820:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.804822:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.804823:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.804824:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.804827:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677473.804828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677473.804829:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.804831:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.804832:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.804833:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.804834:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.804835:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.804836:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.804836:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677473.804837:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.804841:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.804843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.804846:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.804848:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.804850:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.804851:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.804851:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.804853:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.804854:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:24.0:1619677473.804854:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:31.0:1619677473.804855:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:24.0:1619677473.804856:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.804858:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.804860:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.804861:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677473.804865:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:24.0:1619677473.804867:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.804868:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.804868:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.804869:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677473.804870:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.804870:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:24.0:1619677473.804870:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677473.804874:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677473.804875:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.804876:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677473.804876:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:24.0:1619677473.804877:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677473.804877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677473.804879:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:6.0:1619677473.804879:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.804880:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.804881:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677473.804881:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.804882:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677473.804883:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.804884:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.804885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677473.804885:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.804886:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.804887:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.804891:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.804893:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677473.804893:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:6.0:1619677473.804896:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.804896:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677473.804896:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.804897:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677473.804897:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:2.0:1619677473.804897:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.804899:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.804899:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.804900:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:2.0:1619677473.804900:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.804901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:2.0:1619677473.804903:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677473.804913:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.804915:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.804916:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.804918:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.804918:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.804920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.805386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.805394:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.805398:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.805400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.805403:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.805406:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.805408:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.805410:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.805412:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.805414:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.805416:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.805421:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.805424:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.805427:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.805429:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.805430:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.805432:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.805432:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.805433:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.805434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.805447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.805450:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.805451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.805451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.805453:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.805456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.805592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.805600:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.805603:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.805604:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.805606:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.805609:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.805610:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.805612:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.805613:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.805614:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.805615:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.805618:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.805620:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.805622:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.805623:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.805624:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.805625:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.805625:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.805626:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.805627:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.805631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.805639:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.805643:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.805644:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.805645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.805647:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.805650:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.805761:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.805764:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.805765:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.805767:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.805769:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.805770:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.805772:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.805773:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.805774:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.805775:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.805778:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.805780:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.805783:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.805783:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.805784:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.805786:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.805786:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.805788:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.805789:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.805789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.805799:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.805800:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.805801:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.805802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.805802:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.805802:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.805803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.805804:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.805806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.805807:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.805903:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.805906:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.805907:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.805910:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.805910:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.805911:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.805912:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.805913:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.805914:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.805915:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.805918:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.805920:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.805922:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.805923:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.805923:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.805925:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.805925:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.805926:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.805927:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.805937:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.805940:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.805940:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.805942:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.805943:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.805944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.805958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.805965:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.805967:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677473.805967:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:21.0:1619677473.805970:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.805970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677473.805972:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:21.0:1619677473.805973:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.805973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:21.0:1619677473.805975:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.805977:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.805978:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.805979:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.805980:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.805981:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677473.805982:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.805986:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.805988:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:17.0:1619677473.805989:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:21.0:1619677473.805990:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.805991:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:17.0:1619677473.805991:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.805992:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.805994:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.805994:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.805995:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.805996:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677473.805996:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.805997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677473.805998:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.2:1619677473.806000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:17.0:1619677473.806005:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.806009:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677473.806013:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.806015:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.806015:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.806017:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:1.0:1619677473.806017:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.806018:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677473.806021:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:1.0:1619677473.806021:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.806024:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.806025:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.806025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.806027:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.806027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.806028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677473.806033:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.806036:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.806037:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.806037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.806040:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.806042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.806145:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.806148:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.806150:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.806151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.806152:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.806153:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.806154:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.806156:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.806157:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.806158:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.806161:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.806163:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.806166:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.806166:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.806167:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.806169:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.806169:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.806170:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.806171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.806174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.806180:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.806183:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.806184:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.806184:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.806185:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.806188:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.806193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677473.806197:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:24.0:1619677473.806199:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677473.806200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.806200:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:24.0:1619677473.806202:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.806202:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.806203:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677473.806203:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.806205:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.806205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677473.806206:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.806208:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677473.806209:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.806210:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.806211:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.806212:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677473.806215:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.806217:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677473.806220:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.806220:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677473.806221:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677473.806223:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677473.806223:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.806224:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.806225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677473.806229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.806237:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.806240:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.806241:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.806241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.806243:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.806245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.806286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.806293:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.806297:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.806300:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.806302:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.806305:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.806306:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.806308:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.806309:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.806310:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.806311:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.806314:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.806315:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.806318:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.806319:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.806319:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.806321:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.806322:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.806323:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.806323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.806336:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.806339:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.806340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.806340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.806342:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.806349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.806351:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.806353:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.806354:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.806354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.806355:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.806355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.806462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.806469:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.806472:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.806473:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.806474:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.806477:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.806478:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.806479:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.806481:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.806482:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.806482:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.806485:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.806487:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.806490:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.806491:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.806491:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.806493:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.806493:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.806494:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.806496:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.806496:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.806502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677473.806507:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.806509:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.806509:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.806510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.806511:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677473.806511:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.806512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.806513:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.806513:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.806516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.806530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.806537:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.806540:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.806541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.806543:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.806546:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.806547:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.806549:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.806550:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.806551:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.806552:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.806556:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.806557:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.806560:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.806561:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.806562:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.806564:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.806564:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.806565:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.806566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677473.806576:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.806579:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677473.806581:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.806582:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677473.806583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.806585:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677473.806594:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677473.806597:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677473.806597:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.806598:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677473.806599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.806600:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:22.0:1619677473.806783:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.806786:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.806787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.806789:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.806791:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.806793:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.806794:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.806796:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.806796:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.806797:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.806800:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.806802:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.806804:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.806805:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.806806:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.806807:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.806808:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.806809:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.806810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.806821:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.806823:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.806824:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.806826:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.806827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.806829:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.806842:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.806845:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.806846:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.806847:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.806847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.806848:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.807099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.807107:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.807109:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.807110:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.807112:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.807115:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.807117:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.807118:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.807119:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.807120:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.807121:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.807125:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.807127:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.807130:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.807131:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.807131:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.807133:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.807133:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.807134:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.807135:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.807147:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.807150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.807150:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.807151:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.807153:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.807156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.807240:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.807243:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.807245:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.807247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.807247:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.807249:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677473.807250:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.807251:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.807252:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.807253:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677473.807257:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.807259:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677473.807262:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.807262:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677473.807263:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.807264:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677473.807265:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.807266:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.807268:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.807269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677473.807271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677473.807283:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677473.807283:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.807285:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677473.807286:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.807287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677473.807287:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.807288:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677473.807288:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.807288:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.807290:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.807291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.807291:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:20.0:1619677473.807292:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (24)--
00000800:00000200:20.0:1619677473.807292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.807294:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677473.807355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.807362:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.807365:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.807366:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.807368:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.807371:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.807373:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.807374:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.807375:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.807376:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.807377:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.807380:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.807382:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.807385:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.807385:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.807386:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.807387:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.807388:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.807389:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.807389:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.807394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.807403:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.807405:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.807406:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.807407:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.807409:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.807412:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677473.807673:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.807676:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.807676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.807678:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.807680:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.807682:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.807683:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.807685:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.807686:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.807686:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.807689:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.807691:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.807693:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.807694:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.807695:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.807696:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.807697:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.807698:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.807698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.807710:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.807712:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.807713:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.807715:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.807716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.807717:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.807890:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.807894:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.807895:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.807896:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.807899:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.807901:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.807902:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.807903:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.807904:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.807905:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.807908:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.807910:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.807912:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.807913:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.807914:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.807915:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.807916:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.807916:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.807917:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.2:1619677473.807926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.807931:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677473.807932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677473.807934:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.807935:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.807937:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.807940:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:18.0:1619677473.807940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.807941:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.807942:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.807943:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.807943:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.807944:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.807945:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:18.0:1619677473.807945:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.807946:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:18.0:1619677473.807946:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677473.807948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:18.0:1619677473.807949:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.807951:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.807953:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.807954:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.807955:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.807956:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.807957:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.807958:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.807959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.807963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.807970:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.807972:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.807973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.807973:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.807975:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.807976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.808069:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.808072:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.808074:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.808076:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.808077:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.808078:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.808080:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.808081:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.808082:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.808083:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.808087:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.808089:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.808091:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.808091:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.808092:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.808093:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.808094:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.808095:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.808096:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.808106:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.808108:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.808109:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.808111:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.808112:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.808113:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.808208:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.808211:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.808213:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.808214:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.808215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.808216:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.808217:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.808218:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.808219:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.808220:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.808223:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.808225:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.808227:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.808228:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.808228:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.808230:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.808230:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.808231:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.808231:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.808235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.808242:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.808244:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.808245:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.808245:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.808247:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.808249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.808514:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.808517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.808517:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.808519:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.808521:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.808523:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.808524:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.808526:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.808526:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.808527:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.808530:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.808532:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.808534:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.808535:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.808536:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.808538:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.808538:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.808539:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.808540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.808552:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.808554:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.808555:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.808557:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.808558:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.808559:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.808865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.808873:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.808876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.808876:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.808878:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.808881:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.808882:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.808884:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.808885:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.808886:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.808887:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.808890:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.808892:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.808895:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.808895:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.808896:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.808898:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.808898:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.808899:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.808899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.808904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.808912:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.808915:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.808916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.808917:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.808919:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.808921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.808924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.808925:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.808927:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.808929:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.808929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.808930:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.808930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677473.809549:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.809552:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.809553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.809555:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.809557:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.809558:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677473.809560:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.809561:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.809562:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.809563:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677473.809566:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.809568:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.809570:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.809571:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.809571:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.809572:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.809573:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.809574:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.809574:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677473.809580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.809587:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.809589:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.809590:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.809591:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.809592:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.809593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.809594:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.809596:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.809597:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.809599:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.809602:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.809603:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.809604:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.809606:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.809607:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.809608:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.809611:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.809612:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.809615:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.809616:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.809616:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.809618:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.809619:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.809620:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.809621:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.809622:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.809632:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.809634:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.809634:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.809634:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.809635:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.809636:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.809637:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.809639:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.809640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.809641:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.809709:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.809712:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.809714:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.809716:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.809717:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.809718:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.809719:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.809721:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.809722:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.809723:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.809726:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.809728:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.809730:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.809731:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.809732:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.809733:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.809734:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.809735:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.809735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.809739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.809745:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.809748:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.809749:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.809749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.809751:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.809753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.809775:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.809777:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.809779:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.809781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.809781:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.809782:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.809783:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.809785:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.809785:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.809786:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.809789:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.809790:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.809793:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.809793:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.809794:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.809795:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.809796:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.809797:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.809799:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.809799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.809801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.809807:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.809809:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.809810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.809810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.809812:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.809813:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.0:1619677473.809814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.809815:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.809816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.809817:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:5.2:1619677473.810070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.810075:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.810078:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.810079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.810081:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.810082:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.810084:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.810085:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.810086:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.810087:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.810088:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.810092:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.810093:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.810096:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.810097:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.810097:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.810099:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.810099:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.810101:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.810102:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.810102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.810107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677473.810109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677473.810112:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:22.0:1619677473.810113:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.810113:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677473.810115:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:1.0:1619677473.810115:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.810115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.810116:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.810116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.810117:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.810117:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.810118:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:6.0:1619677473.810118:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:20.0:1619677473.810119:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.810120:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.810121:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.810122:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.810123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:20.0:1619677473.810124:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.810125:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.810126:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.810129:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.810130:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.810133:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.810133:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.810134:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.810136:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.810136:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677473.810137:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677473.810138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.810152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.810160:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.810162:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.810163:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.810165:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.810165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.810167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.810174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.810182:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.810184:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.810185:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.810186:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.810186:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.810188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677473.810236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.810246:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.810250:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.810252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.810255:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.810258:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.810261:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.810263:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.810265:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.810267:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.810269:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.810274:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.810277:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.810280:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.810281:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.810282:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.810284:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.810285:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.810286:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.810287:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.810290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.810297:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.810299:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.810300:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.810300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.810302:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.810304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.810468:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.810471:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.810472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.810473:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.810475:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.810477:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.810478:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.810479:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.810480:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.810481:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.810484:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.810486:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.810489:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.810490:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.810490:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.810492:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.810493:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.810494:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.810494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.810506:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.810508:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.810509:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.810511:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.810512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.810513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.810525:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.810527:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.810528:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.810529:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.810530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.810530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677473.810761:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.810763:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.810766:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.810768:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.810768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.810769:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.810771:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.810772:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.810773:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.810774:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.810777:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.810779:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.810782:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.810783:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.810783:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.810785:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.810786:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.810787:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.810787:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.810790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.810797:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.810800:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.810801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.810801:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.810803:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.810805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.810812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.810818:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.810821:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.810821:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.810821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.810823:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.810823:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.810924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677473.810932:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677473.810935:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677473.810937:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.810939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677473.810940:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.810941:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677473.810942:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677473.810942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:30.0:1619677473.810944:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.810945:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.810946:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.810946:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.810948:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:30.0:1619677473.810950:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.810951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.810951:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677473.810952:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.810953:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.810955:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.810956:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:30.0:1619677473.810956:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:20.0:1619677473.810956:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.810957:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677473.810958:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:20.0:1619677473.810958:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677473.810959:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000400:00000200:20.0:1619677473.810959:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677473.810960:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000400:00000200:20.0:1619677473.810960:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.810962:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:30.0:1619677473.810963:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:20.0:1619677473.810963:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.2:1619677473.810964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.810965:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.810968:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.810969:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:4.2:1619677473.810969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:20.0:1619677473.810970:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.810972:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.810972:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677473.810973:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.810974:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.810975:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:17.0:1619677473.810975:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.810976:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.810976:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677473.810977:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000400:00000200:17.0:1619677473.810979:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.810980:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677473.810980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677473.810981:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:17.0:1619677473.810982:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677473.810986:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.810989:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.810989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.810990:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.810992:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.810995:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.811201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.811209:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.811211:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.811212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.811214:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.811216:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.811218:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.811219:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.811221:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.811222:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.811223:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.811226:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.811228:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.811231:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.811232:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.811232:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.811234:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.811234:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.811235:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.811235:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.811247:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.811250:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.811251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.811251:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.811253:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.811257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.811259:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.811261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.811261:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.811262:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.811263:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.811264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.811329:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.811331:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.811332:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.811334:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.811336:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.811337:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.811338:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.811340:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.811341:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.811342:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.811345:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.811347:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.811349:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.811350:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.811351:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.811352:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.811353:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.811353:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.811354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.811367:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.811370:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.811370:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.811372:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.811373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.811374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.811636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.811643:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.811646:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.811647:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.811649:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.811652:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.811653:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.811654:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.811656:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.811657:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.811658:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.811661:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.811662:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.811665:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.811666:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.811666:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.811668:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.811669:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.811670:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.811671:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.811672:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.2:1619677473.811673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.811677:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677473.811678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677473.811681:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677473.811681:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677473.811682:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.811683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.811683:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.811684:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:24.0:1619677473.811685:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:20.0:1619677473.811685:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.811685:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.811686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.811686:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.811687:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.811687:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.811687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677473.811687:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677473.811688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677473.811688:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.811689:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677473.811689:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.811690:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.811690:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.811691:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.811692:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.811692:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677473.811692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677473.811693:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.811695:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.811697:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.811697:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677473.811699:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.811700:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.811702:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677473.811703:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.811704:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.811704:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:31.0:1619677473.811706:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.811706:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.811706:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677473.811708:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.811708:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677473.811709:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.811709:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:31.0:1619677473.811710:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.811710:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677473.811711:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.811712:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.811713:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.811714:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677473.811715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677473.811717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.811720:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.811722:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:8.2:1619677473.811723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.811725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.811727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677473.811727:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677473.811728:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.811729:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.811730:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.811730:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.811731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.811731:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.811732:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.811733:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.811734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:8.0:1619677473.811734:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.811737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.811890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.811898:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.811901:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.811902:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.811903:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.811906:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.811907:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.811908:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.811910:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.811911:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.811912:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.811916:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.811917:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.811920:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.811921:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.811922:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.811923:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.811924:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.811925:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.811926:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.811926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.811932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.811937:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.811939:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.811940:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.811941:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.811941:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.811941:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677473.811941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.811942:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.811944:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.811947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.811956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.811958:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.811959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.811959:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.811960:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.811961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.812138:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.812141:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.812142:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.812144:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.812146:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.812148:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677473.812149:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.812151:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.812152:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.812153:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677473.812156:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.812158:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677473.812161:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.812162:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677473.812162:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.812164:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677473.812165:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.812166:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.812166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.812170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.812177:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.812180:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.812180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.812181:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.812183:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.812186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.812190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.812196:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.812198:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.812199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.812199:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.812200:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.812201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.812416:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.812419:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.812420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.812422:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.812424:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.812425:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.812427:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.812428:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.812429:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.812430:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.812434:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.812435:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.812438:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.812439:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.812439:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.812441:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.812441:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.812442:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.812443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.812456:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.812458:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.812459:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.812461:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.812462:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.812463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.812486:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.812489:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.812491:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.812493:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.812493:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.812494:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.812495:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.812496:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.812497:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.812498:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.812501:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.812503:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.812505:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.812506:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.812506:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.812508:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.812509:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.812510:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.812512:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.812512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.812522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.812524:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.812524:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.812526:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.812526:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.812527:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.812527:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.812528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.812529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.812530:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677473.812779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.812785:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.812788:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.812789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.812791:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.812794:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.812795:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.812796:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.812797:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.812799:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.812800:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.812803:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.812805:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.812808:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.812809:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.812810:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.812811:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.812812:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.812814:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.812815:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.812815:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.812820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.812826:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677473.812827:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677473.812828:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.812829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.812829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.812831:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.812832:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.812835:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.812835:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.812838:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677473.813088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.813090:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.813093:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.813095:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.813096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.813097:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.813098:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.813100:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.813101:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.813102:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.813103:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.813107:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.813108:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.813111:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.813112:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.813112:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.813114:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.813115:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.813116:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.813116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.813121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.813130:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.813132:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.813133:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.813133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.813136:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.813139:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.813140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.813148:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.813150:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.813151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.813151:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.813152:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.813152:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677473.813291:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.813299:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.813299:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.813301:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.813303:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.813305:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.813306:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.813307:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.813308:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.813309:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.813313:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.813315:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.813317:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.813318:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.813319:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.813320:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.813321:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.813322:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.813322:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.813326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.813333:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.813336:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.813337:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.813337:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677473.813338:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677473.813339:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.813341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.813341:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.813342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.813344:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.813346:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.813347:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.813349:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.813350:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.813351:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.813352:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.813355:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.813357:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.813359:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.813360:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.813361:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.813362:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.813363:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.813364:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.813365:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.813366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.813376:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.813378:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.813378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.813379:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.813380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.813380:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.813381:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.813382:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.813383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.813384:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.813456:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.813459:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.813461:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.813463:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.813463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.813465:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.813466:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.813467:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.813468:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.813469:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.813473:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.813475:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.813476:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.813477:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.813478:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.813480:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.813480:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.813481:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.813482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.813493:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.813495:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.813496:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.813497:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.813499:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.813500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.813519:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.813521:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.813522:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.813523:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677473.813523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.813524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.813796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.813804:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.813806:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.813807:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.813809:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.813812:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.813813:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.813814:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.813816:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.813816:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.813817:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.813820:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.813822:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.813825:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.813826:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.813826:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.813828:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.813829:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.813829:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.813831:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.813831:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677473.813835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677473.813837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677473.813843:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677473.813844:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677473.813846:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677473.813846:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.813847:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677473.813847:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.813847:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:21.0:1619677473.813849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.813849:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.813852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677473.814099:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.814102:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.814102:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.814104:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.814107:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.814109:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677473.814110:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.814112:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.814113:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.814114:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677473.814117:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.814119:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677473.814121:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.814122:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677473.814123:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677473.814125:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677473.814125:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.814126:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677473.814127:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677473.814128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677473.814131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.814137:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:1.0:1619677473.814139:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:26.0:1619677473.814140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677473.814140:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.814141:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.814141:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.814142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:10.0:1619677473.814143:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.814143:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:10.0:1619677473.814146:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677473.814235:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.814237:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.814238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.814240:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.814243:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.814244:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.814245:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.814247:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.814248:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.814249:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.814252:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.814254:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.814256:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.814257:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.814258:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.814259:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.814260:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.814261:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.814261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.814273:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.814275:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.814276:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.814278:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.814278:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.814280:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.814294:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.814300:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.814301:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.814302:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.814302:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.814303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.814460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677473.814467:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.814470:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677473.814471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677473.814473:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.814476:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.814478:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677473.814479:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.814480:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.814481:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.814482:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677473.814485:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.814487:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677473.814489:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.814490:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677473.814491:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677473.814492:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677473.814493:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.814494:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677473.814495:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677473.814496:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.2:1619677473.814498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677473.814501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:26.0:1619677473.814505:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.814507:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.814508:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.814508:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:10.0:1619677473.814508:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.814510:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.814510:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677473.814511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:26.0:1619677473.814512:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:24.0:1619677473.814512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677473.814523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677473.814529:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.814532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.814532:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677473.814533:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.814533:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677473.814534:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.2:1619677473.814721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.814729:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.814731:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677473.814734:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.115@o2ib7 : GET - for me
00000800:00000200:20.0:1619677473.814734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677473.814737:0:19935:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.115@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:21.0:1619677473.814739:0:19935:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.115@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:21.0:1619677473.814740:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.814741:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677473.814743:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677473.814746:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : REPLY try# 0
00000800:00000200:21.0:1619677473.814748:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.814750:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.814750:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.814751:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.814752:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.814753:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677473.814759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677473.814763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.814768:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.814770:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.814770:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:20.0:1619677473.814770:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.814771:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.814773:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.814773:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: REPLY: OK
00000400:00000200:21.0:1619677473.814776:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.814776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.814776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:21.0:1619677473.814778:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677473.814778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.814779:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.814780:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.814781:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.814782:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.814786:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.814787:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.814789:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.814790:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.814790:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.814792:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.814792:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.814793:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.814794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.814806:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.814809:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.814810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.814810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.814812:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.814814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.814840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.814845:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.814848:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.814849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.814851:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.814854:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.814855:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.814857:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.814858:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.814859:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.814860:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.814863:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.814864:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.814867:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.814868:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.814868:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.814870:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.814870:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.814871:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.814871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.814883:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.814886:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.814887:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.814887:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.814889:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.814891:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.814916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677473.814924:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:30.0:1619677473.814926:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677473.814927:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:30.0:1619677473.814929:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:12.0:1619677473.814929:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.814931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:30.0:1619677473.814932:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677473.814932:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.814933:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.814934:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:12.0:1619677473.814934:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:30.0:1619677473.814935:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.814936:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.814936:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.814938:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.814938:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.814940:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.814940:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.814942:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677473.814942:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:30.0:1619677473.814947:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677473.814947:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.814950:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677473.814950:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677473.814952:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.814953:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:12.0:1619677473.814953:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.814954:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:12.0:1619677473.814954:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677473.814954:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:30.0:1619677473.814956:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.814956:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677473.814956:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:30.0:1619677473.814957:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677473.814957:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:30.0:1619677473.814958:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677473.814958:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.814958:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.814962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.814968:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.814970:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.814971:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.814971:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.814973:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.814975:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:31.0:1619677473.814978:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.814981:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.814981:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.814983:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.814985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.814986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.815004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.815011:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.815014:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.815015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.815016:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.815018:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.815020:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.815021:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.815022:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.815023:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.815024:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.815027:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.815029:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.815031:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.815032:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.815032:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.815033:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.815034:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.815035:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.815035:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.815039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.815045:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.815047:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.815048:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.815048:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.815050:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.815053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.815320:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.815323:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.815323:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.815325:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.815327:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.815329:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.815330:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.815332:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.815332:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.815334:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.815337:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.815339:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.815341:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.815342:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.815343:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.815344:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.815345:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.815346:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.815346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.815359:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.815361:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.815362:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.815364:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.815365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.815366:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.815417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.815422:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.815425:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.815427:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.815427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.815429:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.815431:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.815432:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.815434:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.815435:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.815436:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.815439:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.815440:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.815443:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.815443:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.815444:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.815446:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.815446:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.815447:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.815447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.815452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.815460:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.815462:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.815463:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.815464:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.815465:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.815466:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677473.815478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.815484:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.815487:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.815488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.815490:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.815492:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.815494:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.815495:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.815496:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.815497:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.815498:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.815501:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.815503:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.815505:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.815506:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.815507:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.815509:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.815509:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.815510:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.815510:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.815515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.815522:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.815525:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.815526:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.815526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.815529:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.815532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677473.815666:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.815669:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.815669:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677473.815671:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.815674:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.815675:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677473.815677:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.815678:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.815679:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.815680:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677473.815683:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.815685:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.815687:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.815687:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.815688:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.815689:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.815690:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.815691:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.815691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677473.815696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.815703:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.815705:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.815705:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.815707:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.815708:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.815709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677473.815882:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.815885:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.815887:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.815889:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.815889:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.815891:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677473.815892:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.815893:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.815894:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.815895:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677473.815899:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.815901:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677473.815904:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.815904:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677473.815905:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.815907:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677473.815907:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.815908:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.815909:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677473.815913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.815921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.815924:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.815925:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.815926:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.815927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.815929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.816077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.816083:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.816086:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.816087:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.816088:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.816091:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.816092:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.816093:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.816095:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.816096:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.816097:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.816100:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.816101:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.816103:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.816104:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.816105:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.816106:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.816107:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.816108:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.816109:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:4.2:1619677473.816110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677473.816110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.816112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.816115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677473.816117:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.816120:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677473.816121:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:10.0:1619677473.816121:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677473.816122:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:26.0:1619677473.816123:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.816123:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.816123:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.816124:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.816124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677473.816125:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.816125:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.816126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.816126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:26.0:1619677473.816127:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677473.816127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677473.816128:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.816129:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.816130:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.816131:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.816134:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.816136:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.816139:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.816139:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.816140:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.816142:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.816142:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.816143:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.816144:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.816156:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.816158:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.816159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.816160:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.816162:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.816165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.816244:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.816247:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.816247:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.816249:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.816251:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.816253:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.816254:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.816255:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.816256:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.816257:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.816260:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.816261:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.816264:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.816265:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.816265:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.816267:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.816268:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.816268:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.816269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.816280:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.816282:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.816283:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.816285:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.816286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.816287:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.816527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.816534:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.816537:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.816538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.816540:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.816543:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.816544:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.816546:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.816547:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.816548:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.816549:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.816552:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.816554:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.816557:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.816558:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.816558:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.816560:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.816561:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.816561:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.816562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.816567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.816576:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.816579:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.816580:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.816580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.816582:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.816586:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.816724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.816732:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.816735:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.816735:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.816737:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.816740:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.816741:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.816743:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.816744:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.816745:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.816746:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.816749:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.816751:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.816753:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.816754:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.816754:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.816756:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.816756:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.816757:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.816758:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.816770:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.816773:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.816774:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.816774:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.816776:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.816780:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.817060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677473.817066:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.817069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.817069:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:2.0:1619677473.817072:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.817074:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.817075:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677473.817077:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.817078:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.817079:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.817080:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677473.817083:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.817085:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677473.817088:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.817089:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677473.817089:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677473.817091:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677473.817092:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.817092:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:31.0:1619677473.817093:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677473.817093:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:31.0:1619677473.817096:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677473.817097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:31.0:1619677473.817098:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.817100:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.817100:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.817102:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.817103:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.817104:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.817105:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.817105:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:31.0:1619677473.817106:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.817109:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.817111:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677473.817111:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.817111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.817112:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.817114:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.817114:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677473.817114:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.817115:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.817115:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.817117:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677473.817117:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.0:1619677473.817117:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:31.0:1619677473.817118:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677473.817118:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677473.817119:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:22.0:1619677473.817119:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.817121:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.817121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677473.817122:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.817123:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.817124:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.817125:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.817126:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.817128:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677473.817131:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:22.0:1619677473.817131:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.817133:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:22.0:1619677473.817133:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:30.0:1619677473.817134:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.817135:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.817136:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:22.0:1619677473.817136:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.817137:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.817137:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.817137:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.817137:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677473.817138:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.817139:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.817139:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.817140:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.817140:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.817142:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.817143:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.817143:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.817152:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.817153:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.817155:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.817155:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677473.817156:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.817156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.817157:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.817157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.817158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.817159:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.817405:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.817408:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.817410:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.817412:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.817413:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.817414:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.817415:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.817417:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.817417:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.817419:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.817422:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.817424:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.817426:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.817427:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.817428:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.817429:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.817430:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.817431:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.817431:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.817447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.817454:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.817457:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.817458:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.817458:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.817460:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.817462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.817807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.817813:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.817815:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.817817:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.817818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.817819:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.817821:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.817822:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.817823:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.817824:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.817825:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.817828:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.817830:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.817832:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.817833:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.817833:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.817835:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.817835:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.817836:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.817837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.817841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.817848:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.817850:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.817851:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.817853:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.817854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.817855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677473.818097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.818103:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.818105:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.818107:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.818108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.818109:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.818111:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.818112:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.818113:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.818114:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.818115:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.818117:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.818119:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.818121:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.818122:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.818123:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.818124:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.818125:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.818126:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.818127:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.818128:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677473.818130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677473.818133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.818136:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677473.818136:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.818139:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:8.0:1619677473.818139:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.818140:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677473.818140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.818141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677473.818141:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.818143:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.818146:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.818214:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.818216:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.818217:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.818219:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.818221:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.818223:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.818224:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.818225:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.818226:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.818227:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.818230:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.818231:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.818234:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.818234:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.818235:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.818237:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.818237:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.818238:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.818239:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.818250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.818252:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.818253:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.818255:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.818256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.818257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.818271:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.818273:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.818274:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.818275:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.818276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.818277:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.818388:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.818391:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.818393:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.818395:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.818396:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.818397:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.818398:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.818399:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.818401:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.818401:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.818405:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.818407:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.818409:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.818410:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.818410:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.818412:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.818412:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.818413:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.818414:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.818424:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.818427:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.818428:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.818429:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.818431:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.818432:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:1.0:1619677473.818541:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.818543:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677473.818544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677473.818546:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.818548:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.818549:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677473.818551:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.818552:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.818553:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.818554:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677473.818557:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.818558:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677473.818561:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.818562:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677473.818562:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677473.818564:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677473.818564:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677473.818565:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677473.818565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677473.818570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.818578:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.818580:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.818582:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.818583:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.818584:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.818585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.818592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.818600:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.818602:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.818603:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.818604:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.818604:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.818606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.819021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.819026:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.819028:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.819030:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.819031:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.819033:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.819034:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.819035:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.819037:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.819038:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.819039:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.819042:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.819044:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.819046:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.819047:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.819047:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.819049:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.819050:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.819051:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.819052:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.819053:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.819055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677473.819056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677473.819059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.819062:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677473.819063:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677473.819064:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677473.819064:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.819065:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.819066:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677473.819066:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:26.0:1619677473.819067:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:17.0:1619677473.819067:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.819067:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.819068:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:17.0:1619677473.819070:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.819071:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.819073:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:26.0:1619677473.819074:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.819076:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.819077:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.819078:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.819079:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.819080:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.819084:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.819086:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.819088:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.819089:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.819090:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.819091:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.819092:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.819092:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.819093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.819106:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.819109:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.819110:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.819112:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.819114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.819116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.819125:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.819127:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.819128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.819130:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.819132:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.819134:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.819135:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.819137:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.819137:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.819139:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.819142:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.819144:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.819146:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.819147:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.819148:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.819149:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.819150:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.819151:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.819151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.819162:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.819165:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.819165:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.819167:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.819168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.819169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.819242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.819249:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.819252:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.819252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.819254:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.819257:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.819258:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.819260:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.819261:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.819262:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.819263:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.819266:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.819268:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.819270:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.819271:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.819271:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.819273:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.819274:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.819275:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.819275:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.819279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.819287:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.819289:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.819290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.819290:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.819292:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.819301:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677473.819428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677473.819436:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677473.819439:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677473.819441:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.819443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677473.819443:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.819445:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.819446:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.819447:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.819448:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.819449:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.819453:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.819454:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.819457:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.819458:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.819458:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.819460:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.819460:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.819460:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677473.819461:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677473.819462:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677473.819462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000400:00000200:31.0:1619677473.819463:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.819463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.819465:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.819466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:31.0:1619677473.819467:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.819468:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.819469:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.819470:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.819471:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.819473:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.819473:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:31.0:1619677473.819475:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.819475:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.819476:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.819477:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677473.819477:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.819478:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.819479:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:24.0:1619677473.819479:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.819481:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.819481:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677473.819481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.819482:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.819484:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.819484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.819486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.819492:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.819495:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.819495:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.819496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.819497:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.819498:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.0:1619677473.819499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.819500:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.819501:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.819502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.819508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:5.2:1619677473.819513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677473.819514:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.819516:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.819517:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.819517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.819518:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.819519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677473.819519:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.819521:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.819522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.819524:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.819526:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.819528:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.819529:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.819530:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.819532:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.819532:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.819536:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.819537:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.819540:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.819541:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.819542:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.819543:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.819544:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.819545:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.819545:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.819558:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.819561:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.819562:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.819562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.819564:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.819566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.819742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.819750:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.819752:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.819753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.819755:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.819758:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.819759:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.819760:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.819762:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.819763:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677473.819764:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677473.819764:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.819766:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677473.819767:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677473.819768:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.819769:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.819770:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.819770:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.819771:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677473.819772:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677473.819772:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:30.0:1619677473.819773:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.819773:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:30.0:1619677473.819774:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.819774:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:30.0:1619677473.819775:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.819775:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.819775:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677473.819776:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.819776:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677473.819778:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.819780:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:9.2:1619677473.819780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.819782:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.819782:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.819783:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.819784:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.819785:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.819786:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.819786:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677473.819786:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.819788:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.819789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.819789:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.819791:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.819793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.819796:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.819799:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.819800:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.819801:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.819803:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.819804:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.819820:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.819822:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.819823:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.819823:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.819824:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.819825:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677473.820023:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.820026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.820026:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.820028:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.820030:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.820032:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.820033:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.820035:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.820036:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.820037:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.820040:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.820041:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.820044:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.820044:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.820045:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.820047:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.820048:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.820048:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.820049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.820061:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.820063:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.820064:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.820065:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.820066:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.820067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677473.820083:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.820085:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677473.820086:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677473.820088:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.820091:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.820092:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.820094:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.820095:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.820096:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.820097:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.820100:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.820102:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.820104:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.820105:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.820105:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.820107:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.820108:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.820109:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677473.820109:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677473.820130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.820138:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.820140:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.820140:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.820141:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.820142:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.820143:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.820167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.820174:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.820176:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.820177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.820178:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.820178:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.820179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.820832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.820840:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.820843:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.820844:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.820846:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.820848:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.820850:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.820851:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.820852:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.820853:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.820854:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.820858:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.820859:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.820862:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.820863:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.820864:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.820865:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.820866:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.820866:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.820867:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.820881:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.820883:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.820884:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.820885:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.820887:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.820892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.820894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.820896:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.820897:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.820897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.820898:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.820898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.821131:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.821133:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.821134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.821136:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.821137:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.821139:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.821140:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.821141:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.821142:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.821143:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.821147:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.821149:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.821151:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.821152:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.821153:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.821154:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.821154:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.821156:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.821157:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.821157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677473.821163:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.821165:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677473.821167:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:31.0:1619677473.821168:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.821168:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:31.0:1619677473.821169:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.821169:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:30.0:1619677473.821170:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677473.821170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677473.821171:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:22.0:1619677473.821171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.821171:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.821171:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.821172:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.821173:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:31.0:1619677473.821174:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.821174:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677473.821175:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.821175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677473.821176:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.821179:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.821181:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.821184:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.821184:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.821185:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.821187:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.821187:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.821188:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.821189:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.821192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.821199:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.821202:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.821203:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.821203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.821205:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.821207:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.821212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.821218:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.821220:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.821221:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.821221:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.821222:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.821223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.821364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.821371:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.821373:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.821375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.821376:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.821379:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.821380:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.821381:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.821382:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.821383:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.821384:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.821387:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.821389:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.821392:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.821393:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.821393:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.821395:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.821396:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.821396:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.821397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.821410:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.821413:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.821413:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.821414:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.821416:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.821420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.821446:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.821449:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.821451:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.821453:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.821453:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.821455:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.821456:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.821457:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.821458:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.821459:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.821462:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.821464:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.821466:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.821467:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.821468:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.821469:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.821470:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.821471:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.821471:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.821475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.821481:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.821484:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.821485:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.821485:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.821487:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.821489:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.821521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.821529:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.821532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.821532:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.821534:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.821537:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.821539:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.821540:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.821541:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.821542:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.821543:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.821546:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.821547:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.821551:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.821551:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.821552:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.821554:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.821554:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.821555:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.821557:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.821557:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677473.821560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677473.821564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:1.0:1619677473.821568:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677473.821571:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:1.0:1619677473.821571:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.821572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.821572:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.821573:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.821574:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.821575:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:1.0:1619677473.821575:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.821578:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.821584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.821591:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.821594:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.821594:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.821596:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.821599:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.821600:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.821602:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.821603:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.821604:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.821605:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.821608:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.821610:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.821612:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.821613:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.821613:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.821615:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.821616:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.821617:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.821618:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.821618:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.821624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.821629:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.821631:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.821632:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.821633:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677473.821633:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.821633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.821634:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.821634:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.821636:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.821642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.821644:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.821646:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.821647:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.821647:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.821648:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.821649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.821912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677473.821919:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:18.0:1619677473.821920:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:24.0:1619677473.821922:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:18.0:1619677473.821923:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:8.0:1619677473.821923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:2.0:1619677473.821923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:24.0:1619677473.821926:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.821927:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.821929:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.821929:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.821932:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677473.821932:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677473.821934:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.821934:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.821936:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.821937:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.821937:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.821938:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.821939:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677473.821940:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677473.821944:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.821946:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.821947:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677473.821948:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677473.821950:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.821951:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677473.821952:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.821952:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677473.821953:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677473.821954:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:18.0:1619677473.821954:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677473.821954:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:24.0:1619677473.821955:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:18.0:1619677473.821955:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:24.0:1619677473.821956:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:18.0:1619677473.821956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:24.0:1619677473.821957:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677473.821957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677473.821960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.2:1619677473.821961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677473.821966:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.821969:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:17.0:1619677473.821969:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:10.0:1619677473.821970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.821971:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.821972:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:26.0:1619677473.821973:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:17.0:1619677473.821973:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.821973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.821975:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.821976:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:17.0:1619677473.821978:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.822033:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.822036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.822036:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.822038:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.822040:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.822042:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.822043:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.822045:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.822046:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.822047:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.822050:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.822052:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.822055:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.822055:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.822056:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.822058:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.822058:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.822059:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.822060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.822072:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.822074:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.822075:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.822077:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.822077:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.822079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.822089:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.822091:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.822092:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.822093:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.822094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.822094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677473.822568:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.822571:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.822573:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.822575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.822576:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.822577:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.822578:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.822579:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.822581:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.822582:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.822585:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.822587:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.822589:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.822590:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.822590:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.822592:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.822593:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.822594:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.822595:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.822596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.822607:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.822608:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.822609:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.822611:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677473.822611:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.822612:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.822612:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.822613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.822614:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.822615:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677473.822706:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.822709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.822709:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677473.822711:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.822713:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.822715:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677473.822716:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.822718:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.822719:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.822720:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677473.822723:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.822725:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677473.822728:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.822728:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677473.822729:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.822731:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677473.822731:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.822732:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.822732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.822737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.822744:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.822746:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.822747:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.822748:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.822750:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.822753:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.822887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.822890:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.822892:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.822894:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.822895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.822897:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.822899:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.822900:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.822901:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.822902:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.822903:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.822907:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.822908:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.822911:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.822912:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.822912:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.822914:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.822915:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.822915:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.822916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.822920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.822932:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:22.0:1619677473.822934:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677473.822934:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.822935:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.822937:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:21.0:1619677473.822937:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.822937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:6.0:1619677473.822937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.822939:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:22.0:1619677473.822940:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.822942:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.822943:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.822945:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.822946:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.822947:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.822948:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.822951:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.822953:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.822956:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.822956:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.822957:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.822959:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.822960:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.822960:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.822961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.822972:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.822974:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.822975:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.822977:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.822978:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.822979:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.823085:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.823088:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.823090:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.823092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.823093:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.823094:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.823095:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.823097:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.823098:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.823099:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.823102:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.823104:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.823106:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.823107:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.823108:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.823110:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.823110:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.823111:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.823112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.823115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.823121:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.823124:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.823125:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.823125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.823126:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.823129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.823189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.823195:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.823198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.823198:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677473.823201:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.823204:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.823205:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.823207:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.823208:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.823209:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.823210:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.823213:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.823215:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.823216:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.823217:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.823218:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.823220:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.823220:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.823221:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.823222:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.823226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.823233:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.823235:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.823235:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.823236:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.823237:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.823238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.823894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.823902:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.823905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.823905:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677473.823907:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.823910:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.823912:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.823913:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.823914:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.823915:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.823916:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.823919:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.823921:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.823924:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.823925:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.823926:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.823927:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.823928:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.823928:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.823929:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.823934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.823943:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.823945:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.823946:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.823948:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.823949:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.823951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.823954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.823962:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.823965:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.823966:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.823966:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.823967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.823968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.823994:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.823997:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.823997:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.823999:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.824002:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.824003:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.824004:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.824005:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.824006:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.824007:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.824011:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.824012:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.824015:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.824016:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.824016:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.824018:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.824019:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.824020:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.824020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.824033:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.824034:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.824035:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.824037:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.824038:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.824039:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.824063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.824070:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.824073:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.824074:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.824075:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.824077:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.824079:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.824080:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.824082:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.824083:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.824084:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.824087:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.824089:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.824092:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.824092:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.824093:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.824094:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.824095:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.824096:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.824096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.824100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677473.824103:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.824106:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.824108:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.824108:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:31.0:1619677473.824110:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.824110:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.824111:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:30.0:1619677473.824112:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677473.824112:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.824112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:30.0:1619677473.824113:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.824114:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:30.0:1619677473.824115:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.824116:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677473.824116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:30.0:1619677473.824117:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.824120:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677473.824121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:30.0:1619677473.824122:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.824124:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.824125:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.824126:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.824127:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.824128:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677473.824128:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677473.824129:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.824129:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677473.824131:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.824132:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.824132:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:9.2:1619677473.824133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677473.824133:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.824134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677473.824139:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.824141:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.824142:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.824143:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.824145:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.824147:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.824259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.824264:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.824267:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.824268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.824270:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.824272:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.824274:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.824275:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.824276:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.824277:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.824278:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.824282:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.824284:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.824286:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.824287:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.824288:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.824290:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.824290:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.824291:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.824291:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.824306:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.824309:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.824310:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.824310:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.824312:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.824314:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.824437:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.824440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.824440:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677473.824443:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.824445:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.824446:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677473.824447:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.824449:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.824450:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.824451:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677473.824454:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677473.824455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677473.824456:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677473.824459:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.824460:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:6.0:1619677473.824461:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.824463:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.824463:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677473.824463:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.824464:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:26.0:1619677473.824465:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:6.0:1619677473.824465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:10.0:1619677473.824467:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.824468:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.824471:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.824472:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.824473:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.824475:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.824476:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.824477:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:12.0:1619677473.824477:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.824479:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:26.0:1619677473.824480:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677473.824480:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.824482:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:12.0:1619677473.824482:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.824484:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.824484:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.824485:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:12.0:1619677473.824485:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.824486:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.824487:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.824488:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.824488:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.824489:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.824493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.824500:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:13.0:1619677473.824500:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.824502:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677473.824503:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:13.0:1619677473.824503:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:24.0:1619677473.824504:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.824504:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.824504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.0:1619677473.824504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.824505:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.824506:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677473.824508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.824508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.824513:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.824516:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.824518:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.824519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.824520:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.824522:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.824523:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.824525:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.824526:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.824527:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.824530:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.824532:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.824534:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.824535:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.824536:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.824537:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.824538:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.824539:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.824540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.824544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.824552:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.824555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.824555:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.824556:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.824558:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.824560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.824563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.824571:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.824573:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.824574:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.824574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.824575:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.824576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.824602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.824609:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.824612:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.824612:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.824614:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.824617:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.824618:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.824619:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.824620:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.824621:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.824622:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.824625:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.824627:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.824630:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.824630:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.824631:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.824633:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.824633:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.824634:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.824635:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.824639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.824647:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.824650:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.824650:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.824651:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.824653:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.824656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677473.824912:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.824914:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.824915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.824917:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.824920:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.824922:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.824923:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.824924:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.824925:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.824926:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.824930:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.824931:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.824934:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.824935:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.824935:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.824937:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.824937:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.824939:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.824940:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.824941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677473.824949:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677473.824950:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.824951:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:31.0:1619677473.824952:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677473.824952:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.824953:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.824953:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.824953:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.824954:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:31.0:1619677473.824955:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.824955:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.824956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677473.824957:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.824957:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677473.824958:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677473.824959:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.824960:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.824961:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.824963:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.824964:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.824967:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.824968:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.824971:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.824972:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.824972:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.824974:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.824975:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.824976:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.824976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.824980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.824986:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.824988:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.824989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.824989:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.824991:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.824993:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.825046:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.825048:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.825051:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.825052:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.825052:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.825054:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.825055:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.825056:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.825057:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.825058:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.825061:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.825062:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.825064:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.825065:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.825066:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.825067:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.825068:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.825069:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.825069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.825073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.825079:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.825081:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.825082:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.825082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.825084:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.825086:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.825089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:9.2:1619677473.825091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.825096:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:8.0:1619677473.825097:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.825098:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:24.0:1619677473.825100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.825100:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.825101:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.825101:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.825102:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:31.0:1619677473.825103:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.825103:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:8.0:1619677473.825103:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:31.0:1619677473.825105:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.825106:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.825107:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.825108:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.825109:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.825111:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.825113:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.825115:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.825116:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.825116:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.825118:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.825119:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677473.825119:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677473.825120:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677473.825123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.825130:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.825132:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.825133:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.825133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.825135:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.825137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.825147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.825154:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.825157:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.825157:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.825159:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.825162:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.825163:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.825164:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.825166:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.825166:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.825167:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.825170:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.825172:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.825175:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.825175:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.825176:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.825178:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.825178:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.825179:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.825181:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.825181:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.825187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677473.825192:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.825194:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.825194:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.825195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.825196:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677473.825196:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.825197:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.825198:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.825198:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.825201:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.825261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677473.825267:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.825269:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.825270:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.825272:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.825274:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.825275:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677473.825276:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.825278:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.825279:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.825280:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677473.825283:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.825285:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677473.825288:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.825289:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677473.825290:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677473.825291:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677473.825292:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:9.2:1619677473.825298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677473.825299:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677473.825300:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:26.0:1619677473.825306:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.825308:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677473.825309:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.825310:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.825312:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.825315:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677473.825429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.825435:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.825438:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.825440:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.825440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.825442:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.825444:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.825445:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.825446:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.825447:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.825448:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.825451:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.825453:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.825455:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.825456:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.825457:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.825458:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.825459:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.825460:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.825460:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.825472:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.825475:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.825476:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.825476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.825478:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.825480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.825491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.825499:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.825502:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.825503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.825504:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.825507:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.825509:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.825510:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.825511:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.825512:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.825513:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.825516:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.825518:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.825521:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.825522:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.825522:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.825524:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.825524:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.825526:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.825527:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.825527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.825530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.825533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677473.825537:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.825540:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.825540:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.825541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.825542:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677473.825542:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.825543:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.825544:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677473.825544:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.825547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.825825:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.825828:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.825829:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.825831:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.825833:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.825835:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.825836:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.825837:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.825838:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.825839:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.825843:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.825844:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.825847:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.825847:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.825848:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.825850:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.825851:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.825851:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.825852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.825863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.825866:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.825867:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.825868:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.825869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.825871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.825883:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.825885:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.825886:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.825886:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.825887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.825888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.825990:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.825993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.825993:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677473.825996:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.825998:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.825999:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.826001:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.826002:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.826003:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.826004:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.826008:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.826010:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.826012:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.826013:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.826014:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.826015:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.826016:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.826016:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.826017:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677473.826029:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.826031:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.826032:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.826032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.826035:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.826037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.826082:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.826085:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.826088:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.826089:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.826090:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.826091:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.826093:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.826094:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.826095:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.826096:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.826099:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.826100:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.826103:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.826104:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.826105:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.826106:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.826107:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.826108:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.826108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.826112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.826118:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.826120:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.826121:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.826121:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.826123:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.826126:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.826131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.826136:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.826139:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.826139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.826140:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.826141:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.826141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.826367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.826375:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.826377:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.826378:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.826379:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.826382:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.826384:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.826385:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.826386:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.826387:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.826388:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.826391:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.826393:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.826395:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.826396:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.826397:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.826398:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.826399:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.826400:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.826400:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.826413:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.826416:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.826417:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.826417:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.826420:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.826423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.826659:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.826662:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:4.2:1619677473.826662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677473.826664:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.826667:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.826667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.826668:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.826669:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.826669:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677473.826671:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.826671:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.826672:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:31.0:1619677473.826673:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.826673:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.826675:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.826676:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.826678:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.826679:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.826681:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.826681:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:8.0:1619677473.826681:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:31.0:1619677473.826682:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:8.0:1619677473.826682:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96507416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677473.826683:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.826683:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677473.826684:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:8.0:1619677473.826684:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677473.826685:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677473.826685:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677473.826687:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.826687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:8.0:1619677473.826688:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677473.826689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.826690:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.826693:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.826694:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.826695:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.826696:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.826697:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.826697:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.826697:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677473.826698:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.826698:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.826699:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:24.0:1619677473.826700:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.826700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.826701:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677473.826702:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.826703:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:9.2:1619677473.826703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677473.826704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.826705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:2.0:1619677473.826711:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.826713:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.826714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.826715:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.826717:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.826720:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677473.826813:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.826816:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.826818:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.826820:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.826820:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.826821:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677473.826823:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.826824:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.826825:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.826826:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677473.826829:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.826831:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677473.826834:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.826835:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677473.826835:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.826837:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677473.826838:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.826839:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.826840:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.826841:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677473.826843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.826852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677473.826854:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677473.826854:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.826855:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.826857:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677473.826857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.826857:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.826858:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.826859:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677473.826860:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.826933:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.826935:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.826936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.826938:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.826940:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.826942:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.826943:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.826944:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.826945:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.826947:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.826950:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.826952:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.826954:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.826955:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.826956:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.826957:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.826958:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.826959:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.826959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.826971:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.826973:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.826974:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.826976:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.826976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.826978:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677473.827115:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.827118:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.827119:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.827121:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.827123:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.827125:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677473.827126:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.827128:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.827129:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.827130:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.827133:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.827134:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.827137:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.827137:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.827138:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.827140:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.827140:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.827142:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677473.827143:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677473.827143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:6.0:1619677473.827154:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.827156:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.827157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.827157:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.827159:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.827162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.827171:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.827174:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.827174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.827175:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.827176:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.827176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.827410:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677473.827412:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.827413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.827414:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:31.0:1619677473.827465:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.827468:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.827471:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.827473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.827474:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.827475:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.827477:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.827478:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.827479:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.827480:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.827484:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.827485:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.827488:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.827489:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.827489:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.827491:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.827491:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.827492:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.827493:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.827504:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.827507:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.827507:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.827509:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.827510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.827512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677473.827822:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.827825:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.827826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.827828:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.827830:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.827831:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.827832:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.827834:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.827835:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.827836:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.827838:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.827840:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.827843:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.827844:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.827844:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.827846:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.827847:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.827848:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.827848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.827859:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.827862:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.827863:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.827864:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.827866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.827867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.828101:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.828103:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.828104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.828106:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.828109:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.828110:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677473.828112:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.828113:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.828114:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.828115:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677473.828118:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.828119:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677473.828123:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.828123:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677473.828124:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677473.828126:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677473.828126:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.828127:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.828127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:31.0:1619677473.828138:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.828141:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.828142:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.828144:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.828145:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.828147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677473.828613:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.828616:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.828618:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.828619:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.828620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.828621:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.828622:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.828623:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.828624:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.828625:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.828628:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.828630:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.828632:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.828633:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.828633:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.828635:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.828636:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.828636:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.828637:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.828643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.828650:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.828652:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.828653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.828653:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.828655:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.828658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.828715:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.828718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.828718:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.828721:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.828724:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.828725:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.828726:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.828727:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.828728:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.828729:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.828732:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.828734:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.828737:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.828737:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.828738:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.828740:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.828740:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.828742:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.828743:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.828743:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.828753:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.828755:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.828755:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.828756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.828757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.828757:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.828758:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.828760:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.828761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.828762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.829083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.829089:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.829092:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.829093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.829095:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.829097:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.829098:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.829099:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.829101:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.829101:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.829102:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.829106:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.829107:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.829110:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.829110:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.829111:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.829113:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.829113:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.829114:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.829114:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.829115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.829120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.829125:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.829126:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.829127:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.829127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.829128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.829129:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.829130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.829130:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.829132:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.829135:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.829222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677473.829226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:26.0:1619677473.829230:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677473.829230:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.829233:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:26.0:1619677473.829234:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.829234:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677473.829236:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.829237:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.829238:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.829239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.829239:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677473.829240:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677473.829240:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.829241:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677473.829241:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.829243:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677473.829244:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677473.829244:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.829244:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.829245:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.829246:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.829247:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.829247:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677473.829247:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.829249:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.829250:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.829250:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677473.829251:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.829254:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96507404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.829254:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.829254:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.829255:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.829256:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.829257:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.829257:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677473.829258:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677473.829258:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.829259:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.829259:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.829260:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.829260:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677473.829261:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.829261:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:31.0:1619677473.829262:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.829262:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677473.829262:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:26.0:1619677473.829263:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677473.829263:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:31.0:1619677473.829264:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677473.829264:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.829264:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677473.829266:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.829267:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.829267:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:9.2:1619677473.829268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.829269:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.829269:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.829270:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.829271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677473.829275:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.829277:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677473.829277:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.829278:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.829278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.829281:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:22.0:1619677473.829281:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:30.0:1619677473.829282:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:24.0:1619677473.829282:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.829283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.829283:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.829284:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.829285:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.829287:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.829287:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.829288:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.829289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:24.0:1619677473.829290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677473.829720:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.829723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.829723:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677473.829726:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.829728:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.829730:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677473.829731:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.829732:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.829733:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.829734:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677473.829738:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.829739:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677473.829742:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.829743:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677473.829744:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677473.829745:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677473.829746:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.829746:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.829747:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677473.829753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.829763:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.829765:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.829766:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.829767:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.829768:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.829770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.829792:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.829795:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.829796:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.829798:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.829800:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.829801:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.829803:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.829804:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.829805:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.829806:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.829809:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.829811:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.829813:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.829814:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.829815:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.829816:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.829817:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.829818:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.829818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.829831:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.829833:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.829834:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.829835:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.829836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.829837:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.829849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.829850:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.829852:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.829853:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.829853:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.829854:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.829854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.829854:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677473.829855:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.829856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.829856:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.829858:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.829859:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.829861:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.829862:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.829863:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.829864:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.829867:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.829869:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.829871:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.829872:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.829873:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.829874:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.829875:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.829875:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.829876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.829881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.829889:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.829891:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.829892:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.829892:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.829894:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.829896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677473.829921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.829926:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.829929:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.829931:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.829931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.829934:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.829936:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677473.829937:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.829938:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.829939:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.829940:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677473.829944:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.829945:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677473.829948:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.829949:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677473.829950:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677473.829952:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677473.829952:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.829953:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677473.829954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:9.2:1619677473.829958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:4.2:1619677473.829960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.829967:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.829969:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.829970:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677473.829970:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.829972:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.829972:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:26.0:1619677473.829974:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.829976:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.829977:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.829977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.829978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.829979:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.829980:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.829982:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.829983:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.829984:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.829987:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677473.829987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.829990:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.829993:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.829994:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.829995:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.829995:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.829996:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.829996:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.829996:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.829997:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.829998:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.829998:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.829998:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677473.829999:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.830000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.830012:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.830014:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.830015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.830016:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.830018:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.830023:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.830025:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.830027:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.830028:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.830028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.830029:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.830030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.830223:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.830225:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.830228:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.830230:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.830230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.830231:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.830232:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.830233:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.830234:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.830235:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.830238:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.830239:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.830242:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.830242:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.830243:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.830244:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.830245:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.830246:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.830247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.830250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.830257:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.830259:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.830260:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.830260:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.830262:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.830264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.830436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677473.830444:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677473.830447:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677473.830449:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.830451:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677473.830451:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.830452:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677473.830454:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.830455:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.830456:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.830457:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677473.830460:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.830462:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677473.830465:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.830465:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677473.830466:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677473.830468:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677473.830468:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677473.830469:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677473.830470:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677473.830475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.830483:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.830486:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.830487:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.830487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.830489:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.830493:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.830494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677473.830502:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.830505:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.830505:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.830506:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.830507:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.830507:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677473.830688:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.830691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.830691:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.830694:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.830696:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.830697:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.830698:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.830699:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.830701:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.830702:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.830705:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.830707:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.830709:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.830710:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.830711:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.830713:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.830713:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.830714:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.830715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.830726:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.830728:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.830729:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.830731:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.830732:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.830733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.830807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.830813:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.830816:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.830816:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.830819:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.830821:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.830823:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.830824:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.830825:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.830827:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.830828:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.830831:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.830832:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.830836:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.830837:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.830838:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.830839:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.830840:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.830841:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.830841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.830853:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.830856:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.830857:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677473.830857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:12.0:1619677473.830861:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.830862:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.830864:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677473.830865:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.830868:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.830869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.830871:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.830874:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.830875:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.830877:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.830878:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.830879:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.830880:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.830883:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.830884:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.830887:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.830888:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.830889:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.830890:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.830891:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.830892:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.830892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.830904:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.830907:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.830908:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.830908:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.830910:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.830914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.831152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.831160:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.831163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.831163:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.2:1619677473.831165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677473.831166:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.831169:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.831170:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677473.831170:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.831171:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.831173:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:10.0:1619677473.831173:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.831174:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.831175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.831176:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.831176:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.831179:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.831180:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677473.831180:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677473.831182:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.831183:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.831184:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.831184:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.831186:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677473.831186:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.831187:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.831187:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677473.831189:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.831189:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.831190:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.831190:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:21.0:1619677473.831191:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677473.831192:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.831192:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677473.831194:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.831194:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.831195:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.831197:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.831197:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677473.831198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677473.831199:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.831200:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.831200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.831203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677473.831203:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.831205:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:20.2:1619677473.831206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677473.831206:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.831207:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677473.831207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.831207:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.831208:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.831209:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.831209:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.831213:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:6.0:1619677473.831213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677473.831214:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.831214:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677473.831215:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.831215:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.831216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.831217:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.831217:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.831218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677473.831219:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677473.831618:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.831620:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.831621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.831623:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.831625:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.831627:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.831628:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.831629:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.831630:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.831631:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.831635:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.831636:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.831639:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.831640:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.831640:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.831642:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.831643:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.831644:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.831644:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.831656:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677473.831658:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677473.831658:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.831659:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.831661:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677473.831661:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.831662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.831663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677473.831664:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.831665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.831667:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.831668:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.831669:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.831671:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.831672:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.831673:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.831676:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.831678:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.831680:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.831681:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.831682:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.831683:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.831683:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.831685:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.831686:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.831687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.831697:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.831699:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.831700:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.831701:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677473.831701:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.831702:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.831702:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.831703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.831704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.831705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.831713:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.831715:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.831716:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.831717:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.831717:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.831718:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.832037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.832043:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.832046:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.832047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.832049:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.832052:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.832054:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.832055:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.832056:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.832057:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.832058:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.832061:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.832063:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.832064:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.832065:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.832066:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.832068:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.832068:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.832069:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.832069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.832074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.832081:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.832083:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.832084:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.832084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.832085:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.832087:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.832094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.832101:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.832103:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.832104:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.832104:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.832105:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.832105:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.832411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.832419:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.832421:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.832422:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.832424:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.832427:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.832428:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.832429:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.832430:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.832431:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.832432:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.832435:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.832437:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.832439:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.832440:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.832440:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.832442:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.832442:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.832443:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.832444:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.832456:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.832459:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.832459:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.832461:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.832463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.832464:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.832667:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.832670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.832670:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.832672:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.832675:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.832676:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.832677:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.832678:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.832679:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.832680:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.832683:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.832685:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.832687:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.832688:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.832689:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.832690:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.832691:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.832692:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.832694:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.832694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.832705:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.832707:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.832707:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.832708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.832709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.832709:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.832710:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.832711:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.832712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.832713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.832848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677473.832856:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.832859:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677473.832860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677473.832862:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.832865:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.832866:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.832868:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.832869:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.832870:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.832871:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.832874:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.832876:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.832879:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.832880:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.832881:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.832882:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.832883:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677473.832884:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677473.832884:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677473.832888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.832897:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.832899:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.832900:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.832900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.832903:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.832906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.833548:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.833551:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.833553:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.833555:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.833555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.833556:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.833558:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.833559:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.833560:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.833561:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.833564:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.833565:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.833568:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.833568:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.833569:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.833570:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.833571:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.833572:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.833572:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.833577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.833584:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.833587:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.833588:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.833588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.833590:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.833592:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.833594:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:9.2:1619677473.833595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:22.0:1619677473.833597:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.833597:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.833599:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.833601:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.833602:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677473.833603:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.833604:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.833605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.833605:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.833606:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.833606:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:6.0:1619677473.833606:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.833607:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677473.833607:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.833608:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.833610:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.833612:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.833615:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.833615:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.833616:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.833618:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.833618:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.833619:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.833620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.833632:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.833634:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.833635:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.833637:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.833638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.833639:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.833648:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.833650:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.833651:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.833651:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.833652:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.833653:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.833778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.833784:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.833787:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.833788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.833790:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.833792:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.833794:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.833795:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.833797:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.833798:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.833799:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.833802:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.833803:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.833806:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.833807:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.833807:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.833809:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.833809:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.833810:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.833810:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.833823:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.833825:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.833826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.833827:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.833829:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.833831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.833956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.833964:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.833967:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.833968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.833970:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.833972:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.833973:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.833975:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.833976:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.833977:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.833978:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.833981:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.833982:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.833985:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.833985:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.833986:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.833987:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.833988:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.833988:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.833989:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.833993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.833999:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.834002:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.834002:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.834003:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.834004:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.834006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.834054:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.834057:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.834059:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.834061:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.834061:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.834063:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.834064:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.834065:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.834066:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.834067:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.834070:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.834071:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.834073:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.834074:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.834074:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.834075:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.834076:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.834077:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.834077:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677473.834082:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.834085:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.834085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.834087:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.834088:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.834090:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:17.0:1619677473.834090:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.834091:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.834091:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.834093:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:17.0:1619677473.834093:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677473.834094:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:17.0:1619677473.834094:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677473.834095:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:17.0:1619677473.834095:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.834096:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.834099:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.834100:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.834103:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.834104:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.834104:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.834106:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.834106:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.834107:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.834108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677473.834120:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.834122:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.834123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.834124:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.834126:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.834128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.834139:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.834141:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.834142:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.834142:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.834143:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.834144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.834500:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.834503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.834503:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.834505:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.834507:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.834509:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.834510:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.834511:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.834512:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.834513:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.834516:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.834518:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.834521:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.834521:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.834522:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.834524:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.834524:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.834525:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.834525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.834539:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.834541:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.834542:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.834543:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.834545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.834545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.834699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.834706:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.834709:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.834710:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.834711:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.834714:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.834715:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.834716:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.834718:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.834719:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.834720:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.834723:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.834725:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.834728:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.834729:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.834730:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.834732:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.834732:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.834733:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.834733:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.834745:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.834748:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.834749:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.834749:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.834752:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.834755:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677473.835498:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.835500:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.835502:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.835504:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.835505:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.835506:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.835507:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.835509:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.835510:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.835511:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.835514:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.835516:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.835518:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.835519:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.835519:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.835521:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.835521:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.835522:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.835523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677473.835527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:9.2:1619677473.835528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:12.0:1619677473.835535:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:8.0:1619677473.835535:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.835537:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.835538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:12.0:1619677473.835538:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:8.0:1619677473.835538:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.835541:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.835541:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.835543:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.835544:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.835544:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:12.0:1619677473.835545:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:9.2:1619677473.835545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:12.0:1619677473.835546:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.835548:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.835549:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.835550:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:24.0:1619677473.835551:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:12.0:1619677473.835553:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.835554:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.835554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.835555:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:12.0:1619677473.835555:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677473.835556:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.835556:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677473.835558:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.835558:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677473.835559:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677473.835559:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:12.0:1619677473.835560:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677473.835561:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.835562:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.835562:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:6.0:1619677473.835562:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.835563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.835565:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677473.835566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:6.0:1619677473.835567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.835568:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.835569:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.835570:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.835571:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.835572:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.835573:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.835575:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:6.0:1619677473.835575:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677473.835576:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.835577:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:6.0:1619677473.835577:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.835578:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.835580:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.835580:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.835581:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677473.835581:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.835583:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.835584:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.835585:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.835585:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:9.2:1619677473.835588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677473.835595:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.835598:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:21.0:1619677473.835598:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677473.835598:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.835599:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.835600:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.835600:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.0:1619677473.835603:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.835605:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677473.835606:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.835607:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.835608:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677473.835609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.835611:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.835613:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.835615:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.835616:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.835617:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.835618:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.835619:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.835620:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.835621:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.835625:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.835626:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.835629:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.835630:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.835630:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.835632:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.835632:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.835633:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.835634:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.835637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.835644:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.835646:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.835647:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.835647:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.835649:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.835651:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.835943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.835951:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.835953:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.835954:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.835955:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.835958:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.835960:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.835961:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.835962:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.835963:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.835964:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.835967:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.835968:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.835971:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.835972:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.835972:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.835974:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.835975:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.835975:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.835976:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.835988:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.835991:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.835992:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.835992:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.835994:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.835998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.836026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.836032:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.836035:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.836036:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.836037:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.836040:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.836042:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.836043:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.836044:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.836045:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.836046:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.836049:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.836050:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.836053:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.836054:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.836054:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.836056:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.836057:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.836057:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.836058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.836070:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.836072:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.836073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.836074:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.836076:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.836079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.836459:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.836462:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.836462:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.836464:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.836466:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.836468:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.836469:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.836471:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.836472:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.836473:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.836476:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.836478:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.836480:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.836481:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.836482:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.836483:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.836484:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.836485:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.836486:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.836486:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.836497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.836498:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.836499:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.836499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.836500:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.836500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.836501:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.836503:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.836504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.836505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.836672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.836678:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.836681:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.836682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.836684:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.836686:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.836688:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.836689:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.836691:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.836692:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.836693:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.836696:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.836698:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.836701:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.836701:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.836702:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.836704:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.836704:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.836706:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.836707:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.836707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.836711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.836712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.836718:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.836720:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.836721:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677473.836721:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:18.0:1619677473.836722:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677473.836722:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.836723:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.836723:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.836724:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.836726:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.836726:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:17.0:1619677473.836729:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.836729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.836730:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.836731:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677473.836732:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.836732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:17.0:1619677473.836734:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.836735:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.836736:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677473.836739:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.836741:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.836743:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677473.836744:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.836745:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677473.836745:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.836746:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677473.836747:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:30.0:1619677473.836748:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.836748:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.836748:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.836749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:31.0:1619677473.836750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.836750:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.836751:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.836753:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.836753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:30.0:1619677473.836754:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.836755:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.836756:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.836759:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.836759:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:30.0:1619677473.836760:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.836762:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.836762:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677473.836763:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:24.0:1619677473.836763:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.836763:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.836764:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.836765:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:8.0:1619677473.836765:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.836766:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.836767:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.836767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677473.836769:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.836770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.836776:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677473.836778:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.0:1619677473.836779:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677473.836780:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.836781:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.836781:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.836781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.836782:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.836783:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.836784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677473.836784:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677473.836785:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:21.0:1619677473.836785:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.836787:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.836788:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.836789:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.836792:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.836795:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.836797:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.836798:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.836798:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.836800:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.836800:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.836802:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.836802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.836806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.836814:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.836817:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.836818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.836818:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.836820:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.836823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.837081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.837088:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.837090:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.837091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.837093:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.837096:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.837097:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.837098:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.837100:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.837101:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.837102:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.837105:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.837107:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.837110:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.837111:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.837111:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.837113:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.837113:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.837114:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.837116:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.837116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.837119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.837122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677473.837127:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.837128:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.837130:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.837131:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.837131:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:2.0:1619677473.837131:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.837132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.837133:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677473.837133:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.837137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.837176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.837184:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.837186:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.837187:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.837189:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.837191:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.837193:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.837194:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.837195:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.837196:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.837197:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.837200:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.837202:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.837204:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.837205:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.837206:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.837207:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.837208:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.837209:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.837210:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.837211:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.837213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.837216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677473.837221:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.837223:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:18.0:1619677473.837224:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.837224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.837225:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.837226:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.837227:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677473.837227:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:10.0:1619677473.837227:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677473.837230:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:30.0:1619677473.837246:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.837249:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.837251:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.837253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.837253:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.837255:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.837256:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.837257:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.837258:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.837259:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.837262:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.837264:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.837266:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.837267:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.837267:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.837269:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.837270:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.837271:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.837273:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.837273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.837275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.837281:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.837283:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.837284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.837284:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.837286:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677473.837286:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.837288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.837289:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.837290:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.837290:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.837349:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.837351:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.837352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.837354:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.837356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.837358:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.837359:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.837360:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.837361:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.837362:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.837365:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.837366:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.837369:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.837370:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.837370:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.837372:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.837373:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.837374:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.837374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.837386:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.837389:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.837390:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.837391:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.837392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.837393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.837404:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.837407:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.837407:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.837408:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.837409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.837410:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.837770:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.837773:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.837776:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.837778:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.837778:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.837779:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677473.837781:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.837782:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.837783:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.837784:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677473.837787:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.837789:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677473.837792:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.837793:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677473.837794:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677473.837795:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677473.837796:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.837797:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.837798:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.837799:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677473.837802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.837810:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677473.837812:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677473.837813:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.837813:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.837815:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:20.0:1619677473.837815:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.837816:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677473.837816:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677473.837817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677473.837818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.838495:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.838498:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.838499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.838500:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.838503:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.838504:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.838505:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.838506:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.838507:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.838508:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.838511:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.838513:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.838515:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.838516:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.838516:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.838518:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.838519:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.838520:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.838520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.838533:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.838535:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.838536:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.838538:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.838539:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.838540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.838660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.838668:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.838671:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.838672:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.838673:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.838676:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.838677:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.838679:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.838680:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.838681:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.838682:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.838685:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.838686:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.838689:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.838689:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.838690:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.838691:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.838692:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.838693:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.838693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.838697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.838704:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.838706:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.838707:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.838707:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.838709:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.838711:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.839060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.839066:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.839069:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.839070:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.839072:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.839074:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.839076:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.839077:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.839078:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.839079:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.839080:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.839083:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.839085:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.839088:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.839088:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.839089:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.839091:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.839091:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.839092:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.839092:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.839105:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.839108:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.839109:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.839109:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.839111:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.839114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677473.839118:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.839120:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.839123:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.839125:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.839125:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.839127:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.839128:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.839130:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.839131:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.839132:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.839135:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.839136:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.839139:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.839140:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.839141:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.839142:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.839143:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.839144:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.839144:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.839148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.839154:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.839156:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.839157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.839157:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.839159:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.839161:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.839170:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.839172:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.839174:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.839176:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.839177:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.839178:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.839179:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.839181:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.839182:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.839182:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.839185:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.839187:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.839189:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.839190:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.839190:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.839192:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.839192:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.839193:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.839194:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.839198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.839204:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.839206:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.839207:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.839207:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.839209:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.839211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.839216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.839222:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.839224:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.839225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.839225:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.839226:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.839227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.839363:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.839366:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.839367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.839368:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.839370:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.839372:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.839373:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.839374:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.839375:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.839376:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.839379:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.839381:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.839383:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.839384:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.839385:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.839386:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.839387:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.839388:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.839388:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.839401:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.839404:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.839404:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.839406:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.839407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.839408:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.839501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.839508:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.839511:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.839512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.839514:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.839516:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.839518:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.839519:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.839520:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.839521:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.839523:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.839526:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.839528:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.839531:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.839531:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.839532:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.839534:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.839534:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.839535:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.839535:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.839548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.839551:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.839552:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.839552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.839554:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.839557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.839566:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.839569:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.839570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.839570:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.839571:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.839572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.840194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.840196:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.840197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.840199:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.840201:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.840203:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.840204:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.840205:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.840207:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.840207:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.840211:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.840213:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.840215:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.840216:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.840216:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.840218:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.840218:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.840220:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.840221:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.840221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.840232:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.840233:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.840233:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.840234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.840235:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.840235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.840236:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.840238:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.840239:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.840240:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.840348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677473.840356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.840359:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.840361:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.840363:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.840363:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.840365:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677473.840366:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.840368:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.840369:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.840370:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:13.0:1619677473.840373:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.840375:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677473.840377:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.840378:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:13.0:1619677473.840379:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:13.0:1619677473.840381:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:13.0:1619677473.840381:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.840382:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677473.840384:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677473.840384:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:4.2:1619677473.840391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:30.0:1619677473.840395:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.840397:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.840398:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.840398:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:30.0:1619677473.840399:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.840400:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:30.0:1619677473.840401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:12.0:1619677473.840401:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:31.0:1619677473.840402:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:12.0:1619677473.840402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677473.840887:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.840890:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.840891:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.840893:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.840895:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.840897:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677473.840898:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.840899:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.840900:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.840901:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677473.840904:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.840905:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:1.0:1619677473.840905:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.840907:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.840908:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:1.0:1619677473.840908:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.840908:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:30.0:1619677473.840910:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.840910:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.840910:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:31.0:1619677473.840911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:1.0:1619677473.840911:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:30.0:1619677473.840912:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.840912:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:30.0:1619677473.840914:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.840915:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.840916:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677473.840916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:30.0:1619677473.840917:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.840918:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.840922:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.840923:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677473.840924:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:30.0:1619677473.840925:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.840926:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677473.840926:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:30.0:1619677473.840927:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:17.0:1619677473.840927:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.840928:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677473.840928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:30.0:1619677473.840929:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:17.0:1619677473.840929:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.840930:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.840931:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:17.0:1619677473.840931:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.840934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.840941:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.840943:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.840944:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.840944:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.840946:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.840948:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.841102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.841108:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.841111:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.841112:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.841113:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.841116:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.841118:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.841119:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.841120:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.841121:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.841122:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.841125:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.841126:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.841129:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.841130:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.841131:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.841132:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.841133:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.841134:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.841134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.841147:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.841149:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.841150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.841151:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.841153:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.841156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.841218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.841226:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.841228:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.841229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.841231:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.841233:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.841235:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.841236:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.841237:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.841238:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.841239:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.841242:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.841244:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.841246:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.841247:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.841248:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.841249:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.841250:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.841251:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.841252:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.841252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.841258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.841263:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.841265:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.841266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.841266:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.841267:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677473.841267:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.841268:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.841269:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677473.841269:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.841272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.841285:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.841286:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.841287:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.841288:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.841288:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.841289:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.841289:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.841290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:6.0:1619677473.841290:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.841291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.841292:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.841300:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.841302:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.841303:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.841304:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.841305:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.841308:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.841309:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.841312:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.841312:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.841313:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.841315:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.841315:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.841316:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.841316:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.841330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.841332:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.841333:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.841335:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.841336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.841337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.841347:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.841350:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.841351:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.841351:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.841352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.841353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.841373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677473.841381:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.841383:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.841384:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.841386:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.841389:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.841391:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677473.841392:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.841394:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.841395:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.841396:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677473.841399:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.841401:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677473.841403:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.841404:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677473.841405:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677473.841406:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677473.841407:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.841407:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677473.841408:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:31.0:1619677473.841410:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.841412:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.2:1619677473.841413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000400:00000200:31.0:1619677473.841414:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.841416:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.841416:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:30.0:1619677473.841417:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.841418:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.841419:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.841419:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677473.841419:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.841420:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.841420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:31.0:1619677473.841421:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.841422:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677473.841422:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.841425:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000400:00000200:31.0:1619677473.841426:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.841428:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.841430:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.841430:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.841431:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.841432:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.841433:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.841434:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.841434:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.841445:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.841447:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.841448:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.841449:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.841451:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.841452:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.841461:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.841464:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.841465:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.841465:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677473.841466:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.841467:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.841467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.841474:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.841477:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.841478:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.841480:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.841483:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.841484:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.841486:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.841487:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.841488:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.841490:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.841492:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.841494:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.841497:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.841498:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.841499:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.841501:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.841501:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.841502:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.841503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677473.841513:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.841516:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677473.841517:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.841518:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677473.841520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.841521:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677473.841522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.841526:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.841529:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.841531:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.841532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.841533:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.841535:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.841536:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.841538:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.841539:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.841540:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.841543:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.841545:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.841547:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.841548:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.841549:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.841550:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.841551:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.841552:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.841552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.841564:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.841567:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.841568:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.841568:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.841570:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.841572:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.841872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.841880:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.841883:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.841884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.841886:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.841888:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.841890:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:4.2:1619677473.841890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677473.841891:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.841892:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.841893:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.841894:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.841897:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.841897:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677473.841899:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677473.841900:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:26.0:1619677473.841901:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.841901:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677473.841902:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.841903:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:8.0:1619677473.841903:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.841905:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.841905:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000400:00000200:8.0:1619677473.841905:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.841906:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.841906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:8.0:1619677473.841907:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.841908:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96507342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.841909:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.841910:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.841911:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.841914:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.841915:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677473.841918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:8.0:1619677473.841918:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.841919:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677473.841920:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:8.0:1619677473.841920:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:12.0:1619677473.841921:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.841921:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:12.0:1619677473.841922:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.841922:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.841922:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.841923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677473.841924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.841926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:9.2:1619677473.841928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.841935:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.841938:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.841939:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.841939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.841941:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.841945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.841955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.841960:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.841963:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.841963:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677473.841965:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.841967:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.841969:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.841970:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.841972:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.841972:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.841974:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.841976:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.841978:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.841980:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.841981:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.841981:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.841983:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.841984:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.841984:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.841985:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.841989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.841995:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.841997:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.841998:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.841999:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.841999:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.842001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.842005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.842013:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.842015:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.842016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.842016:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.842017:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.842017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.842151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677473.842159:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.842162:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.842164:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.842166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:6.0:1619677473.842166:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:13.0:1619677473.842167:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.842169:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677473.842169:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.842170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:13.0:1619677473.842171:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.842172:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.842173:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.842174:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.842174:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.842175:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677473.842176:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.842177:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.842179:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.842180:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.842181:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.842182:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:13.0:1619677473.842184:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677473.842186:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.842188:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.842188:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:13.0:1619677473.842189:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677473.842189:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677473.842191:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677473.842191:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.842192:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.842192:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:6.0:1619677473.842192:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.842192:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:13.0:1619677473.842193:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:6.0:1619677473.842194:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.842195:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.842196:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.2:1619677473.842197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677473.842197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:1.0:1619677473.842206:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:21.0:1619677473.842208:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:1.0:1619677473.842209:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:21.0:1619677473.842210:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:17.0:1619677473.842210:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.842210:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.842211:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.842213:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:21.0:1619677473.842214:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.842214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.842216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.842216:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677473.842930:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.842933:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.842935:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.842937:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.842938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.842939:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.842940:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.842942:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.842943:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.842944:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.842947:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.842949:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.842952:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.842952:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.842953:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.842955:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.842955:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.842956:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.842957:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.842961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.842968:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.842971:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.842972:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.842972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.842974:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.842976:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.843043:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.843046:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.843047:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.843049:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.843051:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.843053:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.843054:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.843056:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.843057:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.843058:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.843061:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.843062:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.843065:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.843065:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.843066:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.843068:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.843068:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.843069:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.843070:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.843081:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.843083:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.843084:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.843086:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.843087:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.843088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.843259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.843264:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.843267:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.843269:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.843269:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.843271:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.843273:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.843274:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.843275:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.843276:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.843277:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.843280:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.843282:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.843284:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.843285:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.843286:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.843287:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.843288:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.843288:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.843289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.843293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.843302:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.843304:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.843305:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.843306:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.843307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.843308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677473.843319:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.843322:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.843324:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.843327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.843327:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.843328:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.843329:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.843331:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.843332:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.843333:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.843336:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.843337:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.843339:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.843340:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.843341:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.843343:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.843343:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.843344:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.843345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.843349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.843356:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.843358:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.843359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.843359:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.843361:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.843363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.843697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.843704:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.843706:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.843707:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.843709:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.843711:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.843712:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.843713:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.843715:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.843716:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.843717:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.843720:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.843722:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.843724:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.843725:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.843725:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.843727:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.843727:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.843728:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.843729:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.843742:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.843744:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.843745:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.843745:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.843747:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.843750:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.843843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.843851:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.843854:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.843854:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.843856:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.843859:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.843860:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.843861:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677473.843861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677473.843863:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.843864:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.843865:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.843869:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677473.843870:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.843871:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.843872:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.843873:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.843874:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.843876:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:10.0:1619677473.843876:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.843877:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.843877:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.843878:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:26.0:1619677473.843879:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.843880:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.843880:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:26.0:1619677473.843881:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.843881:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.843882:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:26.0:1619677473.843884:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.843884:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.843885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.843886:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.843886:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.843889:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.843890:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.843890:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.843890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.843892:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.843892:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.843894:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.843894:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.843894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.843896:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.843896:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.843897:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:9.2:1619677473.843897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.843897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.843898:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.843898:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.843898:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677473.843900:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.843903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.843904:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.843906:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.843907:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.843907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.843909:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.843912:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.843915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.843921:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.843923:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.843924:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.843924:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.843925:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.843926:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.844121:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.844123:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.844124:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.844126:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.844128:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.844129:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.844130:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.844132:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.844133:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.844134:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.844137:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.844139:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.844141:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.844142:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.844143:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.844144:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.844145:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.844146:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.844147:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.844148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.844158:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.844160:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.844160:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.844161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.844162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.844163:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.844163:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.844165:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.844166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.844167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.844324:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.844327:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.844329:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.844332:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.844332:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.844333:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.844334:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.844336:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.844337:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.844338:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.844341:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.844343:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.844346:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.844346:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.844347:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.844349:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.844349:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.844351:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.844352:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.844353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.844364:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.844365:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.844366:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.844368:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.844368:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677473.844368:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.844369:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.844369:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.844370:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.844371:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.844662:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.844665:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.844667:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.844669:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.844669:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.844670:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.844671:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.844673:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.844674:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.844675:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.844677:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.844679:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.844681:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.844682:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.844682:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.844684:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.844684:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.844685:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.844686:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.844687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.844689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.844697:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.844699:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677473.844700:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677473.844700:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.844700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.844702:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:24.0:1619677473.844702:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.844703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.844704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677473.844704:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.844921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.844930:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.844932:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.844933:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.844935:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.844937:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.844938:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.844939:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.844941:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.844941:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.844942:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.844945:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.844947:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.844949:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.844950:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.844950:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.844952:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.844952:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.844953:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.844953:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.844992:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.844994:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.844995:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.844996:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.2:1619677473.844997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:12.0:1619677473.844999:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.845000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:18.0:1619677473.845004:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.845006:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.845007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.845009:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.845009:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:18.0:1619677473.845011:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.845012:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.845012:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:6.0:1619677473.845012:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:18.0:1619677473.845013:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:12.0:1619677473.845013:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:18.0:1619677473.845014:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677473.845014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.845015:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677473.845015:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.845016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:18.0:1619677473.845016:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.845017:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.845018:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677473.845019:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.845021:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.845021:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.845022:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:18.0:1619677473.845023:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:6.0:1619677473.845023:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.845025:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.845026:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677473.845027:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.845028:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:6.0:1619677473.845028:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.845029:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677473.845032:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677473.845032:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:6.0:1619677473.845032:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.845033:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.845034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:6.0:1619677473.845034:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.845036:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677473.845037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.845037:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.845037:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.845039:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.845040:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.845041:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.845041:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677473.845044:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.845046:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.845047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.845048:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.845050:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.845052:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.0:1619677473.845052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.845055:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.845055:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.845057:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.845058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.845059:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.845070:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.845072:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.845074:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.845074:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.845075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.845076:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.845593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.845601:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.845603:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.845604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.845606:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.845608:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.845610:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.845611:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.845613:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.845614:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.845615:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.845618:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.845619:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.845622:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.845623:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.845624:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.845625:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.845626:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.845626:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.845627:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.845640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.845643:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.845644:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.845644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.845646:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.845650:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.845883:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.845886:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.845887:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.845888:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.845890:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.845891:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.845893:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.845894:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.845895:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.845896:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.845900:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.845901:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.845904:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.845904:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.845905:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.845907:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.845907:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.845908:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.845909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.845920:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.845922:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.845923:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.845925:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.845926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.845927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.846003:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.846006:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.846007:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.846009:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.846011:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.846013:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.846014:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.846015:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.846016:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.846017:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.846021:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.846022:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.846025:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.846026:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.846026:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.846028:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.846029:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.846030:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677473.846031:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.846031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:4.2:1619677473.846036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677473.846042:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.846043:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.846044:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677473.846044:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:22.0:1619677473.846045:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.846045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.846046:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677473.846046:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677473.846047:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:10.0:1619677473.846047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.846048:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.846048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:26.0:1619677473.846050:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.846050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677473.846052:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.846054:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.846055:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.846057:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.846057:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.846059:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.846060:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677473.846062:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.846063:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677473.846064:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.846065:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.846067:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.846067:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.846067:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.846068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677473.846068:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677473.846069:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677473.846070:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.846070:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.846070:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677473.846071:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.846071:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677473.846072:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.846072:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677473.846073:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677473.846076:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.846077:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677473.846080:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.846081:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677473.846082:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.846083:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677473.846083:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677473.846084:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.846085:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.846085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677473.846086:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.846087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.846087:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.846089:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.846090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677473.846092:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.846098:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.846100:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:22.0:1619677473.846101:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677473.846101:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.846103:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.846104:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.846104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:22.0:1619677473.846105:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.846105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.846106:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.846106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.846107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677473.846107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.846115:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.846118:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.846119:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.846120:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.846120:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.846121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:31.0:1619677473.846622:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.846625:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.846628:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.846630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.846631:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.846632:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.846633:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.846635:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.846635:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.846636:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.846640:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.846641:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.846644:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.846645:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.846645:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.846647:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.846647:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.846648:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.846649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.846657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.846664:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.846667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.846667:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.846668:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.846670:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.846672:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.846683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.846688:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.846691:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.846692:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.846692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.846693:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.846694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.846842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.846847:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.846850:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.846852:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.846853:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.846855:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.846856:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.846858:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.846859:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.846860:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.846861:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.846864:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.846866:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.846868:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.846869:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.846869:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.846871:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.846872:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.846873:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.846874:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.846875:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677473.846880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.846884:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.846886:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:6.0:1619677473.846886:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.846887:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.846887:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.846888:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.846889:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.846890:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.846891:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.846894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.846914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.846921:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.846924:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.846925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.846927:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.846930:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.846931:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.846932:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.846934:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.846935:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.846936:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.846939:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.846941:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.846944:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.846944:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.846945:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.846947:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.846947:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.846948:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.846950:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.846950:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.846956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677473.846962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.846963:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.846964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677473.846965:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677473.846965:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677473.846966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677473.846966:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.846966:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:8.0:1619677473.846967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677473.846968:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.846969:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.846972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.846972:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.846973:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:6.0:1619677473.846973:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677473.846975:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.846975:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.846976:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.846978:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.846979:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.846982:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.846983:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.846986:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.846987:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.846987:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.846989:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.846989:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.846990:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.846991:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677473.847000:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677473.847002:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677473.847003:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.847005:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.847005:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.847005:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.847006:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677473.847007:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:30.0:1619677473.847008:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.847008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:30.0:1619677473.847009:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.847010:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.847010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677473.847011:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.847012:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.847013:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.847016:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.847018:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.847020:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.847021:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.847021:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.847023:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.847023:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.847024:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.847025:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.847035:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.847038:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.847039:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.847040:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.847042:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.847043:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.847320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677473.847326:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.847329:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.847330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.847332:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.847335:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.847337:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677473.847338:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.847340:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.847341:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.847342:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677473.847345:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.847346:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677473.847350:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.847350:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677473.847351:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677473.847353:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677473.847354:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.847355:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677473.847356:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677473.847356:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677473.847359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677473.847362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677473.847367:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:18.0:1619677473.847368:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:26.0:1619677473.847370:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:18.0:1619677473.847370:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:10.0:1619677473.847370:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:18.0:1619677473.847371:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:10.0:1619677473.847372:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.847372:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:10.0:1619677473.847374:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.847377:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677473.847579:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.847582:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677473.847582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677473.847584:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.847587:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.847588:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677473.847590:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.847591:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.847592:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.847593:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677473.847596:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.847598:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677473.847600:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.847601:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677473.847601:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677473.847603:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677473.847604:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.847605:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677473.847605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677473.847609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.847617:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.847619:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.847620:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.847620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.847623:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.847625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677473.847863:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.847866:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.847867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.847868:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.847871:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.847872:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.847873:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.847874:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.847875:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.847876:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.847878:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.847880:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.847882:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.847883:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.847883:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.847885:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.847885:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.847887:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.847888:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.847888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.847899:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.847901:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.847901:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.847902:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.847903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.847903:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.847904:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.847906:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.847907:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.847908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.848258:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.848261:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.848263:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.848265:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.848265:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.848267:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.848268:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.848269:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.848270:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.848271:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.848274:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.848276:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.848278:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.848279:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.848280:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.848281:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.848282:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.848283:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.848284:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.848287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.848294:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.848300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.848302:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (22)--
00000800:00000200:4.2:1619677473.848302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:24.0:1619677473.848303:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.848304:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.848306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677473.848310:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.848314:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.848314:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.848316:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.848318:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.848319:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.848320:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.848322:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.848322:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.848323:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.848326:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.848328:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.848330:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.848331:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.848331:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.848333:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.848333:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.848334:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.848334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:13.0:1619677473.848348:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.848350:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.848351:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.848353:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.848355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.848356:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677473.848453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.848460:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.848462:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.848463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.848466:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.848468:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.848469:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.848471:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.848472:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.848473:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.848474:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.848477:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.848478:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.848481:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.848481:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.848482:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.848484:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.848484:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.848485:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.848486:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.848486:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.848491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.848497:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.848498:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677473.848498:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.848499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.848500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.848501:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.848502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.848502:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.848504:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.848507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.848573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.848579:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.848582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.848582:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677473.848585:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.848587:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.848589:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.848590:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.848591:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.848593:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.848593:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.848597:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.848598:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.848600:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.848601:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:4.2:1619677473.848602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677473.848602:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.848603:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.848604:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.848605:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.848606:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.848606:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677473.848609:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677473.848609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.848612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677473.848612:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.848613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.848615:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.848617:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.848618:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:10.0:1619677473.848618:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:18.0:1619677473.848619:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.848619:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677473.848620:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.848620:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677473.848620:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.848620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677473.848621:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677473.848621:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.848622:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.848622:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677473.848623:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677473.848624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:10.0:1619677473.848624:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.848627:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.848629:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.848632:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.848633:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.848633:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.848635:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.848635:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.848636:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.848637:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.848649:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.848652:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.848652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.848653:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.848655:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.848658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.848666:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.848669:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.848670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.848670:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.848671:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.848672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.848715:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.848718:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.848720:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.848722:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.848722:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.848724:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.848725:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.848726:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.848727:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.848728:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.848731:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.848732:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.848735:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.848735:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.848736:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.848736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677473.848737:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.848738:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.848739:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.848740:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.848743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.848744:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.2:1619677473.848744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677473.848746:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677473.848747:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:22.0:1619677473.848748:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:10.0:1619677473.848748:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.848749:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677473.848749:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677473.848750:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677473.848750:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:26.0:1619677473.848752:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.848752:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.848752:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.848753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.848753:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.848753:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677473.848754:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.848754:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677473.848755:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677473.848756:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677473.848756:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.848756:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.848757:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.848758:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.848758:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.848759:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.848759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677473.848760:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.848760:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.848761:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.848761:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.848762:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.848763:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677473.848763:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.848764:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677473.848765:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.848766:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677473.848768:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.848771:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.848772:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677473.848772:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677473.848774:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.848775:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677473.848775:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.848775:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677473.848776:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677473.848777:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.848777:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677473.848778:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.848778:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.848779:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677473.848779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:2.0:1619677473.848779:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.848780:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677473.848780:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.848781:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677473.848781:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.848782:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.848782:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:2.0:1619677473.848782:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.848783:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677473.848783:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.848784:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.848785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677473.848789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677473.848790:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.848792:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.848793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.848795:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:20.0:1619677473.848795:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.848796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:13.0:1619677473.848796:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:24.0:1619677473.848797:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:20.0:1619677473.848798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:24.0:1619677473.848799:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.848799:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:8.0:1619677473.848799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:13.0:1619677473.848800:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.848801:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:13.0:1619677473.848801:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.848803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677473.848803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.848804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.0:1619677473.848810:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.848813:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.848814:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.848815:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.848815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.848816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.849082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677473.849087:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.849089:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.849091:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.849092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.849094:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.849095:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.849097:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.849098:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.849099:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.849100:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.849104:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.849106:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.849109:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.849110:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.849110:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.849112:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.849113:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.849114:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677473.849115:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677473.849115:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:4.2:1619677473.849121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.849124:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677473.849125:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677473.849127:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:30.0:1619677473.849128:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677473.849128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.849128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677473.849129:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.849131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:30.0:1619677473.849131:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.849134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677473.849258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.849261:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.849263:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677473.849266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.849266:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.849268:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.849270:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.849271:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.849273:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.849274:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.849275:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.849278:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.849280:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.849282:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.849283:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.849284:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.849285:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.849286:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.849287:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.849287:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.849292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.849301:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.849303:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.849304:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.849304:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.849307:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.849309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677473.849417:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.849420:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.849422:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.849424:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.849424:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.849425:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.849427:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.849428:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.849429:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.849430:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.849433:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.849435:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.849437:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.849438:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.849439:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.849440:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.849441:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.849442:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.849444:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.849444:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.849454:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.849456:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.849456:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.849458:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.849459:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.849459:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.849459:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.849460:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.849461:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.849462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677473.849821:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.849824:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.849825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.849827:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.849829:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.849830:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.849832:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.849833:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.849834:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.849835:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.849838:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.849840:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.849842:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.849843:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.849844:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.849845:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.849846:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.849847:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.849847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.849859:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.849861:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.849862:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.849863:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.849864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.849866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.849993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.849997:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.849999:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.850002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.850002:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.850004:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.850005:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.850007:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.850008:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.850009:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.850010:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.850013:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:4.2:1619677473.850014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677473.850015:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.850018:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.850019:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.850019:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.850021:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.850022:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.850022:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:10.0:1619677473.850022:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677473.850023:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:10.0:1619677473.850025:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.850026:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.850027:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.850030:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.850031:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.850033:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.850034:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677473.850035:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677473.850035:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.850036:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677473.850036:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677473.850038:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677473.850039:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.850040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.850040:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.850040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677473.850041:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.850042:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.850042:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.850042:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:22.0:1619677473.850043:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:30.0:1619677473.850044:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.850045:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.850045:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.850046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.850046:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:30.0:1619677473.850047:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.850047:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:30.0:1619677473.850048:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.850048:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:30.0:1619677473.850049:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.850049:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.850049:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.850050:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:30.0:1619677473.850052:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.850054:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:9.2:1619677473.850054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677473.850056:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.850057:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.850057:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.850059:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.850059:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.850060:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.850061:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677473.850062:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.850065:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:9.2:1619677473.850065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677473.850066:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.850066:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.850069:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.850071:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677473.850072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677473.850074:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.850075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.850075:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.850077:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.850079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.850083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.850089:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.850091:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.850092:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.850092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.850093:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.850094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.850670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.850678:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.850681:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.850682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.850683:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.850686:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.850688:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.850689:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.850690:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.850691:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.850692:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.850695:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.850697:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.850700:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.850701:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.850701:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.850703:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.850703:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.850704:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.850704:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.850719:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.850721:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.850722:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.850722:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.850723:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.850725:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.850725:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.850729:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.850729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.850730:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677473.850731:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.850732:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.850732:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.850732:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:22.0:1619677473.850734:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.850734:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.850734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.850735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.850736:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.850736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:22.0:1619677473.850737:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.850741:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.850743:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.850745:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.850746:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.850747:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.850748:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.850749:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.850749:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.850750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.850762:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.850764:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.850765:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.850767:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.850767:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.850769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.850811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.850819:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.850822:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.850823:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.850825:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.850828:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.850829:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.850830:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.850832:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.850833:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.850834:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.850837:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.850839:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.850842:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.850842:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.850843:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.850845:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.850845:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.850846:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.850847:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.850848:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.850853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.850858:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.850861:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.850861:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.850862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.850862:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.850863:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.850864:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677473.850864:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677473.850864:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.850868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.851031:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.851034:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.851036:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.851038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.851039:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.851040:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.851041:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.851043:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.851044:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.851045:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.851048:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.851049:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.851052:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.851052:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.851053:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.851055:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.851055:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.851056:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.851056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.851060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.851067:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.851069:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.851070:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.851070:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.851072:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.851075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.851552:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.851554:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.851556:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.851558:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.851559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.851560:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.851561:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.851562:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.851563:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.851564:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.851566:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.851568:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.851570:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.851571:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.851571:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.851573:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.851573:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.851574:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.851575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.851579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.851586:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.851588:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.851589:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.851590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.851591:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.851593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.851644:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.851647:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.851648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.851650:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.851652:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.851653:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.851654:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.851656:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.851657:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.851658:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.851661:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.851663:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.851665:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.851666:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.851667:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.851668:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.851669:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.851671:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.851672:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.851672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.851682:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.851682:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.851684:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677473.851685:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.851685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.851686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.851686:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.851687:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.851688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677473.851690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:20.0:1619677473.851691:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:12.0:1619677473.851698:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.851701:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677473.851703:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677473.851705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677473.851706:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.851707:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677473.851709:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677473.851710:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677473.851711:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677473.851712:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677473.851715:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677473.851717:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677473.851720:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677473.851720:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677473.851721:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677473.851723:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677473.851724:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677473.851725:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.851725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.851729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.851738:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.851741:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.851742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.851742:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.851744:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.851748:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.852659:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.852662:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.852664:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.852666:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.852667:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.852668:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.852669:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.852670:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.852672:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.852672:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.852676:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.852678:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.852680:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.852681:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.852682:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.852683:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.852684:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.852685:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.852687:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.852687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.852689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.852697:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.852699:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.852700:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.852700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.852702:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677473.852702:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.852704:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677473.852704:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.852705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.852706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677473.852707:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.852710:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.852711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.852713:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.852715:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.852716:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.852718:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.852719:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.852720:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.852721:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.852725:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.852726:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.852729:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.852730:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.852730:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.852732:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.852733:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.852733:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.852734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.852746:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.852748:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.852749:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.852751:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.852752:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.852753:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.852767:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.852770:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.852771:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.852772:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.852772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.852773:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.852801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.852804:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.852806:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.852808:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.852809:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.852810:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.852812:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.852813:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.852814:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.852815:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.852816:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.852819:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.852820:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.852822:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.852823:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.852824:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.852826:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.852826:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.852827:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.852827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677473.852839:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.852842:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.852843:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.852843:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.852845:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.852847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.852882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.852890:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.852893:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.852894:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.852896:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.852898:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.852900:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.852901:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.852903:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.852904:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.852905:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.852908:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.852910:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.852913:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.852914:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.852914:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.852916:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.852916:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.852917:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.852918:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.852930:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.852932:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.852933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.852933:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.852935:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.852939:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.853215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.853222:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.853225:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.853225:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.853227:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.853229:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.853231:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.853232:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.853233:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.853234:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.853235:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.853239:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.853240:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.853243:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.853244:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.853244:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.853246:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.853246:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.853247:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.853248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.853260:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.853263:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.853263:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.853264:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.853266:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.853268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.853491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.853499:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.853502:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.853503:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.853505:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.853507:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.853509:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.853510:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.853512:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.853512:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.853514:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.853517:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.853518:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.853520:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.853521:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.853522:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.853523:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.853524:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.853525:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.853525:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.853530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.853537:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.853539:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.853540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.853540:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.853542:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.853544:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.853617:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.853620:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.853621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.853622:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.853624:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.853626:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.853627:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.853629:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.853630:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.853631:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.853634:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.853636:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.853638:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.853639:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.853639:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.853641:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.853642:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.853643:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.853643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.853655:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.853657:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.853658:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.853659:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.853660:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.853662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.853892:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.853894:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.853897:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.853899:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.853899:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.853900:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.853901:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.853903:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.853904:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.853905:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.853908:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.853909:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.853911:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.853912:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.853913:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.853914:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.853915:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.853916:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.853916:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.853926:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.853929:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.853930:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.853931:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.853933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.853934:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.853950:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.853953:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.853953:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.853954:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.853955:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.853956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.853973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.853980:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.853983:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.853984:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.853985:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.853988:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.853989:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.853991:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.853992:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.853993:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.853994:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.853997:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.853999:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.854001:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.854002:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.854002:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.854004:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.854005:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.854006:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.854006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.854010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.854018:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.854021:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.854022:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.854022:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.854024:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.854026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.854030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.854037:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.854040:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.854041:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.854041:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.854042:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.854042:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.854504:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.854506:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.854507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:1.0:1619677473.854507:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:22.0:1619677473.854508:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.854510:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:22.0:1619677473.854511:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.854511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.854512:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.854513:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.854514:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.854515:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.854515:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.854516:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.854517:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677473.854518:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677473.854519:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.854521:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.854522:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677473.854522:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.854523:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.854524:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:1.0:1619677473.854527:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.854529:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.854530:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.854531:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:1.0:1619677473.854531:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.854532:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677473.854532:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.854533:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.854534:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.854534:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.854535:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:1.0:1619677473.854535:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.854535:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:22.0:1619677473.854536:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:1.0:1619677473.854536:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.854537:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.0:1619677473.854548:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.854550:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.854551:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.854552:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677473.854553:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.854554:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.854555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.854555:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.854556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.854556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.854558:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.854560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.854681:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.854683:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.854686:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.854688:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.854688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.854689:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.854690:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.854692:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.854693:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.854694:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.854697:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.854699:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.854702:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.854703:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.854703:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.854705:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.854706:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.854707:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.854707:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.854711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.854718:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.854721:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.854721:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.854722:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.854723:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.854725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.854730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.854736:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.854739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.854739:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.854739:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.854740:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.854741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.854833:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.854835:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.854838:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.854839:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.854840:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.854841:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.854842:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.854844:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.854845:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.854845:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.854849:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.854850:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.854852:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.854853:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.854853:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.854855:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.854855:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.854857:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.854858:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.854859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.854860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.854866:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.854869:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.854870:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.854870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.854871:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.854872:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677473.854873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.854874:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.854875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.854876:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677473.855314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.855322:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.855324:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.855325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.855327:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.855329:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.855331:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.855332:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.855333:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.855334:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.855335:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.855338:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.855339:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.855342:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.855343:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.855343:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.855345:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.855346:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.855346:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.855347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.855360:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.855362:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.855363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.855364:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.855366:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.855369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.855378:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.855381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.855381:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.855382:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.855383:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.855383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.855578:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.855580:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.855581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.855582:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.855584:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.855585:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.855586:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.855588:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.855589:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.855590:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.855592:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.855594:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.855596:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.855597:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.855598:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.855599:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.855600:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.855601:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.855602:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.855603:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.855613:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.855615:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.855615:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.855615:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.855616:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.855617:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.855618:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.855619:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.855620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.855621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.855641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.855649:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.855652:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.855653:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.855655:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.855657:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.855659:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.855660:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.855661:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.855662:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.855663:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.855666:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.855668:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.855671:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.855672:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.855672:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.855674:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.855674:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.855675:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.855675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.855687:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.855690:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.855691:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.855691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.855693:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.855697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.855755:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.855758:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.855759:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.855762:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.855762:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.855763:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.855765:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.855766:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.855767:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.855768:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.855771:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.855773:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.855775:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.855776:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.855776:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.855778:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.855778:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.855779:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.855780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.855783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.855789:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.855792:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.855793:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.855793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.855795:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.855797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.855801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.855807:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.855809:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.855810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.855810:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.855811:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.855811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.855831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.855835:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.855837:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.855839:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.855840:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.855841:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.855843:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.855844:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.855846:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.855847:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.855848:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.855851:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.855853:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.855855:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.855856:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.855857:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.855859:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.855859:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.855860:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.855861:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.855864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.855870:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.855873:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.855874:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.855874:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.855876:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.855879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.856350:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.856353:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.856355:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.856357:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.856357:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.856359:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.856360:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.856361:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.856362:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.856363:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.856367:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.856368:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.856371:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.856372:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.856372:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.856374:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.856374:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.856375:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.856376:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.856380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.856387:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.856390:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.856391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.856391:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.856392:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.856395:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.856473:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.856475:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.856476:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.856478:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.856480:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.856481:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.856483:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.856484:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.856485:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.856486:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.856490:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.856491:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.856494:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.856495:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.856495:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.856497:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.856498:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.856499:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.856499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.856511:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.856514:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.856514:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.856516:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.856517:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.856519:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.856531:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.856533:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.856534:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.856535:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.856535:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.856536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.856645:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.856648:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.856650:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.856651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.856652:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.856654:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.856655:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.856656:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.856657:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.856658:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.856662:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.856664:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.856666:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.856667:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.856668:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.856669:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.856670:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.856671:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.856671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.856675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.856676:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6ca00] (21)++
00000800:00000200:22.0:1619677473.856679:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.14@o2ib3
00000800:00000200:6.0:1619677473.856679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6ca00] (22)--
00000800:00000200:24.0:1619677473.856681:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:22.0:1619677473.856682:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.14@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677473.856683:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.856684:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.856684:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.856684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:22.0:1619677473.856687:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677473.856688:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.856689:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.856690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677473.856691:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.856692:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.856693:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677473.856697:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677473.856698:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677473.856702:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.856702:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:22.0:1619677473.856703:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:22.0:1619677473.856705:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:22.0:1619677473.856706:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6ca00] (21)++
00000800:00000200:22.0:1619677473.856707:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6ca00] (22)++
00000800:00000200:22.0:1619677473.856708:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6ca00] (23)--
00000800:00000200:22.0:1619677473.856709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6ca00] (22)--
00000800:00000200:6.0:1619677473.856719:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6ca00] (22)++
00000800:00000200:13.0:1619677473.856721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:6.0:1619677473.856721:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6ca00] (23)--
00000800:00000200:6.0:1619677473.856721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6ca00] (22)--
00000800:00000200:6.0:1619677473.856722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6ca00] (21)--
00000800:00000200:13.0:1619677473.856724:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.856725:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.856728:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.856728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.856732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:12.0:1619677473.856741:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677473.856744:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677473.856745:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.856746:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677473.856746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.856747:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:4.2:1619677473.856776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.856783:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.856786:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.856787:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677473.856788:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:8.0:1619677473.856788:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.856791:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:8.0:1619677473.856791:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.856791:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:8.0:1619677473.856794:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677473.856795:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.856795:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.856796:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.856797:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.856797:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.856798:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.856799:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677473.856800:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.856802:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.856804:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.856804:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677473.856805:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.0:1619677473.856807:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.856809:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.856811:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677473.856811:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.856811:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.856812:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.856813:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.856814:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:8.0:1619677473.856814:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.856814:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677473.856815:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:8.0:1619677473.856815:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.856816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677473.856817:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677473.856817:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.856818:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677473.856819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677473.856823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677473.856828:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.856831:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.856831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:2.0:1619677473.856831:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:22.0:1619677473.856832:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.856833:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:22.0:1619677473.856834:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.856834:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.856835:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.856837:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.856838:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677473.856840:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.856842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.856850:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.856852:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.856853:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.856853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.856854:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.856855:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677473.857089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.857094:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.857097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.857097:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677473.857099:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.857101:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.857102:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.857104:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.857105:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.857106:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.857107:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.857111:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.857112:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.857114:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.857115:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.857115:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.857117:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.857118:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.857118:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.857119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.857123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.857131:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.857132:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.857133:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.857134:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.857135:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.857136:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.857370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.857375:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677473.857375:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.857379:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.857379:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:6.0:1619677473.857379:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:20.0:1619677473.857380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.857383:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:6.0:1619677473.857383:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.857385:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.857386:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.857387:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:6.0:1619677473.857388:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677473.857389:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.857389:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.857391:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.857392:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.857393:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.857393:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.857395:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677473.857395:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677473.857400:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:6.0:1619677473.857401:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.857402:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677473.857403:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.857405:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.857406:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677473.857406:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.857407:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677473.857407:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.857408:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.857409:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.857410:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677473.857410:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.857410:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677473.857411:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677473.857411:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.857412:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:6.0:1619677473.857412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.857413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677473.857416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677473.857420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677473.857424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677473.857425:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:24.0:1619677473.857427:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677473.857427:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.857427:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.857428:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:20.0:1619677473.857428:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.857428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.857429:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.857429:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.857430:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.857431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.857431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:24.0:1619677473.857432:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:20.0:1619677473.857433:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677473.857434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.857769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.857777:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.857780:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.857781:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.857783:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.857785:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.857787:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.857788:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.857790:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.857791:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.857792:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.857796:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.857797:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.857800:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.857801:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.857802:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.857804:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.857804:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.857805:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.857806:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.857818:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.857821:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.857821:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.857822:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.857824:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.857827:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677473.857913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.857919:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.857922:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.857923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.857925:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.857927:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.857929:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.857930:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.857931:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.857933:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.857934:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.857937:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.857939:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.857941:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.857942:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.857943:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.857944:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.857945:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.857946:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.857946:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.857958:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.857961:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.857962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.857962:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.857964:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.857966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.857976:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.857979:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.857979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.857980:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.857981:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.857981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.858113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.858120:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.858123:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.858124:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.858126:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.858129:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.858130:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.858132:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.858133:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.858134:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.858135:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.858138:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.858139:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.858142:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.858143:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.858144:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.858145:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.858146:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.858146:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.858147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.858158:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.858160:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.858161:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.858163:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.858165:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.858166:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677473.858245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.858252:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.858255:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.858256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.858257:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.858260:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.858261:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.858262:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.858264:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.858264:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.858265:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.858268:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.858269:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.858271:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.858272:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.858272:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.858274:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.858274:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.858275:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.858276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.858280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.858286:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.858288:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.858289:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.858289:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.858291:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.858298:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.858388:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.858391:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.858393:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.858395:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.858396:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.858397:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.858398:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.858399:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.858400:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.858401:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.858404:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.858405:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.858408:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.858408:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.858409:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.858410:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.858411:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.858412:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.858412:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.858417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.858424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.858426:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.858427:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.858427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.858429:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.858431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.858435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.858441:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677473.858442:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.0:1619677473.858443:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.858444:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.858444:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.858445:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677473.858445:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.858446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677473.858446:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677473.858448:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.858450:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.858451:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.858452:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.858454:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.858454:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.858455:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.2:1619677473.858458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:6.0:1619677473.858458:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.858460:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.858462:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.858463:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.858464:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677473.858464:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677473.858465:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.858466:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.858467:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:2.0:1619677473.858467:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.858468:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677473.858468:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:2.0:1619677473.858469:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.858472:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.858473:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.858475:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.858476:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.858477:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.858478:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.858479:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.858481:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.858482:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.858482:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:21.0:1619677473.858484:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.858484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677473.858484:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.858486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677473.858487:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.858488:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.858489:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.858491:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.858491:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.858493:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.858494:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.858494:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.858499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677473.858503:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677473.858505:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677473.858506:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.858507:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.858508:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.858508:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677473.858509:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.858509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677473.858510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.858512:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677473.858523:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677473.858526:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677473.858527:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.858528:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677473.858528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.858530:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677473.858850:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.858853:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.858855:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.858857:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.858857:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.858858:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.858860:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.858861:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.858862:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.858863:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.858866:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.858868:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.858870:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.858871:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.858871:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.858873:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.858873:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.858874:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.858875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.858885:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.858887:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.858888:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.858890:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.858892:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.858893:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.859344:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.859347:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.859348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.859350:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.859352:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.859353:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.859355:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.859356:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.859357:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.859358:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.859362:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.859363:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.859366:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.859366:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.859367:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.859369:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.859369:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.859371:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.859372:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.859372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.859383:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.859383:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.859386:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.859386:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.859387:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.859387:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.859387:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.859388:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.859389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.859391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.859710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.859717:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.859720:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.859721:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.859722:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.859725:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.859726:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.859728:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.859729:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.859730:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.859731:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.859734:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.859736:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.859739:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.859740:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.859740:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.859742:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.859743:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.859744:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.859745:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.859745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.859748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.859751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677473.859756:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.859758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.859759:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.859760:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.859760:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677473.859760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.859761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.859762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677473.859763:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.859766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.859769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.859777:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.859779:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.859780:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.859780:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.859782:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.859782:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.860023:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.860026:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.860028:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.860030:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.860030:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.860032:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.860033:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.860034:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.860035:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.860036:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.860040:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.860041:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.860043:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.860044:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.860045:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.860046:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.860047:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.860048:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.860050:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.860050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.860052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.860058:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.860061:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.860062:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.860062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.860064:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677473.860064:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.860066:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677473.860066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.860067:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.860068:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677473.860237:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.860240:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.860241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677473.860242:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677473.860244:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.860245:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677473.860246:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.860247:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.860247:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677473.860249:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.860249:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.860249:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.860250:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677473.860251:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.860253:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.860253:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.860255:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.860255:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:30.0:1619677473.860257:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.860259:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677473.860260:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677473.860263:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.860263:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677473.860266:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677473.860266:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.860267:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677473.860268:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.860268:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677473.860269:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677473.860269:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677473.860270:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.860270:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677473.860271:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.860271:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677473.860272:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677473.860272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677473.860273:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.860274:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.860275:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.860277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.860284:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.860286:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.860287:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.860287:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.860289:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677473.860289:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.860291:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677473.860291:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677473.860291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:21.0:1619677473.860292:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.860297:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677473.860300:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.860301:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.860302:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677473.860302:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.860303:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.860304:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.860304:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (23)++
00000800:00000200:20.0:1619677473.860305:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (24)--
00000800:00000200:20.0:1619677473.860305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.860307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.860319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.860327:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.860330:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.860331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.860333:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.860335:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.860337:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677473.860339:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.860340:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.860341:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.860342:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677473.860345:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.860347:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677473.860350:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.860351:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677473.860351:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677473.860353:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677473.860353:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.860355:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.860356:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.860356:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677473.860360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677473.860362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.860369:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:17.0:1619677473.860370:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677473.860371:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.860372:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.860373:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677473.860374:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.860376:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.860377:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.860379:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.860379:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.860381:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.860384:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:1.0:1619677473.860385:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.860386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.860388:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.860392:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677473.860393:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.860396:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.860397:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.860399:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677473.860404:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.860407:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677473.860411:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.860413:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677473.860414:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677473.860416:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677473.860417:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.860418:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.860419:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.860422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.860429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.860432:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.860433:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.860433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.860435:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.860438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.860485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.860492:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.860495:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.860495:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677473.860498:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.860500:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.860502:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.860503:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.860504:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.860505:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.860506:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.860509:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.860511:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.860514:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.860515:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.860516:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.860517:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.860518:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.860519:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.860520:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.860520:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.860526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.860532:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.860533:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.860535:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.860535:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.860535:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.860536:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.860536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.860537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.860539:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.860542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.860969:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.860972:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.860974:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.860976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.860977:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.860978:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.860979:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.860980:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.860982:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.860982:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.860986:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.860987:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.860989:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.860990:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.860991:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.860992:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.860993:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.860994:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.860994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.860998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.861005:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.861007:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.861008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.861008:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.861010:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.861012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.861017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.861023:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.861025:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.861026:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.861027:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.861027:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.861028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.861599:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.861602:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.861604:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.861606:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.861606:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.861608:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.861609:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.861610:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.861611:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.861612:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.861614:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.861616:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.861618:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.861619:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.861620:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.861621:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.861622:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.861622:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.861623:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.861628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.861635:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.861637:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.861638:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.861638:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.861640:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.861642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.861736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.861742:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.861745:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.861746:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.861747:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.861750:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.861751:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.861752:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.861754:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.861754:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.861755:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.861758:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.861760:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.861762:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.861763:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.861764:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.861766:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.861766:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.861766:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.861767:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.861780:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.861783:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.861784:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.861784:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.861786:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.861790:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.861800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.861803:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.861804:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.861804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.861805:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.861806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.862008:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.862010:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.862011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.862013:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.862015:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.862016:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.862018:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.862019:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.862020:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.862021:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.862024:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.862026:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.862029:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.862029:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.862030:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.862032:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.862032:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.862033:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.862033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.862045:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.862048:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.862049:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.862050:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.862051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.862053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.862461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:30.0:1619677473.862462:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.862465:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677473.862466:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677473.862467:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.862469:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.862469:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.862469:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:30.0:1619677473.862471:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:21.0:1619677473.862471:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.862471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677473.862472:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.862474:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.862474:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.862475:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.862476:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:21.0:1619677473.862476:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.862477:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.862479:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.862481:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677473.862481:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.862482:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677473.862484:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:21.0:1619677473.862486:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.862488:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677473.862489:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.862490:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.862491:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677473.862491:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.862492:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677473.862492:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677473.862493:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677473.862493:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677473.862494:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.862495:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677473.862495:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.862496:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.862497:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.862497:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677473.862499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.862507:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.862509:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:31.0:1619677473.862510:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:24.0:1619677473.862510:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.862510:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.862512:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.862513:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.862513:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:31.0:1619677473.862515:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.862515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.862516:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.862518:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677473.862555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.862562:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.862565:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.862566:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.862568:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.862571:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.862572:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.862573:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.862575:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.862576:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.862577:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.862580:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.862582:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.862585:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.862586:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.862586:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.862588:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.862589:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.862589:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.862590:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.862591:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.862596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.862603:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.862603:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.862605:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.862606:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677473.862606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.862607:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.862607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.862608:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677473.862609:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.862612:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.863009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.863016:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.863019:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.863020:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.863021:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.863024:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.863025:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.863027:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.863028:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.863029:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.863030:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.863032:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.863034:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.863036:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.863036:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.863037:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.863038:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.863039:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.863039:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.863040:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.863044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.863050:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.863053:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.863054:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.863054:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.863056:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.863058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.863160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.863166:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.863169:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.863170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.863173:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.863176:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.863177:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.863179:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.863181:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.863182:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.863183:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.863186:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.863187:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.863190:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.863191:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.863191:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.863193:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.863194:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.863194:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.863195:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.863208:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.863210:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.863211:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.863211:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.863214:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.863216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.863310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677473.863318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677473.863321:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677473.863323:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.863325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677473.863325:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.863326:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.863328:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.863329:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.863330:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.863331:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.863334:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.863336:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.863338:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.863339:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.863339:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.863341:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.863341:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677473.863342:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677473.863343:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677473.863347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.863354:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.863356:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:31.0:1619677473.863357:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:8.0:1619677473.863357:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.863358:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.863359:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.863360:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:8.0:1619677473.863361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:31.0:1619677473.863362:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.863364:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.863364:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.863365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:31.0:1619677473.863366:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.863367:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.863368:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.863369:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.863370:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.863371:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:31.0:1619677473.863373:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.863373:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.863374:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.863374:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.863375:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.863375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.863376:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.863377:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.863377:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.863378:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.863380:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.863380:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.863381:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.863382:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.863385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.863391:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.863394:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.863395:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.863395:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.863396:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.863398:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677473.864041:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.864044:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.864045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.864047:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.864049:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.864050:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.864052:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.864053:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.864054:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.864055:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.864058:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.864060:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.864062:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.864063:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.864063:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.864065:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.864066:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.864067:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.864067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.864078:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.864081:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.864082:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.864083:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.864084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.864086:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.864149:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.864151:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.864153:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.864155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.864156:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.864157:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.864158:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.864160:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.864161:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.864162:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.864164:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.864166:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.864169:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.864169:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.864170:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.864171:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.864172:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.864173:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.864174:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.864184:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.864186:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.864187:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.864188:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.864190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.864191:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.864355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.864362:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.864365:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.864365:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.864367:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.864369:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.864371:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.864372:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.864373:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.864374:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.864375:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.864378:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.864379:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.864382:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.864383:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.864384:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.864385:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.864386:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.864387:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.864388:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.864388:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.864392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.864395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677473.864401:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.864402:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:18.0:1619677473.864403:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:10.0:1619677473.864404:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:2.0:1619677473.864404:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.864405:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.864405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.864406:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677473.864407:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.864411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677473.864832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.864838:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.864840:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.864842:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.864843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.864845:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.864847:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.864848:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.864849:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.864850:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.864851:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.864855:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.864857:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.864859:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.864860:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.864861:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.864862:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.864863:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.864864:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.864865:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.864866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.864869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677473.864870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.864873:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.864876:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.864876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.864877:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677473.864878:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.864880:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.864882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.864882:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.864883:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.864886:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677473.864889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:8.2:1619677473.864890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.864892:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.864895:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677473.864897:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.864897:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.864898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.864899:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.864900:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.864901:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677473.864901:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.864901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:20.0:1619677473.864902:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.864902:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.864902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:20.0:1619677473.864904:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.864905:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.864906:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.864909:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.864911:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.864913:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.864914:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.864915:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.864917:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.864917:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.864918:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.864919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677473.864932:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.864934:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.864935:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.864935:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.864937:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.864940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.865202:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.865204:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.865207:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.865209:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.865209:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.865210:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.865212:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.865213:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.865214:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.865215:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.865218:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.865220:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.865222:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.865223:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.865224:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.865226:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.865226:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.865228:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.865229:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.865230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.865231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.865238:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.865240:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.865241:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.865242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.865243:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677473.865244:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.865246:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.865246:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677473.865246:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.865247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.865274:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.865276:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.865278:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.865280:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.865280:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.865282:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.865283:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.865284:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.865285:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.865286:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.865289:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.865290:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.865293:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.865293:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.865299:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.865300:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.865301:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.865302:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.865302:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.865307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.865314:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.865316:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.865317:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.865317:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.865319:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.865321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.865394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.865402:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.865404:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.865405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.865407:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.865410:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.865411:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.865412:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.865414:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.865415:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.865415:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.865418:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.865420:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.865423:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.865424:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.865424:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.865426:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.865426:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.865427:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.865427:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.865440:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.865443:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.865443:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.865444:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.865447:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.865450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.865500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.865506:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.865509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.865509:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677473.865512:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.865514:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.865515:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.865517:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.865518:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.865519:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.865520:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.865523:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.865525:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.865527:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.865528:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.865528:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.865530:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.865531:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.865531:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.865532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.865536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.865543:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.865545:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.865546:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.865546:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.865547:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.865548:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.865568:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.865570:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.865571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.865573:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.865575:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.865576:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.865578:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.865579:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.865580:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.865581:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.865584:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.865586:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.865588:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.865589:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.865590:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.865591:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.865592:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.865593:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.865594:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.865595:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.865606:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.865607:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.865608:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.865608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.865609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.865609:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.865610:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.865612:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.865612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.865614:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.865628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.865634:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.865637:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.865638:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.865640:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.865643:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.865644:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.865646:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.865647:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.865648:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.865649:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.865652:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.865654:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.865657:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.865657:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.865658:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.865660:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.865660:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.865661:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.865662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.865666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.865674:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.865677:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.865678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.865678:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.865681:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.865684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677473.865981:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.865984:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.865986:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.865988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.865988:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.865990:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.865991:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.865992:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.865993:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.865994:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.865997:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.865999:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.866001:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.866002:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.866003:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.866004:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.866005:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.866006:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.866006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.866010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.866016:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.866019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.866019:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.866020:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.866022:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.866024:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.866031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.866037:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.866039:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.866040:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.866040:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.866041:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.866041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.866253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.866259:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677473.866260:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:18.0:1619677473.866262:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.866262:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677473.866263:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:18.0:1619677473.866265:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677473.866266:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.866266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:18.0:1619677473.866267:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.866268:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.866269:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677473.866270:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677473.866270:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.866272:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.866272:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.866273:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.866273:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.866275:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.866275:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677473.866276:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677473.866280:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677473.866282:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.866283:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677473.866284:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677473.866286:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.866287:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.866287:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:31.0:1619677473.866288:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677473.866288:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:31.0:1619677473.866289:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.866290:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677473.866290:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677473.866291:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:18.0:1619677473.866291:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.866292:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.866293:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:9.2:1619677473.866297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.866299:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.866301:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.866302:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677473.866305:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:12.0:1619677473.866306:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:8.0:1619677473.866308:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.866309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:12.0:1619677473.866309:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:8.0:1619677473.866309:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.866310:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.866312:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:8.0:1619677473.866312:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.866313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.0:1619677473.866314:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677473.866315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677473.866316:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.866318:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.866319:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.866320:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:13.0:1619677473.866323:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.866326:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.866327:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.866328:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.866329:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.866330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677473.866704:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.866707:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.866708:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.866710:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.866713:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.866714:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677473.866715:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.866717:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.866718:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.866719:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677473.866722:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.866724:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677473.866727:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.866728:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677473.866728:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677473.866730:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.866730:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.866731:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.866732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677473.866744:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.866747:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.866748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.866748:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.866750:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.866753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.866771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.866778:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.866781:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.866782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.866784:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.866787:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.866789:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.866790:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.866791:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.866792:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.866793:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.866796:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.866797:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.866801:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.866802:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.866802:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.866804:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.866804:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.866805:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.866805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.866810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.866818:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.866821:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.866821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.866822:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.866824:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.866827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677473.867304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.867311:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.867314:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.867315:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.867317:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.867320:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.867322:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.867323:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.867324:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.867325:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.867326:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.867330:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.867331:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.867334:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.867335:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.867335:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.867337:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.867337:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.867338:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.867339:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.867352:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.867355:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.867356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.867356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.867358:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.867362:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677473.867379:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.867382:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.867384:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.867387:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.867387:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.867388:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677473.867390:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.867391:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.867392:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.867393:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677473.867396:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.867398:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677473.867401:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.867402:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677473.867402:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.867404:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677473.867405:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.867405:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.867406:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677473.867414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.867423:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.867425:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.867426:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.867428:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.867429:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.867431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.867465:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.867467:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.867468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.867470:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.867472:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.867473:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.867475:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.867476:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.867477:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.867478:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.867482:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.867483:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.867486:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.867487:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.867488:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.867489:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.867490:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.867491:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.867491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.867503:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.867505:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.867506:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.867508:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.867508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.867510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.867536:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.867539:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.867540:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.867541:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.867541:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.867542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.867762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.867769:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.867772:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.867773:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.867775:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.867777:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.867779:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.867780:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.867781:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.867782:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.867784:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.867787:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.867789:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.867791:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.867792:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.867792:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.867794:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.867795:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.867795:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.867796:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.867800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.867806:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.867809:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.867809:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.867810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.867811:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.867814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.867853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.867859:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.867862:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.867863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.867865:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.867867:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.867868:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.867870:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.867871:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.867872:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.867873:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.867876:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.867877:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.867880:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.867881:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.867881:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.867883:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.867883:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.867884:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.867885:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.867886:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.867891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.867896:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.867897:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.867898:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.867898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.867899:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.867899:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.867900:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.867901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.867902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.867905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.867914:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.867916:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.867917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.867918:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.867919:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.867919:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.868074:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.868077:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.868079:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.868081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.868082:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.868084:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.868085:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.868086:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.868087:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.868088:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.868092:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.868094:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.868096:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.868096:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.868097:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.868098:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.868099:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.868100:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.868101:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.868111:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.868113:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.868114:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.868115:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.868117:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.868118:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.868958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.868966:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.868968:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.868969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.868971:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.868975:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.868976:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.868977:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.868978:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.868979:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.868980:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.868983:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.868985:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.868988:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.868989:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.868990:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.868991:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.868992:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.868993:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.868994:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.868995:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.868998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.869001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677473.869006:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.869008:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.869009:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.869010:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.869010:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:2.0:1619677473.869010:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.869011:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.869012:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677473.869013:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.869016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.869164:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.869167:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.869169:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.869171:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.869171:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.869172:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.869173:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.869175:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.869176:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.869177:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.869180:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.869182:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.869184:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.869184:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.869185:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.869186:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.869187:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.869188:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.869188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.869199:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.869202:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.869202:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.869204:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.869205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.869206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.869211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.869218:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.869220:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:30.0:1619677473.869221:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677473.869221:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.869223:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.869224:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.869224:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.869225:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677473.869225:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:10.0:1619677473.869225:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.869226:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:10.0:1619677473.869227:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.869228:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.869229:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.869230:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.869231:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.869234:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.869236:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.869239:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.869240:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.869240:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.869242:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.869242:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.869243:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.869244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.869259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.869267:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.869269:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.869270:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.869271:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.869273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.869275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.869425:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.869428:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.869429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.869430:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.869433:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.869434:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.869435:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.869437:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.869438:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.869439:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.869441:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.869443:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.869445:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.869446:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.869447:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.869449:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.869449:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.869450:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.869451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.869464:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.869466:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.869466:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.869468:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.869469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.869470:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.869552:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.869555:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.869556:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.869558:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.869560:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.869562:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677473.869563:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.869564:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.869565:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.869566:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677473.869570:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.869572:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677473.869575:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.869576:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:1.0:1619677473.869577:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677473.869579:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677473.869579:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.869580:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677473.869581:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.869581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.869592:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677473.869594:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677473.869594:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:12.0:1619677473.869594:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:17.0:1619677473.869595:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:12.0:1619677473.869598:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.869599:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.869602:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.869604:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.869607:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677473.869696:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.869699:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.869701:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.869703:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.869703:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.869705:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.869706:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.869707:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.869708:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.869709:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.869713:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.869714:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.869717:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.869717:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.869718:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.869720:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.869720:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.869721:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.869722:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.869725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.869733:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.869735:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.869736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.869736:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.869738:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.869741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.869745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.869751:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.869754:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.869754:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.869755:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.869755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.869756:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.869922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.869929:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.869932:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.869932:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.869934:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.869937:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.869938:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677473.869939:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.869941:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.869941:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.869942:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677473.869945:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.869947:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677473.869949:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.869949:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677473.869950:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677473.869952:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677473.869952:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.869953:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.869953:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677473.869964:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.869967:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.869967:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.869969:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.869971:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.869972:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677473.869997:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.870000:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.870002:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.870003:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.870004:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.870005:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.870006:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.870007:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677473.870007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:30.0:1619677473.870008:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.870009:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.870011:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.870013:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.870014:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:30.0:1619677473.870015:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.870015:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.870016:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677473.870016:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:30.0:1619677473.870017:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677473.870017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677473.870018:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.870019:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677473.870019:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.870021:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.870021:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677473.870021:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.870022:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677473.870023:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.870025:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.870025:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.870026:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677473.870028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:10.0:1619677473.870029:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.870031:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:10.0:1619677473.870031:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.870033:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.870033:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.870033:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.870034:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677473.870034:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)++
00000400:00000200:31.0:1619677473.870035:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.870035:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:10.0:1619677473.870035:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (23)++
00000800:00000200:30.0:1619677473.870036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677473.870036:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:31.0:1619677473.870037:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (24)--
00000800:00000200:26.0:1619677473.870037:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.870037:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (24)--
00000800:00000200:30.0:1619677473.870038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000800:00000200:10.0:1619677473.870038:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)++
00000800:00000200:31.0:1619677473.870039:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677473.870039:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (24)--
00000400:00000200:26.0:1619677473.870040:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.870040:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:26.0:1619677473.870041:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.870041:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.870043:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.870044:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.870046:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.870046:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.870047:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.870049:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:26.0:1619677473.870050:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.870051:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:26.0:1619677473.870051:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.870052:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.870054:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.870054:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.870055:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677473.870055:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.870055:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.870057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:26.0:1619677473.870057:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.870058:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.870059:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.870060:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.870060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677473.870066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:6.0:1619677473.870071:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.870073:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:6.0:1619677473.870073:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.870074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.870075:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:6.0:1619677473.870075:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.870076:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (23)++
00000800:00000200:26.0:1619677473.870077:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (24)--
00000800:00000200:26.0:1619677473.870077:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.870078:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:6.0:1619677473.870078:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.870079:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.870082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677473.870083:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.870085:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677473.870086:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.870087:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.870088:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.870089:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677473.870091:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677473.870092:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.870093:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677473.870094:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.870094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.870095:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.870095:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.870096:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.870096:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677473.870096:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677473.870097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.870098:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677473.870098:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.870099:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.870100:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677473.870109:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.870112:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.870113:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677473.870113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:31.0:1619677473.870114:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.870116:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.870117:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:26.0:1619677473.870119:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.870122:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.870123:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.870124:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.870126:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.870128:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.870129:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.870130:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.870131:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.870132:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.870135:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.870136:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.870139:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.870139:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.870140:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.870141:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.870142:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.870143:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.870143:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.870148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.870150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:18.0:1619677473.870156:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.870157:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677473.870159:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:10.0:1619677473.870159:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.870160:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677473.870160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.870161:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.870161:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.870163:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.870164:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.870165:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.870167:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.870168:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.870169:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.870169:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677473.870172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:10.0:1619677473.870172:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.870174:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.870177:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.870178:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.870178:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.870180:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.870180:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.870181:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.870182:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.870186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677473.870192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:1.0:1619677473.870194:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.870197:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.870198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.870198:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.870199:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:1.0:1619677473.870200:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.870201:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.870202:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.870203:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.870204:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:26.0:1619677473.870207:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.870208:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677473.870210:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.870211:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.870212:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.870213:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677473.870216:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.870217:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677473.870219:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.870220:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:26.0:1619677473.870221:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677473.870222:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677473.870223:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.870224:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.870225:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.870226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677473.870231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:13.0:1619677473.870234:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:4.2:1619677473.870234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677473.870237:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677473.870237:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.870239:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.870240:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677473.870240:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677473.870241:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:10.0:1619677473.870241:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.870242:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.870244:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.870244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.870244:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.870246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.870246:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.870247:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:10.0:1619677473.870248:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.870249:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.870251:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.870252:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.870252:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.870256:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.870257:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.870259:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.870260:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.870261:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.870262:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.870263:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.870264:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.870265:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.870265:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.870271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.870276:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.870278:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677473.870279:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.870280:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.870280:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.870281:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.870282:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677473.870282:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.870283:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.870286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.870720:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.870722:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.870724:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.870726:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.870727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.870728:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.870729:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.870730:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.870731:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.870732:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.870735:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.870736:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.870738:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.870739:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.870740:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.870741:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.870742:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.870743:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.870743:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.870748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.870755:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.870757:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.870758:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.870758:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.870760:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.870762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.870948:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.870951:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.870952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.870953:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.870956:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.870957:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.870958:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.870960:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.870961:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.870962:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.870964:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.870966:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.870968:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.870969:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.870970:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.870971:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.870972:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.870973:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.870973:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.870985:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.870987:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.870988:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.870990:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.870991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.870992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.871027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.871033:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.871035:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.871036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.871038:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.871041:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.871042:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.871043:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.871045:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.871046:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.871047:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.871051:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.871053:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.871056:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.871057:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.871057:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.871059:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.871060:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.871061:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.871061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677473.871072:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.871075:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.871076:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.871078:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.871078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.871081:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677473.871096:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677473.871099:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677473.871100:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.871100:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677473.871101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.871102:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677473.871325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.871332:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.871335:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.871336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.871337:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.871340:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.871342:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.871343:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.871344:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.871345:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.871346:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.871350:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.871352:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.871355:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.871356:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.871356:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.871358:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.871358:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.871359:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.871360:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.871373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.871376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.871376:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.871377:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.871379:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.871383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.871516:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.871519:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.871521:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.871523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.871524:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.871525:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.871526:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.871527:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.871529:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.871529:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.871533:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.871535:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.871537:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.871538:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.871539:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.871540:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.871541:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.871542:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.871542:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.871546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.871552:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.871555:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.871556:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.871556:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.871558:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.871560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.871667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.871675:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.871677:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.871678:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.871680:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.871683:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.871684:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.871686:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.871687:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.871688:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.871689:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.871692:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.871694:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.871697:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.871697:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.871698:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.871700:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.871700:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.871701:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.871701:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.871713:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.871716:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.871717:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.871717:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.871719:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.871722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.872281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677473.872283:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.872285:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.872287:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677473.872287:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677473.872289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.872290:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.872290:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:30.0:1619677473.872291:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:2.0:1619677473.872291:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:30.0:1619677473.872298:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.872300:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.872301:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.872302:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.872303:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677473.872303:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.872305:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677473.872307:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677473.872307:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.872308:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677473.872310:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:18.0:1619677473.872310:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.872311:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677473.872313:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.872314:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.872314:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677473.872315:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.872316:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.872317:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.872317:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677473.872317:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677473.872318:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677473.872319:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.872320:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.872321:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.872322:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:9.2:1619677473.872322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677473.872323:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.872324:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.872324:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.872329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.872330:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.872332:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.872333:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.872333:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.872335:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.872336:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.872337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677473.872338:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.872339:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.872339:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.872341:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.872343:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.872348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.872356:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.872358:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.872359:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.872359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.872360:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.872360:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.872394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677473.872400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.872402:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.872404:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.872405:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.872407:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.872409:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677473.872410:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.872412:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.872413:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.872414:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677473.872418:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.872420:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677473.872423:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.872423:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677473.872424:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677473.872426:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677473.872427:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.872427:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.872428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677473.872441:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.872443:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.872444:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.872446:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.872447:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.872449:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677473.872603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.872611:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.872614:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.872614:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.872616:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.872619:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.872621:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.872622:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.872623:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.872624:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.872625:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.872628:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.872630:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.872632:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.872633:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.872633:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.872635:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.872635:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.872636:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.872636:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.872640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.872647:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.872649:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.872650:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.872650:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.872651:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.872653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.872658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.872664:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.872667:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.872668:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.872668:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.872668:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.872669:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.872711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.872717:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.872720:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.872720:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.872722:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.872725:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.872726:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.872727:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.872728:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.872730:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.872731:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.872734:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.872736:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.872738:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.872739:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.872740:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.872741:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.872742:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.872742:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.872743:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.872756:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.872758:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.872759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.872759:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.872761:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.872764:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.872786:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.872789:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.872789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.872791:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.872793:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.872794:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.872796:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.872797:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.872798:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.872799:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.872803:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.872805:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.872807:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.872808:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.872808:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.872810:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.872811:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.872812:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.872813:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.872813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.872823:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.872824:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.872826:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.872826:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.872827:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.872827:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.872827:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.872829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.872829:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.872831:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.873519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.873527:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.873530:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.873531:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.873533:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.873535:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.873536:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.873538:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.873539:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.873540:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.873541:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.873544:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.873546:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.873549:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.873550:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.873550:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.873552:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.873552:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.873553:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.873553:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.873559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.873567:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.873570:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.873571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.873571:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.873574:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677473.873577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:30.0:1619677473.873625:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.873628:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.873630:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.873632:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.873633:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.873634:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.873635:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.873637:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.873638:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.873639:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.873642:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.873644:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.873646:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.873647:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.873647:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.873649:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.873649:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.873651:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.873652:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.873653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.873667:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.873669:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.873669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.873670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.873731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.873738:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.873741:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.873742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.873742:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.873744:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.873746:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.873951:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.873954:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.873956:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.873958:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.873958:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.873960:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.873961:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.873963:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.873964:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.873965:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.873968:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.873970:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.873972:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.873973:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.873973:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.873974:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.873975:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.873976:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.873976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.873987:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.873989:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.873990:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.873992:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.873993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.873994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.874632:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.874635:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.874637:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.874639:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.874640:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677473.874640:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.874641:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.874642:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.874643:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.874644:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677473.874647:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.874649:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677473.874651:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.874651:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:31.0:1619677473.874652:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:31.0:1619677473.874653:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677473.874654:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.874655:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.874656:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.874657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.874671:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:9.2:1619677473.874671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:30.0:1619677473.874673:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.874674:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.874675:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677473.874678:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.874680:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.874681:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.874681:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.874683:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.874686:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677473.874733:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.874736:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.874737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.874739:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.874741:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.874742:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.874744:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.874745:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.874746:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.874747:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.874751:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.874752:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.874755:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.874755:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.874756:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.874758:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.874758:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.874759:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.874760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.874772:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.874774:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.874775:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.874777:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.874778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.874779:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.874791:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.874794:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.874795:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.874795:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.874796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.874797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.875103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.875110:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.875113:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.875114:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.875115:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.875117:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.875119:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.875120:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.875122:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.875123:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.875124:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.875127:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.875129:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.875132:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.875133:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.875133:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.875135:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.875135:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.875136:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.875137:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.875149:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.875152:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.875153:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.875153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.875155:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.875158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.875173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677473.875179:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677473.875182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677473.875182:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000400:00000200:1.0:1619677473.875185:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.875187:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.875189:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677473.875190:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.875191:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.875192:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.875193:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677473.875197:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.875199:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677473.875202:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.875203:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677473.875203:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677473.875205:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677473.875206:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.875206:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677473.875207:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677473.875211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.875218:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.875220:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.875221:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.875222:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.875224:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.875227:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677473.875326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677473.875333:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677473.875337:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677473.875337:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000400:00000200:10.0:1619677473.875339:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.875341:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.875343:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677473.875344:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.875346:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.875346:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.875348:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677473.875351:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.875353:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.875356:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.875356:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677473.875357:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677473.875359:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.875359:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677473.875360:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677473.875361:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:20.0:1619677473.875374:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.875376:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.875377:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.875379:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:21.0:1619677473.875380:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.875382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.875706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.875709:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.875711:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.875714:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.875714:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.875715:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.875716:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.875717:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.875718:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.875719:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.875722:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.875724:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.875725:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.875726:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.875726:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.875726:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.875728:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677473.875728:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.875728:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677473.875729:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.875729:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.875730:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.875731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.875732:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.875733:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.875734:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.875735:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.875736:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.875737:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.875740:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.875740:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.875740:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.875741:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.875741:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.875742:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.875743:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.875744:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.875744:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.875745:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.875745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.875746:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.875747:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.875748:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.875748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.875753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.875759:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677473.875760:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:30.0:1619677473.875762:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677473.875762:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677473.875763:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.875763:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000400:00000200:24.0:1619677473.875763:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.875764:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677473.875764:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.875765:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:24.0:1619677473.875765:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.875768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.875939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.875946:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.875948:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.875949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.875951:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.875954:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.875955:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.875957:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.875958:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.875959:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.875960:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677473.875963:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.875965:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677473.875968:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.875969:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.875970:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677473.875972:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677473.875972:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.875973:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.875973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677473.875985:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.875988:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.875989:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.875991:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677473.875991:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.875994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677473.876058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.876065:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.876068:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.876068:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.876070:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.876073:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.876075:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.876076:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.876077:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.876078:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.876079:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.876082:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.876083:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.876086:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.876087:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.876088:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.876089:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.876090:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.876091:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.876091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.876104:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.876106:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.876107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.876108:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.876110:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.876113:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.876124:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.876126:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.876127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.876128:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.876129:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.876129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.876283:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.876285:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.876286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.876288:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.876289:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.876291:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.876292:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.876299:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.876299:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.876300:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.876304:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.876306:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.876308:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.876309:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.876310:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.876312:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.876312:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.876313:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.876313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.876327:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.876329:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.876330:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.876332:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.876333:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.876334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.876531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.876538:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.876541:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.876542:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.876544:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.876546:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.876548:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.876549:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.876551:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.876552:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.876553:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.876555:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.876557:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.876560:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.876561:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.876562:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.876563:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.876564:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.876565:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.876565:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.876578:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.876581:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.876582:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.876582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.876584:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.876587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677473.876600:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.876602:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.876604:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.876606:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.876607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.876607:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.876609:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.876610:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.876611:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.876612:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.876615:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.876617:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.876619:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.876619:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.876620:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.876621:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.876622:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.876623:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.876623:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.876633:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.876636:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.876636:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.876638:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.876640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.876640:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.877276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.877283:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.877285:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.877286:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.877288:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.877291:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.877292:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.877300:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.877301:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.877302:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.877303:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.877305:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.877307:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.877309:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.877310:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.877310:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.877312:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.877312:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.877312:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.877313:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.877318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.877325:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.877328:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.877329:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.877329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.877330:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.877333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677473.877389:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.877392:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.877392:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.877395:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.877397:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.877398:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677473.877400:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.877401:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.877402:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.877403:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677473.877406:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.877407:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.877409:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.877410:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.877411:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.877412:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.877412:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.877413:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.877414:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677473.877418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.877426:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.877428:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.877428:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.877429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.877430:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.877431:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677473.877644:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.877647:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.877649:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.877651:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.877651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.877652:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.877653:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.877655:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.877656:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.877657:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.877660:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.877662:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.877664:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.877665:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.877666:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.877667:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.877668:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.877669:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.877669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.877673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.877679:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.877682:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.877683:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.877683:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.877685:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.877686:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.877695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.877701:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.877704:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.877705:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.877705:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.877706:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.877706:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.878006:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.878009:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.878011:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.878013:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.878013:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.878014:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.878015:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.878016:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.878017:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.878018:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.878021:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677473.878021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677473.878023:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.878024:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.878025:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.878026:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677473.878026:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677473.878027:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.878028:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.878028:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.878029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677473.878029:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.878030:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.878032:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.878033:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.878035:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.878036:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.878037:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.878038:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677473.878039:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:17.0:1619677473.878039:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677473.878042:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.878042:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.878042:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.878044:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677473.878045:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.878045:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.878047:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:17.0:1619677473.878047:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.878048:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.878048:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.878050:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.878051:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.878051:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.878052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.878064:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.878067:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.878068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.878068:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.878070:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.878073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.878084:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.878087:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.878087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.878088:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.878089:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.878089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.878167:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.878170:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.878171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.878172:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.878174:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.878176:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.878177:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.878179:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.878180:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.878181:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.878184:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.878186:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.878189:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.878189:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.878190:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.878191:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.878192:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.878193:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.878193:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.878205:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.878207:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.878208:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.878210:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.878211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.878212:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.879041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.879049:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.879052:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.879053:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.879055:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.879058:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.879059:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.879060:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.879062:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.879063:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.879064:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677473.879066:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:26.0:1619677473.879067:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.879069:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:26.0:1619677473.879069:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677473.879071:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.879072:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.879072:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677473.879073:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.879073:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.879073:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:30.0:1619677473.879074:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.879075:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.879075:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.879075:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000400:00000200:30.0:1619677473.879076:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.879076:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.879076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:30.0:1619677473.879077:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.879078:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.879081:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677473.879082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.879083:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.879085:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.879085:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.879086:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.879088:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.879088:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.879089:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.879089:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677473.879090:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.879092:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.879093:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.879094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:21.0:1619677473.879095:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.879096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.879098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677473.879101:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.879103:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.879104:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.879105:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.879106:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.879109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.879372:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.879375:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.879377:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.879379:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.879379:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.879380:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.879381:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.879383:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.879384:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.879385:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.879388:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.879389:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.879391:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.879392:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.879392:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.879394:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.879395:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.879397:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.879398:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.879399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.879400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.879407:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.879409:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.879410:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.879410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.879412:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.879413:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677473.879414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.879415:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.879416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.879417:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677473.879755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.879763:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.879765:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.879766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.879768:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.879771:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.879772:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.879773:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.879774:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.879775:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.879776:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.879779:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.879780:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.879783:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.879784:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.879784:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.879786:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.879787:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.879787:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.879788:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.879793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.879801:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.879804:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.879805:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.879806:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677473.879807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.879810:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.879814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.879822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.879824:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.879825:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.879826:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.879827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.879828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:30.0:1619677473.879926:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.879929:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.879931:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.879933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.879934:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.879935:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.879936:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.879937:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.879939:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.879939:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.879943:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.879944:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.879947:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.879947:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.879948:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.879949:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.879950:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.879951:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.879952:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.879953:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.879955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.879962:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.879964:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.879965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.879965:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.879967:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677473.879967:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.879969:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677473.879969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.879970:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.879971:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.880104:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.880106:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.880107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.880109:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.880111:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.880113:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.880114:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.880115:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.880117:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.880118:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.880121:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.880122:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.880125:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.880126:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.880126:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.880128:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.880128:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.880130:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.880131:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.880131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.880142:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.880143:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.880144:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.880144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.880145:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.880146:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.880146:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.880148:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.880149:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.880150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.880160:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.880162:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.880163:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.880164:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.880164:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.880166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.880436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.880443:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.880446:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.880447:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.880449:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.880451:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.880453:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.880454:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.880456:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.880457:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.880458:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.880461:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.880463:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.880465:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.880466:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.880467:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.880468:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.880469:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.880470:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.880471:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.880472:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.880475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:4.2:1619677473.880479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.880484:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.880484:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.880485:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:22.0:1619677473.880487:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.880488:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677473.880488:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.880488:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.880488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.880489:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.880490:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.880491:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:22.0:1619677473.880491:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.880491:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.880493:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677473.880493:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.880495:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.880496:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.880497:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.880498:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677473.880500:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677473.880501:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.880502:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.880505:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.880506:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.880507:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.880508:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.880509:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.880509:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.880510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.880515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.880524:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.880527:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.880527:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.880528:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.880530:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.880533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677473.880775:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.880778:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.880780:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.880782:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.880783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.880784:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.880785:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.880786:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.880787:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.880788:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.880791:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.880792:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.880794:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.880795:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.880795:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.880797:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.880797:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.880798:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.880799:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.880809:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.880812:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.880812:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.880814:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.880816:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.880817:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:24.2:1619677473.881155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677473.881158:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677473.881161:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.54@o2ib7
00000800:00000200:8.0:1619677473.881162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677473.881163:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.54@o2ib7 : PUT - routed
00000800:00000200:24.0:1619677473.881165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:30.0:1619677473.881333:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.881336:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.881338:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.881339:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.881340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.881340:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.881341:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.881343:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.881344:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.881344:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.881347:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.881349:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.881351:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.881351:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.881352:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.881353:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.881354:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.881355:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.881355:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.881360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.881367:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.881369:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.881370:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.881370:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.881372:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.881374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.881481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.881488:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.881490:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.881491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.881492:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.881495:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.881496:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.881497:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.881498:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.881499:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.881500:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.881503:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.881504:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.881507:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.881508:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.881508:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.881510:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.881511:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.881511:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.881512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.881524:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.881527:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.881528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.881528:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.881530:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.881534:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.881591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.881593:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.881594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.881596:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.881598:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.881599:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.881600:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.881602:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.881603:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.881604:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.881607:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.881609:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.881611:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.881612:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.881613:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.881615:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.881615:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.881616:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.881617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.881629:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.881631:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.881632:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.881633:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.881634:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.881635:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.881758:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.881761:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.881763:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.881764:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.881765:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.881767:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.881768:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.881770:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.881771:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.881772:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.881775:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.881777:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.881780:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.881780:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.881781:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.881783:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.881783:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.881784:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.881785:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.881788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.881795:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.881797:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.881798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.881798:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.881800:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.881803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.881977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.881985:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.881988:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.881988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.881990:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.881993:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.881995:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.881996:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.881997:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.881998:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.881999:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.882002:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.882003:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.882005:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.882006:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.882006:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.882008:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.882008:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.882009:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.882009:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.882013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.882019:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.882022:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.882022:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.882023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.882024:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.882026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.882031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.882037:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.882040:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.882041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.882041:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.882042:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.882042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.882115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.882122:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.882124:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.882125:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.882126:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.882129:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.882130:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.882131:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.882132:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.882133:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.882134:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.882136:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.882138:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.882140:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.882141:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.882141:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.882143:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.882144:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.882145:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.882145:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677473.882149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.882158:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.882161:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.882162:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.882162:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.882164:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.882167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677473.882192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.882198:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.882201:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.882202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.882204:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.882207:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.882208:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.882209:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.882211:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.882212:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.882213:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.882216:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.882218:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.882219:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.882220:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.882221:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.882222:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.882223:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.882224:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.882225:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.882226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.882228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.882231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.882237:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.882238:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.882238:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.882238:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.882238:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677473.882239:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.882240:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.882240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.882241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.882241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677473.882418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.882425:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.882428:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.882429:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.882431:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.882433:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.882435:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.882436:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.882437:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.882438:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.882439:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.882442:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.882444:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.882447:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.882447:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.882448:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.882450:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.882450:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.882451:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.882453:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.882453:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.882459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677473.882465:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.882466:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.882468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.882468:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677473.882469:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.882469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677473.882469:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.882470:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.882471:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.882474:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677473.882986:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.882989:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.882991:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.882993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.882993:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.882994:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.882995:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.882997:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.882998:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.882998:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.883001:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.883003:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.883005:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.883005:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.883006:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.883007:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.883008:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.883009:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.883009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.883013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.883021:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.883023:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.883024:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.883024:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.883026:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.883028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.883138:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.883141:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.883143:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.883145:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.883145:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.883147:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.883148:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.883149:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.883150:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.883150:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.883153:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.883155:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.883157:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.883158:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.883158:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.883160:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.883160:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.883161:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.883162:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.883176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.883182:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.883185:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.883186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.883186:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.883187:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.883189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.883458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677473.883465:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.883468:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677473.883469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677473.883470:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.883473:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.883474:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677473.883475:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.883476:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.883477:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.883478:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.883481:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.883483:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.883485:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.883485:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677473.883486:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677473.883488:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677473.883488:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.883489:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677473.883489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:30.0:1619677473.883502:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.883504:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677473.883505:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.883507:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.883509:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.883510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:22.0:1619677473.883532:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.883535:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.883536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.883538:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.883540:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.883541:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.883542:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.883544:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.883545:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.883546:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.883549:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.883550:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.883553:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.883553:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.883554:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.883556:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.883556:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.883557:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.883558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.883569:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.883572:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.883573:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.883574:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.883575:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.883577:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.883907:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.883909:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677473.883910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677473.883912:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.883915:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.883916:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677473.883918:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.883919:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.883920:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.883921:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677473.883924:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.883925:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677473.883928:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.883929:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677473.883930:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677473.883932:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677473.883932:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.883933:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.883933:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:31.0:1619677473.883944:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.883947:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.883948:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.883950:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.883951:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.883953:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677473.883963:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677473.883966:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677473.883967:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.883967:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677473.883968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.883969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677473.884087:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.884090:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.884092:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.884094:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.884094:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.884095:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677473.884096:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.884097:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.884098:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.884099:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677473.884102:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.884104:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677473.884107:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.884108:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677473.884109:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677473.884110:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677473.884111:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.884112:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.884114:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.884114:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677473.884117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.884128:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677473.884129:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677473.884130:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.884131:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677473.884131:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677473.884132:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677473.884132:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.884134:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.884134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.884137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.884231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.884237:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.884240:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.884241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.884243:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.884246:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.884247:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.884248:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.884250:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.884251:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.884252:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.884256:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.884257:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.884260:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.884261:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.884262:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.884264:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.884265:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.884266:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.884267:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.884267:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.884272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.884278:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677473.884278:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677473.884280:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.884281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677473.884281:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677473.884282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677473.884282:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.884284:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.884287:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.884288:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677473.884635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919078fb5400] (20)++
00000800:00000200:2.0:1619677473.884641:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919078fb5400] (21)++
00000800:00000200:18.0:1619677473.884644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (22)--
00000800:00000200:2.0:1619677473.884644:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.6@o2ib3
00000400:00000200:2.0:1619677473.884646:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.6@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.884649:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.884651:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677473.884652:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.884654:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.884655:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.884656:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677473.884659:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.884660:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677473.884663:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.884664:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677473.884664:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677473.884666:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677473.884666:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919078fb5400] (21)++
00000800:00000200:2.0:1619677473.884667:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919078fb5400] (22)--
00000800:00000200:2.0:1619677473.884667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919078fb5400] (21)--
00000800:00000200:20.0:1619677473.884680:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.884682:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.884683:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.884685:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.884686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.884688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.884869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.884875:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.884878:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.884879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.884881:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.884883:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.884884:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.884886:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.884887:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.884888:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.884889:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.884893:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.884894:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.884897:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.884898:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.884898:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.884900:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.884901:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.884901:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.884902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.884915:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.884917:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.884918:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.884918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.884920:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.884923:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.885258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.885266:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.885269:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.885270:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.885272:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.885274:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.885276:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.885277:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.885279:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.885280:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.885281:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.885284:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.885286:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.885289:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.885290:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.885290:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.885292:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.885293:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.885299:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.885300:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.885307:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.885309:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.885310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.885310:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.885313:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.885316:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.885361:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.885364:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.885366:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.885369:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.885369:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.885371:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677473.885372:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.885373:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.885374:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.885375:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:30.0:1619677473.885378:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.885380:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677473.885383:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.885384:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677473.885384:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677473.885386:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677473.885387:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.885388:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.885388:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677473.885400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.885402:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.885403:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.885405:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.885407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.885408:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.885423:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.885425:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.885426:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.885427:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.885427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.885429:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.885518:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.885521:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.885522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.885524:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.885526:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.885528:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.885529:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.885531:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.885532:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.885533:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.885535:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.885537:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.885540:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.885540:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.885541:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.885543:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.885543:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.885544:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.885545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:4.2:1619677473.885550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:21.0:1619677473.885556:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677473.885557:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.0:1619677473.885558:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.885559:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.885560:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:21.0:1619677473.885560:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.885560:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:20.0:1619677473.885562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677473.885562:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.885563:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677473.885565:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.885567:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.885568:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.885569:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.885570:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.885571:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677473.885574:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.885576:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.885579:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.885580:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677473.885580:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.885582:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.885582:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.885583:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.885584:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.885585:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677473.885589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677473.885591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677473.885597:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677473.885598:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:20.0:1619677473.885599:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677473.885600:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:20.0:1619677473.885600:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.885601:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.885602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:20.0:1619677473.885602:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.885603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.885605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:31.0:1619677473.885943:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.885946:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.885948:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.885949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.885951:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.885952:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677473.885954:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.885955:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.885956:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.885957:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677473.885959:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.885961:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.2:1619677473.885961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.885963:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.885964:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:31.0:1619677473.885964:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:31.0:1619677473.885966:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677473.885966:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.885967:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677473.885967:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.885968:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677473.885970:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.885971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.885971:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.885973:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.885973:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677473.885979:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.885981:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.885982:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.885983:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.885985:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.885986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677473.886170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.886177:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.886179:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.886180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.886182:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.886185:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.886186:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.886187:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.886189:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.886190:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.886191:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.886194:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.886195:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.886198:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.886199:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.886199:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.886201:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.886201:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.886202:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.886204:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.886204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.886210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.886215:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.886217:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.886217:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.886218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.886219:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:6.0:1619677473.886219:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.886220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.886220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677473.886221:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.886224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.886236:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.886239:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.886239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.886240:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.886241:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.886241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.886417:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.886419:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.886422:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.886424:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.886424:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.886425:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.886426:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.886428:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.886429:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.886429:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.886432:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.886434:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.886436:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.886437:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.886437:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.886439:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.886439:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.886440:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.886441:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.886452:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.886454:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.886455:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.886456:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.886458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.886459:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.886473:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.886476:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.886476:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.886477:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677473.886478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.886479:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.886736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677473.886743:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677473.886746:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677473.886747:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.886749:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.886750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677473.886751:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677473.886752:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.886753:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.886754:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.886755:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677473.886758:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.886759:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677473.886762:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.886763:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677473.886763:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677473.886765:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677473.886765:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677473.886766:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677473.886767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677473.886781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.886789:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.886792:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.886793:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.886793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.886795:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.886799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.887072:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.887075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.887075:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.887077:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.887079:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.887081:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.887082:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.887084:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.887085:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.887086:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.887088:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.887090:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.887093:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.887093:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.887094:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.887096:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.887096:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.887098:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.887099:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.887099:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.887110:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.887111:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.887112:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.887112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.887113:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.887114:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.887114:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.887116:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.887117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.887118:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.887257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.887265:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.887267:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.887268:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.887270:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.887273:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.887275:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.887276:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.887277:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.887278:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677473.887278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:10.0:1619677473.887279:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.887282:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.887283:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.887286:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.887286:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.887286:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:10.0:1619677473.887287:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.887288:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.887288:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.887289:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:10.0:1619677473.887289:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.887290:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.887290:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677473.887291:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:9.2:1619677473.887295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:8.0:1619677473.887298:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.887300:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.887301:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96507129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.887302:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:8.0:1619677473.887302:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.887303:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.887303:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.887304:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.887304:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.887306:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.887308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.887308:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.887309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.887310:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.887313:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.887314:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.887314:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.887316:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.887316:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.887317:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.887317:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:4.2:1619677473.887324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677473.887330:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.887331:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677473.887333:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.887334:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677473.887334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.887334:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.887335:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677473.887336:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:26.0:1619677473.887337:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.887339:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.887340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677473.887341:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.887342:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.887344:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.887344:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.887345:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.887348:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.887350:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.887353:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.887353:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.887354:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.887356:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.887356:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.887357:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.887357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.887380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.887382:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.887383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.887384:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.887386:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.887388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.887489:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.887492:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.887494:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.887496:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.887497:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.887498:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.887500:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.887501:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.887502:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.887503:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.887505:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.887507:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.887509:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.887510:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.887511:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.887512:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.887513:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.887514:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.887516:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.887516:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.887517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.887524:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.887526:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.887527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.887527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.887529:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.887530:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677473.887531:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.887532:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.887533:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.887533:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677473.887559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.887565:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.887568:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.887569:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.887571:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.887574:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.887575:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.887576:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.887577:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.887578:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.887579:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.887582:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.887584:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.887586:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.887587:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.887587:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.887589:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.887589:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.887590:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.887590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.887602:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.887604:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.887605:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.887606:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.887609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.887610:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.887621:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677473.887623:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.887624:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.887625:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.887625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.887626:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677473.888315:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.888317:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.888319:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.888322:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.888322:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.888323:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.888324:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.888325:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.888326:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.888327:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.888330:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.888331:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.888334:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.888334:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.888335:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.888336:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.888337:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.888338:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.888338:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.888343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.888350:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.888353:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.888354:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.888354:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.888355:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.888357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.888398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677473.888406:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677473.888408:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677473.888410:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.888412:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677473.888413:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.888414:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677473.888415:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.888417:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.888418:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.888418:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677473.888421:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.888423:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677473.888425:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.888426:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:31.0:1619677473.888426:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677473.888428:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:31.0:1619677473.888428:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677473.888430:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:31.0:1619677473.888431:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:31.0:1619677473.888432:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:20.2:1619677473.888438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:12.0:1619677473.888441:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.888443:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.888444:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.888445:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000400:00000200:12.0:1619677473.888445:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.888447:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:30.0:1619677473.888448:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:13.0:1619677473.888448:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:30.0:1619677473.888449:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677473.888449:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:30.0:1619677473.888596:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.888598:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.888600:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.888602:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.888603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.888603:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.888605:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.888606:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.888607:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.888608:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.888611:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.888612:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.888614:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.888614:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.888615:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.888616:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.888617:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.888618:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.888618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.888628:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.888631:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.888632:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.888633:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.888635:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.888636:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.888727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.888733:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.888736:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.888737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.888739:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.888742:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.888743:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.888745:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.888746:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.888747:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.888748:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.888751:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.888752:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.888755:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.888756:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.888757:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.888758:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.888759:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.888760:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.888761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677473.888771:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.888774:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.888775:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.888777:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677473.888778:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.888780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:6.0:1619677473.888805:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.888807:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.888808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.888810:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.888812:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.888813:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.888815:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.888816:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.888817:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.888818:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.888821:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.888823:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.888825:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.888826:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.888827:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.888829:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.888829:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.888831:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.888831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.888842:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.888845:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.888845:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.888847:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.888849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.888850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677473.889307:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.889311:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.889313:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.889317:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.889320:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.889322:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677473.889324:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.889326:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.889328:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.889329:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677473.889335:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.889338:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677473.889342:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.889343:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677473.889344:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677473.889346:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677473.889347:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.889348:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.889349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.889353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.889360:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.889363:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.889364:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.889364:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.889366:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.889370:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677473.889374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.889380:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.889383:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.889383:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.889384:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.889385:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.889385:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:30.0:1619677473.890014:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.890017:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.890019:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.890021:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.890022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.890023:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.890024:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.890025:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.890026:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.890027:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.890030:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.890032:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.890034:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.890035:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.890036:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.890037:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.890038:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.890039:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.890040:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.890041:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.890043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.890050:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.890053:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.890054:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.890054:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.890055:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677473.890056:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.890057:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.890057:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677473.890058:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.890059:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677473.890119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.890127:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.890130:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.890131:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.890132:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.890134:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.890136:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.890137:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.890139:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.890140:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.890141:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.890144:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.890146:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.890149:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.890149:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.890150:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.890152:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.890152:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.890153:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.890154:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.890155:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.890161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.890167:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.890168:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.890170:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.890170:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677473.890171:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.890172:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.890172:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.890172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.890175:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.890179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.890280:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.890283:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.890285:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.890287:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.890287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.890289:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.890290:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.890291:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.890292:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.890298:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.890302:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.890303:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.890305:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.890305:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.890306:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.890308:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.890308:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.890309:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.890310:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.890322:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.890325:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.890325:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.890327:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.890329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.890329:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:5.2:1619677473.890573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.890580:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.890583:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.890583:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.890585:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.890588:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.890590:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.890591:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.890592:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.890593:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.890594:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.890598:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.890600:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.890603:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.890603:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.890604:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.890606:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.890606:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.890607:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.890608:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.890609:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.890614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.890619:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.890620:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.890621:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.890621:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.890622:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.890622:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.890623:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.890624:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.890625:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.890627:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.890642:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.890645:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.890646:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.890646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.890647:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.890648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.890666:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.890668:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.890669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.890671:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.890673:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.890674:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.890676:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.890677:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.890679:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.890679:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.890683:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.890685:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.890687:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.890688:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.890689:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.890691:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.890691:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.890692:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.890692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.890705:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.890707:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.890708:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.890710:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.890710:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.890712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.890728:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.890731:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.890732:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.890732:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.890733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.890734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.891051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677473.891059:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.891061:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677473.891062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677473.891064:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677473.891066:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.891068:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.891069:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.891071:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.891072:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.891073:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.891076:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.891077:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.891080:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.891081:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.891081:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.891083:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.891084:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.891085:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.891085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677473.891108:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.891111:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.891111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.891112:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.891114:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.891117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.891266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677473.891272:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677473.891273:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677473.891275:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677473.891276:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.891277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677473.891278:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.891279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677473.891279:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.891280:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.891281:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677473.891282:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.891283:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.891283:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.891284:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.891285:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.891286:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.891286:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:24.0:1619677473.891286:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.891287:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.891287:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.891291:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.891292:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.891299:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.891300:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.891302:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.891303:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677473.891303:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677473.891303:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.891304:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.891305:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677473.891305:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.891306:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.891306:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677473.891306:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.891307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677473.891307:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.891308:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.891308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677473.891320:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.891323:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.891324:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.891325:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.891326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.891328:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.891330:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.891333:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.891334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.891334:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.891336:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.891339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.891349:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.891351:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.891352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.891353:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.891354:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.891354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.891792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.891800:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.891804:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.891804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.891806:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.891809:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.891811:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.891812:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.891813:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.891814:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.891815:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.891818:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.891819:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.891822:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.891823:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.891823:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.891825:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.891826:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.891826:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.891827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.891839:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.891842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.891842:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.891843:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.891845:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.891847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.891970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.891978:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.891980:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.891981:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.891983:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.891985:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.891987:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.891988:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.891989:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.891990:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.891991:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.891993:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.891995:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.891997:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.891997:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.891998:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.891999:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.892000:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.892000:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.892001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.892005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.892011:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.892014:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.892015:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.892015:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.892017:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.892019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.892024:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.2:1619677473.892025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:9.2:1619677473.892026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.892027:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.892029:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.892031:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.892032:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.892032:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677473.892033:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:24.0:1619677473.892033:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:31.0:1619677473.892034:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677473.892035:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:31.0:1619677473.892036:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.892036:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.892036:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677473.892036:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677473.892037:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.892037:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:31.0:1619677473.892038:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.892038:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.892039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:18.0:1619677473.892040:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.892042:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677473.892043:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.892044:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:31.0:1619677473.892045:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:18.0:1619677473.892045:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677473.892047:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677473.892047:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677473.892048:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:18.0:1619677473.892048:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677473.892049:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:18.0:1619677473.892049:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677473.892050:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.892051:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.892052:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.892052:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:18.0:1619677473.892052:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.892054:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.892056:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677473.892056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677473.892057:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.892057:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.892059:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.892060:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.892061:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.892061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677473.892062:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.892064:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:9.2:1619677473.892065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:8.0:1619677473.892065:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.892066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.892067:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.892069:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677473.892072:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.892074:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:22.0:1619677473.892075:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:2.0:1619677473.892075:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.892076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.892077:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.892079:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:2.0:1619677473.892079:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.892080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.892081:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.892084:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.892085:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.892087:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.892088:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.892089:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.892090:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.892094:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.892096:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.892098:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.892099:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.892100:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.892101:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.892102:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.892103:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.892103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.892115:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.892117:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.892118:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.892120:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.892121:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.892123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.892366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.892374:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.892376:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.892377:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.892379:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.892381:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.892383:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.892384:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.892385:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.892386:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.892387:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.892391:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.892392:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.892394:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.892395:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.892396:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.892397:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.892397:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.892398:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.892399:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.892403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.892409:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.892411:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677473.892412:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.892413:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.892414:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.892415:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:30.0:1619677473.892817:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.892820:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.892822:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.892824:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.892824:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.892826:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.892827:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.892828:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.892829:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.892830:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.892833:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.892835:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.892837:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.892838:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.892839:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.892841:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.892841:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.892842:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.892844:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.892844:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.892846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.892853:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.892856:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.892857:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.892857:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.892858:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677473.892859:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.892860:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.892861:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677473.892861:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.892862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.893523:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.893526:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.893528:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.893530:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.893531:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.893532:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.893533:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.893534:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.893535:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.893535:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.893538:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.893540:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.893542:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.893543:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.893543:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.893545:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.893545:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.893546:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.893547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.893561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.893568:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.893571:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.893572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.893572:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.893573:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.893576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.893586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.893593:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.893596:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.893596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.893598:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.893601:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.893602:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.893604:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.893605:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.893606:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.893607:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.893610:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.893612:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.893615:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.893616:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.893616:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.893618:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.893619:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.893620:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.893620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.893632:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.893635:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.893635:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.893637:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.893639:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.893640:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677473.893744:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.893747:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.893749:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.893750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.893751:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.893752:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.893753:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.893755:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.893756:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.893757:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.893760:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.893762:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.893765:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.893765:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.893766:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.893768:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.893768:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.893769:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.893769:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.893784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.893790:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.893793:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.893794:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.893794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.893796:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.893798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.893805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.893811:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.893814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.893814:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.893815:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.893816:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.893816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.893946:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.893949:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.893950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.893952:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.893954:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.893955:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.893957:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.893958:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.893959:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.893960:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.893963:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.893965:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.893967:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.893968:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.893968:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.893970:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.893971:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.893972:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.893973:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.893974:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.893984:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.893986:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.893986:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.893987:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.893988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.893988:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.893989:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.893991:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.893992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.893993:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.894072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677473.894079:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.894081:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677473.894082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677473.894084:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.894087:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.894088:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.894089:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.894091:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.894091:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.894092:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.894095:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.894097:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.894098:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.894099:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.894100:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.894101:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.894102:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.894102:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.894103:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677473.894106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.894113:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.894114:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.894115:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.894116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.894117:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.894119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.894524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677473.894528:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.894530:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677473.894533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.894533:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.894536:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.894537:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.894539:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.894540:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.894541:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.894542:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.894545:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.894547:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.894550:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.894551:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.894551:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.894553:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.894554:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.894554:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.894555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.894560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.894568:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.894570:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.894571:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.894573:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.894574:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.894575:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.894581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.894589:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.894592:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.894592:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.894593:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.894593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.894595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677473.894854:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.894856:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.894858:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.894861:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.894861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.894863:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677473.894864:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.894865:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.894866:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.894867:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677473.894870:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.894872:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677473.894875:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.894876:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677473.894877:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677473.894879:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677473.894879:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.894880:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.894881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677473.894885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.894893:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.894895:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.894896:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.894897:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.894898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.894900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677473.894940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.894948:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.894951:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.894951:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.894953:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.894955:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.894956:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.894958:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.894959:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.894960:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.894961:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.894964:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.894965:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.894968:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.894969:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.894969:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.894971:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.894971:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.894973:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.894974:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.894974:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.894978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.894980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.894987:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677473.894987:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677473.894989:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.894990:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677473.894990:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.894991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.894991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677473.894991:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.894993:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.894997:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.894999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.895006:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.895009:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.895010:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.895010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.895011:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.895012:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677473.895169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677473.895176:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677473.895179:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677473.895179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677473.895181:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.895184:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.895185:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677473.895187:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.895188:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.895189:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.895190:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677473.895193:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.895194:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677473.895196:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.895196:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677473.895197:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677473.895199:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677473.895199:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677473.895200:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677473.895201:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:30.0:1619677473.895211:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.895213:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677473.895214:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.895216:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.895217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.895218:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:5.2:1619677473.895374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.895380:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.895382:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.895383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.895386:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.895388:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.895389:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.895391:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.895392:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.895393:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.895394:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.895397:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.895398:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.895401:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.895402:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.895402:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.895404:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.895405:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.895405:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.895405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.895419:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.895422:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.895423:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.895423:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.895425:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.895428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.895480:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.895483:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.895485:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.895488:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.895488:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.895490:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.895491:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.895492:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.895493:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.895494:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.895497:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.895498:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.895501:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.895502:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.895502:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.895503:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.895504:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.895505:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.895506:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.895516:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.895518:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.895519:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.895520:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.895522:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.895523:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.895618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.895625:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.895628:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.895629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.895630:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.895633:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.895634:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.895635:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.895636:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.895637:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.895638:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.895641:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.895643:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.895645:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.895646:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.895647:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.895648:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.895649:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.895649:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.895650:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.895662:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.895665:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.895666:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.895666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.895668:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.895672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.895688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.895691:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.895694:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.895696:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.895697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.895698:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.895700:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.895701:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.895702:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.895704:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.895705:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.895708:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.895710:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.895712:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.895713:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.895714:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.895716:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.895716:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.895717:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.895718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.895722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.895728:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.895731:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.895732:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.895732:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.895735:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.895737:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.895767:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.895770:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.895772:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.895774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.895774:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.895775:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.895777:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.895778:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.895779:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.895780:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.895783:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.895785:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.895788:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.895789:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.895789:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.895791:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.895791:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.895792:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.895793:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.895796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.895803:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.895805:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.895806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.895806:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.895808:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.895810:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.895845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677473.895852:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.895855:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677473.895856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677473.895857:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.895859:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.895860:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677473.895861:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.895862:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.895863:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.895864:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677473.895867:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.895868:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.895870:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.895870:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.895871:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.895872:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.895872:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.895873:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677473.895873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677473.895877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.895883:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.895885:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.895885:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.895886:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.895887:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.895888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.895898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.895904:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.895906:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.895907:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.895907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.895908:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.895908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677473.896004:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.896007:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.896008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.896009:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.896011:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.896013:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.896014:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.896016:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.896017:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.896018:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.896021:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.896023:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.896026:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.896026:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.896027:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.896029:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.896030:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.896031:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.896031:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.896043:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.896045:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.896046:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.896047:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.896048:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.896050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.896062:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.896065:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.896065:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.896066:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.896067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.896068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.896468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.896476:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.896479:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.896480:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.896482:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.896485:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.896486:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677473.896487:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.896489:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.896490:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.896491:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677473.896494:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.896495:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.896497:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.896498:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677473.896498:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.896500:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.896500:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.896501:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.896501:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.896506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.896512:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.896514:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.896515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.896515:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.896516:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.896518:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677473.896564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.896571:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.896573:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.896574:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.896575:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.896577:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.896578:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.896579:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.896580:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.896581:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.896582:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.896585:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.896587:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:4.2:1619677473.896588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677473.896589:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.896590:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.896590:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.896592:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.896592:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.896593:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.896594:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677473.896596:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.896599:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.896600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.896601:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.896604:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.896605:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.896606:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96507086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.896607:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:8.0:1619677473.896608:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.896609:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.896610:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:8.0:1619677473.896610:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677473.896611:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.896611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.896614:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.896615:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.896616:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.896618:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677473.896619:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.896620:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.896621:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.896622:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.896623:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.896624:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.896624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.896636:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677473.896637:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677473.896639:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677473.896640:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677473.896640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.896640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.896642:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677473.896643:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.896644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677473.896645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677473.896646:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.896647:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677473.896648:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.896650:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.896651:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.896651:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677473.896654:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.896656:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677473.896659:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.896660:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677473.896660:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677473.896662:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677473.896663:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.896664:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.896665:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.896666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677473.896668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.896680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677473.896680:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677473.896682:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.896683:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677473.896683:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677473.896684:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677473.896684:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.896685:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.896686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.896688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.896697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677473.896705:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.896707:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677473.896708:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.896709:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.896709:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.896710:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677473.896780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.896787:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.896789:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.896790:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.896792:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.896795:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.896796:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.896797:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.896799:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.896799:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.896800:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.896803:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.896805:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.896808:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.896809:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.896809:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.896811:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.896811:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.896812:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.896812:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.896825:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.896827:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.896828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.896829:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.896831:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.896834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677473.896927:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.896930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677473.896930:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000400:00000200:2.0:1619677473.896932:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.896935:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.896936:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677473.896937:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.896939:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.896940:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.896941:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677473.896944:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.896946:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.896948:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.896949:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677473.896950:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677473.896952:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677473.896952:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.896954:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677473.896955:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677473.896955:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:6.0:1619677473.896965:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677473.896966:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:6.0:1619677473.896967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:18.0:1619677473.896968:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:22.0:1619677473.896969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677473.896969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:6.0:1619677473.896969:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.896970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:6.0:1619677473.896971:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.896973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.896985:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.896988:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.896988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.896989:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.896990:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.896991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.897571:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.897573:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.897574:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.897576:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.897578:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.897580:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.897581:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.897582:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.897583:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.897584:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.897588:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.897590:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.897592:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.897593:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.897594:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.897596:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.897596:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.897597:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.897597:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.897610:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.897613:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.897614:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.897615:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.897616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.897618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.897725:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.897727:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677473.897729:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677473.897729:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.897730:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.897732:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.897733:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.897733:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.897735:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.897736:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.897737:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.897738:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.897741:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.897743:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.897744:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.897745:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677473.897745:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.897746:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:31.0:1619677473.897747:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677473.897747:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677473.897748:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677473.897748:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677473.897749:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.897750:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.897750:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.897751:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.897752:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.897753:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.897753:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.897754:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.897755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.897756:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.897756:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.897757:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.897757:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.897757:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.897757:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.897759:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:30.0:1619677473.897759:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.897760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.897761:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.897761:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.897762:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.897764:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.897764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.897765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.897772:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.897775:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.897776:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.897776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.897777:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.897780:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677473.897780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.897781:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.897782:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.897783:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.897784:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677473.897785:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.897786:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.897786:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.897786:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.897787:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.897827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.897834:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.897837:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.897838:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.897840:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.897843:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.897844:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.897845:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.897847:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.897848:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.897849:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677473.897851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677473.897853:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.897854:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.897857:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.897858:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.897858:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.897858:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677473.897860:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.897861:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:8.0:1619677473.897861:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.897862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677473.897862:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.897862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:8.0:1619677473.897864:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677473.897866:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:8.0:1619677473.897871:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.897873:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677473.897874:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:8.0:1619677473.897874:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96507078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.897876:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.897876:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677473.897877:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.897878:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.897878:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677473.897878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.897881:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.897883:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.897885:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677473.897886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677473.897888:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.897889:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.2:1619677473.897891:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.897894:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.897896:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.897896:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.897897:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.897898:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677473.897898:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.897901:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.897902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:9.2:1619677473.897902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677473.897902:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.897903:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.897904:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677473.897909:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.897912:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.897913:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.897913:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.897915:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.897918:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.897919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.897927:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.897929:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.897930:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.897930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.897932:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.897932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.898683:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.898686:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677473.898687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677473.898688:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.898690:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.898692:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677473.898692:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.898694:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.898695:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.898696:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677473.898699:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.898701:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.898703:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.898703:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677473.898704:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.898706:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.898706:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677473.898707:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677473.898707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677473.898713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.898720:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.898722:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.898722:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.898724:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.898724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.898725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:31.0:1619677473.898894:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.898897:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.898899:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.898901:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.898902:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.898903:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.898904:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.898905:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.898906:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.898907:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.898911:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.898912:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.898915:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.898916:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.898916:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.898918:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.898918:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.898919:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.898920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.898923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.898930:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.898933:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.898934:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.898934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.898936:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.898938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.898955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.898961:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.898963:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.898965:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.898966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.898968:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.898970:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677473.898971:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.898972:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.898973:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.898974:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677473.898978:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.898979:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.898982:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.898983:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.898983:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.898985:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.898985:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.898986:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677473.898987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677473.898991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.898999:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.899001:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.899001:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.899003:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.899004:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.899005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.899573:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.899575:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.899576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.899578:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.899580:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.899582:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.899583:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.899585:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.899585:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.899586:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.899589:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.899591:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.899593:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.899594:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.899594:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.899596:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.899597:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.899598:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.899598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.899611:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.899613:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.899614:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.899615:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.899617:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.899618:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.899657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.899665:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.899667:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.899668:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.899670:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.899673:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.899674:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.899676:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.899677:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.899678:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.899679:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.899682:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.899683:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.899686:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.899686:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.899687:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.899689:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.899689:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.899690:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.899690:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.899702:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.899704:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.899705:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.899707:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.899709:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.899710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677473.899711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.899715:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.899718:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.899720:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.899722:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.899723:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677473.899724:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.899725:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.899726:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.899727:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677473.899728:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:20.0:1619677473.899728:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.899731:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:20.0:1619677473.899732:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677473.899733:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.899734:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.899735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.899735:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.899736:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677473.899737:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677473.899737:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.899737:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677473.899738:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.899739:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.899739:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.899739:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:30.0:1619677473.899740:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.899740:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000400:00000200:30.0:1619677473.899741:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.899741:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:30.0:1619677473.899745:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.899746:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.899749:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.899750:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.899750:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.899752:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.899752:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.899753:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677473.899753:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677473.899754:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677473.899755:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.899756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.899756:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.899757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677473.899759:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.899761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.899765:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.899767:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.899768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.899768:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.899770:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.899773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.900043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677473.900050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677473.900053:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677473.900054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677473.900055:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.900058:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.900059:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.900061:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677473.900062:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677473.900062:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.900063:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.900064:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677473.900065:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.900068:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.900068:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.900069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677473.900070:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677473.900071:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.900072:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677473.900072:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677473.900073:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.900073:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.900073:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:31.0:1619677473.900074:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.900075:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.900075:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.900075:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:31.0:1619677473.900076:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.900076:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677473.900077:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677473.900078:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:31.0:1619677473.900079:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.900081:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.900083:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.900083:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:4.2:1619677473.900083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:31.0:1619677473.900084:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.900085:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.900086:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.900087:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.900087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677473.900087:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.900089:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:26.0:1619677473.900090:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:13.0:1619677473.900090:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.900091:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.900092:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677473.900093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:12.0:1619677473.900093:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.900094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677473.900095:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:30.0:1619677473.900097:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.900100:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.900101:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.900102:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.900104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.900105:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.900502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.900510:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.900513:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.900514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.900515:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.900518:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.900519:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.900520:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.900522:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.900522:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.900523:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.900526:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.900528:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.900530:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.900531:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.900532:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.900534:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.900534:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.900535:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.900536:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.900536:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.900542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.900548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677473.900550:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677473.900551:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677473.900552:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677473.900552:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.900552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.900553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.900554:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.900554:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.900557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.900661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.900668:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.900671:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.900672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.900674:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.900676:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.900678:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.900679:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.900680:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.900681:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.900682:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.900686:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.900687:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677473.900688:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677473.900690:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.900691:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677473.900691:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.900691:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:31.0:1619677473.900693:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.900693:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.900693:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.900694:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.900694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677473.900695:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.900695:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.900696:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.900697:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.900699:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.900700:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.900701:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.900704:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.900706:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677473.900707:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:31.0:1619677473.900708:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.900709:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677473.900709:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:31.0:1619677473.900710:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.900710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.900710:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.900711:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.900712:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677473.900712:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.900713:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.900715:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:6.0:1619677473.900715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.900716:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.900718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.900725:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677473.900725:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677473.900728:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:22.0:1619677473.900728:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677473.900729:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677473.900729:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.900729:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:8.0:1619677473.900729:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.900729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.900730:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:30.0:1619677473.900731:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:24.0:1619677473.900731:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.900731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677473.900732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.900733:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677473.900734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.900737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.900743:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.900746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.900746:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.900746:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.900747:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.900748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.900852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.900859:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.900862:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.900863:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.900865:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.900867:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.900868:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.900869:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.900871:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.900871:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.900873:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.900876:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.900877:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.900879:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.900879:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.900880:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.900882:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.900882:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.900883:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.900884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.900888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.900895:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.900897:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.900898:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.900898:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.900899:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.900900:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677473.901126:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.901129:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.901131:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.901133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.901133:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.901134:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.901136:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.901137:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.901138:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.901139:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.901142:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.901144:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.901146:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.901147:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.901148:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.901149:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.901149:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.901150:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.901151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.901161:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.901164:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.901164:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.901166:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.901168:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.901169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.901466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.901474:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.901476:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.901477:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.901479:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.901482:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.901484:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.901485:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.901486:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.901487:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.901488:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.901491:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.901492:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.901495:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.901496:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.901497:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.901498:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.901499:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.901499:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.901501:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.901501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.901507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677473.901513:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.901515:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.901516:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.901517:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677473.901517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.901517:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.901518:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.901519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.901519:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.901522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.901548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677473.901552:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677473.901555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677473.901555:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.901556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.901557:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.901558:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:22.0:1619677473.901559:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.901559:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.901561:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.901561:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.901562:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.901564:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.901564:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.901564:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.901566:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677473.901566:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:24.0:1619677473.901568:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96507061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.901569:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.901570:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.901571:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.901572:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.901573:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:24.0:1619677473.901575:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.901577:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.901578:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677473.901578:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.901579:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677473.901580:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.901580:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677473.901581:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:24.0:1619677473.901581:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.901582:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.901582:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.901582:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677473.901583:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.901583:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677473.901584:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.901584:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677473.901584:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:30.0:1619677473.901585:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677473.901585:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677473.901585:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.901585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677473.901586:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.901587:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96507060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.901587:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.901587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677473.901588:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.901589:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.901590:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.901593:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677473.901593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677473.901595:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.901597:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.901597:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677473.901598:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.901598:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677473.901598:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.901599:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677473.901600:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.901600:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677473.901600:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.901600:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:31.0:1619677473.901601:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000400:00000200:21.0:1619677473.901601:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.901601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677473.901602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:21.0:1619677473.901602:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.901602:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677473.901603:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.901603:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677473.901603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677473.901604:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.901605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677473.901605:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.901606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677473.901607:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.901608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:4.2:1619677473.901608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:6.0:1619677473.901609:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.901611:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.901612:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:6.0:1619677473.901614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677473.901615:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677473.901615:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.901616:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.901616:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.901618:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.901618:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:8.0:1619677473.901618:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.901621:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.901621:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:10.0:1619677473.901623:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.901625:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.901626:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.901627:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.901628:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.901629:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.901632:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.901634:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.901636:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.901637:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.901638:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.901639:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.901640:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.901641:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.901641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.901654:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.901656:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.901657:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.901658:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.901660:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.901663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.901673:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.901676:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.901677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.901677:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.901678:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.901679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.901895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.901901:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.901904:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.901905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.901907:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.901909:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.901910:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.901911:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.901912:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.901913:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.901914:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.901917:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.901919:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.901921:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.901922:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.901922:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.901924:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.901925:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.901926:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.901926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.901930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.901939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.901941:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.901942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.901942:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.901944:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.901947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677473.902104:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.902107:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.902109:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.902111:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.902112:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.902113:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677473.902115:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.902116:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.902117:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.902118:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677473.902121:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.902122:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677473.902126:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.902127:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677473.902127:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.902129:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677473.902130:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.902131:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.902131:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677473.902135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.902143:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.902146:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.902147:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.902149:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.902150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.902152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677473.902961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.902968:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.902971:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.902972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.902973:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.902976:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.902978:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.902979:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.902980:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.902981:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.902982:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.902985:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.902987:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.902989:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.902990:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.902991:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.902992:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.902993:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.902993:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.902994:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.903008:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.903010:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.903011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.903012:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.903014:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.903016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.903133:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.903136:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.903136:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.903138:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.903140:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.903141:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.903142:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.903144:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.903144:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.903145:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.903148:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.903150:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.903152:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.903153:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.903154:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.903156:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.903156:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.903157:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.903157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.903169:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.903171:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.903172:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.903174:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.903175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.903176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.903187:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.903189:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.903190:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.903191:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.903192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.903193:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.903528:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.903531:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.903533:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.903535:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.903536:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.903538:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.903539:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.903540:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.903541:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.903542:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.903545:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.903547:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.903549:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.903550:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.903551:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.903552:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.903553:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.903554:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.903554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.903559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.903566:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.903569:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.903570:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.903570:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.903572:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.903574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.903576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.903582:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.903584:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.903585:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.903585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.903586:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.903586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.903931:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.903933:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.903935:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.903937:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.903938:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.903939:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.903940:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.903941:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.903942:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.903943:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.903946:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.903947:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.903950:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.903950:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.903951:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.903952:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.903953:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.903954:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.903954:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.903958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.903964:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.903967:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.903968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.903968:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.903970:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.903972:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.904441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.904445:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.904448:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.904450:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.904450:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.904452:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.904454:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.904456:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.904457:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.904458:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.904459:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.904462:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.904464:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.904467:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.904467:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.904468:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.904470:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.904471:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.904471:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.904472:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.904480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.904487:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.904490:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.904491:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.904491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.904493:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.904496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.904681:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.904683:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.904686:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.904688:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.904688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.904689:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.904690:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.904692:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.904693:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.904694:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.904697:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.904699:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.904700:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.904701:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.904702:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.904703:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.904704:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.904705:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.904707:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.904708:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.904717:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.904719:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.904720:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.904721:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677473.904721:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.904722:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.904723:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.904724:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.904725:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.904726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677473.905079:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.905082:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.905083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.905085:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.905087:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.905089:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.905090:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.905091:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.905093:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.905094:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.905097:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.905099:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.905101:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.905102:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.905102:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.905104:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.905105:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.905106:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.905106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.905118:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.905120:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.905121:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.905122:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.905123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.905125:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.905138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.905141:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.905144:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677473.905146:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.905146:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.905148:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.905150:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.905151:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.905152:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.905153:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.905154:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.905158:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.905159:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.905162:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.905163:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.905163:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.905165:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.905165:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.905166:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.905167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.905171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.905179:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.905181:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.905182:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.905184:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.905185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.905186:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677473.905245:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.905248:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.905251:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.905252:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.905253:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.905254:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.905256:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.905257:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.905258:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.905259:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.905263:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.905264:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.905267:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.905268:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.905269:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.905271:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.905271:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.905272:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.905274:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.905274:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.905287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.905289:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.905292:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.905292:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.905299:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.0:1619677473.905301:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.905303:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.905304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.905304:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.905306:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.905308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.905318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.905327:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.905329:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.905331:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.905332:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.905335:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.905336:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.905337:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.905339:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.905340:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.905340:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.905344:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.905345:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.905348:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.905349:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.905349:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.905351:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.905351:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.905352:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.905352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.905366:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.905369:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.905370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.905370:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.905372:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.905375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.905454:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.905457:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.905459:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.905461:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.905461:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.905462:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.905463:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.905465:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.905465:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.905466:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.905469:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.905471:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.905472:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.905473:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.905474:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:5.2:1619677473.905474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677473.905475:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.905475:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.905476:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.905477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677473.905480:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.905483:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.905483:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.905485:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.905487:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:17.0:1619677473.905488:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.905489:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677473.905490:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.905490:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.905490:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.905492:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.905493:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.905493:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:17.0:1619677473.905493:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.905494:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677473.905495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:17.0:1619677473.905498:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.905499:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.905502:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.905503:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.905503:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677473.905504:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677473.905505:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.905506:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677473.905507:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677473.905507:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.905507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677473.905508:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.905509:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677473.905510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.905511:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.905519:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.905522:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.905523:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.905523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.905525:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.905527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.905820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.905827:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.905830:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.905830:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677473.905832:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.905835:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.905836:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.905838:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.905839:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.905840:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.905841:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.905844:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.905846:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.905849:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.905849:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.905850:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.905852:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.905852:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.905853:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.905853:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.905865:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.905868:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.905869:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.905869:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.905871:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.905874:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.906264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.906267:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.906270:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.906272:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.906273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.906274:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.906276:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.906277:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.906279:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.906280:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.906281:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.906284:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.906285:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.906288:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.906289:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.906289:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.906291:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.906292:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.906293:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.906300:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.906301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677473.906306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:9.2:1619677473.906308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.906312:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677473.906314:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677473.906314:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.906315:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.906315:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677473.906317:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.906318:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.906318:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.906320:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.906322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.906458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.906464:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.906467:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.906467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.906469:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.906472:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.906474:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.906475:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.906476:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.906477:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.906478:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.906481:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.906482:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.906485:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.906485:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.906486:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.906488:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.906488:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.906490:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.906491:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.906491:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.906494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.906496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:4.2:1619677473.906500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677473.906502:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677473.906502:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677473.906504:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.906505:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677473.906505:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.906506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.906506:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677473.906506:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.906508:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677473.906508:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.906510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677473.906511:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.906512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.906513:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.906516:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.906518:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.906519:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.906521:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.906522:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.906523:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.906526:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.906527:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.906530:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.906531:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.906532:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.906534:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.906534:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.906535:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.906535:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.906548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.906551:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.906551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.906552:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.906554:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.906557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.906669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.906677:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.906680:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.906680:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.906682:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.906685:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.906686:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.906687:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.906688:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.906689:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.906690:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.906693:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.906695:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.906697:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.906698:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.906699:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.906700:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.906701:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.906701:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.906702:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.906714:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.906717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.906717:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.906718:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.906720:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.906723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.906731:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.906734:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.906735:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.906735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.906736:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.906737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.906884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.906887:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.906889:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.906890:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.906891:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677473.906891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.906893:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.906894:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.906895:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.906896:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.906899:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.906900:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.906902:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.906903:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.906903:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.906905:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.906905:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.906906:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.906907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.906917:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.906920:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.906920:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.906922:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.906924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.906925:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677473.907119:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.907122:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.907123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.907125:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.907127:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.907128:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:4.2:1619677473.907128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:6.0:1619677473.907129:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.907131:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.907132:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677473.907133:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:6.0:1619677473.907133:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.907136:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:6.0:1619677473.907137:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.907138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677473.907139:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:21.0:1619677473.907140:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677473.907142:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.907143:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.907143:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.907143:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677473.907145:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:6.0:1619677473.907145:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677473.907146:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.907146:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677473.907147:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.907147:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677473.907148:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.907148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.907149:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.907153:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.907154:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.907158:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.907158:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.907159:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.907159:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.907160:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.907161:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.907161:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:20.0:1619677473.907161:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.907162:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.907162:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.907163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:21.0:1619677473.907164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.907165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677473.907167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.907173:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.907175:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.907176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.907177:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.907179:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.907182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.907185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.907191:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.907194:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.907194:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.907195:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.907195:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.907196:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.907396:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.907399:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.907401:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.907404:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.907404:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.907405:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.907407:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.907408:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.907409:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.907410:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.907413:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.907414:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.907417:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.907418:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.907418:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.907420:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.907420:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.907421:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.907422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.907426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.907433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.907435:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.907436:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.907437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.907438:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.907440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.907575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.907583:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.907586:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.907587:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.907588:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.907591:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.907592:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.907593:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.907595:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.907596:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.907596:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.907599:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.907601:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.907603:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.907604:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.907604:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.907606:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.907606:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.907607:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.907607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.907620:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.907622:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.907623:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.907624:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.907627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677473.907627:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:30.0:1619677473.907642:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677473.907642:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:30.0:1619677473.907645:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:12.0:1619677473.907645:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677473.907646:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:30.0:1619677473.907647:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.907647:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677473.907647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677473.907649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677473.907649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:30.0:1619677473.907650:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.907651:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.907652:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.907653:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.907654:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.907655:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.907658:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.907659:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.907661:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.907662:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.907662:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.907664:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.907664:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.907665:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.907666:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677473.907672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677473.907676:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.907679:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.907680:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.907680:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677473.907681:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.907682:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677473.907683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677473.907683:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677473.907684:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:8.0:1619677473.907685:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.907687:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.907689:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.907690:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.907691:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.907692:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.907693:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.907696:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.907697:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.907700:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.907701:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.907701:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.907703:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.907703:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.907704:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.907705:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.907718:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.907721:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.907722:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.907723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.907724:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.907727:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.908467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.908473:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.908475:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.908478:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.908478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.908480:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.908481:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.908483:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.908484:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.908485:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.908486:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.908490:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.908491:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.908494:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.908494:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.908495:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.908497:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.908497:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.908498:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.908499:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.908504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.908513:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.908515:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.908516:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.908517:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.908518:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.908521:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.908523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.908530:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.908532:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.908533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.908534:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.908535:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.908535:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677473.908587:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.908590:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.908592:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.908594:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.908594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.908595:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677473.908596:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.908598:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.908599:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.908599:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677473.908602:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.908604:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677473.908608:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.908608:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677473.908609:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677473.908611:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677473.908611:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.908613:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.908614:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.908615:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677473.908617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.908625:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.908627:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677473.908628:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677473.908628:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.908630:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.908630:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.908631:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.908632:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.908632:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677473.908633:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.908645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.908648:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.908649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.908650:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.908653:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.908654:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.908655:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.908657:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.908658:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.908659:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.908662:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.908663:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.908666:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.908667:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.908667:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.908669:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.908670:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.908671:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.908672:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.908673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.908683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.908685:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.908685:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.908686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.908687:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.908687:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.908688:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.908690:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.908691:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.908692:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.909318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677473.909320:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:30.0:1619677473.909321:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:6.0:1619677473.909323:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:30.0:1619677473.909324:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.909324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:31.0:1619677473.909325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677473.909326:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.909327:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.909328:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.909329:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.909330:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.909330:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677473.909332:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.909332:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.909333:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.909334:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677473.909334:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.909335:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.909336:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677473.909340:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.909341:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.909342:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677473.909343:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677473.909345:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.909345:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.909346:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677473.909346:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.909347:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677473.909347:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677473.909348:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.909348:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.909349:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677473.909349:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677473.909350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677473.909350:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.909351:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677473.909352:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677473.909353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677473.909354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:9.2:1619677473.909359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.909361:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677473.909363:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:24.0:1619677473.909364:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.909365:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.909365:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:8.0:1619677473.909365:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677473.909366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:18.0:1619677473.909366:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:24.0:1619677473.909367:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.909367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:18.0:1619677473.909369:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.909370:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:18.0:1619677473.909370:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.909371:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.909372:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.909375:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.909802:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.909805:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.909807:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.909808:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.909809:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.909811:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.909812:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.909814:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.909815:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.909816:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.909819:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.909821:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.909824:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.909824:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.909825:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.909827:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.909827:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.909828:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.909829:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.909832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.909838:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.909841:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.909842:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.909842:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.909844:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.909846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.910546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.910555:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677473.910555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677473.910559:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677473.910559:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677473.910560:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677473.910561:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677473.910562:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.910563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677473.910564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.910564:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:26.0:1619677473.910566:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.910567:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.910567:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.910569:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.910569:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.910570:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.910572:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677473.910572:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677473.910573:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677473.910573:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.910574:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.910576:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.910576:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.910576:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.910577:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.910578:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.910578:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.910579:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677473.910579:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677473.910580:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.910582:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677473.910585:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677473.910585:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.910589:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677473.910589:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677473.910590:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.910591:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.910592:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.910592:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.910594:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677473.910594:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677473.910595:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.910595:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677473.910596:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.910596:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677473.910596:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.910596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:26.0:1619677473.910598:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.910598:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677473.910599:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677473.910599:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.910600:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.910600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677473.910601:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:9.2:1619677473.910601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.910603:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.910604:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.910605:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.910606:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677473.910609:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.910611:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.910612:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.910612:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677473.910613:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:18.0:1619677473.910614:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.910615:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.910616:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.910616:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:20.0:1619677473.910618:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.910619:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.910620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.910627:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.910630:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.910630:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.910631:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.910633:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.910633:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.910636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.910636:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.910637:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.910638:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.910638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.910640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.910648:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.910651:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.910652:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.910652:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.910653:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.910654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677473.910699:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.910702:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.910704:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.910706:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.910706:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.910708:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.910709:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.910711:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.910712:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.910713:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.910716:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.910718:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.910720:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.910721:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.910722:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.910723:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.910724:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.910725:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.910725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.910729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.910735:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.910738:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.910739:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.910739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.910741:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.910743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.910775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.910781:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.910784:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:4.2:1619677473.910785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677473.910785:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.910787:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.910790:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.910791:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.910792:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.910794:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.910794:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677473.910795:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.910796:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.910799:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.910799:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.910800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677473.910802:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.910806:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.910806:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.910807:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.910807:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.910809:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.910809:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.910810:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:10.0:1619677473.910810:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.910811:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.910812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677473.910812:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.910814:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.910816:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:5.2:1619677473.910816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677473.910818:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.910819:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677473.910822:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.910823:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.910824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677473.910825:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.910825:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677473.910828:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.910831:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.910832:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677473.910833:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677473.910833:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.910835:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.910836:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677473.910836:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677473.910837:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.910837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.910838:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677473.910839:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.910839:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.910840:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.910841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.910842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.910845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677473.910847:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.910850:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.910850:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.910851:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.910853:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.910854:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677473.910855:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677473.910857:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.910859:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.910860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.910878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.910884:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.910886:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.910887:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.910888:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.910888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.910889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.911090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.911097:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.911099:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.911100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.911102:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.911104:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.911106:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.911107:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.911108:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.911109:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.911110:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.911113:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.911114:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.911117:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.911118:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.911118:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.911120:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.911120:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.911121:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.911122:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.911123:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.911126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.911129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677473.911133:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.911135:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.911136:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.911137:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:18.0:1619677473.911137:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.911137:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.911138:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.911139:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677473.911139:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.911142:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.911146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.911153:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.911156:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.911157:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.911157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.911158:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.911159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.911262:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.911265:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.911267:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.911269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.911270:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.911272:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.911273:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.911274:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.911275:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.911276:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.911279:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.911281:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.911283:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.911284:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.911284:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.911286:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.911287:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.911288:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.911288:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.911292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.911301:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.911303:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.911304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.911304:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.911306:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.911308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677473.911458:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.911460:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677473.911461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677473.911463:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.911466:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.911467:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677473.911469:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.911470:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.911471:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.911472:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677473.911475:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.911477:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.911479:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.911480:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.911480:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.911481:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.911482:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.911483:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677473.911483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677473.911488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.911496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.911498:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.911498:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.911499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.911500:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.911502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.911546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.911554:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.911557:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.911558:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.911560:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.911562:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.911563:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.911565:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.911566:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.911567:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.911568:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.911571:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.911573:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.911576:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.911576:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.911577:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.911579:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.911579:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.911580:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.911580:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.911594:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.911597:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.911598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.911598:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.911600:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.911603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.911698:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.911701:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.911702:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.911705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.911705:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.911706:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.911707:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.911709:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.911710:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.911710:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.911713:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.911715:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.911717:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.911717:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.911718:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.911719:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.911720:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.911721:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.911721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.911731:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.911734:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.911735:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.911736:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.911738:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.911739:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.911744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.911751:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.911753:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.911755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.911756:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.911758:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.911759:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.911761:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.911762:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.911763:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.911764:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.911767:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.911768:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.911771:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.911772:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.911773:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.911774:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.911775:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.911776:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.911776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:4.2:1619677473.911779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677473.911784:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.911787:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:13.0:1619677473.911788:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:21.0:1619677473.911789:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.911790:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:13.0:1619677473.911791:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:21.0:1619677473.911792:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.911792:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.911794:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:13.0:1619677473.911795:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.911795:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:21.0:1619677473.911796:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96507014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.911797:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677473.911797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:21.0:1619677473.911798:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.911799:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.911803:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.911805:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.911807:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.911808:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.911809:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.911810:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.911811:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.911812:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677473.911813:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677473.911814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677473.911816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677473.911819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.911826:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677473.911826:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677473.911826:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.911827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.911828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677473.911828:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.911829:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.911830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.911830:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.911833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677473.912317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.912325:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.912328:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.912328:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.912331:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.912333:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.912335:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.912336:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.912337:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.912338:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.912339:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.912343:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.912345:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.912348:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.912348:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.912349:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.912351:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.912351:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.912352:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.912352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.912358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.912366:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.912368:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677473.912369:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.912370:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.912372:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.912376:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677473.912377:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.912379:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677473.912379:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677473.912381:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.912383:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.912385:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677473.912386:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.912387:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.912388:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.912389:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677473.912393:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.912394:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677473.912398:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.912398:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677473.912399:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677473.912401:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677473.912401:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677473.912402:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677473.912403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677473.912407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.912414:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.912416:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.912417:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.912417:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.912420:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.912423:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677473.912448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.912456:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.912459:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.912460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.912461:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.912464:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.912466:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.912467:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.912468:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.912469:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.912470:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.912474:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.912476:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.912478:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.912479:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.912480:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.912482:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.912482:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.912483:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.912483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.912496:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.912499:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.912499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.912500:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.912502:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.912505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.912537:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.912540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.912540:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.912542:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.912544:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.912545:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.912547:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.912548:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.912549:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.912550:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.912554:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.912556:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.912558:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.912559:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.912560:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.912562:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.912562:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.912563:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.912563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.912575:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.912577:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.912578:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.912580:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.912580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.912582:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.912653:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.912656:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.912658:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.912660:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.912660:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.912661:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.912662:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.912664:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.912665:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.912666:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.912669:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.912671:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.912673:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.912674:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.912675:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.912677:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.912677:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.912679:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.912680:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.912681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.912682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.912689:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.912691:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.912692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.912692:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.912694:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677473.912694:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.912696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.912697:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.912697:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.912698:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.912840:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.912842:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.912845:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.912846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.912847:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.912848:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.912849:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.912850:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.912851:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.912852:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.912855:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.912857:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.912859:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.912859:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.912860:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.912861:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.912862:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.912863:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.912864:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.912865:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.912867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.912873:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.912875:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.912876:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.912877:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.912878:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677473.912878:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.912880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677473.912881:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.912881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.912882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.912892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.912898:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.912900:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.912901:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.912901:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.912902:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.912903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.913812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.913819:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.913822:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.913823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.913824:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.913827:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.913828:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.913829:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96507007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.913831:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.913831:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.913832:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.913835:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.913837:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.913840:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.913841:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.913841:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.913843:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.913843:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.913844:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.913844:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.913850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.913858:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.913860:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.913862:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.913862:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.913864:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.913868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.914102:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.914105:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.914105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.914108:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.914110:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.914111:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.914113:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.914114:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.914115:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.914116:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.914120:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.914121:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.914124:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.914125:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.914125:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.914127:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.914127:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.914128:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.914129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.914140:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.914143:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.914143:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.914145:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.914146:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.914148:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.914445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.914450:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.914452:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.914455:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.914455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.914457:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.914458:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.914460:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.914461:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.914462:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.914463:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.914466:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.914468:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.914470:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.914471:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.914471:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.914473:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.914474:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.914474:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.914475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.914488:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.914491:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.914492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.914492:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.914494:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.914496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.914505:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.914507:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.914508:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.914508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.914509:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.914510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.914517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.914520:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.914522:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.914524:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.914525:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.914526:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.914528:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.914529:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.914530:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.914531:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.914532:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.914535:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.914537:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.914539:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.914540:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.914541:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.914542:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.914543:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.914544:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.914544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.914549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.914558:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.914561:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.914562:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.914562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.914564:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.914566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677473.914780:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.914783:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.914785:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.914787:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.914788:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.914789:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.914790:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.914791:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.914792:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.914793:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.914796:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.914797:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.914800:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.914801:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.914801:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.914803:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.914803:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.914805:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.914805:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.914809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.914815:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.914818:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.914819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.914819:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.914821:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.914822:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.915014:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.915017:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.915019:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.915021:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.915022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.915022:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.915024:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96507002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.915025:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.915026:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.915027:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.915030:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.915031:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.915033:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.915034:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.915034:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.915036:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.915036:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.915037:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.915038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.915048:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.915051:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.915052:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.915053:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.915055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.915056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.915069:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.915072:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.915072:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.915073:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677473.915073:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.915074:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677473.915283:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.915286:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677473.915287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677473.915289:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.915291:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.915292:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:4.2:1619677473.915298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000400:00000200:6.0:1619677473.915299:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96507001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.915300:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.915302:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.915303:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.915305:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:6.0:1619677473.915306:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.915308:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:6.0:1619677473.915308:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.915309:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:6.0:1619677473.915310:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.915311:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:26.0:1619677473.915312:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.915312:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:26.0:1619677473.915314:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.915314:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677473.915315:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000400:00000200:26.0:1619677473.915316:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677473.915316:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.915316:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:26.0:1619677473.915317:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96507000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.915318:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.915319:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.915320:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.2:1619677473.915321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:26.0:1619677473.915323:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.915325:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677473.915328:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.915329:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677473.915329:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:1.0:1619677473.915329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677473.915332:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (25)--
00000800:00000200:26.0:1619677473.915332:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:1.0:1619677473.915333:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (24)--
00000800:00000200:26.0:1619677473.915334:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:17.0:1619677473.915334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.915334:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.915335:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:26.0:1619677473.915336:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:1.0:1619677473.915336:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.915338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:4.2:1619677473.915341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:1.0:1619677473.915341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.915342:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.915344:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677473.915345:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.915346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.915346:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.915348:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:1.0:1619677473.915349:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677473.915350:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677473.915351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677473.915352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677473.915359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.915366:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.915368:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.915369:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.915369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.915370:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.915371:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.915555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.915562:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.915564:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.915565:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.915567:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.915569:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.915571:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.915572:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.915574:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.915575:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.915576:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.915579:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.915580:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.915583:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.915584:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.915584:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.915586:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.915587:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.915588:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.915588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.915601:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.915603:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.915604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.915605:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.915607:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.915610:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.915792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.915799:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.915802:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.915803:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.915804:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.915807:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.915808:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.915809:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.915810:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.915811:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.915812:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.915815:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.915816:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.915818:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.915818:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.915819:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.915820:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.915821:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.915822:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.915822:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.915826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.915832:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.915835:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.915836:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.915836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.915838:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.915840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.916094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.916100:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.916103:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.916103:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.916105:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.916107:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.916109:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.916110:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.916111:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.916112:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.916113:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.916116:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:1.0:1619677473.916117:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.916118:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.916119:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:1.0:1619677473.916119:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:6.0:1619677473.916121:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.916121:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.916123:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.916123:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677473.916124:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.916125:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:1.0:1619677473.916125:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.916126:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.916126:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.916126:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.916127:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677473.916128:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.916129:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.916130:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.916133:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.916136:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.916139:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.916140:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.916140:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.916141:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.916141:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:22.0:1619677473.916142:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.916143:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.916143:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:22.0:1619677473.916144:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.916145:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.916147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.916147:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.916147:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.916148:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.916159:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.916160:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.916162:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.916162:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.916162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677473.916163:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.916163:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677473.916164:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.916165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.916167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.916198:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.916201:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.916203:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.916205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.916205:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.916207:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.916208:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.916210:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.916210:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.916212:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.916215:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.916217:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.916219:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.916220:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.916221:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.916222:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.916223:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.916224:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.916224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.916228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.916234:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.916237:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.916238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.916238:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.916240:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.916242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.916377:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.916380:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.916382:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.916384:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.916385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.916386:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.916387:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.916388:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.916389:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.916390:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.916392:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.916394:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.916396:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.916397:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.916397:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.916399:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.916399:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.916400:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.916401:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.916405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.916412:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.916414:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.916415:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.916416:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.916417:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.916419:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.916608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.916616:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.916618:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.916619:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.916621:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.916623:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.916624:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.916625:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.916626:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.916627:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.916628:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.916631:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.916633:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.916635:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.916636:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.916637:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.916638:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.916639:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.916640:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.916640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.916653:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.916655:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.916656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.916657:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.916659:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.916662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.917219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.917226:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.917229:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.917230:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.917231:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.917234:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.917236:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.917237:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.917238:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.917239:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.917240:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.917243:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.917244:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.917247:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.917248:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.917249:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.917250:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.917251:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.917251:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.917252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.917257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.917267:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.917271:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.917273:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.917273:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.917277:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.917281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677473.917326:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.917329:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.917331:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.917333:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.917333:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.917334:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.917336:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.917337:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.917338:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.917339:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.917342:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.917343:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.917345:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.917346:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.917346:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.917348:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.917348:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.917349:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.917350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.917354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.917361:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.917364:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.917365:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.917365:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.917367:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.917369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.917372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.917378:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.917381:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.917381:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.917382:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.917382:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.917383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.918088:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.918091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.918091:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.918093:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.918095:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.918097:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.918098:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.918099:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.918100:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.918101:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.918104:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.918106:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.918108:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.918109:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.918109:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.918111:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.918112:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.918113:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.918113:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.918126:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.918128:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.918129:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.918131:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.918132:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.918133:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.918142:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.918145:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.918145:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.918146:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.918147:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.918148:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.918218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.918225:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.918228:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.918229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.918230:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.918233:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.918234:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.918236:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.918237:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.918238:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.918239:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.918242:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.918244:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.918247:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.918248:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.918249:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.918250:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.918251:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.918252:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.918253:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.918253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.918257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.918259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677473.918264:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.918266:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.918267:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.918268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.918268:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:2.0:1619677473.918268:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.918269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.918270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677473.918270:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.918274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.918303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677473.918310:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677473.918310:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.918313:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.918313:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677473.918314:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677473.918317:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.918318:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.918318:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677473.918320:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.918321:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.918322:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677473.918323:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677473.918323:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.918325:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.918325:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.918326:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.918326:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.918328:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.918328:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677473.918329:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677473.918332:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677473.918334:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.918335:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677473.918336:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677473.918337:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.918337:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677473.918338:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.918338:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677473.918339:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.918340:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677473.918340:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677473.918341:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677473.918341:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677473.918342:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677473.918342:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:31.0:1619677473.918344:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677473.918344:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.918344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677473.918345:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.918346:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.918348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.918351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.918357:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.918358:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.918358:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:30.0:1619677473.918359:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:18.0:1619677473.918359:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.918359:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.918359:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.918360:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.918360:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:18.0:1619677473.918360:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677473.918361:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677473.918361:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.918361:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677473.918361:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.918362:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.918363:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677473.918363:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.918364:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.918365:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:9.2:1619677473.918365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677473.918366:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677473.918367:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.918367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.918368:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677473.918369:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.918369:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.918371:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.918372:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.918373:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677473.918373:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:31.0:1619677473.918374:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.918375:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677473.918376:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.918377:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.918377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.918378:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000400:00000200:31.0:1619677473.918379:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.918379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.918380:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.918383:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.918383:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677473.918383:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:31.0:1619677473.918384:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.918385:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.918386:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677473.918386:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:31.0:1619677473.918387:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.918387:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:1.0:1619677473.918387:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.918389:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.918391:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.918391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:17.0:1619677473.918393:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677473.918395:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.918396:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.918397:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.918398:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:8.0:1619677473.918398:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.918400:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.918401:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:17.0:1619677473.918401:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.918401:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.918404:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:8.0:1619677473.918404:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.918407:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.918407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677473.918408:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677473.918408:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677473.918410:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677473.918410:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.918411:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.918412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677473.918416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.918423:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.918426:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677473.918426:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.918427:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.918429:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.918432:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677473.918530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.918537:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.918539:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.918540:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.918542:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.918545:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.918546:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677473.918547:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.918549:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.918550:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.918551:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677473.918554:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.918556:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677473.918558:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.918559:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677473.918560:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677473.918561:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677473.918562:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.918562:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.918563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:30.0:1619677473.918574:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.918576:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.918577:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.918578:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.918580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.918581:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.918730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.918736:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.918739:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.918740:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.918741:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.918744:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.918745:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.918746:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.918748:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.918749:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.918749:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.918752:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.918753:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.918756:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.918757:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.918758:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.918759:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.918760:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.918761:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.918762:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.918762:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677473.918765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677473.918768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677473.918775:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677473.918775:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677473.918777:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677473.918777:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:20.0:1619677473.918778:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.918778:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.918779:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677473.918780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.918780:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677473.918784:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677473.918786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677473.918793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.918796:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.918797:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.918798:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.918798:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.918799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.919454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.919460:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.919463:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.919464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.919466:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.919469:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.919470:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.919472:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.919473:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.919474:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.919475:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.919479:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.919481:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.919484:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.919484:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.919485:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.919487:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.919488:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.919489:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.919489:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677473.919501:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.919504:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.919505:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.919507:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.919508:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.919510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:22.0:1619677473.919647:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.919650:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.919650:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.919652:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.919655:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.919656:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.919657:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.919659:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.919660:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.919661:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.919664:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.919666:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.919669:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.919670:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.919670:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.919672:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.919673:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.919674:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.919674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.919686:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.919688:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.919689:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.919691:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.919692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.919693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.920024:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.920027:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.920029:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.920031:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.920031:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.920033:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.920034:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.920035:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.920036:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.920037:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.920041:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.920042:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.920045:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.920045:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.920046:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.920048:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.920048:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.920049:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.920050:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.920060:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.920063:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.920063:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.920065:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.920066:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.920067:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.920275:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.920278:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.920280:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.920281:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.920282:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677473.920282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.920283:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.920285:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.920285:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.920286:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.920290:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.920291:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.920299:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.920300:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.920300:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.920302:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.920302:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.920304:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.920305:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.920306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.920308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.920316:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.920318:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.920319:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.920319:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.920320:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677473.920321:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.920323:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677473.920323:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.920324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.920325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.920327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.920333:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.920336:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.920337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.920337:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.920338:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.920338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.920411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.920419:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.920422:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.920423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.920425:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.920428:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.920429:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.920431:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.920432:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.920433:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.920434:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.920437:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.920439:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.920442:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.920442:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.920443:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.920445:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.920445:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.920446:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.920447:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.920448:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.920453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.920460:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677473.920460:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.920462:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.920463:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677473.920463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.920464:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677473.920464:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.920465:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.920466:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.920469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.920480:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.920482:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.920483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.920484:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.920485:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.920485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.920491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.920494:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.920496:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.920498:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.920499:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.920501:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.920502:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.920503:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.920505:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.920506:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.920507:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.920510:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.920512:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.920514:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.920515:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.920516:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.920517:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.920518:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.920519:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.920520:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.920521:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.2:1619677473.920526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.920531:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.920533:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677473.920534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.920535:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.920547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.920555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.920558:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.920559:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.920559:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.920561:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.920563:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677473.920816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.920821:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.920824:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.920824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.920826:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.920828:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.920830:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.920831:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.920832:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.920833:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.920834:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.920837:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.920839:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.920842:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.920842:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.920843:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.920845:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.920845:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.920846:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.920846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.920858:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.920861:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.920862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.920862:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.920864:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.920866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.920868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.920875:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.920878:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.920879:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.920881:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.920883:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.920884:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.920886:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.920888:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.920889:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.920890:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.920893:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.920895:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.920898:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.920899:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.920899:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.920901:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.920901:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.920902:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.920903:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.920904:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677473.920909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677473.920915:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.920917:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677473.920917:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677473.920918:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.920918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.920919:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.920919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.920920:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.920920:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.920924:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.921505:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.921508:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.921510:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.921512:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.921512:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.921514:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.921515:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.921516:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.921517:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.921518:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.921521:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.921523:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.921525:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.921526:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.921526:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.921528:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.921528:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.921529:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.921530:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.921534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.921541:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.921543:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.921544:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.921544:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.921546:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.921548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.921605:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.921608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.921608:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.921610:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.921612:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.921614:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.921615:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.921617:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.921618:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.921619:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.921622:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.921624:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.921627:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.921627:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.921628:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.921630:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.921631:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.921632:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.921632:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.921644:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.921646:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.921647:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.921648:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.921649:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.921650:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.921705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.921712:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.921715:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.921716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.921718:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.921721:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.921722:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.921723:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.921725:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.921726:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.921727:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.921730:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.921732:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.921735:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.921736:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.921736:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.921738:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.921738:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.921739:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.921740:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.921752:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.921755:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.921755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.921756:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.921758:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.921761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.921771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.921779:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.921781:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.921782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.921784:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.921787:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.921789:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.921790:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.921791:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.921792:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.921793:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.921796:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.921798:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.921801:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.921801:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.921802:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.921804:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.921804:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.921805:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.921805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.921817:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.921819:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.921820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.921821:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.921823:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.921826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.921835:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.921837:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.921838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.921839:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.921840:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.921840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.921949:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.921952:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.921954:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.921955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.921956:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.921957:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.921959:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.921960:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.921961:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.921962:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.921964:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.921966:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.921968:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.921969:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.921969:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.921971:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.921972:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.921973:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.921973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.921977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.921983:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.921985:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.921986:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.921987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.921988:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.921990:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.922081:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.922084:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.922086:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.922088:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.922088:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.922089:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.922090:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.922091:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.922092:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.922093:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.922096:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.922098:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.922100:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.922100:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.922101:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.922103:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.922103:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.922104:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.922104:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.922108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.922114:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.922117:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.922118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.922118:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.922120:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.922122:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.922925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.922931:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.922934:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.922935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.922936:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.922939:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.922941:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.922942:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.922943:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.922944:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.922944:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.922947:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.922949:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.922952:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.922953:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.922953:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.922955:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.922955:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.922956:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.922956:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.922970:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.922973:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.922973:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.922974:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.922976:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.922979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.923036:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.923038:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.923041:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.923043:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.923043:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.923044:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.923046:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.923047:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.923048:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.923049:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.923052:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.923053:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.923056:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.923056:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.923057:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.923058:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.923059:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.923061:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.923062:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.923063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.923064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.923070:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.923073:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.923074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.923074:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.923076:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677473.923076:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.923078:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677473.923078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.923079:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.923080:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677473.923624:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.923627:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.923628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.923630:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.923632:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.923633:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.923635:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.923636:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.923637:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.923638:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.923641:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.923643:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.923646:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.923646:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.923647:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.923649:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.923650:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.923651:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.923652:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.923653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.923663:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.923664:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.923666:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.923666:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.923667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.923667:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.923667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.923669:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.923670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.923672:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.923874:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.923877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677473.923877:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677473.923879:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.923881:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.923883:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677473.923884:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.923885:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.923886:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.923887:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677473.923891:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.923892:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677473.923895:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.923896:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677473.923897:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677473.923899:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677473.923899:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677473.923900:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677473.923901:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677473.923902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677473.923905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677473.923912:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:10.0:1619677473.923912:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:22.0:1619677473.923914:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677473.923915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:10.0:1619677473.923915:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.923916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:22.0:1619677473.923916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:10.0:1619677473.923917:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.923919:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.2:1619677473.923920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677473.923922:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:18.0:1619677473.923927:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.923930:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.923931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.923934:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.923937:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.923938:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.923939:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.923941:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.923942:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.923943:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.923946:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.923948:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.923951:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.923951:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.923952:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.923954:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.923954:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.923955:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.923956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.923961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.923971:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.923975:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.923976:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.923977:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.923980:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.923983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677473.924060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.924067:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.924069:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.924070:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.924072:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.924074:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.924076:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677473.924077:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.924078:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.924079:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.924080:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677473.924083:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.924085:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677473.924087:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.924088:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677473.924089:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677473.924090:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677473.924091:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.924091:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.924092:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.924107:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.924110:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.924111:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.924114:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.924116:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.924117:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677473.924682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.924685:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677473.924686:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677473.924687:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.924690:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.924691:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677473.924693:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.924694:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.924695:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.924696:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677473.924699:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.924700:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677473.924703:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.924704:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677473.924705:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677473.924706:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677473.924707:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.924708:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677473.924709:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677473.924710:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:13.0:1619677473.924721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:1.0:1619677473.924721:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:13.0:1619677473.924723:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:17.0:1619677473.924724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:13.0:1619677473.924724:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.924724:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677473.924725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:13.0:1619677473.924726:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.924729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677473.924730:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:30.0:1619677473.924789:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.924791:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.924793:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.924795:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.924796:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.924797:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.924798:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.924799:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.924800:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.924801:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.924804:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.924806:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.924808:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.924809:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.924809:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.924810:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.924811:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.924812:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.924812:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.924823:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.924825:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.924826:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677473.924826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677473.924827:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.924829:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.924830:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:10.0:1619677473.924834:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.924837:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.924838:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.924839:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.924842:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.924843:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677473.924843:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.924844:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677473.924846:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.924846:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000400:00000200:10.0:1619677473.924846:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677473.924847:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677473.924847:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:10.0:1619677473.924847:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677473.924848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:10.0:1619677473.924848:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.924851:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.924852:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.924855:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.924856:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.924856:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.924858:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.924858:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.924859:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.924860:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.924883:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.924885:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.924886:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.924887:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.924889:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.924892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.925197:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.925200:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.925200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.925202:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.925205:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.925206:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.925207:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.925209:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.925210:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.925211:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.925214:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.925216:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.925218:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.925219:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.925220:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.925222:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.925222:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.925223:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.925223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.925235:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.925237:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.925238:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.925240:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.925241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.925242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.925252:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.925255:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.925256:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.925256:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.925257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.925258:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.925524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677473.925532:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.925535:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677473.925537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677473.925538:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.925542:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.925544:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677473.925545:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.925546:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.925548:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.925548:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677473.925552:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.925553:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677473.925556:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.925557:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677473.925558:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677473.925559:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677473.925560:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677473.925561:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677473.925562:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677473.925562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.2:1619677473.925566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677473.925568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:17.0:1619677473.925574:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:8.0:1619677473.925575:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:17.0:1619677473.925577:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:8.0:1619677473.925578:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:1.0:1619677473.925578:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.925579:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.925579:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677473.925580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:17.0:1619677473.925581:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.925584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.925778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.925785:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.925788:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.925789:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.925791:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.925794:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.925795:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.925797:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.925798:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.925799:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.925800:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.925804:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.925805:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.925808:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.925809:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.925809:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.925811:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.925811:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.925812:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.925813:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.925826:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.925830:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.925831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.925831:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.925833:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.925836:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.925961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.925967:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.925971:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.925972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.925974:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.925976:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.925977:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.925979:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.925980:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.925981:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.925982:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.925985:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.925987:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.925989:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.925990:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.925990:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.925992:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.925993:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.925994:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.925994:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.925998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.926006:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.926008:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.926008:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.926009:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.926010:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.926011:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677473.926089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.926095:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.926098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.926098:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.926100:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.926103:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.926104:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.926106:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.926107:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.926108:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.926109:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.926112:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.926113:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.926116:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.926117:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.926117:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.926119:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.926119:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.926120:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.926121:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.926133:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.926136:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.926137:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.926137:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.926139:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.926142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.926530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.926538:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.926540:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.926541:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.926543:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.926546:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.926547:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.926549:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.926550:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.926551:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.926552:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.926555:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.926556:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.926558:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.926559:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.926560:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.926561:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.926562:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.926562:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.926563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677473.926566:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677473.926568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.926569:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.926571:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.926573:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.926574:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.926574:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.926575:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.926575:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:30.0:1619677473.926577:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.926577:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.926578:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.926578:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.926578:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.926579:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:30.0:1619677473.926581:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.926581:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.926584:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.926585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.926586:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.926586:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.926587:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.926589:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.926589:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677473.926589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:30.0:1619677473.926590:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.926591:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677473.926595:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.926598:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.926598:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:8.0:1619677473.926598:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:24.0:1619677473.926599:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (25)--
00000800:00000200:8.0:1619677473.926599:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (25)--
00000800:00000200:24.0:1619677473.926600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.926600:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.926601:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.926602:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.926604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.926613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.926620:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.926622:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.926623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.926624:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.926627:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.926629:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:31.0:1619677473.926630:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:24.0:1619677473.926630:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.926631:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677473.926632:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:24.0:1619677473.926632:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.926633:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677473.926635:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.926636:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:24.0:1619677473.926636:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677473.926637:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.926638:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:24.0:1619677473.926638:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:31.0:1619677473.926640:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.926641:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.926641:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677473.926642:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.926642:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.926642:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:31.0:1619677473.926643:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.926644:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:31.0:1619677473.926646:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.926647:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.926647:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.926647:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.926648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677473.926649:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.926650:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:9.2:1619677473.926650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677473.926651:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.926652:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.926653:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.926654:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.926654:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:2.0:1619677473.926658:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.926661:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.926662:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.926662:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.926664:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.926668:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.926685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.926691:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.926694:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.926695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.926695:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.926697:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.926699:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.926825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677473.926833:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.926835:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.926837:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.926840:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.926840:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.926842:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677473.926843:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.926844:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.926845:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.926846:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677473.926849:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.926851:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677473.926854:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.926854:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:13.0:1619677473.926855:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:13.0:1619677473.926856:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677473.926857:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.926858:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.926858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677473.926863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.926871:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.926874:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.926875:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.926875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.926877:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.926880:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677473.926975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.926978:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.926981:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.926983:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.926984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.926985:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.926987:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.926988:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.926989:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.926990:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.926991:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.926995:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.926997:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.926999:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.927000:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.927001:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.927002:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.927003:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.927004:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.927005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.927008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.927016:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.927019:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.927020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.927020:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.927022:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.927024:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.927027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.927034:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.927037:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.927038:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.927038:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.927039:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.927040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.927164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.927167:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.927168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.927170:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.927172:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.927174:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.927175:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.927176:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.927177:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.927178:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.927181:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.927183:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.927185:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.927186:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.927186:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.927188:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.927189:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.927190:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.927190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.927202:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.927204:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.927205:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.927207:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.927207:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.927209:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.927461:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.927464:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.927466:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.927468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.927468:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.927469:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.927471:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.927472:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.927473:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.927474:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.927477:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.927478:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.927481:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.927481:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.927482:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.927483:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.927484:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.927485:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.927486:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.927490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.927497:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.927499:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.927500:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.927500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.927502:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.927504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.928108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.928115:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.928117:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.928118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.928120:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.928122:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.928123:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.928124:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.928126:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.928126:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.928127:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.928130:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.928132:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.928135:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.928135:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.928136:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.928138:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.928138:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.928139:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.928141:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.928141:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.928144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.928146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677473.928152:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.928153:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677473.928154:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.928155:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677473.928155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.928156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677473.928156:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.928157:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677473.928158:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.928161:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.928452:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.928455:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.928457:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.928459:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.928459:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.928460:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.928462:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.928463:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.928464:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.928465:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.928468:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.928469:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.928472:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.928472:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.928473:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.928474:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.928475:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.928477:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.928478:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.928479:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.928480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.928486:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.928489:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.928490:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.928490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.928492:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677473.928492:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.928494:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677473.928494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.928495:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.928496:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.928708:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.928711:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.928713:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.928715:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.928715:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.928716:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.928717:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.928718:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.928719:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.928720:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.928723:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.928724:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677473.928727:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.928727:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.928728:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.928729:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.928730:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.928730:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.928731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.928735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.928741:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.928744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.928744:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.928745:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.928746:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.928748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.929046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677473.929050:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.929053:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677473.929055:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.929056:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.929057:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.929059:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.929060:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.929061:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.929062:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.929063:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.929067:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.929068:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.929071:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.929071:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.929072:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.929073:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.929074:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.929075:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.929075:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.929080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.929088:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.929090:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.929090:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.929092:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.929093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.929094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677473.929201:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.929203:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.929204:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.929206:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.929209:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.929210:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.929211:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.929213:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.929214:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.929215:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.929218:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.929219:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.929222:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.929223:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.929223:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.929225:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.929226:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.929227:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.929227:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.929238:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.929241:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.929242:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.929243:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.929244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.929246:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.929454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.929460:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.929463:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.929464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.929466:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.929468:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.929469:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.929470:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.929472:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.929473:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.929474:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.929477:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.929479:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.929482:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.929483:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.929484:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.929486:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.929486:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.929488:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.929489:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.929490:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677473.929495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.929500:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677473.929500:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677473.929503:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:12.0:1619677473.929503:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677473.929504:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:12.0:1619677473.929504:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.929504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677473.929506:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.929507:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.929509:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677473.929628:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.929631:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.929634:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.929636:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.929636:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.929637:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.929638:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.929639:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.929641:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.929642:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.929645:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.929647:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.929648:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.929649:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.929650:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.929652:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.929652:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.929653:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.929654:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.929664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.929666:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.929667:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.929668:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.929670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.929671:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.929754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.929762:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.929765:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.929766:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.929768:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.929770:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.929771:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.929773:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.929774:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.929775:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.929776:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.929779:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.929780:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.929783:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.929784:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.929785:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.929786:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.929787:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.929787:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.929788:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.929801:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.929803:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.929804:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.929805:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.929807:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.929810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.929821:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.929823:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.929824:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.929824:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.929825:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.929826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.929938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.929945:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.929948:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.929949:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.929950:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.929953:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.929954:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.929956:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.929957:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.929958:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.929959:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.929962:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.929964:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.929967:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.929968:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.929968:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.929970:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.929970:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.929971:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.929971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.929976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.929984:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.929987:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.929987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.929988:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.929990:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.929993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.929995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.930003:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.930006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.930006:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.930006:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.930008:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.930008:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.930665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.930673:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.930676:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.930677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.930679:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.930681:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.930682:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.930684:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.930685:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.930686:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.930687:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.930690:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.930691:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.930693:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.930694:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.930695:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.930696:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.930696:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.930697:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.930699:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.930699:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.930702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.930705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677473.930709:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.930711:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677473.930712:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677473.930712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.930712:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.930714:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677473.930714:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.930715:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.930716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.930716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.930726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.930731:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.930733:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.930734:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.930735:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.930735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.930736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.930784:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.930787:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.930788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.930789:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.930792:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.930793:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.930794:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.930796:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.930797:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.930798:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.930801:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.930803:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.930805:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.930806:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.930807:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.930808:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.930809:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.930810:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.930811:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.930812:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.930823:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.930824:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.930825:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.930825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.930826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.930827:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.930827:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.930829:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.930830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.930831:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.931146:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.931149:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.931151:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.931153:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.931154:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.931155:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.931156:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.931158:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.931159:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.931160:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.931163:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.931165:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.931167:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.931168:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.931169:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.931171:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.931171:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.931173:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.931175:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.931175:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677473.931176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.931183:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.931185:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677473.931186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.931186:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.931188:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.931189:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.0:1619677473.931190:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.931191:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.931192:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.931192:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.931200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677473.931206:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.931208:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.931209:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677473.931209:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.931210:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.931211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677473.931309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.931315:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.931318:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.931319:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.931321:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.931323:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.931324:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.931326:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.931327:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.931328:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.931329:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.931332:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.931334:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.931337:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.931338:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.931338:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.931340:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.931340:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.931342:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677473.931343:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.931343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.931348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.931353:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.931355:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.931356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677473.931356:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677473.931357:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.931358:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677473.931359:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.931360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.931361:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.931364:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.931578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.931586:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.931589:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.931590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.931592:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.931594:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.931596:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.931597:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.931598:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.931599:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.931600:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.931604:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.931605:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.931608:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.931609:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.931610:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677473.931610:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.931611:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.931612:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677473.931612:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.931612:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.931613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677473.931614:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.931616:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.931617:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.931617:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.931618:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.931620:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.931620:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.931621:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.931624:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.931625:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677473.931626:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.931627:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.931627:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677473.931628:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:6.0:1619677473.931628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.931629:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:22.0:1619677473.931629:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.931630:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.931630:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.931631:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677473.931631:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.931632:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677473.931635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.931642:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.931645:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.931646:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.931648:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.931649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.931649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677473.932043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.932050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.932053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.932053:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677473.932055:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.932058:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.932060:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.932061:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.932062:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.932063:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.932064:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.932067:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.932068:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.932071:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.932071:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.932072:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.932073:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.932074:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.932074:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.932075:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.932086:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.932089:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.932089:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.932091:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.932093:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.932095:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.932107:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.932109:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.932110:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.932111:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.932111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.932112:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677473.932362:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.932365:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.932367:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.932369:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.932370:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.932371:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677473.932372:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.932373:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.932374:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.932375:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677473.932378:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.932379:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677473.932382:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.932383:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677473.932384:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.932385:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677473.932386:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.932387:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.932387:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677473.932392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.932401:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.932403:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.932404:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.932406:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.932407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.932409:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677473.932634:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.932637:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.932638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.932639:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.932642:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.932643:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.932644:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.932645:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.932646:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.932647:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.932650:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.932652:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.932654:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.932655:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.932656:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.932658:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.932658:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.932659:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.932660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.932676:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.932678:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.932679:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.932679:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:21.0:1619677473.932680:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.932681:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.932682:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:20.0:1619677473.932683:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:4.2:1619677473.932683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.0:1619677473.932684:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000400:00000200:1.0:1619677473.932684:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.932685:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.932686:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:1.0:1619677473.932686:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.932687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.932687:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.932687:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (24)--
00000800:00000200:20.0:1619677473.932688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:1.0:1619677473.932688:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:21.0:1619677473.932689:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677473.932689:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.932691:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677473.932691:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.932692:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677473.932693:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:1.0:1619677473.932693:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677473.932694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677473.932696:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.932699:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.932700:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.932702:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677473.932703:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.932703:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677473.932703:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:8.0:1619677473.932704:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677473.932705:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677473.932706:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.932706:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:8.0:1619677473.932707:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677473.932707:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:8.0:1619677473.932708:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677473.932708:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677473.932708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:8.0:1619677473.932709:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.2:1619677473.932711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:8.0:1619677473.932713:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.932715:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.932718:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.932719:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:17.0:1619677473.932719:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:8.0:1619677473.932719:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.932720:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.932721:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:17.0:1619677473.932721:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:24.0:1619677473.932722:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.932722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.0:1619677473.932722:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677473.932723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677473.932723:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:24.0:1619677473.932724:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.932724:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.932725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.932726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.932727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.932728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.932736:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.932739:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.932740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.932741:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.932743:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.932745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.932851:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.932854:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.932856:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.932857:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.932859:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.932860:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.932862:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.932863:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.932864:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.932865:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.932868:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.932870:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.932872:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.932873:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.932873:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.932875:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.932875:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.932876:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.932877:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.932880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.932887:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.932890:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.932890:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.932891:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.932893:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.932895:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677473.932988:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.932990:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.932992:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.932994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.932994:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.932996:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.932997:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.932998:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.932999:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.933000:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.933003:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.933004:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.933006:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.933007:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.933008:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.933009:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.933010:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.933011:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.933012:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.933013:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.933015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.933021:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.933023:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.933024:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.933024:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.933026:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677473.933026:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.933028:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677473.933028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.933029:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.933030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677473.933058:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.933060:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677473.933061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677473.933064:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.933066:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.933068:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677473.933069:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.933070:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.933071:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.933072:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677473.933075:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.933077:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677473.933079:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.933080:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.933080:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.933082:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.933082:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677473.933083:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677473.933083:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677473.933088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.933096:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.933098:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.933098:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.933099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.933100:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.933102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677473.933616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677473.933622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.933624:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677473.933625:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677473.933626:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677473.933629:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.933630:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677473.933631:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.933632:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.933633:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.933634:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677473.933637:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677473.933639:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677473.933641:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.933642:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677473.933643:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677473.933645:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:1.0:1619677473.933645:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677473.933646:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677473.933647:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677473.933647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:5.2:1619677473.933656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:31.0:1619677473.933659:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:17.0:1619677473.933660:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:31.0:1619677473.933661:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:17.0:1619677473.933661:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:31.0:1619677473.933662:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.933662:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677473.933663:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000400:00000200:31.0:1619677473.933664:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000400:00000200:17.0:1619677473.933664:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.933664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677473.933665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:30.0:1619677473.933666:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:1.0:1619677473.933666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000400:00000200:17.0:1619677473.933667:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.933669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:17.0:1619677473.933670:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677473.933672:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.933673:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.933674:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.933675:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.933678:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.933679:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677473.933679:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.933682:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677473.933682:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.933683:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:17.0:1619677473.933683:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.933683:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.933684:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677473.933685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677473.933685:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.933686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:17.0:1619677473.933686:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.933686:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.933687:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:30.0:1619677473.933697:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677473.933699:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677473.933700:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.933702:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.933702:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.933703:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.933703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.933704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.933704:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.933705:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677473.933705:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677473.933707:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.933708:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.933708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677473.933709:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.933710:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.933713:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.933715:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677473.933717:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.933718:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.933718:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.933719:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677473.933720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.933720:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.933721:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.933721:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677473.933721:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.933722:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.933723:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677473.933723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.933724:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.933733:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.933735:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.933736:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.933738:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.933739:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.933740:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.934048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.934057:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.934059:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.934060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.934062:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.934065:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.934066:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.934067:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.934069:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.934070:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.934070:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.934074:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.934075:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.934078:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.934079:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.934079:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.934081:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.934081:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.934082:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.934083:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.934087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.934096:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.934098:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.934099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.934099:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.934102:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.934104:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.934519:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.934522:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.934523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.934525:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.934527:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.934528:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.934530:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.934531:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.934532:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.934533:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.934536:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.934538:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.934541:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.934542:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.934542:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.934544:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.934545:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.934545:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.934546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.934558:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.934561:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.934561:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.934563:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.934564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.934565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.934636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.934644:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.934647:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.934648:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.934650:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.934653:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.934654:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.934656:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.934657:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.934658:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.934659:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.934662:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.934664:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.934666:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.934667:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.934668:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.934670:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.934670:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.934671:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.934672:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.934684:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.934687:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.934688:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.934688:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.934690:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.934693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677473.934809:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.934812:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.934815:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.934816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.934817:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.934818:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677473.934820:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.934821:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.934822:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.934823:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677473.934826:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.934827:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.934827:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.934828:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.2:1619677473.934829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677473.934830:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677473.934831:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.934831:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.934831:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:31.0:1619677473.934833:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.934833:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:31.0:1619677473.934834:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.934835:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677473.934835:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677473.934836:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.934837:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.934838:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:18.0:1619677473.934838:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:30.0:1619677473.934839:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:9.2:1619677473.934839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677473.934839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677473.934840:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.934840:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677473.934842:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677473.934842:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.934843:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677473.934844:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.934845:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.934845:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:18.0:1619677473.934846:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:31.0:1619677473.934847:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677473.934847:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96506915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677473.934848:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.934848:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.934848:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.934849:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.934849:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.934849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.934850:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:18.0:1619677473.934850:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677473.934851:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.934851:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677473.934851:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677473.934852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:24.0:1619677473.934852:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.934854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:18.0:1619677473.934854:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677473.934855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677473.934856:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.934858:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.934859:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.934860:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.934862:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.934862:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677473.934862:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677473.934863:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.934864:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677473.934864:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.934865:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.934865:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.934867:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.934868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.934869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677473.934875:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.934877:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.934878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.934878:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.934880:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.934882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.935326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.935332:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.935334:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.935337:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.935337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.935339:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.935341:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.935342:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.935343:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.935345:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.935346:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.935349:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.935350:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.935354:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.935354:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.935355:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.935357:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.935357:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.935358:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.935359:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.935363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.935371:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.935373:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.935374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.935375:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.935377:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.935379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.935394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.935401:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.935404:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.935405:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.935407:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.935410:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.935411:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.935412:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.935413:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.935414:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.935415:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.935418:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.935419:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.935422:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.935422:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:9.2:1619677473.935422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:10.0:1619677473.935423:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.935424:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.935425:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.935425:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.935426:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.935428:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:9.2:1619677473.935430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.935430:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.935431:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.935432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.935432:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.935433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677473.935434:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.935435:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.935436:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.935437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.935438:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.935440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.936006:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.936009:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.936010:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.936011:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.936014:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.936015:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.936016:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.936018:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.936019:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.936020:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.936023:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.936024:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.936026:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.936027:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.936028:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.936029:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.936030:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.936031:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.936031:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:5.2:1619677473.936042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677473.936043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.936045:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.936046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.936048:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.936048:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.0:1619677473.936050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.936051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.936051:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.936052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.936054:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.936057:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.936058:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.936060:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.936061:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.936062:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.936063:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.936066:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.936068:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.936070:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.936071:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.936072:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.936073:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.936074:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.936075:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.936075:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.936088:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.936090:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.936091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.936091:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.936093:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.936096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.936396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.936403:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.936406:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.936407:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.936409:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.936411:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.936413:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.936414:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.936415:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.936416:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.936417:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.936420:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.936422:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.936425:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.936426:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.936426:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.936428:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.936428:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.936429:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.936430:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.936443:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.936446:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.936446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.936447:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.936449:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.936452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.936491:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.936494:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.936496:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.936498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.936498:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.936499:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.936501:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.936502:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.936503:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.936504:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.936507:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.936509:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.936511:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.936512:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.936513:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.936514:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.936515:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.936516:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.936516:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677473.936528:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.936531:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.936532:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.936532:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.936534:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.936538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.936670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677473.936677:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.936679:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.936680:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.936682:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.936685:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.936686:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.936687:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.936688:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.936689:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.936690:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.936694:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.936695:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.936699:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.936699:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.936700:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.936702:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.936702:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.936703:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.936704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:13.0:1619677473.936716:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.936719:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.936720:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.936721:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.936723:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.936724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677473.936735:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.936738:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.936740:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.936741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.936742:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.936744:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.936745:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.936746:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.936747:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.936748:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.936751:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.936753:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.936755:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.936755:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.936756:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.936758:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.936758:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.936759:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.936760:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.936770:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.936772:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.936773:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.936775:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.936776:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.936777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.936787:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.936790:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.936790:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.936791:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677473.936792:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.936793:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.937081:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.937084:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677473.937085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677473.937087:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.937089:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.937091:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677473.937092:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.937093:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.937094:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.937095:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677473.937099:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.937101:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677473.937104:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.937105:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677473.937106:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677473.937108:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677473.937108:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677473.937109:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677473.937110:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:30.0:1619677473.937121:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.937123:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677473.937124:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.937126:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.937127:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.937129:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677473.937195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.937200:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.937203:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.937204:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.937206:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.937209:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.937210:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.937212:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.937213:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.937214:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.937215:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.937218:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.937220:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.937223:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.937223:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.937224:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.937226:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.937226:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.937227:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.937228:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677473.937239:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.937241:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.937242:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.937244:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677473.937245:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:4.2:1619677473.937246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677473.937247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:8.0:1619677473.937254:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.937257:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.937258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.937259:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.937263:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.937264:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.937265:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.937267:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.937268:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.937268:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.937271:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.937273:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.937276:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.937277:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.937277:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.937279:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.937280:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.937281:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.937282:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.937282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.937288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.937295:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.937299:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677473.937300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677473.937301:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677473.937302:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677473.937302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.937302:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.937303:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677473.937305:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.937309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677473.937814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07dd000] (20)++
00000800:00000200:17.0:1619677473.937820:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07dd000] (21)++
00000800:00000200:17.0:1619677473.937823:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.55@o2ib3
00000800:00000200:1.0:1619677473.937824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dd000] (22)--
00000400:00000200:17.0:1619677473.937826:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.55@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.937829:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.937830:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677473.937831:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.937833:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.937834:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.937835:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677473.937838:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.55@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.937839:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677473.937842:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.937842:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677473.937843:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677473.937845:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677473.937845:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07dd000] (21)++
00000800:00000200:17.0:1619677473.937846:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07dd000] (22)--
00000800:00000200:17.0:1619677473.937846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07dd000] (21)--
00000800:00000200:21.0:1619677473.937862:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.937863:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.937865:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.0:1619677473.937865:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.937865:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.937866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677473.937867:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.937868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:22.0:1619677473.937869:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.937870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:22.0:1619677473.937872:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.937874:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.937875:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.937877:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.937878:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.937879:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.937882:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.937884:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.937886:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.937887:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.937887:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.937889:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.937890:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.937892:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.937892:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.937893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.937902:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.937903:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.937904:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.937905:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.937905:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.937906:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.937907:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.937907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.937909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.937910:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677473.937912:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.937915:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677473.937916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677473.937918:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.937920:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.937921:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:1.0:1619677473.937921:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677473.937923:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677473.937923:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.937924:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000400:00000200:1.0:1619677473.937924:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.937925:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000400:00000200:1.0:1619677473.937925:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677473.937926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.937927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677473.937927:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677473.937929:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.937931:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.937933:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.937933:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.937934:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.937935:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.937936:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677473.937937:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677473.937937:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677473.937942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.937950:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.937952:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.937952:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.937953:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.937954:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.937956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.937963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677473.937970:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.937972:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.937973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.937973:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.937974:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677473.937974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677473.938033:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.938036:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.938038:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.938040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.938040:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.938041:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.938043:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.938044:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.938045:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.938046:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.938050:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.938052:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.938054:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.938055:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.938056:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.938057:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.938058:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.938059:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.938061:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.938061:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.938071:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.938073:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.938073:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.938075:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.938075:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.938076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.938076:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.938078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.938079:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.938080:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677473.938221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.938228:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.938230:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.938231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.938233:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.938235:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.938237:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.938238:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.938239:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.938240:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.938241:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.938244:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.938245:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.938248:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.938249:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.938249:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.938251:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.938251:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.938252:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.938252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.938265:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.938268:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.938269:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.938269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.938272:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.938277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.938279:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.938281:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.938282:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.938283:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.938283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.938284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.938439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.938445:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.938447:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.938449:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.938450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.938452:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.938453:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.938455:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.938456:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.938457:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.938458:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.938460:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.938462:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.938464:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.938465:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.938466:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.938467:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.938468:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.938470:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.938471:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.938471:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677473.938476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.938480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.938482:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.938483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677473.938483:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.938484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.938485:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.938486:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.938487:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.938488:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.938491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.938785:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.938787:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677473.938788:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:31.0:1619677473.938789:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.938789:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.938790:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.938791:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.938792:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677473.938793:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.938794:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.938795:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.938796:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677473.938798:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.938799:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677473.938800:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677473.938801:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677473.938801:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.938802:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.938803:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.938804:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.938807:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677473.938807:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.938808:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677473.938809:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.938809:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.938810:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677473.938811:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.938811:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.938812:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.938812:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.938813:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.938813:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.938814:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.938814:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.938815:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.938815:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.938816:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.938818:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.938818:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.2:1619677473.938818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677473.938819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677473.938826:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677473.938827:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:22.0:1619677473.938828:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.938829:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.938830:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.938830:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.938831:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:22.0:1619677473.938831:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677473.938832:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677473.938832:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.938832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.938832:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.938832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.938835:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.938835:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677473.938835:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.938835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:10.0:1619677473.938835:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677473.938837:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:10.0:1619677473.938838:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.938839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677473.938840:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677473.938841:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.938842:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.938843:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.938844:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.938848:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.938849:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677473.938852:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.938853:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.938853:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.938855:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.938855:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.938856:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.938857:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677473.938861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.938869:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.938872:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.938873:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.938873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.938876:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.938879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677473.939536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.939544:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.939547:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.939547:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.939549:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.939552:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.939553:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.939555:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.939556:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.939557:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.939557:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.939560:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.939561:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677473.939562:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677473.939564:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677473.939565:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.939565:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:31.0:1619677473.939566:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.939566:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.939567:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.939568:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677473.939569:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.939569:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677473.939569:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:31.0:1619677473.939570:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677473.939570:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:31.0:1619677473.939571:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.939571:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677473.939572:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.939573:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.939574:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677473.939577:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677473.939577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677473.939579:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677473.939582:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.939583:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677473.939583:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677473.939583:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:31.0:1619677473.939585:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.939585:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677473.939586:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677473.939586:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.939586:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:31.0:1619677473.939587:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.939587:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677473.939587:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.939587:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677473.939587:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.939589:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.939589:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.2:1619677473.939592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677473.939592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677473.939600:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.939603:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:22.0:1619677473.939604:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:21.0:1619677473.939604:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.939605:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.939606:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:22.0:1619677473.939607:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.939607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.939608:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.939609:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677473.939609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.939610:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677473.939714:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.939717:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677473.939718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677473.939720:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.939722:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.939723:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677473.939724:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.939726:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.939727:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.939727:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677473.939731:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.939732:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.939734:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.939735:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.939735:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.939736:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.939737:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677473.939738:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677473.939738:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677473.939743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.939750:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.939752:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.939753:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.939753:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.939754:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.939756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677473.939767:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.939770:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.939771:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.939773:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.939775:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.939776:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.939778:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.939779:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.939780:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.939781:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.939784:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.939786:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.939789:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.939790:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.939790:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.939792:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.939793:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.939794:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.939794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.939805:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.939808:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.939808:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.939810:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.939811:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.939812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677473.939948:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.939950:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677473.939951:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677473.939953:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.939954:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.939956:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677473.939957:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.939958:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.939959:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.939960:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677473.939963:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.939965:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677473.939967:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.939968:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677473.939968:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677473.939970:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677473.939971:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677473.939971:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677473.939972:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677473.939984:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.939987:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.939988:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.939988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.939990:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.939993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.940098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.940106:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.940109:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.940110:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.940112:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.940114:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.940116:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.940117:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.940118:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.940119:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.940120:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.940123:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.940124:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.940127:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.940127:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.940128:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.940130:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.940130:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.940131:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.940132:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.940132:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.940135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.940138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677473.940142:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.940144:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.940145:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.940145:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677473.940145:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.940146:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.940147:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.940148:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677473.940149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.940149:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.940155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.940161:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.940163:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.940164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.940164:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.940165:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.940165:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.940453:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.940456:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.940458:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.940460:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.940461:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.940461:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.940462:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.940464:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.940465:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.940465:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.940468:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.940470:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.940471:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.940472:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.940473:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.940474:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.940475:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.940476:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.940476:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.940488:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.940490:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.940491:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.940492:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.940494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.940495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.940629:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.940631:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.940633:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.940635:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.940635:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.940637:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.940638:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.940639:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.940640:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.940640:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.940643:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.940645:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.940647:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.940648:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.940649:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.940650:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.940651:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.940651:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.940652:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677473.940662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:6.0:1619677473.940665:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677473.940667:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677473.940667:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.940669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.940669:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.940670:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:6.0:1619677473.940671:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:21.0:1619677473.940672:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.940672:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.940675:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.940675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:21.0:1619677473.940676:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.940677:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.940678:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.940679:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.940680:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.940683:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677473.940684:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.940684:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.940686:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.940687:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.940687:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.940688:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (23)++
00000800:00000200:6.0:1619677473.940688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.940689:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.940690:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (24)--
00000800:00000200:21.0:1619677473.940690:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677473.940691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677473.940691:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.940692:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677473.940693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.940704:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.940706:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.940707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.940707:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.940709:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.940711:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.941172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.941180:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.941183:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.941184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.941186:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.941188:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.941189:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.941191:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.941192:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.941193:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.941194:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.941197:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.941198:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.941201:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.941202:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.941202:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.941204:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.941204:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.941205:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.941206:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.941210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.941216:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.941219:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.941219:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.941220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.941222:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.941223:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.941371:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.941374:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.941374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.941377:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.941379:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.941380:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.941381:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.941383:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.941384:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.941385:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.941388:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.941390:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.941392:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.941393:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.941394:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.941395:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.941396:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.941397:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.941397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:5.2:1619677473.941398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.941404:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.941407:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.941408:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677473.941410:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:1.0:1619677473.941410:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.941412:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:1.0:1619677473.941412:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.941413:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.941413:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677473.941414:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.941415:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.941416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:1.0:1619677473.941416:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.941417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677473.941417:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.941418:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.941421:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.941423:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.941426:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.941427:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.941427:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.941429:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.941429:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.941430:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.941431:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.941443:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.941445:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.941446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.941446:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.941449:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.941451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.941536:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.941539:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.941541:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.941543:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.941544:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.941545:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.941546:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.941548:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.941549:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.941550:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.941553:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.941555:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.941557:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.941558:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.941559:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.941560:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.941561:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.941562:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.941564:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.941564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.941574:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.941575:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.941576:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.941578:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677473.941578:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.941579:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.941579:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.941580:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.941581:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.941582:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677473.941959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.941965:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.941968:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.941969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.941971:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.941974:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.941975:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.941977:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.941978:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.941979:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.941980:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.941983:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.941985:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.941988:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.941989:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.941990:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.941992:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.941992:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.941993:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.941994:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677473.942004:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.942007:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.942008:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.942009:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677473.942011:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.942012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677473.942352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.942360:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.942362:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.942363:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.942365:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.942367:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.942368:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677473.942369:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.942371:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.942372:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.942372:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677473.942375:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.942377:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677473.942379:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.942380:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677473.942381:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677473.942383:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677473.942383:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.942384:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677473.942386:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.942386:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:4.2:1619677473.942392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677473.942397:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.942399:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677473.942400:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.942400:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:30.0:1619677473.942401:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.942401:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.942402:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:30.0:1619677473.942403:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677473.942403:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677473.942404:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677473.942414:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677473.942417:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677473.942417:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.942418:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677473.942419:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.942420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677473.942584:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.942587:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.942589:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.942590:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.942591:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.942592:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.942593:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.942594:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.942595:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.942596:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.942599:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.942600:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.942603:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.942604:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.942605:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.942606:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.942607:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.942607:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.942608:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.942621:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.942623:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.942624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.942625:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.942627:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.942630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.942666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.942673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.942676:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.942677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.942679:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.942682:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.942683:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.942684:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.942686:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.942687:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.942687:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.942690:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.942692:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.942695:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.942696:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.942696:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.942698:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.942698:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.942699:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.942699:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.942711:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.942714:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.942715:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.942715:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.942717:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.942720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.943362:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.943365:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.943366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.943367:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.943369:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.943371:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.943372:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.943374:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.943375:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.943376:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.943378:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.943380:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.943382:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.943383:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.943384:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.943385:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.943386:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.943387:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.943387:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.943400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.943402:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.943403:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.943404:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.943406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.943407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.943503:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.943505:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.943508:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.943510:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.943510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.943512:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.943513:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.943515:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.943516:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.943517:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.943519:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.943521:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.943523:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.943524:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.943525:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.943526:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.943527:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.943528:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.943528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.943538:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.943541:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.943541:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.943543:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.943544:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.943545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.943557:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.943559:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.943560:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.943561:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677473.943561:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.943562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.943655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.943659:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.943662:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677473.943664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.943664:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677473.943666:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.943668:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.943669:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.943670:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.943672:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.943672:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.943676:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.943678:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.943680:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.943681:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.943682:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.943683:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.943684:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.943685:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.943686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.943697:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.943700:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.943701:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.943701:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.943703:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.943705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.943714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.943716:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.943717:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.943718:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.943718:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.943719:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.944375:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.944378:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.944380:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.944382:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.944382:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.944383:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.944384:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.944386:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677473.944386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677473.944387:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.944388:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.944391:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.944393:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.944394:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677473.944396:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.944397:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.944397:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.944397:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.944398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677473.944399:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.944399:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:10.0:1619677473.944399:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.944400:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.944401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:10.0:1619677473.944402:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.944403:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.944404:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.944406:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.944406:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.944407:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.944410:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.944412:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.944414:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677473.944415:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.944416:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.944416:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.944416:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.944417:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.944418:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.944418:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677473.944418:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.944419:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.944419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677473.944420:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.944422:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.944431:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.944434:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.944434:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.944434:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.944436:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.944439:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.944444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.944451:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.944454:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.944455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.944456:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.944459:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.944461:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.944462:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.944463:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.944464:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.944465:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.944468:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.944469:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.944472:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.944472:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.944473:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.944474:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.944475:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.944476:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.944476:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677473.944488:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.944491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.944491:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.944492:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.944494:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.944497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.944613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.944620:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.944623:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.944624:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.944625:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.944628:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.944630:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.944631:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.944632:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.944633:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.944634:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.944636:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.944638:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.944640:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.944641:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.944641:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.944643:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.944643:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.944644:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677473.944645:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.944646:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677473.944651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677473.944656:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677473.944658:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677473.944659:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.944659:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.944660:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.944661:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:12.0:1619677473.944661:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677473.944662:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.944663:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.944665:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677473.944680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.944687:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.944690:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.944691:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.944692:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.944695:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.944697:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.944698:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.944699:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.944700:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.944701:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.944704:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.944705:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.944708:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.944709:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.944710:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.944711:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.944712:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.944712:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.944713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.944717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.944726:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.944729:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.944730:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.944730:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.944732:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.944735:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.944745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.944752:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.944755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.944755:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.944755:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.944756:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.944757:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.945203:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.945206:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.945208:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.945210:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.945210:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.945212:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.945213:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.945214:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.945215:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.945216:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.945219:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.945220:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.945223:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.945223:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.945224:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.945225:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.945226:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.945227:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.945227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.945231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.945237:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.945240:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.945241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.945241:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.945243:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.945245:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.945337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.945345:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.945347:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.945348:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.945350:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.945352:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.945353:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:22.0:1619677473.945354:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677473.945354:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.945356:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.945357:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:10.0:1619677473.945357:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.945357:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677473.945358:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677473.945361:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.945363:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.945363:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.945365:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677473.945365:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:22.0:1619677473.945366:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.945368:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677473.945368:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677473.945369:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.945369:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:22.0:1619677473.945370:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677473.945370:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.945371:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.945371:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.945372:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.945373:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.945373:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.945375:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.945377:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677473.945377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.945378:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.945379:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.945381:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.945381:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.945382:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.945383:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:24.0:1619677473.945384:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677473.945384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677473.945386:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.945387:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.945388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.945389:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.945391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.945395:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677473.945397:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.945397:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.945397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.945398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.945399:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.945399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.945401:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.945403:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.945404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.945405:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.945406:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.945406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.945406:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.945407:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (24)--
00000800:00000200:20.0:1619677473.945407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.945409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.945779:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.945782:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.945784:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.945786:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.945787:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.945789:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.945790:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.945791:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.945792:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.945793:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.945797:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.945799:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.945801:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.945802:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.945802:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.945804:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.945804:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.945805:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.945806:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.945809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.945816:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.945818:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.945819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.945819:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.945821:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.945823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.945829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.945835:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.945837:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.945838:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.945838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:4.2:1619677473.945838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:24.0:1619677473.945839:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.945840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677473.945842:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.945845:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677473.945847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.945847:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.945849:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.945851:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.945852:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.945853:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.945854:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.945855:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.945859:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.945861:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.945863:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.945864:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.945865:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.945866:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.945867:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.945867:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677473.945868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.945872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.945881:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.945883:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.945884:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.945885:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.945886:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.945887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.945902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.945910:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.945913:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.945914:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.945914:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677473.945915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.945916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677473.946002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677473.946007:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.946010:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677473.946012:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.946012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677473.946013:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.946015:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.946016:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.946017:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.946018:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.946019:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.946022:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.946024:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.946025:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.946026:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.946027:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.946028:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.946029:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.946030:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:21.0:1619677473.946031:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677473.946032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677473.946034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677473.946038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677473.946044:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.946045:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677473.946045:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677473.946045:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:21.0:1619677473.946046:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.946047:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.946048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000800:00000200:21.0:1619677473.946049:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677473.946050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677473.946051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677473.946052:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:18.0:1619677473.946052:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.946054:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.946055:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.946057:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.946060:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.946061:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.946062:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.946064:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.946065:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.946066:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.946069:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.946071:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.946073:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.946073:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.946074:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.946075:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.946076:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.946077:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.946077:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.946082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.946089:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.946091:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.946091:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.946092:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.946093:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.946094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.946190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.946197:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.946200:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.946201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.946203:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.946205:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.946206:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.946208:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.946209:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.946210:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.946211:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.946214:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.946216:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.946219:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.946219:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.946220:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.946222:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.946222:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.946223:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.946223:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.946236:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.946239:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.946240:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.946240:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.946242:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.946246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.946279:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.946282:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.946284:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.946286:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.946287:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.946288:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.946290:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.946291:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.946292:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.946298:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.946302:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.946303:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.946306:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.946307:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.946308:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.946309:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.946310:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.946311:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.946313:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.946313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677473.946325:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.946327:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.946328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.946329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677473.946329:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.946330:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677473.946330:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.946331:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.946333:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677473.946333:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.946334:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677473.946334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.946335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.946336:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.946339:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.946341:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677473.946342:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.946343:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.946343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677473.946344:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.946345:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677473.946346:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.946347:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.946347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677473.946348:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.946348:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.946349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.946350:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677473.946353:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.946354:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677473.946354:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.946356:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677473.946356:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.946357:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.946359:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.946360:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677473.946362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.946371:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677473.946373:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677473.946373:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.946374:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.946376:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.946376:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677473.946376:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.946377:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677473.946377:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.946379:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.946380:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:20.0:1619677473.946380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.946381:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677473.946382:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (23)++
00000800:00000200:20.0:1619677473.946382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677473.946383:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677473.946384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677473.946712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.946718:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.946721:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.946722:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.946724:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.946726:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.946728:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.946729:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.946730:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.946731:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.946732:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.946735:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.946737:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.946739:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.946740:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.946741:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.946742:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.946743:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.946744:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.946744:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.946757:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.946759:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.946760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.946760:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.946762:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.946765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.946873:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.946876:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.946878:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.946880:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.946881:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.946883:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.946884:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.946885:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.946886:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.946887:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.946891:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.946893:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.946895:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.946896:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.946896:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.946898:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.946899:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.946900:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.946900:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.946910:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:22.0:1619677473.946912:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677473.946913:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.946913:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.946915:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.946915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:30.0:1619677473.946916:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.946917:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.946918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:22.0:1619677473.946918:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.946920:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.946922:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.946923:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.946924:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.946925:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.946926:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.946930:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.946931:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.946933:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.946934:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.946935:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.946937:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.946937:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.946938:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.946939:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.946950:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.946952:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.946953:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.946955:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.946956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.946957:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.947173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.947177:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.947179:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.947181:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.947182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.947184:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.947185:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.947186:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.947188:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.947189:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.947190:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.947193:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.947195:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.947197:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.947197:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.947198:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.947200:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.947201:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.947201:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.947202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.947213:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.947216:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.947217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.947217:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.947219:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.947221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.947498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.947505:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.947508:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.947508:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.947510:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.947513:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.947514:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.947516:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.947517:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.947518:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.947519:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.947522:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.947523:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.947526:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.947527:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.947528:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.947530:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.947530:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.947531:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.947532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.947544:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.947547:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.947548:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.947550:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.947552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.947553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677473.947768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677473.947774:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.947776:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677473.947778:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677473.947779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677473.947780:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.947781:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677473.947783:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.947784:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.947785:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.947786:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677473.947789:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677473.947791:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677473.947794:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.947795:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677473.947795:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677473.947797:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677473.947798:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677473.947799:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677473.947800:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677473.947801:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677473.947806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.947810:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677473.947812:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677473.947812:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.947813:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.947814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.947821:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.947824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.947824:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677473.947825:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.947827:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.947830:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.947969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.947976:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.947979:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.947980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.947981:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.947983:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.947984:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.947985:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.947987:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.947988:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.947989:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.947993:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.947994:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.947997:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.947998:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.947998:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.948000:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.948001:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.948002:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.948003:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.948004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.948007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.948009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677473.948014:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.948016:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.948017:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.948018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.948018:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:2.0:1619677473.948018:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.948019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.948020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677473.948020:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.948023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.948038:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.948042:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.948044:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.948046:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.948046:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.948047:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.948049:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.948050:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.948051:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.948052:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.948056:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.948057:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.948060:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.948061:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.948062:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.948064:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.948064:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.948065:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.948066:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.948079:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.948081:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.948082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.948083:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.948085:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.948088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.948328:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.948331:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.948333:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.948335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.948335:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.948337:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.948338:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.948339:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.948340:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.948341:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.948344:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.948346:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.948348:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.948348:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.948349:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.948350:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.948351:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.948351:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.948352:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.948363:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.948366:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.948366:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.948368:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.948369:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.948370:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.948600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.948604:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.948606:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.948609:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.948610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.948612:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.948613:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.948614:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.948616:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.948617:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.948618:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.948621:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.948622:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.948624:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.948625:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.948626:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.948627:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.948628:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.948629:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.948629:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.948641:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.948644:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.948645:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.948645:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.948647:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.948649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.948687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.948695:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.948698:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.948699:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.948701:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.948704:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.948705:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.948707:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.948708:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.948709:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.948710:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.948714:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.948715:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.948718:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.948719:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.948720:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.948722:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.948722:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.948723:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.948723:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.948736:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.948738:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.948739:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.948739:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.948741:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.948744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.948753:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677473.948754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677473.948755:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.948756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.948757:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.948758:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.948758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677473.948762:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.948764:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.948765:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.948767:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.948770:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.948772:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.948773:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.948775:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.948776:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.948776:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.948780:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.948781:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.948784:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.948785:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.948785:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.948787:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.948787:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.948788:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.948788:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.948793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.948802:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.948804:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.948805:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.948806:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.948808:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.948811:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.948928:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.948931:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.948932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.948934:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.948936:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.948938:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.948939:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.948941:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.948942:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.948943:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.948946:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.948948:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.948950:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.948951:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.948951:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.948953:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.948954:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.948955:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.948955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.948967:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.948969:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.948970:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.948972:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.948972:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.948974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.949141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677473.949150:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677473.949152:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677473.949154:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.949156:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677473.949156:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.949158:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677473.949159:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.949160:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.949161:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.949162:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677473.949165:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.949167:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677473.949170:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.949171:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677473.949172:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.949174:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677473.949174:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677473.949175:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677473.949176:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677473.949180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677473.949188:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.949190:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.949191:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.949193:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.949193:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.949196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.949781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.949787:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.949789:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.949791:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.949792:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.949794:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.949795:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.949797:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.949798:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.949799:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.949800:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.949803:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.949805:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.949808:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.949809:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.949809:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.949811:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.949811:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.949812:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.949813:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.949818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.949827:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.949829:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.949830:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.949830:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.949832:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.949834:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.950076:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.950079:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.950080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.950082:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.950084:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.950085:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:4.2:1619677473.950085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:6.0:1619677473.950086:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.950088:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.950089:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.950090:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677473.950093:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677473.950093:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.950095:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.950096:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.950096:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677473.950098:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677473.950099:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.950099:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.950099:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.950101:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677473.950102:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.950102:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677473.950103:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677473.950103:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.950103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:4.2:1619677473.950103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:8.0:1619677473.950105:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.950106:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.950107:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.950108:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.950110:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.950111:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677473.950112:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677473.950113:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677473.950115:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677473.950115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677473.950115:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677473.950116:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.950116:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.950117:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677473.950118:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.950118:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.950118:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:26.0:1619677473.950119:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.950119:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.950119:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677473.950120:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.950120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677473.950121:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:21.0:1619677473.950121:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677473.950122:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96506842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.950123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677473.950124:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677473.950124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:26.0:1619677473.950125:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.950126:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.950129:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.950131:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677473.950132:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677473.950133:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.950134:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677473.950134:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:26.0:1619677473.950135:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677473.950135:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:26.0:1619677473.950136:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.950136:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677473.950136:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.950137:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:1.0:1619677473.950138:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.950139:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.950140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677473.950141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.950142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.950145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677473.950148:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.950150:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.950151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.950151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.950152:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677473.950153:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.950154:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.950155:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677473.950156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677473.950156:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677473.950198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.950205:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.950207:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.950208:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.950210:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.950212:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.950213:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.950214:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.950215:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.950216:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.950217:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.950220:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.950222:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.950224:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.950225:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.950226:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.950227:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.950228:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.950228:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.950229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677473.950233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.950242:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.950244:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.950245:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.950247:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677473.950248:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.950250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677473.950377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677473.950385:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677473.950388:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677473.950390:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.950393:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677473.950394:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.950395:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677473.950396:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.950398:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.950399:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.950400:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677473.950401:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.950402:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.950404:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.950405:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677473.950405:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.950407:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.950409:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677473.950409:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677473.950410:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677473.950410:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:30.0:1619677473.950410:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.950411:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677473.950412:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:30.0:1619677473.950412:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.950413:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.950414:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677473.950415:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000400:00000200:30.0:1619677473.950416:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.950417:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677473.950418:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677473.950418:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677473.950418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677473.950419:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.950421:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.950421:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.950422:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.950423:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.950424:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:4.2:1619677473.950424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677473.950425:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.950426:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:18.0:1619677473.950427:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:30.0:1619677473.950428:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677473.950430:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:18.0:1619677473.950430:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:30.0:1619677473.950431:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:2.0:1619677473.950431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677473.950432:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000400:00000200:18.0:1619677473.950432:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.950434:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.950437:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.950438:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.950441:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.950442:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.950442:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.950444:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.950450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.950452:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.950454:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.950455:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.950455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.950456:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.950457:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.950655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.950658:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.950660:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.950661:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.950662:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.950663:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.950665:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.950666:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.950667:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.950668:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.950670:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.950672:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.950673:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.950674:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.950674:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.950676:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.950676:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.950677:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.950678:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.950688:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.950690:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.950691:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.950693:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.950694:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.950695:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677473.950705:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677473.950708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.950708:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677473.950710:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.950712:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.950714:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677473.950715:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.950716:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.950717:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.950718:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677473.950722:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.950723:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677473.950726:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.950727:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677473.950727:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677473.950729:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677473.950730:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677473.950731:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677473.950731:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677473.950735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.950743:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.950745:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.950747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.950747:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.950749:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.950751:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677473.950755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677473.950762:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.950765:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.950765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.950766:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.950767:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.950768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677473.951080:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.951082:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.951083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.951085:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.951087:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.951088:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.951090:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.951091:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.951092:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.951093:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.951096:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.951097:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.951100:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.951100:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.951101:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.951103:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.951103:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.951104:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.951105:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.951106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.951116:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.951116:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.951118:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.951118:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.951118:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.951119:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.951119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.951121:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.951122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.951123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.951134:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.951137:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.951138:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.951138:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.951139:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.951140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.951162:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.951165:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.951167:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.951168:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.951169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.951170:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677473.951171:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.951173:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.951174:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.951175:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677473.951178:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.951179:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677473.951183:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.951184:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677473.951184:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677473.951186:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:4.2:1619677473.951186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677473.951187:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.951188:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.951190:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.951190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677473.951192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677473.951193:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.951196:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.951197:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.951199:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.951201:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.951202:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.951203:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677473.951204:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677473.951204:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:10.0:1619677473.951205:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.951206:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:10.0:1619677473.951206:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677473.951207:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:10.0:1619677473.951207:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.951208:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677473.951208:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.951209:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677473.951209:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.951210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:10.0:1619677473.951211:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.951213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677473.951213:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.951215:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.951216:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.951216:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.951218:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.951218:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.951219:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.951220:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.951223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.951230:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.951232:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.951233:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.951233:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.951235:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.951237:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.951670:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.951672:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.951675:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.951677:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.951677:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.951679:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.951680:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.951681:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.951682:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.951683:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.951686:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.951688:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.951690:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.951690:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.951691:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.951692:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.951693:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.951694:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.951694:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.951706:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.951708:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.951709:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.951710:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.951712:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.951713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.951722:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.951725:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.951726:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.951726:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.951727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.951728:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677473.951980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.951986:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.951989:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.951990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.951992:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.951994:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.951996:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.951997:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.951998:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.951999:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.952000:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.952003:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.952004:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.952007:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.952007:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.952008:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.952010:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.952010:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.952012:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.952013:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.952013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.952018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.952023:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677473.952024:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.952025:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.952026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677473.952027:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.952027:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.952028:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.952028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.952030:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.952032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677473.952054:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.952057:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.952059:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.952060:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.952061:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.952062:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.952064:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.952065:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.952066:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.952067:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.952070:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.952071:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.952075:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.952075:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.952076:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.952078:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.952078:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.952079:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.952080:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677473.952091:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.952094:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.952095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.952095:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.952097:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.952100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.952114:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.952117:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.952120:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.952121:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.952123:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.952124:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.952125:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.952126:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.952127:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.952128:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.952131:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.952133:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.952134:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.952135:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.952136:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.952137:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.952137:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.952138:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.952139:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.952149:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.952151:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.952152:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.952154:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.952155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.952157:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.952175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.952183:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.952185:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.952186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.952188:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.952190:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.952191:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.952193:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.952194:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.952195:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.952196:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.952199:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.952200:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677473.952203:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.952204:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.952204:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.952206:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.952206:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.952207:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.952207:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677473.952220:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.952222:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.952223:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.952223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.952226:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.952228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.952304:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.952307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.952307:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.952309:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.952311:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.952312:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.952313:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.952315:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.952316:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.952317:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.952320:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.952321:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.952324:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.952325:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.952325:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.952327:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.952328:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.952328:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.952329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.952342:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.952344:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.952345:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.952347:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.952348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.952349:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.952422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.952429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.952432:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.952433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.952434:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.952438:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.952439:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.952440:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.952442:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.952443:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.952444:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.952447:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.952448:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.952451:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.952452:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.952453:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.952454:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.952455:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.952455:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.952456:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.952461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.952469:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.952472:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.952473:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.952473:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.952475:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.952478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.952480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.952488:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.952490:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.952491:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.952491:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.952492:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.952493:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677473.952715:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9190633ddc00] (21)++
00000800:00000200:2.0:1619677473.952717:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.20@o2ib3
00000800:00000200:18.0:1619677473.952718:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9190633ddc00] (22)--
00000400:00000200:2.0:1619677473.952720:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.13.20@o2ib3 : GET - for me
00000400:00000200:2.0:1619677473.952723:0:19932:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.13.20@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:2.0:1619677473.952726:0:19932:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.13.20@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:2.0:1619677473.952727:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9190633ddc00] (21)++
00000800:00000200:2.0:1619677473.952728:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9190633ddc00] (22)--
00000400:00000200:2.0:1619677473.952729:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.13.20@o2ib3 local destination
00000400:00000200:2.0:1619677473.952734:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.13.20@o2ib3(10.51.13.20@o2ib3:10.51.13.20@o2ib3) : REPLY try# 0
00000800:00000200:2.0:1619677473.952735:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.13.20@o2ib3
00000800:00000200:2.0:1619677473.952737:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919064fef980] -> 10.51.13.20@o2ib3 (2) version: 12
00000800:00000200:2.0:1619677473.952738:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9190633ddc00] (21)++
00000800:00000200:2.0:1619677473.952739:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9190633ddc00] (22)++
00000800:00000200:2.0:1619677473.952740:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9190633ddc00] (23)--
00000800:00000200:2.0:1619677473.952741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9190633ddc00] (22)--
00000800:00000200:18.0:1619677473.952751:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9190633ddc00] (22)++
00000800:00000200:18.0:1619677473.952753:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9190633ddc00] (23)--
00000400:00000200:18.0:1619677473.952753:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.952754:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9190633ddc00] (22)--
00000400:00000200:18.0:1619677473.952755:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.13.20@o2ib3: REPLY: OK
00000800:00000200:18.0:1619677473.952756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9190633ddc00] (21)--
00000800:00000200:4.2:1619677473.952912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37fe00] (20)++
00000800:00000200:10.0:1619677473.952921:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37fe00] (21)++
00000800:00000200:26.0:1619677473.952924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37fe00] (22)--
00000800:00000200:10.0:1619677473.952924:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.67@o2ib3
00000400:00000200:10.0:1619677473.952927:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.67@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.952930:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.952931:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677473.952933:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.952934:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.952935:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.952936:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677473.952939:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.952941:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677473.952944:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.952944:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677473.952945:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677473.952947:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677473.952947:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37fe00] (21)++
00000800:00000200:10.0:1619677473.952948:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37fe00] (22)--
00000800:00000200:10.0:1619677473.952948:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37fe00] (21)--
00000800:00000200:8.2:1619677473.952953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.952962:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.952964:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.952965:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.952965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.952968:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.952970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677473.953115:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.953118:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.953120:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.953122:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.953122:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.953123:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.953125:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.953126:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.953127:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.953128:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.953132:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.953134:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.953136:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.953137:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.953137:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.953139:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.953139:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.953140:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.953140:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.953151:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.953153:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.953154:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.953155:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.953157:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.953158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.953284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.953291:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.953297:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.953299:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.953302:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.953304:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.953306:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.953307:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.953308:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.953309:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.953310:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.953313:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.953315:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.953317:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.953318:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.953319:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.953321:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.953321:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.953322:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.953323:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.953324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.953328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.953330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677473.953336:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.953338:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.953338:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677473.953340:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:18.0:1619677473.953340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.953340:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.953341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.953342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677473.953342:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.953345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677473.953402:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.953405:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.953406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.953408:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.953410:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.953411:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.953413:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.953414:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.953415:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.953416:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.953420:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.953421:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.953424:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.953424:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.953425:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.953427:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.953427:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.953428:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.953429:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.953441:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.953443:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.953444:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.953446:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.953446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.953448:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.953464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.953471:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.953474:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.953475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.953477:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.953480:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.953481:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.953483:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.953484:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.953485:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.953486:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.953489:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.953490:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.953493:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.953494:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.953495:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.953496:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.953497:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.953497:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.953498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.953510:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.953513:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.953514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.953514:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.953516:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.953519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.953740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.953746:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.953749:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.953750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.953752:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.953754:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.953755:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.953757:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.953758:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.953760:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.953761:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.953763:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.953765:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.953767:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.953768:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.953768:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.953770:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.953771:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.953772:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677473.953773:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.953773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.953776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.953779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.953784:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.953786:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.953786:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.953787:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.953787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.953787:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.953788:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.953788:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.953789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677473.953790:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677473.953986:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.953988:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.953990:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.953992:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.953993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.953994:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.953995:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.953996:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.953997:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.953998:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.954001:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.954003:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.954005:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.954006:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.954007:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.954009:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.954009:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.954010:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.954012:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.954012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677473.954023:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.954025:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677473.954026:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677473.954026:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.954027:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.954029:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677473.954029:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.954030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.954031:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.954032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.954429:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.954432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.954432:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.954434:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.954436:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.954437:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.954439:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.954440:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.954441:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.954442:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.954445:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.954447:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.954450:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.954450:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.954451:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.954453:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.954454:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.954454:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.954455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.954467:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.954470:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.954471:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.954472:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.954473:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.954475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.954655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.954661:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.954664:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.954665:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.954667:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.954669:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.954671:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677473.954672:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.954673:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.954674:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.954675:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677473.954678:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.954680:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677473.954682:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.954683:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.954684:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677473.954685:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.954686:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.954687:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677473.954688:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677473.954688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677473.954692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677473.954693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.954699:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.954701:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677473.954701:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677473.954702:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.954703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677473.954703:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677473.954704:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677473.954705:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.954706:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.954708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.954713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.954721:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.954723:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677473.954724:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.954725:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.954726:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.954726:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677473.954746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.954752:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.954755:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[2] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.954756:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.954758:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.1.56@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.109@o2ib7 : PUT - routed
00000800:00000200:8.0:1619677473.954761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.955617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.955624:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.955627:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.955628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.955630:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.955633:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.955634:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:10.0:1619677473.955634:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.955635:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677473.955636:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:10.0:1619677473.955637:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.955638:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.955639:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.955639:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.955641:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.955642:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.955642:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677473.955643:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.955644:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.955644:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.955645:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.955647:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.955647:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677473.955648:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.955648:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.955648:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.955650:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.955650:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677473.955651:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.955651:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.955652:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677473.955653:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.955656:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.955657:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.955657:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.955659:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.955659:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.955660:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.955661:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677473.955666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.955674:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677473.955675:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.955676:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.955677:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.955677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.955678:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:24.0:1619677473.955679:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677473.955679:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.955679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.955681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677473.955682:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.955686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.955688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.955690:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.955691:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.955691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.955692:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.955695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.955697:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.955698:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.955699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.955701:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.955703:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.955704:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.955705:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.955707:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.955708:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.955709:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.955712:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.955714:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.955716:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.955717:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.955718:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.955719:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.955720:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.955721:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.955722:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.955722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.955733:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.955734:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.955735:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.955735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.955736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.955737:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.955737:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.955739:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.955740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.955742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.955757:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.955759:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.955760:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.955761:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.955761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.955762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.955953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.955960:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.955963:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.955963:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.955965:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.955968:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.955969:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.955971:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.955972:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.955973:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.955974:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.955977:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.955978:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.955981:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.955982:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.955983:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.955984:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.955985:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.955986:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.955986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.955991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.955999:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.956002:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.956003:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.956003:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.956005:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.956008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677473.956321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.956323:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.956326:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677473.956328:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.956329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.956331:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.956332:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.956334:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.956335:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.956336:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.956337:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.956341:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.956342:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.956345:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.956346:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.956346:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.956348:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677473.956349:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.956349:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677473.956350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.956355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.956363:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.956366:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.956367:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.956367:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.956369:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.956371:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.956375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677473.956382:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.956385:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.956386:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.956386:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.956387:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.956387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.956491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.956498:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.956501:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.956502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.956504:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.956506:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.956507:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.956509:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.956510:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.956511:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.956512:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.956515:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.956516:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.956518:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.956519:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.956520:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.956521:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.956522:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.956523:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.956523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677473.956526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.956533:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:9.2:1619677473.956533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.956535:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.956536:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.956537:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.956539:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.956540:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:10.0:1619677473.956541:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.956541:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.956542:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.956543:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.956543:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.956543:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677473.956544:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.956545:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.956545:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677473.956547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:10.0:1619677473.956548:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.956550:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.956552:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.956553:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.956553:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.956555:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.956555:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.956556:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.956556:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.956568:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.956571:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.956572:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.956573:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.956575:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.956577:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677473.956590:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677473.956593:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677473.956594:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.956595:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.956596:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.956598:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677473.956708:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.956711:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.956713:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.956715:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.956715:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.956717:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.956718:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.956719:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.956720:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.956721:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.956724:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.956725:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.956727:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.956728:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.956728:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.956730:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.956730:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.956732:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.956733:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.956734:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.956744:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.956746:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.956746:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.956748:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.956748:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.956749:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.956749:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.956750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.956751:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.956751:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.956859:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.956862:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.956862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.956864:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.956866:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.956868:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.956869:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.956871:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.956872:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.956873:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.956876:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.956878:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.956880:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.956881:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.956882:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.956883:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.956884:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.956885:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.956885:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.956897:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.956899:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.956900:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.956901:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.956902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.956904:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.957269:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.957271:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.957274:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.957275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.957276:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.957277:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.957278:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677473.957278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677473.957280:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.957281:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.957282:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677473.957283:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677473.957285:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.957286:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:30.0:1619677473.957287:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677473.957287:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677473.957289:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677473.957289:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.957290:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.957291:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:1.0:1619677473.957291:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.957292:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.957299:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.957300:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.957301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677473.957301:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677473.957302:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:1.0:1619677473.957302:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677473.957304:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.957304:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.957304:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.957304:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.957306:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:1.0:1619677473.957306:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677473.957307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.957308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:1.0:1619677473.957310:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.957312:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.957314:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.957315:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.957316:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.957318:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.957318:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.957319:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.957320:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677473.957324:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677473.957325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677473.957327:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.957328:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.957329:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.957329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.957331:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677473.957332:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:8.0:1619677473.957333:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.957335:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.957336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:8.0:1619677473.957336:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:6.0:1619677473.957336:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.957337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677473.957338:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.957340:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.957341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:8.0:1619677473.957343:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.957344:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.957345:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.957347:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.957348:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.957349:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.957352:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.957354:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.957357:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.957358:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.957358:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.957360:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.957360:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.957361:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.957362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.957366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.957375:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.957378:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.957379:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.957379:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.957382:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.957384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.957385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.957393:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.957395:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.957396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.957397:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.957398:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.957398:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677473.957658:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.957661:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.957663:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.957665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.957665:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.957666:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.957667:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.957669:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.957670:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.957671:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.957674:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.957676:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.957679:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.957679:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.957680:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.957682:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.957682:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.957683:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.957684:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677473.957685:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.957686:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677473.957688:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.957688:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.957690:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.957691:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.957692:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.957693:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.957693:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.957694:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677473.957696:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677473.957697:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.957698:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.957699:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.957699:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.957699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.957700:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:22.0:1619677473.957700:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.957701:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.957702:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:22.0:1619677473.957702:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.957703:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.957703:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.957704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677473.957705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.957714:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.957717:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.957718:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.957719:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.957721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.957722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677473.957812:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.957815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.957815:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.957817:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.957819:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.957820:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.957822:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.957823:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.957824:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.957825:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.957828:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.957829:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.957832:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.957833:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.957833:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.957835:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.957836:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.957837:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.957837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.957848:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.957851:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.957852:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.957853:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.957854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.957856:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.958152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.958159:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.958162:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.958163:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.958165:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.958167:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.958169:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677473.958170:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.958172:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.958173:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.958174:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677473.958177:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.958178:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677473.958180:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.958181:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677473.958182:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677473.958183:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677473.958184:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.958184:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677473.958185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:30.0:1619677473.958196:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.958199:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.958200:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.958201:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.958203:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.958203:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.958320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677473.958325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677473.958328:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.958330:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.958331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677473.958332:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677473.958332:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.958335:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677473.958335:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.958336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677473.958337:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677473.958338:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.958338:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.958339:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.958340:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.958341:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.958341:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677473.958342:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.958344:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.958345:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.958345:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677473.958346:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.958346:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677473.958347:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.958349:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677473.958350:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.958350:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.958350:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677473.958352:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.958352:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.958353:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677473.958354:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.958354:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:24.0:1619677473.958355:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.958355:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:24.0:1619677473.958356:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.958356:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677473.958358:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.958358:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.958359:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.958359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677473.958361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.958368:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677473.958370:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677473.958370:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.958371:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.958372:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.958372:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.958373:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.958373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:22.0:1619677473.958375:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.958375:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.958376:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.958377:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.958379:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.958380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677473.958381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.958382:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.958382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.958383:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.958383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.958384:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.958385:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.958515:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.958517:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.958520:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.958521:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.958522:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.958523:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.958524:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.958525:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.958526:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.958527:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.958530:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.958532:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.958534:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.958535:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.958535:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.958536:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.958537:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.958538:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.958539:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.958542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.958549:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.958551:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.958552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.958552:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.958555:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.958557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.958923:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.958925:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.958926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.958928:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.958930:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.958932:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.958933:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.958935:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.958936:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.958937:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.958940:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.958942:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.958944:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.958945:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.958946:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.958948:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.958948:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.958949:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.958950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.958961:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.958964:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.958964:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.958966:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.958967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.958968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.959531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.959537:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.959540:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.959541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.959543:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.959545:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.959547:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.959548:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.959549:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.959550:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.959551:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.959554:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.959556:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.959559:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.959560:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.959561:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.959562:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.959563:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.959564:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.959565:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677473.959577:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.959580:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677473.959581:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.959582:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677473.959584:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.959585:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677473.959634:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.959636:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.959639:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.959640:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.959641:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.959643:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.959644:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.959645:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.959646:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.959647:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.959650:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.959652:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.959655:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.959656:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.959657:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.959658:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.959659:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.959660:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.959661:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.959673:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.959676:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.959676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.959677:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.959679:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.959682:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.960055:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.960058:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.960059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.960061:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.960062:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.960063:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.960065:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.960066:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.960067:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.960068:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.960072:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.960073:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.960076:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.960076:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.960077:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.960079:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.960080:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.960081:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.960082:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.960082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.960092:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.960092:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.960094:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.960095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.960095:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.960096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.960096:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.960097:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.960099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.960100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.960108:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.960111:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.960112:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.960113:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.960113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.960114:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.960617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.960625:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.960628:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.960629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.960630:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.960633:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.960635:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.960636:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.960637:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.960638:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.960639:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.960642:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.960644:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.960646:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.960647:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.960647:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.960649:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.960649:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.960650:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.960651:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.960652:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.960655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.960658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677473.960662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:26.0:1619677473.960665:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677473.960665:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.960665:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.960666:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.960667:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.960667:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677473.960667:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.960669:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677473.960669:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.960673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.960679:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.960681:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.960682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.960682:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.960683:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.960684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.960838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.960845:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.960847:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.960848:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.960851:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.960853:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.960855:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.960856:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.960857:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.960858:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.960859:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.960862:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.960864:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.960865:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.960866:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.960867:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.960868:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.960869:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.960870:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.960870:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.960875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.960882:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.960884:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.960885:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.960885:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.960886:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.960887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.961089:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.961092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.961092:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.961094:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.961097:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.961098:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.961099:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.961101:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.961102:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.961103:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.961105:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.961107:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.2:1619677473.961108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:6.0:1619677473.961109:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.961110:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.961111:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.961112:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.961113:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:18.0:1619677473.961114:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:6.0:1619677473.961114:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.961115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:18.0:1619677473.961117:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677473.961118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.961120:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.961121:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.961123:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677473.961124:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.961125:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677473.961126:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:18.0:1619677473.961126:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.961127:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.961128:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.961129:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.961131:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.961131:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:18.0:1619677473.961132:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.961134:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.961135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677473.961137:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.961138:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677473.961139:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677473.961140:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677473.961141:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.961142:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.961143:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677473.961174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677473.961180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677473.961184:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:24.0:1619677473.961187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677473.961187:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.961187:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.961189:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.961190:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:21.0:1619677473.961190:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677473.961191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677473.961192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:24.0:1619677473.961193:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.961196:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.961197:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.961199:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.961200:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.961202:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677473.961202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000400:00000200:24.0:1619677473.961203:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.961208:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.961210:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677473.961210:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677473.961213:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.961213:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677473.961214:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677473.961214:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:24.0:1619677473.961215:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677473.961215:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:24.0:1619677473.961216:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677473.961216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:24.0:1619677473.961217:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677473.961217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677473.961218:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.961219:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.961220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.961222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.961225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677473.961229:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677473.961232:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677473.961232:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.961233:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.961233:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.961234:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.961235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.961236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677473.961236:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.961239:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.961318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.961325:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.961328:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.961329:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.961330:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.961333:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.961334:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.961335:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.961337:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.961338:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.961338:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.961341:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.961343:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.961345:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.961345:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.961346:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.961347:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.961348:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.961348:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.961349:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.961353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.961360:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.961362:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.961363:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.961363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.961365:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.961367:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.961614:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.961617:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.961619:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.961621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.961621:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.961623:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.961624:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.961625:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.961626:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.961627:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.961630:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.961631:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.961634:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.961635:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.961635:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.961637:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.961638:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.961639:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.961640:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.961641:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677473.961651:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677473.961654:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677473.961654:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.961655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.961656:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.961657:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.961658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677473.961658:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.961659:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.961661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.961670:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.961672:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.961673:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.961674:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.961675:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.961675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.961760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677473.961768:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677473.961771:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677473.961773:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.961775:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677473.961776:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.961777:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677473.961778:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.961780:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.961781:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.961781:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677473.961785:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.961786:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677473.961788:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.961789:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677473.961789:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677473.961791:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.961791:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677473.961792:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677473.961793:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:30.0:1619677473.961803:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.961805:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677473.961806:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.961808:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.961809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.961810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677473.961884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.961886:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.961888:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.961890:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.961890:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.961891:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.961893:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.961894:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.961895:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.961895:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.961898:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.961899:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.961901:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.961902:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.961902:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.961904:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.961904:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.961906:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.961907:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.961908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.961918:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.961919:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.961919:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.961921:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677473.961921:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.961922:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.961923:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.961924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.961925:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.961925:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.962379:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.962382:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.962383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.2:1619677473.962384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:22.0:1619677473.962385:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.962387:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.962388:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.962390:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.962391:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.962392:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677473.962393:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000400:00000200:22.0:1619677473.962393:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677473.962395:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:22.0:1619677473.962398:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677473.962399:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.962400:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677473.962401:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.962401:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:30.0:1619677473.962403:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:22.0:1619677473.962403:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677473.962404:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.962404:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.962405:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677473.962406:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.962407:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.962407:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.962407:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677473.962408:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677473.962408:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.962409:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:30.0:1619677473.962411:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.962413:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677473.962416:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.962417:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677473.962417:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677473.962419:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677473.962420:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677473.962420:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677473.962421:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:20.0:1619677473.962421:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.962423:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.962424:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.962425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:20.0:1619677473.962426:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.962428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.962429:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677473.962433:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.962436:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.962437:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.962437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.962439:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.962443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:31.0:1619677473.962470:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.962473:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.962475:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.962477:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.962478:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.962479:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.962480:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.962481:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.962482:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.962483:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.962486:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.962488:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.962489:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.962490:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.962490:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.962492:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.962492:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.962493:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.962494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.962504:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.962506:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.962507:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.962509:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.962510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.962511:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.962516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677473.962521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677473.962523:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677473.962524:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.962524:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.962525:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.962525:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.962526:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.0:1619677473.962526:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.962527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.962528:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.962532:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.962533:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.962535:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.962536:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.962537:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.962538:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.962540:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.962542:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.962545:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.962546:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.962546:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.962548:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.962548:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.962549:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.962549:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.962554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677473.962557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:17.0:1619677473.962563:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:31.0:1619677473.962564:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:17.0:1619677473.962566:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677473.962566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:31.0:1619677473.962567:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:17.0:1619677473.962567:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.962569:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.962570:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.962571:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677473.962572:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.962573:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:17.0:1619677473.962573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:31.0:1619677473.962575:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.962576:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.962577:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.962578:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677473.962581:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.962583:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677473.962585:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.962586:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677473.962586:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677473.962588:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.962588:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677473.962589:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677473.962590:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:30.0:1619677473.962599:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.962602:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677473.962603:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.962604:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.962606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.962607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677473.963122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677473.963128:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.963130:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677473.963132:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.963133:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677473.963135:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.963136:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.963137:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.963139:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.963140:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.963141:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.963144:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.963146:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.963148:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.963149:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.963150:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.963151:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.963152:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677473.963152:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.963153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677473.963157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.963164:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.963166:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.963167:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.963169:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.963170:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.963171:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677473.963233:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.963236:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.963238:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.963240:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.963240:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.963242:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.963243:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.963244:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.963245:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.963246:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.963250:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.963252:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.963254:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.963255:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.963255:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.963257:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.963258:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.963259:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.963259:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.963263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.963269:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.963271:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.963272:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.963272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.963274:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.963276:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.963318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.963321:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.963323:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.963325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.963325:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.963326:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.963327:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.963328:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.963329:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.963330:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.963333:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.963334:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.963337:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.963338:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.963339:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.963340:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.963341:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.963342:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.963342:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677473.963356:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.963359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.963359:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.963360:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.963362:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.963365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.963496:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.963498:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.963499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.963501:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.963502:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677473.963502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677473.963504:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.963505:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.963506:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.963507:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.963508:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677473.963510:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677473.963511:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.963513:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677473.963513:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.963514:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.963515:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.963516:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.963517:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.963517:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677473.963518:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.963519:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.963520:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677473.963520:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677473.963521:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.963521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:8.0:1619677473.963521:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.963522:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.963523:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.963524:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.963528:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.963530:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.963532:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.0:1619677473.963532:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.963533:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.963534:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.963535:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677473.963535:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:21.0:1619677473.963536:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.963536:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677473.963537:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.963537:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.963537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677473.963538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.963540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677473.963549:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.963552:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.963553:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.963553:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.963555:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.963558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.963870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677473.963877:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.963879:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677473.963880:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677473.963882:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.963884:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.963886:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677473.963887:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.963889:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.963890:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.963890:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.963894:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.963895:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.963897:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.963897:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677473.963898:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677473.963899:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677473.963900:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677473.963901:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677473.963902:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677473.963903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:4.2:1619677473.963909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000800:00000200:31.0:1619677473.963912:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.963914:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.963915:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.963916:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000400:00000200:31.0:1619677473.963917:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:8.0:1619677473.963917:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:30.0:1619677473.963918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677473.963918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:31.0:1619677473.963919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:8.0:1619677473.963919:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:30.0:1619677473.963928:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677473.963931:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677473.963932:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.963932:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677473.963933:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.963934:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677473.964190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.964197:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.964199:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.964200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.964202:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.964204:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.964206:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.964207:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.964208:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.964209:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.964210:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.964213:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.964214:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.964217:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.964218:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.964219:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.964220:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.964221:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.964221:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.964222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.964234:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.964236:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.964237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.964238:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.964240:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.964242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.964494:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.964497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.964497:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677473.964500:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.964501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.964503:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.964504:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.964506:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.964507:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.964508:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.964511:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.964513:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.964515:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.964516:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.964517:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.964519:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.964519:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.964520:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.964521:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.964522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.964533:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.964533:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.964535:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.964535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.964535:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.964536:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.964536:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.964537:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.964540:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.964540:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.964546:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.964548:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.964550:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.964551:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677473.964553:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.964553:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.964553:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.964554:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000400:00000200:30.0:1619677473.964555:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677473.964555:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000400:00000200:30.0:1619677473.964556:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677473.964556:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:30.0:1619677473.964557:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677473.964557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677473.964558:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.964559:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.964563:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.964564:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.964567:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.964567:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.964568:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.964570:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.964570:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.964571:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.964572:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.964575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.964582:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.964585:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.964586:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.964586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.964588:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.964590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677473.964662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.964669:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.964672:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.964672:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677473.964674:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.964677:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.964678:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.964679:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.964681:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.964682:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.964683:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.964686:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.964688:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.964691:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.964691:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.964692:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.964694:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.964694:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.964695:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.964696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.964719:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.964721:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.964722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.964722:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677473.964722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:6.0:1619677473.964724:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.964727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.964730:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.964732:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.964733:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.964735:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.964738:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.964739:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.964740:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.964742:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.964743:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.964744:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.964747:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.964749:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.964752:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.964752:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.964753:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.964755:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.964755:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.964756:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.964756:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677473.964770:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.964773:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.964774:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.964777:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677473.964779:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.964781:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677473.965071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.965078:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.965081:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.965082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.965083:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.965087:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.965088:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677473.965090:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.965091:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.965092:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.965093:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677473.965097:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.965098:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677473.965102:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.965102:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677473.965103:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677473.965105:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677473.965106:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.965107:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.965108:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677473.965119:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677473.965122:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677473.965124:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.965126:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677473.965129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.965130:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677473.965267:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.965270:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.965272:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.965273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.965274:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.965276:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.965277:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.965278:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.965279:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.965280:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.965284:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.965285:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.965289:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.965290:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.965290:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.965292:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.965292:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.965298:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.965298:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.965307:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.965310:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.965310:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.965311:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.965313:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.965315:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.965348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.965356:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.965359:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.965359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.965361:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.965363:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.965365:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.965366:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.965367:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.965368:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677473.965368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677473.965369:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.965372:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.965373:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.965375:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.965375:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677473.965376:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.965377:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.965378:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.965378:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:26.0:1619677473.965379:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.965379:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677473.965379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677473.965380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677473.965380:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.965383:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.965384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:8.0:1619677473.965384:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.965386:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.965387:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.965388:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677473.965390:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:8.0:1619677473.965390:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.965391:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.965392:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.965394:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.965394:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.965395:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.965397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.965398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.965399:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.965400:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.965400:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:9.2:1619677473.965402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.965402:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.965403:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.965403:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.965405:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677473.965406:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:9.2:1619677473.965408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.965408:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677473.965408:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.965409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.965409:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.965410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677473.965416:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.965418:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.965419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.965420:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.965422:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.965425:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.965752:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.965754:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.965755:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.965757:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.965759:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.965760:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.965761:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.965763:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.965764:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.965765:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.965768:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.965769:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.965772:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.965773:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.965773:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.965775:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.965776:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.965777:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.965777:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.965789:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.965791:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.965792:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.965794:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.965795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.965796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.966053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.966060:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.966063:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.966064:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.966066:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.966068:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.966069:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.966070:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.966071:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.966073:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.966074:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.966077:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.966079:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.966081:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.966082:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.966082:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.966084:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.966084:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.966085:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.966085:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.966089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.966095:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.966098:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.966098:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.966099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.966100:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.966102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.966230:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.966233:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.966235:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.966237:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.966237:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.966239:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.966240:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.966241:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.966242:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.966243:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.966246:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.966247:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.966249:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.966250:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.966251:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.966252:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.966252:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.966254:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.966255:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.966256:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.966266:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.966267:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.966268:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.966269:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.966269:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.966270:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.966271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.966271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.966272:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.966273:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.966793:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.966796:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.966798:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.966800:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.966801:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.966801:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.966802:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.966803:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.966804:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.966805:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.966807:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.966809:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.966810:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.966811:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.966811:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.966813:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.966813:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.966814:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.966815:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.966826:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.966829:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.966829:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.966831:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.966832:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.966833:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.966870:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.966873:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.966873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.966876:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.966878:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.966879:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.966881:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.966882:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.966883:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.966884:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.966887:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.966888:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.966891:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.966891:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.966892:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.966894:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.966894:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.966895:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.966896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.966907:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.966910:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.966910:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.966912:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.966913:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.966914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.966953:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.966956:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.966958:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.966960:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.966960:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.966962:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.966963:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.966964:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.966965:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.966966:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.966970:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.966971:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.966974:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.966975:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.966976:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.966977:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.966978:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.966979:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.966980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677473.966991:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.966994:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.966995:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.966995:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.966998:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.967001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.967016:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.967018:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.967019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.967020:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.967021:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.967021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.967447:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.967450:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.967452:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.967455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.967455:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.967457:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.967458:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.967459:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.967460:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.967461:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.967464:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.967466:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.967467:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.967468:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.967469:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.967470:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.967470:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.967471:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.967472:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.967483:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.967485:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.967486:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.967488:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.967489:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.967490:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677473.967883:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.967886:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.967887:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.967889:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.967891:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.967892:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.967893:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.967895:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.967896:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.967897:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.967900:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.967901:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.967904:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.967905:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.967905:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.967907:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.967907:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.967908:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.967909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.967921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.967923:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.967924:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.967925:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:4.2:1619677473.967925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677473.967928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.967929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.967933:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.967936:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.967937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.967938:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.967942:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.967943:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.967945:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.967946:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.967947:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.967948:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.967951:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.967953:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.967956:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.967956:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.967957:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.967959:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.967959:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.967960:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.967961:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.967961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.967967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677473.967973:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.967974:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677473.967975:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677473.967976:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677473.967976:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.967976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.967977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.967978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677473.967979:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.967982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.968166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677473.968173:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.968176:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677473.968177:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677473.968179:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.968182:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.968183:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677473.968184:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.968186:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.968187:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.968188:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677473.968191:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.968192:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677473.968194:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.968195:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:26.0:1619677473.968196:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677473.968197:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677473.968198:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677473.968199:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677473.968200:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677473.968200:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:4.2:1619677473.968206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:12.0:1619677473.968210:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677473.968213:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677473.968213:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.968213:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:12.0:1619677473.968215:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.968215:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677473.968215:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:12.0:1619677473.968217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677473.968217:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677473.968218:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677473.968230:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677473.968232:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677473.968233:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677473.968234:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677473.968234:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677473.968236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677473.968485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677473.968491:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.968494:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.968495:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677473.968497:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.968500:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.968501:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677473.968503:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.968504:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.968505:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.968506:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677473.968509:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.968511:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.968513:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.968514:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.968514:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.968516:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.968517:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.968518:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.968518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.968523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.968525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.968528:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.968531:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677473.968531:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:20.0:1619677473.968533:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677473.968533:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677473.968534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677473.968534:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.968534:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:20.0:1619677473.968535:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.968536:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:20.0:1619677473.968537:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:18.0:1619677473.968537:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:20.0:1619677473.968538:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.968539:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.968540:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.968541:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:9.2:1619677473.968541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000400:00000200:20.0:1619677473.968545:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.968546:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677473.968548:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677473.968549:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.968550:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.968550:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:2.0:1619677473.968551:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677473.968552:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:18.0:1619677473.968552:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.968552:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.968552:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677473.968553:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677473.968553:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677473.968554:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.968555:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677473.968556:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677473.968559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677473.968561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.968566:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677473.968568:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677473.968568:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677473.968569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.968570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677473.968571:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.968571:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.968572:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.968574:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.968576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.968771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:8.0:1619677473.968778:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677473.968781:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.3@o2ib3
00000800:00000200:24.0:1619677473.968782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:8.0:1619677473.968784:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : GET - for me
00000400:00000200:8.0:1619677473.968787:0:19938:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.3.3@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:8.0:1619677473.968789:0:19938:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.3.3@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:8.0:1619677473.968791:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677473.968792:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000400:00000200:8.0:1619677473.968793:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.3.3@o2ib3 local destination
00000400:00000200:8.0:1619677473.968797:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.3.3@o2ib3(10.51.3.3@o2ib3:10.51.3.3@o2ib3) : REPLY try# 0
00000800:00000200:8.0:1619677473.968799:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.3.3@o2ib3
00000800:00000200:8.0:1619677473.968800:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919fce371880] -> 10.51.3.3@o2ib3 (2) version: 12
00000800:00000200:8.0:1619677473.968801:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677473.968802:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39c600] (22)++
00000800:00000200:8.0:1619677473.968803:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919fee39c600] (23)--
00000800:00000200:8.0:1619677473.968804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:4.2:1619677473.968810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677473.968816:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:24.0:1619677473.968818:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39c600] (23)--
00000400:00000200:24.0:1619677473.968819:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.968820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:24.0:1619677473.968821:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.3.3@o2ib3: REPLY: OK
00000800:00000200:24.0:1619677473.968822:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:4.2:1619677473.968992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677473.969000:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677473.969003:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677473.969004:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.969006:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.969008:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677473.969008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677473.969009:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677473.969009:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:31.0:1619677473.969011:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677473.969011:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.969012:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.969013:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677473.969015:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.969017:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677473.969018:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.969019:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677473.969019:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.969020:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.969022:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.969022:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.969023:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.969027:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677473.969027:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.969028:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677473.969028:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677473.969028:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.969030:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677473.969031:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677473.969032:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677473.969032:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:30.0:1619677473.969032:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.969032:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.969033:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.969034:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.969035:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.969035:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.969036:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.2:1619677473.969036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677473.969038:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.969038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:1.0:1619677473.969044:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.969047:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.969048:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.969048:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.969050:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677473.969050:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.969052:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677473.969052:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.969053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677473.969053:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677473.969054:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677473.969054:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.969056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.969056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:6.0:1619677473.969056:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.969059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.969149:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.969152:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.969152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.969155:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.969156:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.969158:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.969159:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.969161:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.969162:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.969163:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.969166:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.969168:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.969170:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.969171:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.969171:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.969173:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677473.969173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677473.969174:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.969175:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.969176:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.969177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.969180:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.969183:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.969184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.969186:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677473.969186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677473.969187:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:26.0:1619677473.969188:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.969189:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.969189:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:26.0:1619677473.969190:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677473.969190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677473.969191:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.969191:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.969191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:26.0:1619677473.969192:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.969192:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.969194:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.969194:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.969194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677473.969194:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677473.969195:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677473.969196:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.969197:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677473.969200:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.969200:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.969202:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677473.969202:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.969203:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677473.969205:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.969205:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:21.0:1619677473.969205:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677473.969205:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.969206:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:8.0:1619677473.969206:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.969207:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.969208:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.969208:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677473.969208:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677473.969209:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.969210:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.0:1619677473.969211:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677473.969211:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.969212:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677473.969214:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677473.969215:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677473.969215:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.969215:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677473.969215:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.969216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677473.969216:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.969216:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677473.969217:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.969218:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677473.969218:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.969219:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.969219:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677473.969220:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.969220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677473.969222:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677473.969222:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:31.0:1619677473.969223:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.969224:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:9.2:1619677473.969224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:31.0:1619677473.969225:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96506748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.969225:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.969225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:31.0:1619677473.969226:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.969226:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.969228:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.969228:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677473.969229:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677473.969231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:31.0:1619677473.969232:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.969232:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:31.0:1619677473.969234:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677473.969235:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:31.0:1619677473.969236:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677473.969236:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.969236:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677473.969237:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.969238:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677473.969238:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.969239:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.969240:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.969241:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677473.969241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.969242:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.969245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.969251:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.969254:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.969255:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.969255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.969257:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.969259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.969342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677473.969347:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.969350:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.969352:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.969352:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.969354:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.969356:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677473.969357:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.969358:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.969359:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.969361:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677473.969364:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.969366:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.969367:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.969368:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677473.969369:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.969371:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.969371:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.969372:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677473.969374:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677473.969374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677473.969380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.969386:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.969387:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.969388:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.969388:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000400:00000200:21.0:1619677473.969389:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.969389:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677473.969390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.969390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.969392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.969393:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:5.2:1619677473.970056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.970063:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.970065:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.970066:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.970068:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.970070:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.970072:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.970073:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.970075:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.970075:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.970077:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.970080:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.970082:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.970084:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.970085:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.970086:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.970087:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.970088:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.970088:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.970089:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.970101:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:4.2:1619677473.970102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677473.970104:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.970105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.970105:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.970107:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.970110:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677473.970110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677473.970112:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.970113:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.970115:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.970118:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.970119:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677473.970119:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.970120:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.970122:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.970122:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.970123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.970124:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677473.970124:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677473.970125:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.970125:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:10.0:1619677473.970125:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.970128:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.970130:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.970132:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.970133:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.970134:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.970135:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.970136:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.970137:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.970138:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.970139:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.970141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.970144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677473.970148:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:26.0:1619677473.970151:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677473.970151:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.970152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.970152:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.970153:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.970154:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677473.970154:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.970155:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.970156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.970266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.970269:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.970270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.970272:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.970274:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.970275:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.970277:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.970278:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.970279:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.970280:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.970284:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.970285:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.970288:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.970289:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.970290:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.970291:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.970292:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.970299:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.970300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.970306:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.970308:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.970309:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.970311:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.970312:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.970313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.970351:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.970354:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.970356:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.970359:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.970359:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.970360:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.970361:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.970363:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.970364:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.970364:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.970367:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.970369:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.970372:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.970372:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.970373:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.970374:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.970375:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.970377:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.970378:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.970379:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.970380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.970387:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.970389:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.970390:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.970390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.970392:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677473.970392:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.970394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677473.970395:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.970395:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.970396:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.970397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677473.970403:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.970406:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.970407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.970407:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.970408:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677473.970408:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.970805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.970813:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.970815:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.970816:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.970818:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.970820:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.970821:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.970822:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.970823:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.970824:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.970825:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.970828:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.970829:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.970831:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.970832:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.970833:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.970834:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.970834:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.970835:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.970836:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.970839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.970845:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.2:1619677473.970846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:24.0:1619677473.970847:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.970848:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.970849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.970850:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.970851:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:24.0:1619677473.970852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677473.970853:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677473.970855:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.970856:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677473.970857:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.970859:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677473.970860:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.970861:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.970862:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.970863:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.970866:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.970868:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677473.970870:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.970871:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677473.970871:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677473.970873:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677473.970874:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677473.970874:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677473.970875:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:22.0:1619677473.970887:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.970889:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.970890:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.970890:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.970892:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.970894:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.971099:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:4.2:1619677473.971101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677473.971102:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.971104:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.971106:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.971107:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.971107:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677473.971108:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.971109:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677473.971110:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:31.0:1619677473.971111:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.971111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677473.971112:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.971113:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.971113:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.971116:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.971117:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677473.971117:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:31.0:1619677473.971118:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.971119:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.971120:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.971121:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.971121:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677473.971122:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.971123:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.971123:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.971125:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:8.0:1619677473.971125:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.971126:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677473.971126:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677473.971127:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.971127:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.0:1619677473.971129:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.971130:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.971131:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.971132:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.971133:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.971133:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.971134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.971139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677473.971139:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.971142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.971142:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.971143:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.971145:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.971147:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677473.971148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677473.971150:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.971151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.971151:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.971153:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.971156:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.971157:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.971160:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.971161:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.971161:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.971163:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.971163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.971208:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677473.971210:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677473.971211:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.971213:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.971213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.971213:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:31.0:1619677473.971215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677473.971216:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.971217:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.971218:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:6.0:1619677473.971218:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.971219:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677473.971220:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.971221:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.971222:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.971223:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.971224:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.971224:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.971225:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677473.971226:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:30.0:1619677473.971231:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.971231:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.971233:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677473.971235:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677473.971236:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.971237:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.971237:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677473.971238:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:6.0:1619677473.971238:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677473.971239:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677473.971239:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677473.971240:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677473.971240:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677473.971241:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677473.971241:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677473.971242:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677473.971242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677473.971243:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677473.971253:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.971255:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677473.971256:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:20.0:1619677473.971256:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.971257:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.971258:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:21.0:1619677473.971258:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677473.971259:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.971260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677473.971261:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.971263:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.971264:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.971265:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.971265:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.971265:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.971266:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677473.971266:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (24)--
00000800:00000200:30.0:1619677473.971267:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.971268:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.971741:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.971744:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.971746:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.971748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.971748:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.971749:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.971750:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.971752:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.971753:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.971754:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.971757:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.971758:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.971760:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.971761:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.971761:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.971763:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.971763:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.971764:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.971765:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.971776:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.971778:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.971779:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.971780:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.971782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.971783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.971886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677473.971894:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.971897:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677473.971898:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677473.971899:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.971901:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.971903:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677473.971904:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.971906:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.971907:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.971907:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677473.971910:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.971912:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.971915:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.971916:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677473.971916:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.971918:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677473.971919:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.971919:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677473.971920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677473.971925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.971933:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677473.971936:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.971936:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.971938:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.971939:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.971941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677473.971973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677473.971980:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.971983:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677473.971984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677473.971985:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677473.971988:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.971989:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677473.971990:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.971992:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.971993:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.971994:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.971997:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677473.971999:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.972001:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.972002:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.972003:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.972004:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.972005:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.972006:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.972006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677473.972011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.972019:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.972021:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.972022:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.972022:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.972024:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677473.972027:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677473.972200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677473.972207:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677473.972210:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677473.972212:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.972214:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.972214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677473.972216:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677473.972217:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.972218:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.972219:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.972221:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.972223:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.972225:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677473.972228:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.972228:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677473.972229:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.972231:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.972232:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677473.972233:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677473.972233:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677473.972237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.972244:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.972246:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.972247:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.972247:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.972249:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:4.2:1619677473.972251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:24.0:1619677473.972252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677473.972259:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.972261:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.972262:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.972264:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.972266:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.972267:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677473.972268:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.972270:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.972270:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.972271:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:30.0:1619677473.972272:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677473.972274:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.972275:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677473.972276:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.972277:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677473.972278:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.972278:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:31.0:1619677473.972279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677473.972279:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:30.0:1619677473.972280:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.972280:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.972280:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000400:00000200:30.0:1619677473.972281:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677473.972281:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000400:00000200:30.0:1619677473.972282:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.972282:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:30.0:1619677473.972284:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.972285:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.972285:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677473.972286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:30.0:1619677473.972289:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.972290:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.972292:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.972298:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:30.0:1619677473.972299:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.972299:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677473.972300:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.972302:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677473.972302:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.972303:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.972304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677473.972304:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.972305:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.972306:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.972307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.2:1619677473.972308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.972310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.972312:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.972314:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.972314:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.972315:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.972316:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677473.972316:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.0:1619677473.972316:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.972319:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.972320:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.972322:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.972322:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.972325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677473.972344:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.972347:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.972347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.972350:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.972352:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.972353:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.972355:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.972356:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.972358:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.972358:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.972361:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.972363:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.972365:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.972366:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.972367:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.972369:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.972369:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.972370:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.972371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.972382:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.972384:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.972385:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.972387:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.972388:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.972389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.972402:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.972405:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.972406:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.972406:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.972407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.972408:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.972571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.972579:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.972582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.972582:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.972584:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.972587:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.972588:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.972590:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.972591:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.972592:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.972593:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.972596:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.972598:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.972601:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.972602:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.972602:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.972604:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.972604:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.972605:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.972605:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.972618:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.972621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.972621:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.972622:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.972624:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.972627:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.973160:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.973163:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.973165:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.973166:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.973168:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.973169:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.973170:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.973172:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.973173:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.973174:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.973177:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.973178:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.973182:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.973182:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.973183:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.973185:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.973185:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.973186:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.973186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.973199:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.973201:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.973202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.973203:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.973205:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.973208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.973272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.973280:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.973282:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.973283:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.973285:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.973286:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.973288:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677473.973289:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.973290:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.973291:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.973292:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677473.973300:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.973302:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677473.973305:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.973306:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677473.973306:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677473.973308:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677473.973308:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.973309:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.973310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.973315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677473.973317:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.973320:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.973321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677473.973323:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:6.0:1619677473.973323:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.973325:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.973326:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:6.0:1619677473.973326:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677473.973327:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.973327:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677473.973327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:6.0:1619677473.973329:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.973330:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:6.0:1619677473.973331:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.973332:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677473.973334:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:6.0:1619677473.973334:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.973336:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.973339:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.973340:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.973340:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.973342:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.973342:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.973344:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.973345:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.973346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677473.973352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:22.0:1619677473.973357:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.973358:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.973359:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.973359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677473.973360:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:22.0:1619677473.973360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.973360:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.973361:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.973363:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:20.0:1619677473.973363:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.973364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677473.973364:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677473.973365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677473.973366:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.973368:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.973370:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677473.973371:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.973373:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.973374:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.973375:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677473.973378:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.973380:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677473.973383:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.973383:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.973384:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.973386:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.973386:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.973387:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677473.973389:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.973390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677473.973392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.973395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677473.973399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.973402:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.973403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.973403:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:2.0:1619677473.973403:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.973405:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:2.0:1619677473.973405:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:10.0:1619677473.973406:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.973407:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677473.973409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677473.973805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.973811:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.973814:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.973815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.973817:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.973820:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.973821:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677473.973823:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.973824:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.973825:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.973826:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677473.973828:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.973830:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677473.973833:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.973834:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677473.973834:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677473.973836:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677473.973836:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.973838:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.973839:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.973839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677473.973842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677473.973845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677473.973849:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.973852:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677473.973852:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:18.0:1619677473.973853:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.973853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.973854:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.973855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677473.973855:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.973855:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677473.973859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.973891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677473.973899:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677473.973902:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677473.973904:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.973906:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677473.973907:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.973908:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677473.973910:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.973911:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.973912:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.973913:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677473.973916:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.973918:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677473.973921:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.973921:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677473.973922:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677473.973924:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677473.973924:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677473.973926:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677473.973927:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677473.973928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677473.973929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.973934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:2.0:1619677473.973937:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.973940:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.973941:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.973941:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.973942:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000400:00000200:2.0:1619677473.973943:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.973944:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677473.973945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677473.973946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:2.0:1619677473.973947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677473.974030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.974036:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.974039:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.974040:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.974042:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.974044:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.974046:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.974047:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.974048:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.974049:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.974050:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.974053:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.974055:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.974058:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.974059:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.974059:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.974061:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.974061:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.974062:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.974062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.974075:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.974078:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.974079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.974079:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.974081:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.974085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.974343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.974345:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.974346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.974348:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.974350:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.974351:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.974352:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.974354:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.974355:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.974356:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.974359:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.974361:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.974363:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.974364:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.974364:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.974366:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.974367:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.974367:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.974368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.974380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.974382:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.974383:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.974385:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.974386:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.974387:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.974388:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677473.974390:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.974393:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677473.974395:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.974396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677473.974397:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.974399:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677473.974400:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.974402:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.974403:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.974404:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677473.974407:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.974409:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677473.974412:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.974413:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:20.0:1619677473.974414:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.0:1619677473.974415:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:20.0:1619677473.974416:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677473.974417:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677473.974417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:9.2:1619677473.974421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.974429:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.974431:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.974432:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677473.974433:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.974435:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.974438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:31.0:1619677473.974696:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.974699:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.974701:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.974703:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.974703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.974705:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677473.974706:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.974707:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.974708:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.974710:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677473.974713:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.974715:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677473.974718:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.974719:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677473.974719:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677473.974721:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677473.974722:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.974723:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.974723:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677473.974728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.974737:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.974740:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.974741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.974741:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.974743:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.974746:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.974827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.974835:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.974837:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.974838:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.974839:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.974842:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.974843:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.974844:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.974846:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.974847:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.974848:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.974850:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.974852:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.974854:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.974855:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.974855:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.974857:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.974857:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.974858:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.974858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.974862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.974869:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.974871:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.974872:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.974872:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.974874:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.974876:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.975172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.975177:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.975180:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.975182:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.975183:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.975185:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.975186:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.975187:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.975189:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.975190:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.975191:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.975194:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.975196:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.975199:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.975199:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.975200:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.975202:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.975202:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.975203:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.975204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.975208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.975217:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.975219:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.975220:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.975220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.975222:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.975224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.975228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677473.975235:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677473.975237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.975238:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.975239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677473.975239:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.975240:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.975240:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677473.975241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677473.975242:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.975242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.975244:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.975246:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.975248:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.975249:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.975250:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.975251:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.975252:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.975256:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.975258:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.975260:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.975261:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.975262:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.975263:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.975264:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.975265:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677473.975265:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677473.975266:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677473.975271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.975276:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677473.975277:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677473.975278:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.975279:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677473.975280:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677473.975280:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.975281:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.975281:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.975283:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.975286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677473.975415:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.975418:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.975421:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.975423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.975423:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.975424:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.975426:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.975427:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.975428:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.975429:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.975432:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.975434:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.975436:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.975437:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.975438:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.975439:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.975440:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.975441:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.975443:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.975444:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.975453:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.975455:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.975455:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.975457:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.975457:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.975458:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.975458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.975459:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.975460:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.975461:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677473.975590:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.975593:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.975594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.975595:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.975598:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.975599:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.975600:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.975602:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.975603:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.975604:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.975607:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.975608:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.975611:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.975612:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.975612:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.975614:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.975615:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.975616:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.975616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.975627:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.975630:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.975631:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.975632:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.975634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.975635:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.975688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.975695:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.975698:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.975699:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.975701:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.975703:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.975705:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.975706:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.975707:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.975708:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.975709:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677473.975712:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.975714:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.975717:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.975717:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.975718:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.975720:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.975720:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.975721:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.975721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677473.975734:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.975736:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.975737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.975738:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.975740:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.975744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.975936:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.975938:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.975941:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.975942:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.975943:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.975944:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.975945:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.975946:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.975947:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.975948:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.975951:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.975952:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.975954:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.975955:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.975955:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.975957:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.975957:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.975958:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.975959:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.975969:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.975972:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.975972:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.975974:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.975976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.975976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.976017:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.976019:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.976021:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.976023:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.976023:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.976025:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.976026:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.976027:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.976028:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.976029:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.976032:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.976033:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.976037:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.976037:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.976038:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.976040:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.976040:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.976042:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.976043:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.976044:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677473.976053:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.976056:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677473.976057:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677473.976057:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.976057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.976059:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.976060:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.976061:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.976061:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.976063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.976074:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.976077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.976077:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.976078:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.976079:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.976079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.976401:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.976403:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.976406:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.976408:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.976408:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.976410:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.976411:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.976412:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.976413:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.976414:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.976417:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.976418:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.976421:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.976421:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.976422:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.976423:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.976424:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.976425:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.976426:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.976430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.976437:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.976439:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.976440:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.976441:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.976442:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.976445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.976582:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.976584:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.976585:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.976587:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.976589:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.976590:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.976591:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.976593:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.976594:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.976595:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.976598:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.976599:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.976602:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.976602:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.976603:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.976605:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.976605:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.976606:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.976607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.976619:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.976621:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.976622:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.976623:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.976624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.976625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.976636:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.976638:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.976639:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.976640:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.976640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.976642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.976895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.976902:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.976905:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.976905:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.976908:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.976911:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.976912:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.976913:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.976915:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.976916:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.976917:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.976920:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.976922:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.976925:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.976925:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.976926:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.976928:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.976928:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.976929:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.976929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.976934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.976942:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.976945:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.976946:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.976946:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.976949:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.976952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677473.976955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.976962:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.976965:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.976965:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.976966:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.976967:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.976968:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.977071:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.977074:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.977076:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.977078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.977079:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.977080:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.977081:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.977083:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.977084:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.977085:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.977088:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.977089:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.977091:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.977091:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.977092:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.977093:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.977094:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.977095:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.977096:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.977106:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.977108:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.977109:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.977110:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.977112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.977113:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.977126:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.977129:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.977130:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.977130:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.977131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.977132:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.977325:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.977328:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.977330:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.977332:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.977332:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.977334:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677473.977335:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.977336:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.977337:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.977338:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677473.977341:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.977342:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677473.977345:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.977346:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677473.977347:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677473.977348:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677473.977349:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.977350:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.977350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677473.977356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677473.977365:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677473.977368:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.977369:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.977369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677473.977371:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.977374:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677473.977464:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.977467:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.977468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.977470:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.977472:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.977473:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.977475:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.977476:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.977477:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.977478:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.977481:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.977482:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.977485:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.977486:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.977487:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.977488:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.977489:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.977490:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.977491:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.977492:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.977502:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.977504:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.977505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677473.977505:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.977506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.977507:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.977508:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.977510:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.977511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.977511:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677473.977647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.977654:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:4.2:1619677473.977655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:26.0:1619677473.977657:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.977658:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.977660:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677473.977662:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000400:00000200:26.0:1619677473.977663:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.977664:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677473.977665:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:26.0:1619677473.977665:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.977667:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677473.977667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:30.0:1619677473.977668:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.977668:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677473.977669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:26.0:1619677473.977669:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677473.977670:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.977672:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:26.0:1619677473.977672:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677473.977673:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.977674:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677473.977674:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:30.0:1619677473.977675:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.977675:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.977677:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677473.977677:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.977677:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.977677:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677473.977678:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677473.977679:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677473.977681:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677473.977681:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:18.0:1619677473.977681:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677473.977682:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.977682:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.977682:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677473.977683:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.977684:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:18.0:1619677473.977685:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677473.977686:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96506702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677473.977687:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.977687:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:18.0:1619677473.977687:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677473.977687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677473.977688:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:18.0:1619677473.977689:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677473.977690:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.977690:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000400:00000200:18.0:1619677473.977690:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677473.977691:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677473.977692:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000400:00000200:18.0:1619677473.977695:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:9.2:1619677473.977695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677473.977696:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.977697:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.977697:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677473.977699:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.977700:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.977701:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.977702:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:18.0:1619677473.977702:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677473.977703:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.977703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:24.0:1619677473.977704:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:2.0:1619677473.977704:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000400:00000200:24.0:1619677473.977705:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.977705:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677473.977705:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677473.977706:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677473.977706:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677473.977707:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.977707:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677473.977707:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.977707:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.977708:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677473.977708:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677473.977709:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.977709:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677473.977710:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677473.977714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677473.977717:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.977720:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:18.0:1619677473.977720:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:31.0:1619677473.977721:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.977722:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:31.0:1619677473.977723:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:18.0:1619677473.977723:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677473.977724:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.977724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677473.977726:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677473.977910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.977916:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.977918:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.977919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.977921:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.977924:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.977925:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.977926:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.977928:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.977929:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.977929:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.977932:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.977933:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.977935:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.977935:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.977936:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.977937:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.977938:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.977938:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.977939:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677473.977943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.977950:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.977952:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.977953:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.977954:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.977955:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677473.977956:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.977958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.977965:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.977968:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.977969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.977970:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.977973:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.977975:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.977976:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.977978:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.977978:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.977979:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.977982:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.977984:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.977987:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.977988:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.977988:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.977990:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.977990:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.977991:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.977991:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.978004:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.978007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.978007:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.978008:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.978010:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.978013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.978399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.978407:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.978410:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.978411:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.978412:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.978416:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.978417:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.978418:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.978420:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.978421:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.978421:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.978425:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.978426:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.978429:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.978430:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.978430:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.978432:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.978432:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.978433:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677473.978435:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.978435:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677473.978441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677473.978447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.978448:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.978449:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677473.978450:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.978451:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677473.978451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.978451:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.978451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.978453:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.978456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.978521:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.978523:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.978524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.978526:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.978527:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.978529:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.978530:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.978532:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.978533:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.978534:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.978537:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.978539:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.978541:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.978542:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.978543:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.978544:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.978545:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.978546:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.978546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.978558:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.978560:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.978561:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.978563:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.978564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.978565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.978775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.978782:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.978785:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.978786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.978788:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.978791:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.978792:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.978793:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.978795:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.978796:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.978797:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.978800:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.978802:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.978805:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.978806:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.978806:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.978808:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.978808:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.978809:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.978809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.978829:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.978832:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.978833:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.978833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.978836:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.978839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.979281:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.979284:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.979286:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.979289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.979289:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.979290:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.979292:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.979299:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.979300:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.979301:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.979304:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.979306:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.979309:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.979310:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.979310:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.979312:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.979313:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.979313:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.979314:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677473.979328:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.979330:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.979331:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.979332:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.979334:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:4.2:1619677473.979334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:6.0:1619677473.979337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.979342:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.979345:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677473.979345:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677473.979347:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677473.979350:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.979352:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.979353:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.979354:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.979355:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.979356:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.979359:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677473.979360:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.979363:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.979364:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.979364:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.979366:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.979366:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677473.979367:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677473.979368:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677473.979369:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:4.2:1619677473.979374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:22.0:1619677473.979379:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.979381:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:22.0:1619677473.979382:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.979382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.979383:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.979384:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677473.979384:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.979385:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:22.0:1619677473.979385:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677473.979388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.979399:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.979402:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.979402:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.979403:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.979404:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.979405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.979443:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.979445:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.979446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.979448:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.979450:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.979451:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.979452:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.979454:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.979455:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.979456:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.979459:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.979461:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.979463:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.979464:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.979465:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.979467:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.979467:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.979468:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.979468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.979481:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.979483:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.979484:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.979485:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.979486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.979488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.979583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.979591:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.979593:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.979594:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.979596:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.979599:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.979600:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677473.979601:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.979603:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.979603:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.979604:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677473.979608:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.979609:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.979611:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.979612:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.979613:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.979615:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.979615:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.979616:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.979617:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.979618:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.979620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.979623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677473.979627:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.979629:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.979630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.979630:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677473.979630:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.979632:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677473.979632:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.979633:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.979634:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.979634:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.979638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.979644:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.979646:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.979647:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.979648:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.979648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.979648:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:5.2:1619677473.979960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.979966:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.979969:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.979970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.979972:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.979974:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.979976:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.979977:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.979978:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.979979:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.979980:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.979984:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.979986:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677473.979988:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.979989:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.979990:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.979991:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.979992:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.979993:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677473.979993:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.980006:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.980008:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.980009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.980009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.980011:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.980014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677473.980023:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.980026:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.980026:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.980027:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.980028:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.980029:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.980139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.980143:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.980145:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.980148:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.980149:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.980150:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.980152:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677473.980153:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.980154:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.980156:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.980156:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.980160:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.980162:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677473.980164:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.980165:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677473.980166:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.980167:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.980168:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.980169:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.980169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677473.980181:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.980184:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.980185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.980185:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.980187:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.980189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.980333:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.980336:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.980336:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.980339:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.980340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.980342:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.980343:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.980345:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.980346:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.980347:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.980350:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.980352:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.980354:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.980355:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.980356:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.980357:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.980358:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.980359:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.980359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.980372:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.980374:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.980375:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.980377:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.980378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.980379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.980397:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.980399:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.980400:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.980401:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.980402:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.980403:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.980500:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.980503:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.980506:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.980508:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.980509:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.980511:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677473.980512:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.980513:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.980514:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.980515:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.980518:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.980520:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677473.980523:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.980523:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.980524:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.980525:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.980526:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.980528:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677473.980529:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.980530:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677473.980532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.980539:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.980541:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.980542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.980542:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.980544:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677473.980544:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.980546:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677473.980546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.980547:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.980548:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677473.980651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.980659:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.980662:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.980663:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.980664:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.980666:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.980668:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677473.980669:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.980671:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.980671:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.980672:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677473.980675:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.980677:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677473.980677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:26.0:1619677473.980680:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.980680:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677473.980681:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677473.980681:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677473.980683:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677473.980683:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677473.980683:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:26.0:1619677473.980684:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.980685:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677473.980685:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677473.980686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677473.980687:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.980689:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677473.980690:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.980691:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.980692:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.980693:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677473.980697:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.980697:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.980699:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677473.980700:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.980701:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677473.980701:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677473.980701:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.980702:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677473.980702:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677473.980703:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.980704:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.980704:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.980705:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:6.0:1619677473.980706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677473.980707:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677473.980707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677473.980710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677473.980713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677473.980718:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677473.980719:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677473.980719:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677473.980720:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:20.0:1619677473.980720:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.980721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677473.980722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677473.980722:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.980723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677473.980724:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677473.980981:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.980983:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.980986:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.980987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.980989:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.980990:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.980991:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.980993:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.980994:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.980995:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.980998:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.980999:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.981002:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.981003:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.981004:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.981006:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.981006:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.981007:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.981007:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.981020:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.981022:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.981023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.981024:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.981026:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.981029:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.981041:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.981044:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.981045:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.981048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.981049:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.981050:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.981051:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.981052:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.981053:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.981054:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.981057:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.981059:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.981061:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.981061:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.981062:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.981063:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.981064:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.981065:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.981065:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.981075:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.981078:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.981079:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.981080:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.981082:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.981082:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677473.981443:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.981446:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.981447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.981448:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.981450:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.981452:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.981453:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.981455:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.981456:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.981457:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.981459:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.981461:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.981463:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.981464:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.981465:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.981467:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.981467:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.981469:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.981470:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.981470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.981481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.981481:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.981483:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.981483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.981483:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.981484:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.981484:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.981486:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.981487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.981488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.981661:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.981664:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.981666:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.981668:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.981669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.981669:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.981670:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.981672:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.981673:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.981674:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.981678:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.981680:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.981682:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.981683:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.981683:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.981685:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.981685:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.981686:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.981687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.981690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.981697:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.981700:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.981701:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.981701:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677473.981703:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.981705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.982397:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.982399:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.982400:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.982402:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.982404:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.982406:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.982407:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.982408:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.982409:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.982410:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.982413:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.982415:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.982417:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.982418:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.982419:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.982420:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.982421:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.982422:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.982423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.982436:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.982438:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.982439:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.982441:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.982442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.982443:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.982732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.982740:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.982743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.982743:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.982745:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.982748:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.982750:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.982751:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.982752:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.982753:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.982754:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.982758:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677473.982758:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.982759:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.982760:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.982763:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.982763:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.982764:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.982764:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677473.982765:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.982765:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677473.982766:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.982766:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677473.982767:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.0:1619677473.982767:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:31.0:1619677473.982768:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.982769:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.982769:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.982769:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677473.982770:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.982771:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677473.982773:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.982775:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:4.2:1619677473.982775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677473.982777:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.982778:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677473.982778:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677473.982780:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677473.982780:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.982781:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.982782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677473.982782:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677473.982783:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677473.982784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.982785:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677473.982786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.982790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.982793:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677473.982793:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:22.0:1619677473.982794:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.982794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.982795:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.982796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.982796:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.982797:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.982799:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.982800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.982802:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.982810:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.982813:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.982814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.982814:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.982816:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.982816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.982957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677473.982964:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.982967:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677473.982968:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.982970:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677473.982973:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.982974:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677473.982975:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.982976:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.982977:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.982978:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677473.982982:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.982983:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677473.982986:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.982987:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677473.982988:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677473.982990:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677473.982990:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.982991:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677473.982992:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677473.983003:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.983006:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.983007:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.983009:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.983010:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.983012:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677473.983092:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.983095:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.983097:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.983099:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.983099:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.983100:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.983102:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.983103:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.983104:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.983105:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.983108:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.983109:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.983112:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.983113:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.983114:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.983116:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.983116:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.983117:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677473.983119:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.983119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677473.983129:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:4.2:1619677473.983130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:31.0:1619677473.983132:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677473.983132:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.983133:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.983133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.983135:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.983136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677473.983136:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:22.0:1619677473.983136:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.983137:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677473.983139:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:22.0:1619677473.983139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.983140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677473.983142:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677473.983144:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.983146:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.983147:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.983148:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.983150:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.983151:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.983154:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677473.983156:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.983158:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.983159:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.983160:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.983161:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.983162:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677473.983163:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677473.983163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677473.983175:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.983178:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.983179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.983179:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.983181:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.983184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.983328:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.983331:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.983332:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.983333:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.983335:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.983337:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.983338:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.983339:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.983340:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.983341:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.983344:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.983345:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.983348:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.983349:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.983350:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.983351:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.983352:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.983353:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.983353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.983366:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.983368:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.983369:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.983370:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.983372:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.983373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.983379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677473.983387:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.983389:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677473.983390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677473.983392:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.983395:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.983396:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677473.983398:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.983399:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.983400:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.983401:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677473.983404:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.983406:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677473.983409:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.983410:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.983411:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.983412:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.983413:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.983413:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.983414:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677473.983426:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.983429:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.983430:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.983430:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.983432:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.983435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677473.983911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:8.0:1619677473.983919:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677473.983921:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.3.3@o2ib3
00000800:00000200:24.0:1619677473.983922:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:8.0:1619677473.983925:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.983928:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.983929:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677473.983930:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.983931:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.983933:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.983933:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677473.983936:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.983938:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.983939:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.983940:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677473.983940:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.983942:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677473.983943:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677473.983943:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000800:00000200:8.0:1619677473.983943:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:9.2:1619677473.983948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.983953:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.983955:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.983956:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.983957:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.983957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.983958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677473.983966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:4.2:1619677473.983969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:8.0:1619677473.983974:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677473.983975:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677473.983976:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.983977:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677473.983977:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677473.983977:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677473.983978:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.3@o2ib3
00000800:00000200:8.0:1619677473.983978:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677473.983979:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:24.0:1619677473.983980:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.983982:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.983984:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677473.983985:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.983986:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.983987:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.983987:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677473.983989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000400:00000200:24.0:1619677473.983990:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.983992:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677473.983995:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.983996:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:24.0:1619677473.983997:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:8.0:1619677473.983997:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:8.0:1619677473.983998:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.3@o2ib3
00000800:00000200:24.0:1619677473.983999:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:24.0:1619677473.984000:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (23)++
00000800:00000200:24.0:1619677473.984000:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (24)--
00000400:00000200:8.0:1619677473.984000:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.984001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:24.0:1619677473.984002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:8.0:1619677473.984002:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.984004:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677473.984005:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.984006:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.984007:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.984007:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677473.984010:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.984012:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:8.0:1619677473.984012:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677473.984014:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:8.0:1619677473.984014:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677473.984015:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.984015:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:13.0:1619677473.984016:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.984016:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677473.984017:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677473.984018:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:8.0:1619677473.984018:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677473.984019:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000800:00000200:8.0:1619677473.984019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:13.0:1619677473.984020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.2:1619677473.984023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677473.984028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:17.0:1619677473.984032:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:24.0:1619677473.984034:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:17.0:1619677473.984035:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677473.984036:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.984036:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677473.984037:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.3@o2ib3
00000400:00000200:17.0:1619677473.984038:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.984038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:24.0:1619677473.984040:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677473.984042:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:24.0:1619677473.984044:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.984046:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677473.984047:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677473.984047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:24.0:1619677473.984048:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.984050:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.984051:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677473.984054:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.984055:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.984056:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677473.984056:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677473.984058:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.984059:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.984060:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:24.0:1619677473.984061:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:8.0:1619677473.984061:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.984063:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:8.0:1619677473.984063:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.984064:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.984065:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.984066:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.984066:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.984067:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677473.984068:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39c600] (22)++
00000800:00000200:8.2:1619677473.984069:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677473.984070:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (23)--
00000800:00000200:24.0:1619677473.984070:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:24.0:1619677473.984072:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.984075:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677473.984075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000800:00000200:21.0:1619677473.984077:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677473.984077:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.0:1619677473.984080:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:8.0:1619677473.984080:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677473.984081:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.984081:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.984081:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:21.0:1619677473.984082:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.984083:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:20.0:1619677473.984083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677473.984084:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39c600] (23)--
00000800:00000200:8.0:1619677473.984084:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.984084:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.984085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:8.0:1619677473.984085:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677473.984086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677473.984086:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677473.984087:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677473.984089:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.3@o2ib3
00000800:00000200:8.2:1619677473.984089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:24.0:1619677473.984091:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.984094:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677473.984095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:24.0:1619677473.984096:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677473.984098:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677473.984098:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:24.0:1619677473.984100:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677473.984100:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:1.0:1619677473.984100:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:24.0:1619677473.984101:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677473.984101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.984101:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.984102:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677473.984103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:1.0:1619677473.984104:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.984106:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.984106:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677473.984107:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.984107:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677473.984107:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677473.984108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.984109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677473.984109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.984110:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677473.984113:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.984114:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:24.0:1619677473.984114:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677473.984116:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:24.0:1619677473.984117:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:17.0:1619677473.984117:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.984118:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000800:00000200:24.0:1619677473.984119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:17.0:1619677473.984120:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677473.984120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:1.0:1619677473.984120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.984121:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.984122:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.984122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677473.984127:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677473.984130:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:12.0:1619677473.984130:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:8.0:1619677473.984130:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.3@o2ib3
00000400:00000200:8.0:1619677473.984132:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.984133:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.984134:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.984134:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677473.984137:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677473.984137:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.984137:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677473.984138:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.984139:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.984140:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.984141:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677473.984143:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:8.0:1619677473.984145:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.984147:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677473.984150:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.984151:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677473.984151:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677473.984151:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.984153:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677473.984154:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677473.984154:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:13.0:1619677473.984155:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:8.0:1619677473.984155:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000800:00000200:8.0:1619677473.984155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:13.0:1619677473.984156:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677473.984157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677473.984157:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677473.984158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.984166:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.984169:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.984170:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.984170:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.984173:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.984175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677473.984176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:24.0:1619677473.984183:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677473.984185:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.3@o2ib3
00000800:00000200:8.0:1619677473.984186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:24.0:1619677473.984188:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.984191:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.984192:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677473.984194:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.984195:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.984196:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.984197:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677473.984199:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.984201:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677473.984204:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.984204:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677473.984205:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677473.984207:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677473.984208:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677473.984208:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000800:00000200:24.0:1619677473.984208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:8.2:1619677473.984216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.984224:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:4.2:1619677473.984225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:21.0:1619677473.984227:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677473.984227:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.984229:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.984231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.984232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677473.984233:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677473.984236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:8.0:1619677473.984236:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.3@o2ib3
00000400:00000200:8.0:1619677473.984239:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.984242:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.984243:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677473.984245:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.984246:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.984247:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.984248:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677473.984251:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.984252:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677473.984254:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.984255:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677473.984255:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677473.984257:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677473.984257:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677473.984259:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39c600] (22)++
00000800:00000200:8.0:1619677473.984260:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (23)--
00000800:00000200:8.0:1619677473.984260:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:4.2:1619677473.984263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000800:00000200:31.0:1619677473.984270:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:24.0:1619677473.984270:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:24.0:1619677473.984272:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.3@o2ib3
00000800:00000200:31.0:1619677473.984273:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.984273:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.984274:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (23)++
00000400:00000200:31.0:1619677473.984275:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.984275:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.984275:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39c600] (24)--
00000800:00000200:8.0:1619677473.984275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:8.0:1619677473.984276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:24.0:1619677473.984277:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.984278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677473.984278:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.984279:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677473.984279:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677473.984281:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677473.984281:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.984282:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.984283:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.984284:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.984284:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677473.984285:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677473.984285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677473.984287:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.984288:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677473.984289:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677473.984289:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:30.0:1619677473.984290:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.984291:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.984292:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.984292:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677473.984293:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.984293:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:30.0:1619677473.984301:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677473.984301:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677473.984304:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.984305:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677473.984306:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677473.984306:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677473.984307:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:26.0:1619677473.984308:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.984308:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:30.0:1619677473.984309:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.984309:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000800:00000200:24.0:1619677473.984309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:30.0:1619677473.984310:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:26.0:1619677473.984310:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.984311:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:26.0:1619677473.984311:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677473.984312:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:26.0:1619677473.984312:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96506663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677473.984313:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.984314:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677473.984314:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.984315:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.984315:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.984316:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.984316:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677473.984318:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677473.984318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.984320:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.984322:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.984323:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.984323:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677473.984323:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:4.2:1619677473.984324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:26.0:1619677473.984325:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.984325:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.984326:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.984326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.984326:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.984327:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.984327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677473.984328:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:22.0:1619677473.984330:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.984331:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677473.984331:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:9.2:1619677473.984331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:20.0:1619677473.984332:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.984333:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:22.0:1619677473.984333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.984334:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677473.984334:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.984334:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:21.0:1619677473.984335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677473.984336:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677473.984337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677473.984338:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.984339:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.2:1619677473.984339:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000400:00000200:24.0:1619677473.984341:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.984343:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.984343:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.3@o2ib3
00000800:00000200:24.0:1619677473.984346:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.984346:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.984347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:21.0:1619677473.984347:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677473.984348:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:8.0:1619677473.984349:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.984350:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:21.0:1619677473.984350:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677473.984350:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000400:00000200:8.0:1619677473.984350:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677473.984351:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677473.984351:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:24.0:1619677473.984352:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:21.0:1619677473.984352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:8.0:1619677473.984352:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677473.984353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677473.984353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:8.0:1619677473.984353:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677473.984355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:8.0:1619677473.984355:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.984356:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677473.984359:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.984361:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677473.984363:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.984363:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677473.984364:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677473.984365:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677473.984366:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677473.984367:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000800:00000200:8.0:1619677473.984367:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:31.0:1619677473.984378:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.984380:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.984381:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.984382:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.984384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.984385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.984400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:24.0:1619677473.984407:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677473.984409:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.3@o2ib3
00000800:00000200:8.0:1619677473.984410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:24.0:1619677473.984411:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.984414:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.984415:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677473.984416:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.984418:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.984419:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.984419:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677473.984422:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.984423:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677473.984425:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.984426:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:24.0:1619677473.984427:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:24.0:1619677473.984428:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:24.0:1619677473.984428:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677473.984429:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000800:00000200:24.0:1619677473.984429:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:9.2:1619677473.984434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.984440:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.984442:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.984443:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.984443:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.984445:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.984447:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677473.984504:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.984507:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.984508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.984510:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.984512:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.984513:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.984515:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.984516:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.984517:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.984518:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.984521:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.984523:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.984526:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.984526:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.984527:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.984529:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.984529:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.984530:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.984531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.984543:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.984545:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.984546:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.984547:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.984549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.984550:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.984562:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.984565:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.984566:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.984567:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.984567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.984568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.984775:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.984778:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.984780:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.984781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.984782:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.984784:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.984785:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.984786:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.984787:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.984788:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.984791:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.984793:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.984796:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.984797:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.984798:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.984799:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.984800:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.984801:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.984801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.984813:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.984816:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.984817:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.984817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.984819:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.984822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.985189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677473.985196:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.985199:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.985201:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677473.985203:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.985204:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677473.985205:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677473.985207:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.985208:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.985209:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.985210:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677473.985213:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.985215:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677473.985218:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.985218:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:13.0:1619677473.985219:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:13.0:1619677473.985221:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677473.985221:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.985223:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677473.985225:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677473.985225:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677473.985227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.985231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:2.0:1619677473.985235:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:12.0:1619677473.985238:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:2.0:1619677473.985238:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.985239:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.985239:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677473.985240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:12.0:1619677473.985241:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:2.0:1619677473.985241:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677473.985242:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677473.985243:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677473.985245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677473.985246:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.985249:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.985250:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.985252:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.985254:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.985255:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.985257:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.985258:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.985259:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.985260:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.985262:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.985264:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.985267:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.985267:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.985268:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.985270:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.985270:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.985271:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.985272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677473.985284:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677473.985286:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677473.985287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677473.985287:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.985289:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.985291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677473.985402:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.985405:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.985407:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.985409:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.985410:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.985411:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.985412:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.985414:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.985415:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.985416:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.985420:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.985421:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.985424:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.985425:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.985425:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.985427:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.985427:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.985428:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.985429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.985440:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.985442:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.985443:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.985445:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.985446:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.985447:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677473.985457:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.985460:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.985461:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.985461:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.985462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.985463:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.985471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677473.985479:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.985481:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.985482:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677473.985484:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.985485:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677473.985486:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677473.985487:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:26.0:1619677473.985488:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677473.985488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677473.985489:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.985490:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.985491:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.985492:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.985493:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677473.985493:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.985496:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.985497:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677473.985498:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.985499:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.985500:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.985501:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:22.0:1619677473.985501:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.985502:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677473.985502:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.985503:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.985503:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.985504:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.985505:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.985505:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677473.985506:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.985508:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.985511:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.985511:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.985512:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.985515:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.985515:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.985517:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.985518:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.985518:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.985519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.985519:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.985520:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.985520:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.985522:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.985526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.985527:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.985528:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677473.985528:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.985529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677473.985530:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677473.985531:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677473.985531:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.985531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677473.985533:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677473.985533:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.985534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.985534:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.985535:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677473.985535:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677473.985536:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.985536:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.985582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677473.985589:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.985592:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677473.985593:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677473.985595:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677473.985598:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.985599:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677473.985600:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.985602:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.985603:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.985604:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677473.985607:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677473.985609:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677473.985611:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.985612:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677473.985613:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677473.985614:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677473.985615:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.985615:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677473.985616:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677473.985627:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677473.985630:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677473.985630:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677473.985632:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677473.985634:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677473.985635:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677473.986093:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.986096:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.986098:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.986099:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.986100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.986101:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.986102:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.986103:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.986104:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.986105:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.986109:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.986110:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.986112:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.986113:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.986113:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.986115:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.986115:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.986116:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.986117:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.986127:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.986129:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.986130:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.986131:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.986133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.986134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677473.986168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677473.986175:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.986177:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677473.986178:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677473.986180:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.986182:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.986183:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677473.986185:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.986186:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.986187:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.986188:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677473.986191:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.986192:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677473.986194:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.986195:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677473.986195:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.986197:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.986198:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677473.986199:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677473.986200:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.986200:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677473.986203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677473.986206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.986212:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.986213:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677473.986213:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677473.986213:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.986214:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677473.986214:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677473.986215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.986215:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.986216:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677473.986217:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.986409:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.986411:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.986412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.986414:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.986416:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.986417:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.986418:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.986420:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.986421:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.986422:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.986425:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.986427:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.986429:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.986430:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.986431:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.986433:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.986433:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.986434:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.986435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.986447:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.986450:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.986450:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.986452:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.986453:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.986454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.986794:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.986797:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.986799:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.986801:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.986801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.986803:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.986804:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.986805:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.986806:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.986807:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.986811:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.986813:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.986816:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.986817:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.986817:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.986819:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.986820:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.986821:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.986821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.986833:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.986836:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.986836:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.986837:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.986839:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.986842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.987479:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.987482:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.987483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.987484:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.987486:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.987488:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.987489:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.987491:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.987492:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.987493:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.987496:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.987497:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.987500:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.987501:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.987501:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.987503:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.987504:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.987505:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.987505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.987518:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.987520:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.987521:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.987523:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.987524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.987525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.987883:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.987886:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.987888:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.987891:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.987891:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.987892:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.987894:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.987895:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.987896:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.987897:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.987900:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.987902:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.987904:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.987905:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.987906:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.987907:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.987908:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.987909:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.987909:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.987913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.987920:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.987923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.987923:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677473.987924:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.987925:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.987928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677473.988088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.988094:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.988096:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.988098:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.988099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.988101:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.988102:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677473.988104:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.988105:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.988106:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.988108:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677473.988111:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.988113:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677473.988115:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.988116:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677473.988117:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677473.988118:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.988119:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.988120:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.988120:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.988124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677473.988132:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677473.988135:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677473.988135:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.988137:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.988138:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677473.988139:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677473.988239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.988247:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.988250:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.988250:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.988252:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.988255:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.988257:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.988258:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.988259:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.988260:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.988261:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.988265:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.988266:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.988269:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.988270:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.988270:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.988272:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.988273:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.988273:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.988274:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.988286:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.988289:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.988290:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.988290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.988292:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.988303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.988305:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677473.988307:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677473.988308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.988309:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.988310:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677473.988310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.988351:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.988353:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.988354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.988356:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.988358:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.988359:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.988360:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.988362:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.988363:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.988364:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.988367:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.988369:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.988371:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.988372:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.988373:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.988375:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.988375:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.988375:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.988376:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.988388:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.988390:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.988391:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.988393:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.988394:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.988395:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677473.988406:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677473.988409:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677473.988409:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.988410:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677473.988411:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.988412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.988476:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.988479:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.988481:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.988483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.988484:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.988485:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.988487:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.988488:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.988489:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.988490:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.988493:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.988495:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.988498:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.988499:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.988499:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.988501:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.988502:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.988503:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.988503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.988515:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.988518:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.988519:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.988519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.988521:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.988524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.988579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.988586:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.988589:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.988591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.988592:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.988594:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.988595:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.988597:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.988598:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.988599:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.988600:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677473.988603:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.988604:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.988607:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.988608:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.988609:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.988610:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.988611:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.988611:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.988612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677473.988624:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.988627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.988627:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.988628:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.988630:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.988633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.988727:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.988730:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.988732:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.988734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.988735:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.988736:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677473.988738:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.988739:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.988740:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.988741:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677473.988744:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.988745:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677473.988748:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.988749:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677473.988750:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677473.988751:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677473.988752:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.988753:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.988755:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677473.988756:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677473.988758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677473.988769:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677473.988770:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677473.988772:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677473.988773:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.988773:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677473.988773:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.988774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677473.988775:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.988775:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677473.988777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677473.989024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.989032:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.989035:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.989036:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.989038:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677473.989039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:10.0:1619677473.989040:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.989042:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.989043:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.989044:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.989045:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677473.989046:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677473.989046:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:18.0:1619677473.989048:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:10.0:1619677473.989050:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.989050:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677473.989051:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677473.989052:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:18.0:1619677473.989054:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.989055:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.989055:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:18.0:1619677473.989056:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:10.0:1619677473.989056:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677473.989057:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677473.989057:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.989057:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677473.989058:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.989059:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677473.989059:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:18.0:1619677473.989060:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:10.0:1619677473.989060:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677473.989060:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677473.989063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677473.989064:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677473.989065:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677473.989066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677473.989068:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.989068:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677473.989069:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.989070:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677473.989071:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677473.989071:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.989072:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677473.989072:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677473.989073:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677473.989073:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677473.989073:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:8.0:1619677473.989073:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.989075:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677473.989075:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.989076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.989077:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677473.989077:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677473.989082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677473.989085:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.989088:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.989089:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.989089:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.989091:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677473.989094:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677473.989131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677473.989136:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.989139:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677473.989140:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677473.989142:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.989144:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.989146:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677473.989147:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.989149:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.989149:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.989151:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677473.989154:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.989155:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677473.989158:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.989159:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677473.989159:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677473.989161:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677473.989161:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677473.989162:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677473.989163:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677473.989164:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677473.989167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.2:1619677473.989169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:24.0:1619677473.989173:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677473.989175:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:24.0:1619677473.989176:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.989177:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.989177:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:8.0:1619677473.989177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:18.0:1619677473.989178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.989178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:24.0:1619677473.989179:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.989182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.989241:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.989243:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.989244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.989246:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.989248:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.989250:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.989251:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.989253:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.989254:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.989255:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.989258:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.989259:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.989262:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.989263:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.989263:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.989265:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.989266:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.989267:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.989268:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.989268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.989278:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677473.989279:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.989281:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677473.989281:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.989281:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677473.989282:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.989282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677473.989284:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.989285:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.989286:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.989608:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.989610:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677473.989612:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.989615:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.989615:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677473.989617:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.989618:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.989620:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.989620:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.989622:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.989625:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.989626:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.989628:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.989629:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677473.989630:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.989631:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677473.989632:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677473.989633:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.989633:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677473.989644:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.989647:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.989648:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.989649:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.989651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.989652:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677473.989989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677473.989995:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.989998:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677473.989999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677473.990001:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677473.990003:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677473.990004:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677473.990005:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677473.990007:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677473.990008:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677473.990009:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677473.990012:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677473.990014:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677473.990016:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677473.990017:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677473.990018:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677473.990019:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677473.990020:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677473.990020:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.990021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677473.990034:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.990036:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.990037:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.990037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677473.990039:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.990042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.990316:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.990319:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.990320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.990322:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.990323:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.990325:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.990326:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.990328:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.990329:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.990330:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.990334:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.990336:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.990339:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.990339:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:4.2:1619677473.990339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677473.990340:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.990342:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.990343:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.990344:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.990344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:10.0:1619677473.990347:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.990350:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.990351:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.990353:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.990355:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.990356:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:10.0:1619677473.990356:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677473.990358:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677473.990359:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.990359:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.990359:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.990360:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.990361:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677473.990362:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.990362:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677473.990365:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.990367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677473.990367:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677473.990370:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.990371:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677473.990371:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677473.990373:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677473.990373:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.990374:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.990375:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677473.990387:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.990390:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677473.990390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.990391:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.990393:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.990396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.990405:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.990408:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.990408:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677473.990409:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.990410:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.990410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677473.990621:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.990623:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.990625:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.990627:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.990628:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.990629:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.990630:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.990631:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.990632:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.990633:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.990636:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.990638:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.990641:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.990642:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.990642:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.990644:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.990645:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.990646:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.990648:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.990648:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677473.990657:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.990660:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677473.990661:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677473.990661:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.990661:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.990663:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.990664:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677473.990665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.990666:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677473.990667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.990671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677473.990678:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677473.990681:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677473.990683:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.990685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677473.990686:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.990687:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677473.990687:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677473.990688:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.990689:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.990690:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.990691:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.990692:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.990694:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677473.990696:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677473.990697:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.990697:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677473.990698:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677473.990699:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.990700:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.990701:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.990702:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677473.990705:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.990707:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.990708:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.990709:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.990709:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.990709:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677473.990709:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677473.990710:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677473.990711:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.990711:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677473.990712:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.990712:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677473.990713:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.990713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.990713:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677473.990714:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:31.0:1619677473.990715:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.990717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677473.990725:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:8.0:1619677473.990725:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:30.0:1619677473.990727:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:8.0:1619677473.990727:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:30.0:1619677473.990728:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.990728:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.990729:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.990729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677473.990731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:8.0:1619677473.990731:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.990733:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.0:1619677473.990733:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677473.991198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677473.991205:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.991208:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677473.991209:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677473.991210:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.991213:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.991213:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677473.991215:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:6.0:1619677473.991215:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:26.0:1619677473.991216:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.991216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677473.991218:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.991219:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.991219:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.991220:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677473.991221:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.991222:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.991223:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.991224:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677473.991225:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.991226:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677473.991227:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677473.991227:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.991230:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.991231:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:6.0:1619677473.991231:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.991232:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677473.991233:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677473.991233:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.991234:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677473.991235:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677473.991235:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:6.0:1619677473.991236:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.991237:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.991237:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.2:1619677473.991239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677473.991239:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.991240:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.991241:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.991241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.991250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677473.991252:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677473.991253:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.991254:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:20.0:1619677473.991254:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.991255:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.991256:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677473.991257:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677473.991258:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.991258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677473.991260:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.991261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.991685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677473.991690:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.991693:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677473.991695:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677473.991695:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677473.991697:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.991699:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677473.991700:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677473.991701:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677473.991702:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677473.991703:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677473.991707:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677473.991709:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677473.991711:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677473.991712:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677473.991713:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677473.991714:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677473.991715:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677473.991716:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.991716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677473.991721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677473.991730:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.991733:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677473.991734:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677473.991734:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.991736:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.991739:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677473.991904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677473.991911:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.991914:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677473.991915:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677473.991917:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.991920:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.991921:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677473.991923:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.991924:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.991925:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.991926:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677473.991929:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.991931:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677473.991934:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.991934:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677473.991935:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677473.991937:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677473.991937:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677473.991938:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.991938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677473.991943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.991950:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677473.991953:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677473.991954:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677473.991954:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677473.991956:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677473.991959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677473.992065:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.992068:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.992069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.992071:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.992073:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.992074:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.992076:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.992077:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.992078:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.992079:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.992082:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.992084:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.992086:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.992087:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.992087:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.992089:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.992090:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.992091:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.992091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.992103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.992105:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.992106:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.992108:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.992109:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.992110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.992123:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.992126:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.992127:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.992127:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.992128:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.992129:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677473.992460:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.992462:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677473.992465:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.992467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.992467:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.992469:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677473.992470:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.992471:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.992472:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.992473:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677473.992476:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.992478:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.992481:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.992482:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677473.992483:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677473.992484:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.992485:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677473.992486:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677473.992486:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677473.992500:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.992503:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.992504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.992504:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.992506:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.992509:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.992554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:10.0:1619677473.992562:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677473.992565:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677473.992565:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:10.0:1619677473.992567:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.992570:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.992572:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677473.992573:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.992574:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.992575:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.992576:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677473.992579:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.992581:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677473.992583:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.992583:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677473.992584:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677473.992585:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677473.992586:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677473.992587:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677473.992587:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:31.0:1619677473.992598:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.992600:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.992601:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.992603:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.992604:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.992606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677473.993010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677473.993017:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.993020:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677473.993021:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677473.993022:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677473.993024:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.993026:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677473.993027:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.993028:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677473.993029:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.993030:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677473.993032:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.993034:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677473.993037:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.993037:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677473.993038:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677473.993039:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.993040:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677473.993041:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677473.993041:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677473.993053:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677473.993056:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.993057:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.993057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677473.993059:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.993062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677473.993149:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.993151:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677473.993152:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677473.993154:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677473.993156:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.993157:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.993158:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677473.993160:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.993161:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.993162:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.993165:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677473.993167:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677473.993170:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677473.993170:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.993171:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677473.993173:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677473.993173:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677473.993175:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.993176:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.993176:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.993187:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.993188:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677473.993188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677473.993189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.993190:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.993191:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.993191:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.993193:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.993194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.993195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.993619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.993626:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.993629:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.993630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.993632:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.993635:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.993637:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.993638:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.993639:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.993640:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.993641:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.993644:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.993646:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.993649:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.993650:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.993650:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.993652:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.993652:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.993654:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677473.993655:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677473.993656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677473.993662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677473.993667:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677473.993669:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677473.993670:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677473.993671:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677473.993671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.993671:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.993672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677473.993673:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677473.993673:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.993676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.993805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677473.993813:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.993816:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677473.993817:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677473.993819:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.993821:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.993823:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677473.993824:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.993826:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.993827:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.993828:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677473.993831:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.993832:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677473.993834:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.993835:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677473.993835:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677473.993837:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677473.993837:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677473.993838:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.993838:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677473.993843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.993849:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.993851:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.993852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.993852:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.993854:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.993856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.993861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.993866:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.993869:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.993870:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677473.993870:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.993870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.993871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677473.993997:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.994000:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.994002:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.994004:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.994004:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.994006:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.994007:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.994008:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.994009:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.994010:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.994013:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.994014:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677473.994016:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.994017:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.994018:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677473.994019:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677473.994020:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.994021:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.994021:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677473.994024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.994031:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.994033:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.994034:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.994034:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.994036:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.994039:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677473.994062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.994064:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.994065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.994067:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.994069:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.994070:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.994072:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.994073:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.994074:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.994075:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.994079:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.994081:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.994083:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.994084:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.994084:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.994086:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.994086:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.994087:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.994088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.994099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.994102:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.994103:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.994104:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.994105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.994106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677473.994503:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.994506:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.994508:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.994510:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.994510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.994512:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.994513:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.994514:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.994515:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.994516:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.994519:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.994521:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.994524:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.994525:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.994526:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.994527:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.994528:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.994529:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.994529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677473.994543:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.994546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.994546:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.994547:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.994549:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.994552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.994561:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677473.994564:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.994565:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.994565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.994566:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677473.994566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677473.994982:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.994985:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677473.994987:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.994989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677473.994990:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.994991:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677473.994992:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.994994:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.994995:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.994996:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677473.994999:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.995001:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677473.995002:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677473.995003:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.995004:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677473.995005:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677473.995005:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:30.0:1619677473.995006:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677473.995006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677473.995007:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677473.995007:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.995008:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677473.995009:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677473.995009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.995010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677473.995010:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677473.995011:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677473.995012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:6.0:1619677473.995013:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677473.995014:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677473.995015:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677473.995018:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.995019:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677473.995020:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677473.995021:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677473.995022:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677473.995022:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.995023:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677473.995023:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.995023:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677473.995023:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:24.0:1619677473.995024:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.995025:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677473.995026:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677473.995026:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677473.995026:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677473.995027:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677473.995027:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.995027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677473.995028:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677473.995038:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.995041:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.995042:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.995043:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.995045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.995046:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677473.995165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677473.995171:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.995173:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677473.995174:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677473.995176:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.995179:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.995180:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677473.995181:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.995183:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.995184:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.995185:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677473.995187:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.995189:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:4.2:1619677473.995190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:1.0:1619677473.995191:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.995192:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677473.995193:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677473.995195:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677473.995195:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677473.995196:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677473.995197:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:1.0:1619677473.995197:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677473.995198:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677473.995200:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677473.995202:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677473.995203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677473.995204:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.995204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677473.995206:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677473.995207:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677473.995208:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677473.995209:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.995209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:31.0:1619677473.995210:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677473.995210:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:31.0:1619677473.995211:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.995211:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677473.995211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677473.995212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677473.995213:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.995213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:4.2:1619677473.995213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677473.995215:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677473.995215:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.995218:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677473.995218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677473.995219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:31.0:1619677473.995221:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.995221:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677473.995221:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677473.995222:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677473.995223:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677473.995224:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677473.995224:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677473.995225:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:2.0:1619677473.995225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677473.995226:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000400:00000200:26.0:1619677473.995226:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.995227:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677473.995228:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:26.0:1619677473.995228:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.995228:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:26.0:1619677473.995229:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677473.995229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677473.995231:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.995232:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.995232:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677473.995233:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.995234:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677473.995234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000400:00000200:2.0:1619677473.995234:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.995236:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.995238:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96506610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677473.995240:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677473.995240:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677473.995241:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000400:00000200:2.0:1619677473.995241:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.995242:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:26.0:1619677473.995243:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677473.995244:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:30.0:1619677473.995245:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677473.995246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:26.0:1619677473.995246:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.995247:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:2.0:1619677473.995247:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677473.995248:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677473.995249:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:9.2:1619677473.995249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677473.995249:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677473.995250:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677473.995250:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677473.995251:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677473.995252:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.995253:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.995254:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.995256:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.995257:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677473.995258:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677473.995258:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677473.995259:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:18.0:1619677473.995260:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.995260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.995261:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.995263:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.995265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677473.995269:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.995272:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677473.995273:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.995274:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:6.0:1619677473.995274:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677473.995275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.995277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:22.0:1619677473.995277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.995277:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.995278:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.995280:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.995282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677473.996133:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.996136:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.996137:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.996139:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.996141:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.996142:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.996143:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.996145:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.996146:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.996147:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.996151:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.996153:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.996155:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.996156:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.996157:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.996159:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.996159:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.996160:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.996161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677473.996173:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677473.996175:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677473.996176:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.996178:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677473.996179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.996180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677473.996191:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.996194:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.996195:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.996195:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.996196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.996197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.996253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677473.996262:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.996265:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677473.996267:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677473.996270:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677473.996270:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677473.996272:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677473.996274:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677473.996275:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677473.996276:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677473.996278:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677473.996282:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677473.996283:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677473.996286:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677473.996287:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677473.996288:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677473.996289:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677473.996290:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677473.996291:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677473.996291:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677473.996306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677473.996313:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.996316:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677473.996317:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677473.996317:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.996319:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.996322:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677473.996330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:4.2:1619677473.996334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677473.996336:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.996339:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677473.996339:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677473.996339:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.996340:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677473.996341:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.0:1619677473.996341:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.996344:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.996345:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.996347:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.996349:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.996351:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.996352:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.996353:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.996354:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.996355:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.996358:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.996359:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.996362:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.996363:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.996364:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.996366:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.996366:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.996367:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.996367:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677473.996370:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.996372:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:9.2:1619677473.996372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:31.0:1619677473.996374:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677473.996377:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.996377:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.996379:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.996380:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677473.996380:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:31.0:1619677473.996381:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.996382:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.996383:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.996383:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677473.996384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677473.996384:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.996387:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677473.996387:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.996389:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677473.996390:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677473.996393:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.996393:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.996394:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:9.2:1619677473.996394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677473.996396:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.996396:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.996397:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677473.996398:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:18.0:1619677473.996402:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.996405:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.996405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.996406:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.996407:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677473.996408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677473.996410:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.996413:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.996414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677473.996414:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.996416:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677473.996419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677473.997021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677473.997028:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.997030:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677473.997031:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677473.997033:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.997035:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.997037:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677473.997038:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.997039:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.997040:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677473.997041:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677473.997043:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.997045:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677473.997046:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.997047:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677473.997048:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.997049:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677473.997050:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677473.997050:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677473.997051:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677473.997055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677473.997061:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.997063:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677473.997064:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.997064:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677473.997065:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.997067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677473.997122:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.997125:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.997125:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.997128:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.997130:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.997131:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.997132:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.997134:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.997135:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.997136:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.997139:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.997140:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.997143:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.997144:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.997144:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.997146:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.997147:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.997148:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677473.997149:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677473.997149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.997160:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677473.997162:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677473.997162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677473.997163:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677473.997164:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.997166:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.997167:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.997168:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.997169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.997170:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677473.997612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677473.997615:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.997618:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677473.997620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677473.997620:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677473.997622:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.997623:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677473.997625:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.997626:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.997627:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.997628:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677473.997631:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.997633:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677473.997635:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.997636:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677473.997637:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.997638:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.2:1619677473.997640:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.0:1619677473.997643:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677473.997644:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677473.997645:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:8.2:1619677473.997645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677473.997646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677473.997647:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.997649:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677473.997650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677473.997651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677473.997652:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677473.997654:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:2.0:1619677473.997655:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.997656:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:2.0:1619677473.997656:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677473.997657:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677473.997658:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677473.997658:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:2.0:1619677473.997658:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.997658:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.997659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677473.997660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:2.0:1619677473.997660:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.997660:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677473.997661:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.997662:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677473.997663:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677473.997664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:2.0:1619677473.997666:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.997667:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677473.997670:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.997670:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677473.997671:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677473.997672:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:2.0:1619677473.997673:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677473.997673:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677473.997674:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677473.997674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677473.997675:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677473.997676:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677473.997676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.997677:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677473.997678:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677473.997680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677473.997687:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677473.997689:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677473.997690:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.997690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.997692:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.997694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677473.997783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907e3cf600] (20)++
00000800:00000200:4.2:1619677473.997787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677473.997788:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)++
00000800:00000200:2.0:1619677473.997791:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.1@o2ib3
00000800:00000200:18.0:1619677473.997792:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000400:00000200:2.0:1619677473.997793:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.1@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677473.997794:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677473.997795:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.997797:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677473.997797:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677473.997797:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677473.997799:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.997801:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677473.997802:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.997802:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.997803:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.997804:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.997807:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677473.997807:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.997808:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677473.997809:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.997810:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677473.997810:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677473.997811:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677473.997811:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677473.997811:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:10.0:1619677473.997812:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677473.997813:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.997813:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907e3cf600] (21)++
00000800:00000200:2.0:1619677473.997814:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907e3cf600] (22)--
00000800:00000200:2.0:1619677473.997815:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)--
00000400:00000200:10.0:1619677473.997816:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677473.997817:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677473.997818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677473.997820:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677473.997821:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677473.997822:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677473.997824:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677473.997824:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677473.997825:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:18.0:1619677473.997826:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.997827:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:10.0:1619677473.997827:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677473.997827:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677473.997828:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.997829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.997830:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.997832:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677473.997833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677473.997838:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677473.997839:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677473.997841:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677473.997841:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677473.997842:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677473.997842:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677473.997842:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.997843:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:4.2:1619677473.997843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907e3cf600] (20)++
00000400:00000200:22.0:1619677473.997844:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.997847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677473.997849:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)++
00000800:00000200:2.0:1619677473.997852:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.1@o2ib3
00000800:00000200:18.0:1619677473.997853:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000400:00000200:2.0:1619677473.997854:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.1@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677473.997856:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:2.0:1619677473.997856:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.997857:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677473.997858:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677473.997859:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677473.997860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677473.997860:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000400:00000200:2.0:1619677473.997860:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677473.997861:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000400:00000200:2.0:1619677473.997861:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677473.997862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:2.0:1619677473.997862:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677473.997865:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.997867:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.997869:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.997869:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677473.997870:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.997871:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677473.997872:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907e3cf600] (21)++
00000800:00000200:2.0:1619677473.997873:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907e3cf600] (22)--
00000800:00000200:2.0:1619677473.997874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)--
00000800:00000200:9.2:1619677473.997877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677473.997885:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.997887:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677473.997887:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677473.997888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677473.997889:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.997890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677473.997914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907e3cf600] (20)++
00000800:00000200:2.0:1619677473.997919:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)++
00000800:00000200:2.0:1619677473.997922:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.1@o2ib3
00000800:00000200:18.0:1619677473.997923:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000400:00000200:2.0:1619677473.997924:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.1@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.997926:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.997927:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677473.997928:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.997929:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.997930:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.997931:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677473.997934:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.997935:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677473.997938:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.997939:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677473.997939:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677473.997941:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677473.997941:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907e3cf600] (21)++
00000800:00000200:2.0:1619677473.997942:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907e3cf600] (22)--
00000800:00000200:2.0:1619677473.997942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)--
00000800:00000200:9.2:1619677473.997946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677473.997947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907e3cf600] (20)++
00000800:00000200:24.0:1619677473.997953:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677473.997953:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)++
00000800:00000200:18.0:1619677473.997955:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.1@o2ib3
00000800:00000200:24.0:1619677473.997956:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:2.0:1619677473.997956:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000400:00000200:24.0:1619677473.997957:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.997957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:18.0:1619677473.997958:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.1@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.997959:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.997960:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:18.0:1619677473.997961:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677473.997963:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.997964:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.997965:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.997966:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677473.997968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:18.0:1619677473.997970:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.997972:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677473.997975:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.997976:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677473.997976:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677473.997978:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677473.997979:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907e3cf600] (21)++
00000800:00000200:18.0:1619677473.997980:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907e3cf600] (22)++
00000800:00000200:18.0:1619677473.997981:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907e3cf600] (23)--
00000800:00000200:18.0:1619677473.997982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000800:00000200:4.2:1619677473.997987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907e3cf600] (21)++
00000800:00000200:30.0:1619677473.997992:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:2.0:1619677473.997992:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)++
00000800:00000200:30.0:1619677473.997995:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:2.0:1619677473.997995:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907e3cf600] (23)--
00000800:00000200:2.0:1619677473.997995:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000400:00000200:30.0:1619677473.997996:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677473.997996:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)--
00000400:00000200:30.0:1619677473.997998:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.997999:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.998001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677473.998007:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.998010:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.998011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.998013:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677473.998015:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.998016:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.998018:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.998019:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677473.998020:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.998021:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677473.998025:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677473.998026:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677473.998029:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.998030:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677473.998030:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677473.998032:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.998033:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.998034:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.998034:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:4.2:1619677473.998035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907e3cf600] (20)++
00000800:00000200:2.0:1619677473.998041:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)++
00000800:00000200:2.0:1619677473.998044:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.1@o2ib3
00000800:00000200:21.0:1619677473.998045:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677473.998045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000800:00000200:21.0:1619677473.998047:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:2.0:1619677473.998047:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.1@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677473.998048:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.998049:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.998050:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.998051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:2.0:1619677473.998051:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:21.0:1619677473.998052:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:2.0:1619677473.998052:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.998053:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.998054:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.998055:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677473.998059:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.998060:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677473.998063:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.998063:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677473.998064:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677473.998066:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677473.998066:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907e3cf600] (21)++
00000800:00000200:2.0:1619677473.998067:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907e3cf600] (22)--
00000800:00000200:2.0:1619677473.998068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)--
00000800:00000200:8.2:1619677473.998072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677473.998080:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.998083:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.998084:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.998084:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.998086:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677473.998088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677473.998094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677473.998101:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677473.998104:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677473.998105:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677473.998105:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.998106:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677473.998106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677473.998128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907e3cf600] (20)++
00000800:00000200:18.0:1619677473.998134:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)++
00000800:00000200:18.0:1619677473.998137:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.1@o2ib3
00000800:00000200:2.0:1619677473.998137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000400:00000200:18.0:1619677473.998139:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.1@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.998141:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.998143:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677473.998144:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.998145:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.998146:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.998147:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677473.998151:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.998152:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677473.998155:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.998156:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677473.998156:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677473.998158:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677473.998158:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907e3cf600] (21)++
00000800:00000200:18.0:1619677473.998159:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907e3cf600] (22)--
00000800:00000200:18.0:1619677473.998160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)--
00000800:00000200:9.2:1619677473.998163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677473.998170:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.998173:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.998174:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.998174:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.998177:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.998180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677473.998181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.998187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.998190:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.998191:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.998191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.998193:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.998193:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.998211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907e3cf600] (20)++
00000800:00000200:2.0:1619677473.998216:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)++
00000800:00000200:2.0:1619677473.998219:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.1@o2ib3
00000800:00000200:18.0:1619677473.998220:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (22)--
00000400:00000200:2.0:1619677473.998222:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.1@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677473.998225:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.998226:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677473.998227:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677473.998228:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677473.998229:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677473.998230:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677473.998234:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.998235:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677473.998238:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677473.998239:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677473.998239:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677473.998241:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677473.998242:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907e3cf600] (21)++
00000800:00000200:2.0:1619677473.998243:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907e3cf600] (22)--
00000800:00000200:2.0:1619677473.998243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cf600] (21)--
00000800:00000200:31.0:1619677473.998253:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677473.998256:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677473.998257:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.998259:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.998261:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677473.998262:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677473.998272:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677473.998274:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677473.998275:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.998276:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677473.998277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.998278:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677473.998329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:31.0:1619677473.998334:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677473.998335:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:31.0:1619677473.998336:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677473.998337:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.998338:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677473.998338:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677473.998338:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:31.0:1619677473.998339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:2.0:1619677473.998339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677473.998341:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677473.998344:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677473.998345:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677473.998347:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677473.998348:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677473.998349:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677473.998350:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677473.998353:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677473.998354:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677473.998357:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677473.998358:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677473.998358:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677473.998360:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677473.998361:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677473.998362:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677473.998363:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677473.998366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:31.0:1619677473.998372:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:26.0:1619677473.998373:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:31.0:1619677473.998374:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677473.998376:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.998376:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677473.998376:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.998377:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677473.998378:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677473.998379:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.998379:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:31.0:1619677473.998380:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677473.998382:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677473.998382:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:31.0:1619677473.998383:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.998384:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.998385:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677473.998388:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.998390:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677473.998393:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.998394:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677473.998394:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677473.998396:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677473.998397:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677473.998398:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.998400:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677473.998400:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677473.998411:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677473.998413:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677473.998413:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677473.998414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677473.998415:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677473.998415:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677473.998416:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677473.998417:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677473.998417:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.998418:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677473.998418:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677473.998420:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677473.998421:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677473.998421:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677473.998423:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.998424:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677473.998426:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677473.998427:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677473.998428:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677473.998429:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677473.998432:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677473.998433:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677473.998436:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677473.998436:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677473.998437:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677473.998438:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.998439:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677473.998440:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677473.998440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677473.998454:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677473.998456:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677473.998457:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.998458:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.998460:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677473.998461:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677473.998489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677473.998497:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.998499:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677473.998500:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677473.998502:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.998505:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.998506:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677473.998508:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.998509:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.998510:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.998511:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677473.998513:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.998515:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677473.998518:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.998518:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677473.998519:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677473.998521:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677473.998521:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677473.998522:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677473.998522:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677473.998534:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.998537:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677473.998538:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677473.998538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677473.998541:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677473.998544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677473.998553:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677473.998555:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677473.998556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.998556:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677473.998557:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.998558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677473.998650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677473.998657:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677473.998660:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677473.998662:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677473.998664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677473.998665:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677473.998666:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677473.998667:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677473.998669:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677473.998670:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677473.998671:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677473.998674:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677473.998675:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677473.998678:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677473.998678:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:31.0:1619677473.998679:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:31.0:1619677473.998681:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677473.998681:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677473.998682:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677473.998683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677473.998686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.998693:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677473.998695:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677473.998696:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.998696:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.998698:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.998700:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677473.998744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677473.998748:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.998751:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677473.998753:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677473.998754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677473.998755:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677473.998757:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677473.998758:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677473.998759:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677473.998760:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677473.998761:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677473.998764:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677473.998765:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677473.998768:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677473.998769:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677473.998770:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677473.998772:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677473.998773:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677473.998773:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677473.998774:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677473.998778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677473.998784:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677473.998787:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677473.998788:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.998788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677473.998790:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677473.998793:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677473.998833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677473.998837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677473.998837:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000400:00000200:1.0:1619677473.998839:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677473.998842:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677473.998843:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677473.998845:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677473.998846:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677473.998847:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677473.998848:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677473.998851:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677473.998852:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677473.998855:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677473.998856:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677473.998857:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677473.998858:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677473.998859:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677473.998860:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677473.998861:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677473.998862:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:9.2:1619677473.998866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677473.998872:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:8.0:1619677473.998873:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba0200] (21)++
00000800:00000200:24.0:1619677473.998874:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677473.998874:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:17.0:1619677473.998874:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677473.998875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:8.0:1619677473.998875:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.33@o2ib3
00000800:00000200:24.0:1619677473.998876:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677473.998877:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.998878:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.1.33@o2ib3 : GET - for me
00000400:00000200:24.0:1619677473.998879:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677473.998882:0:19938:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.1.33@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:24.0:1619677473.998883:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.998884:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba0200] (22)--
00000400:00000200:8.0:1619677473.998884:0:19938:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.1.33@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:24.0:1619677473.998885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677473.998886:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c78ba0200] (21)++
00000800:00000200:8.0:1619677473.998887:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c78ba0200] (22)--
00000400:00000200:8.0:1619677473.998889:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.1.33@o2ib3 local destination
00000400:00000200:8.0:1619677473.998893:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.1.33@o2ib3(10.51.1.33@o2ib3:10.51.1.33@o2ib3) : REPLY try# 0
00000800:00000200:8.0:1619677473.998895:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.1.33@o2ib3
00000800:00000200:8.0:1619677473.998897:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919fedea5d00] -> 10.51.1.33@o2ib3 (2) version: 12
00000800:00000200:8.0:1619677473.998898:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919c78ba0200] (21)++
00000800:00000200:8.0:1619677473.998899:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c78ba0200] (22)++
00000800:00000200:8.0:1619677473.998900:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919c78ba0200] (23)--
00000800:00000200:8.0:1619677473.998901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba0200] (22)--
00000800:00000200:24.0:1619677473.998913:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c78ba0200] (22)++
00000800:00000200:24.0:1619677473.998915:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c78ba0200] (23)--
00000400:00000200:24.0:1619677473.998915:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.998917:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.1.33@o2ib3: REPLY: OK
00000800:00000200:8.0:1619677473.998917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba0200] (22)--
00000800:00000200:24.0:1619677473.998919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c78ba0200] (21)--
00000800:00000200:22.0:1619677473.999067:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677473.999071:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677473.999071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677473.999073:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677473.999073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677473.999076:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677473.999077:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677473.999078:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677473.999080:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677473.999081:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677473.999081:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677473.999082:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:26.0:1619677473.999084:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677473.999084:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677473.999086:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677473.999087:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677473.999088:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:26.0:1619677473.999090:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.999091:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:22.0:1619677473.999091:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677473.999091:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:26.0:1619677473.999092:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677473.999092:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:26.0:1619677473.999094:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677473.999094:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677473.999094:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677473.999095:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677473.999095:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677473.999095:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677473.999096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:26.0:1619677473.999098:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.999100:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677473.999103:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.999104:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677473.999105:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677473.999106:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677473.999107:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677473.999107:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677473.999108:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677473.999109:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677473.999109:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677473.999110:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677473.999110:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677473.999112:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.999112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677473.999113:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677473.999114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677473.999115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677473.999118:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.999120:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.999121:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.999121:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.999122:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677473.999124:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677473.999124:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677473.999125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677473.999126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677473.999126:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.999215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (20)++
00000800:00000200:24.0:1619677473.999223:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999226:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000800:00000200:8.0:1619677473.999227:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000400:00000200:24.0:1619677473.999228:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.999231:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.999232:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677473.999233:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.999235:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.999236:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.999237:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:24.0:1619677473.999240:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.999241:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:4.2:1619677473.999242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999245:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.999246:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677473.999247:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677473.999250:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677473.999250:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (22)++
00000800:00000200:8.0:1619677473.999250:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)++
00000800:00000200:24.0:1619677473.999252:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (24)--
00000800:00000200:8.0:1619677473.999252:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000800:00000200:24.0:1619677473.999253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (23)--
00000800:00000200:24.0:1619677473.999254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000400:00000200:8.0:1619677473.999254:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.999255:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.999257:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677473.999258:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.999259:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.999260:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.999260:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677473.999262:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677473.999263:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677473.999265:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677473.999265:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:20.0:1619677473.999266:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677473.999268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677473.999268:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.999268:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.999269:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:8.0:1619677473.999269:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:8.0:1619677473.999271:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:20.0:1619677473.999272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677473.999272:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (21)++
00000800:00000200:8.0:1619677473.999272:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (22)--
00000800:00000200:8.0:1619677473.999273:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)--
00000800:00000200:9.2:1619677473.999277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677473.999280:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677473.999282:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677473.999283:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677473.999284:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677473.999285:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677473.999285:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677473.999285:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677473.999288:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677473.999289:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.999290:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.999292:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677473.999298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.999299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (20)++
00000800:00000200:18.0:1619677473.999306:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677473.999306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:24.0:1619677473.999307:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)++
00000800:00000200:18.0:1619677473.999308:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677473.999309:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:24.0:1619677473.999310:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000800:00000200:18.0:1619677473.999310:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677473.999311:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:8.0:1619677473.999311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000800:00000200:18.0:1619677473.999312:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:24.0:1619677473.999313:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.999316:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.999317:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677473.999318:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.999319:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.999320:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.999321:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677473.999324:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.999325:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677473.999328:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.999329:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:24.0:1619677473.999329:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.999329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999331:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:24.0:1619677473.999331:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (22)++
00000800:00000200:24.0:1619677473.999333:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d1400] (23)++
00000800:00000200:24.0:1619677473.999335:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (24)--
00000800:00000200:24.0:1619677473.999336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (23)--
00000800:00000200:9.2:1619677473.999337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:24.0:1619677473.999338:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)++
00000800:00000200:24.0:1619677473.999340:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000800:00000200:8.0:1619677473.999340:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (23)++
00000800:00000200:8.0:1619677473.999341:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d1400] (24)--
00000400:00000200:24.0:1619677473.999342:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677473.999342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (23)--
00000400:00000200:24.0:1619677473.999343:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.999343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000800:00000200:2.0:1619677473.999344:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:24.0:1619677473.999345:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677473.999346:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.999347:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677473.999347:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:24.0:1619677473.999348:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677473.999348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.999348:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.999349:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677473.999351:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677473.999353:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677473.999354:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677473.999355:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677473.999358:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.999358:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677473.999359:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677473.999360:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677473.999361:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999361:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (22)--
00000800:00000200:24.0:1619677473.999362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)--
00000800:00000200:6.0:1619677473.999375:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677473.999379:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677473.999379:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677473.999380:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677473.999382:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:4.2:1619677473.999382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (20)++
00000800:00000200:6.0:1619677473.999385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677473.999389:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)++
00000800:00000200:8.0:1619677473.999392:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000800:00000200:24.0:1619677473.999393:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000400:00000200:8.0:1619677473.999394:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.999397:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.999399:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677473.999400:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.999402:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.999403:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.999403:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677473.999407:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.999408:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677473.999410:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.999411:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677473.999411:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677473.999413:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677473.999414:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (21)++
00000800:00000200:8.0:1619677473.999414:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (22)--
00000800:00000200:8.0:1619677473.999415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)--
00000800:00000200:9.2:1619677473.999419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677473.999426:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677473.999428:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677473.999429:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677473.999429:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677473.999431:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:4.2:1619677473.999431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (20)++
00000800:00000200:26.0:1619677473.999433:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:24.0:1619677473.999438:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999441:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000800:00000200:8.0:1619677473.999442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000400:00000200:24.0:1619677473.999443:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.999445:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.999447:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:24.0:1619677473.999448:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.999449:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.999450:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.999450:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677473.999453:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.999455:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677473.999456:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.999457:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:24.0:1619677473.999458:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677473.999459:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677473.999460:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999460:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (22)--
00000800:00000200:24.0:1619677473.999461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)--
00000800:00000200:4.2:1619677473.999470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (20)++
00000800:00000200:30.0:1619677473.999473:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677473.999476:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677473.999476:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.999477:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)++
00000400:00000200:30.0:1619677473.999478:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677473.999480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677473.999480:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000800:00000200:31.0:1619677473.999481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:24.0:1619677473.999481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000400:00000200:8.0:1619677473.999482:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.999485:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.999486:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677473.999488:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.999489:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.999490:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.999490:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677473.999493:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.999495:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677473.999497:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.999498:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677473.999499:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677473.999500:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677473.999501:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (21)++
00000800:00000200:8.0:1619677473.999502:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d1400] (22)++
00000800:00000200:8.0:1619677473.999503:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (23)--
00000800:00000200:8.0:1619677473.999503:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000800:00000200:8.2:1619677473.999506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677473.999508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (21)++
00000800:00000200:21.0:1619677473.999514:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:24.0:1619677473.999515:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)++
00000800:00000200:24.0:1619677473.999517:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d1400] (23)--
00000800:00000200:21.0:1619677473.999517:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:24.0:1619677473.999518:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000400:00000200:21.0:1619677473.999518:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.999519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)--
00000400:00000200:21.0:1619677473.999520:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.999520:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677473.999523:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677473.999526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (20)++
00000800:00000200:8.0:1619677473.999533:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000800:00000200:8.0:1619677473.999536:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000400:00000200:8.0:1619677473.999538:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677473.999541:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.999542:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677473.999543:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.999545:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.999545:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.999546:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677473.999549:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.999551:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677473.999553:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.999554:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677473.999554:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677473.999556:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677473.999557:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (21)++
00000800:00000200:8.0:1619677473.999557:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (22)--
00000800:00000200:8.0:1619677473.999558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)--
00000800:00000200:4.2:1619677473.999573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (20)++
00000800:00000200:24.0:1619677473.999580:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999582:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000800:00000200:8.0:1619677473.999583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000400:00000200:24.0:1619677473.999584:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.999586:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.999587:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677473.999588:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.999589:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677473.999589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:24.0:1619677473.999590:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.999591:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677473.999593:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.999595:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677473.999596:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.999596:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677473.999597:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677473.999598:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677473.999598:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (21)++
00000800:00000200:18.0:1619677473.999598:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677473.999599:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (22)--
00000800:00000200:24.0:1619677473.999599:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)--
00000800:00000200:18.0:1619677473.999601:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677473.999601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677473.999602:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677473.999603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677473.999604:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677473.999607:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677473.999610:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677473.999612:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677473.999613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677473.999613:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677473.999615:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.999617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677473.999703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (20)++
00000800:00000200:24.0:1619677473.999710:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999712:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000800:00000200:8.0:1619677473.999713:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000400:00000200:24.0:1619677473.999714:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.999716:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.999717:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677473.999718:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.999719:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677473.999719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (21)++
00000400:00000200:24.0:1619677473.999720:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.999721:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677473.999724:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.999726:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677473.999727:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)++
00000800:00000200:8.0:1619677473.999728:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000800:00000200:24.0:1619677473.999729:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.999730:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:24.0:1619677473.999730:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:8.0:1619677473.999730:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677473.999732:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:8.0:1619677473.999732:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.999733:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (23)++
00000800:00000200:24.0:1619677473.999734:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (24)--
00000800:00000200:24.0:1619677473.999734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (23)--
00000400:00000200:8.0:1619677473.999734:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:24.0:1619677473.999736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000400:00000200:8.0:1619677473.999736:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.999737:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.999738:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.999739:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677473.999742:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677473.999743:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677473.999745:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:8.0:1619677473.999746:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677473.999746:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:12.0:1619677473.999747:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:8.0:1619677473.999747:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:12.0:1619677473.999748:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677473.999749:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677473.999749:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (21)++
00000400:00000200:12.0:1619677473.999750:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:8.0:1619677473.999750:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d1400] (22)++
00000800:00000200:8.0:1619677473.999751:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (23)--
00000800:00000200:12.0:1619677473.999752:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.0:1619677473.999752:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000800:00000200:13.0:1619677473.999753:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677473.999755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677473.999756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999763:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)++
00000800:00000200:1.0:1619677473.999763:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677473.999766:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d1400] (23)--
00000800:00000200:24.0:1619677473.999766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000800:00000200:1.0:1619677473.999766:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677473.999767:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677473.999767:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677473.999768:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)++
00000400:00000200:1.0:1619677473.999769:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677473.999771:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000800:00000200:8.0:1619677473.999773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000800:00000200:1.0:1619677473.999773:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:24.0:1619677473.999775:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.999778:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.999779:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677473.999780:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.999782:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.999783:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.999783:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677473.999786:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.999787:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677473.999789:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.999790:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:24.0:1619677473.999790:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:4.2:1619677473.999790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:24.0:1619677473.999792:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:24.0:1619677473.999792:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999793:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (22)--
00000800:00000200:24.0:1619677473.999794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)--
00000800:00000200:4.2:1619677473.999796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (20)++
00000800:00000200:9.2:1619677473.999797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.999800:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677473.999802:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:8.0:1619677473.999803:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)++
00000800:00000200:26.0:1619677473.999804:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:10.0:1619677473.999805:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677473.999806:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:24.0:1619677473.999806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000800:00000200:8.0:1619677473.999806:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000400:00000200:26.0:1619677473.999807:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.999807:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677473.999809:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677473.999809:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677473.999809:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677473.999810:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677473.999810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677473.999812:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.999812:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677473.999813:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.999813:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677473.999814:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677473.999815:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677473.999815:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677473.999816:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.999818:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677473.999818:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.999820:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:8.0:1619677473.999820:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677473.999821:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677473.999821:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677473.999822:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677473.999823:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677473.999823:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677473.999824:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677473.999824:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.999825:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:8.0:1619677473.999825:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677473.999826:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677473.999826:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677473.999826:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:8.0:1619677473.999826:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677473.999827:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677473.999828:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677473.999829:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677473.999829:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:8.0:1619677473.999829:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (21)++
00000800:00000200:10.0:1619677473.999830:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:8.0:1619677473.999830:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (22)--
00000800:00000200:8.0:1619677473.999831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)--
00000800:00000200:10.0:1619677473.999832:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677473.999832:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677473.999834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:9.2:1619677473.999835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677473.999838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677473.999843:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:20.0:1619677473.999844:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677473.999845:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:26.0:1619677473.999846:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677473.999846:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:18.0:1619677473.999847:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677473.999847:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677473.999848:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677473.999848:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677473.999848:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677473.999849:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677473.999850:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677473.999850:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677473.999850:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677473.999854:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:2.0:1619677473.999854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677473.999866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d1400] (20)++
00000800:00000200:24.0:1619677473.999873:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999876:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.64@o2ib3
00000800:00000200:8.0:1619677473.999877:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (22)--
00000400:00000200:24.0:1619677473.999878:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677473.999881:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677473.999882:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677473.999883:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677473.999885:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677473.999886:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677473.999886:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677473.999889:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677473.999891:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677473.999893:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677473.999894:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677473.999894:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677473.999896:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677473.999896:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d1400] (21)++
00000800:00000200:24.0:1619677473.999897:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d1400] (22)--
00000800:00000200:24.0:1619677473.999897:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d1400] (21)--
00000800:00000200:30.0:1619677473.999908:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677473.999911:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677473.999912:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677473.999913:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677473.999915:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677473.999916:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:8.2:1619677474.000026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:5.2:1619677474.000029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.000036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.000036:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.000038:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.000038:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677474.000041:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.113@o2ib7 : GET - for me
00000400:00000200:1.0:1619677474.000041:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.000045:0:19930:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.113@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1619677474.000045:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.000047:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.000048:0:19930:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.113@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000400:00000200:1.0:1619677474.000048:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.000050:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000400:00000200:1.0:1619677474.000050:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.000051:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.000051:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.000052:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.000054:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677474.000056:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.000058:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : REPLY try# 0
00000800:00000200:1.0:1619677474.000058:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.000060:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.000062:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.000062:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.000063:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.000063:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.000064:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (23)++
00000800:00000200:1.0:1619677474.000064:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.000066:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (24)--
00000800:00000200:1.0:1619677474.000066:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.000067:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.000067:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.000067:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.000068:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.000068:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.000069:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677474.000071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.000079:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:22.0:1619677474.000080:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.000081:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.000081:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.000083:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.000083:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.000083:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: REPLY: OK
00000400:00000200:22.0:1619677474.000084:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.000084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.000085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:22.0:1619677474.000086:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.000089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.000130:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.000133:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.000136:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.000137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.000138:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.000140:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.000141:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.000142:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.000144:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.000144:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.000147:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.000149:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.000153:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.000154:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.000154:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.000156:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.000157:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.000158:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.000158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.000170:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.000173:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.000173:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.000174:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.000176:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.000179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.000224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677474.000231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677474.000231:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.000234:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.000235:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.000237:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.000238:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677474.000239:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.000240:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:10.0:1619677474.000241:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677474.000241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677474.000242:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.000244:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.000245:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.000246:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.000247:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.000247:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677474.000249:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.000251:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.000252:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677474.000253:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.000253:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.000254:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677474.000255:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.000258:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.000258:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.000259:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.000259:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677474.000261:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.000261:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.000261:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.000262:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.000263:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677474.000264:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.000265:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.000266:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.000268:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.000268:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.000270:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.000271:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.000272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677474.000275:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.000277:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.2:1619677474.000277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677474.000278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.000279:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.000280:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:22.0:1619677474.000281:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.000283:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:2.0:1619677474.000283:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:22.0:1619677474.000284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:12.0:1619677474.000284:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.000285:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:12.0:1619677474.000286:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.000286:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677474.000287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.000287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677474.000289:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677474.000417:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.000419:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677474.000420:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677474.000422:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.000423:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.000423:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:31.0:1619677474.000424:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.000424:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.000426:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.000427:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.000428:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.000429:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.000430:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.000430:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677474.000431:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.000432:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.000433:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677474.000434:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677474.000434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:6.0:1619677474.000436:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.000437:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:30.0:1619677474.000439:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.000440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:30.0:1619677474.000442:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.000442:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677474.000442:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.000445:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.000445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.000445:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677474.000445:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.000446:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.000446:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.000447:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677474.000448:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.000448:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677474.000448:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.000449:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677474.000449:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.000449:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.000449:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677474.000450:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677474.000450:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.000451:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:6.0:1619677474.000451:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677474.000452:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.000452:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677474.000453:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.000453:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:6.0:1619677474.000453:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:9.2:1619677474.000454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.000454:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:8.0:1619677474.000455:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.000456:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.000456:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.000457:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677474.000458:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.000459:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96506561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.000459:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677474.000461:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.000461:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:26.0:1619677474.000462:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.000462:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:26.0:1619677474.000463:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677474.000463:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.000465:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.000466:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677474.000466:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.000468:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.000468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677474.000468:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677474.000468:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677474.000469:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.000469:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:21.0:1619677474.000469:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.000471:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.000471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677474.000472:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.000473:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.000473:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677474.000473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.000473:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.000473:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677474.000474:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677474.000474:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.000476:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677474.000476:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677474.000477:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.0:1619677474.000477:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677474.000478:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:8.0:1619677474.000478:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677474.000479:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:8.0:1619677474.000479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677474.000480:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.0:1619677474.000480:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.000483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.2:1619677474.000484:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677474.000485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.0:1619677474.000488:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.000488:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.000489:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.000489:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.000492:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:17.0:1619677474.000493:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:10.0:1619677474.000495:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.000495:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.000496:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677474.000496:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.000496:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.000497:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.000499:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.000502:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677474.000504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.000512:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.000515:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.000516:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.000516:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.000518:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.000521:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677474.001030:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.001033:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.001034:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.001035:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.001037:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.001039:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677474.001040:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.001041:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.001042:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.001043:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677474.001047:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.001049:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.001050:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.001051:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.001052:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.001053:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.001053:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.001054:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677474.001055:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677474.001056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.2:1619677474.001059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.001067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.001068:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.001068:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:17.0:1619677474.001069:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.001069:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677474.001069:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.001070:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.001071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.001072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677474.001072:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677474.001771:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.001774:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.001775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677474.001777:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:22.0:1619677474.001777:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.001779:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.001780:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677474.001781:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:22.0:1619677474.001782:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.001784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677474.001784:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.001785:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.001785:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.001786:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677474.001787:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.001789:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677474.001790:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.001791:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.001793:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.001793:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:31.0:1619677474.001794:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.001795:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.001796:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.001796:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.001797:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677474.001799:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.001799:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.001800:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677474.001801:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.001801:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.001802:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.001803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677474.001804:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.001805:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.001806:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.001807:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.001808:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.001809:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.001809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.001813:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.001814:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.001815:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677474.001816:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.001817:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.001817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.001819:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.001819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677474.001820:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.001820:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.001821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677474.001822:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.001823:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.001823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.001825:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.001827:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.001839:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.001842:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.001843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.001843:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.001844:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.001845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.002635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.002643:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.002646:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.002647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.002649:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.002652:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.002653:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.002655:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.002656:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.002657:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.002658:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.002661:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.002662:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.002665:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.002666:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.002666:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.002668:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.002668:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.002669:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.002670:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.002683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.002686:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.002686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.002687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.002689:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.002692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.003134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.003142:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.003145:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.003146:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.003148:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.003151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.003152:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.003153:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.003155:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.003156:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.003156:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.003159:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.003161:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.003164:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.003165:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.003165:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.003167:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.003167:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.003169:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.003170:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.003171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.003176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.003181:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.003183:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.003183:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.003184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.003184:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.003185:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.003186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677474.003186:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.003187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.003190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.003259:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.003262:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.003262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.003264:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.003266:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.003267:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.003268:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.003270:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.003271:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.003272:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.003274:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.003276:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.003278:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.003279:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.003280:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.003281:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.003282:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.003283:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.003283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.003301:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.003303:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.003304:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.003306:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.003306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.003308:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.003473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.003481:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.003484:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.003485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.003487:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.003490:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.003491:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677474.003492:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.003493:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.003494:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.003495:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677474.003498:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.003500:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677474.003503:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.003503:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677474.003504:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.003506:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.003506:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.003507:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.003507:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.003513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.003521:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.003524:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.003525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.003525:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.003527:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.003530:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677474.003743:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.003746:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.003748:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.003750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.003751:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.003753:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.003754:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.003755:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.003756:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.003757:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.003760:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.003762:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.003765:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.003766:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.003766:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.003768:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.003768:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.003769:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.003770:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.003783:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.003786:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.003787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.003788:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.003790:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.003793:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.003842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.003849:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.003852:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.003853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.003855:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.003858:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.003859:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.003860:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.003861:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.003862:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.003863:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.003866:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.003868:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.003870:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.003870:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.003871:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.003872:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.003873:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.003873:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.003874:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.003878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.003884:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.003887:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.003887:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.003888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.003890:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.003892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.004123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.004126:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.004129:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.004131:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.004132:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.004133:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.004135:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.004136:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.004137:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.004138:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.004139:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.004143:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.004144:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.004147:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.004147:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.004148:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.004150:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.004150:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.004151:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.004152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.004156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.004164:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.004167:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.004167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.004168:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.004170:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.004172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.004284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.004291:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.004296:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.004301:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.004304:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.004306:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.004308:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.004309:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.004310:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.004311:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.004312:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.004316:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.004318:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.004321:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.004321:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.004322:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.004324:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.004324:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.004325:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.004325:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.004330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.004339:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.004342:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.004344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.004344:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.004346:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.004349:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.004351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.004359:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.004362:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.004362:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.004362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.004364:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.004364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.004849:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.004853:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.004855:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.004857:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.004858:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.004859:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.004861:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.004862:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.004863:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.004864:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.004867:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.004869:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.004871:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.004871:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.004872:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.004873:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.004874:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.004875:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.004875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.004886:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.004888:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.004889:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.004891:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.004891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.004894:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.004895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.004895:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.004898:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.004900:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.004902:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.004902:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.004903:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.004904:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.004906:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.004906:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.004907:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.004910:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.004911:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.004914:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.004914:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.004915:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.004916:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.004917:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.004918:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.004918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.004922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.004929:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.004932:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.004933:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.004934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.004935:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.004938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677474.005017:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.005020:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.005021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.005023:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.005025:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.005027:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.005028:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.005029:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.005030:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.005031:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.005034:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.005036:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.005038:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.005039:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.005040:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.005041:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.005042:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.005043:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.005043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.005055:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.005057:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.005058:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.005060:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.005061:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.005062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.005063:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.005064:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.005064:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:4.2:1619677474.005064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677474.005066:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.005067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677474.005072:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.005076:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.005078:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.005080:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.005082:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.005083:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.005084:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.005086:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.005087:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.005088:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.005091:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.005093:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.005096:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.005096:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.005097:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.005099:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.005099:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.005100:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.005100:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.005113:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.005115:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.005116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.005117:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.005119:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.005122:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.005320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.005326:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.005330:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.005331:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.005332:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.005334:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.005336:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.005337:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.005338:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.005339:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.005340:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.005344:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.005345:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.005348:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.005349:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.005349:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.005351:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.005351:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.005352:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.005353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.005367:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.005369:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.005370:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.005370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.005373:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.005375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.005598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.005604:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.005607:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.005608:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.005610:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.005613:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.005615:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.005616:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.005617:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.005618:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.005619:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.005623:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.005624:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.005626:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.005627:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.005628:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.005630:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.005630:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.005631:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.005631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.005636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.005645:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.005647:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.005648:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.005648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.005650:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.005652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677474.005739:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.005742:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.005744:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.005746:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.005746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.005748:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.005749:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.005750:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.005751:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.005752:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.005755:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.005757:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.005760:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.005761:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.005762:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.005763:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.005764:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.005765:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.005767:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.005768:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.005777:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.005780:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.005781:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.005781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.005781:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.005783:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677474.005783:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.005784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.005785:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.005786:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.005795:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.005798:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.005799:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.005799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.005800:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.005801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.005828:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.005831:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.005833:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.005835:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.005835:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.005837:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.005838:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.005839:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.005840:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.005841:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.005844:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.005845:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.005847:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.005848:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.005848:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.005850:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.005850:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.005851:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.005853:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.005854:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.005855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.005862:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.005865:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.005866:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.005866:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.005867:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.005867:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.005869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.005870:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.005870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.005871:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677474.006302:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.006305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.006305:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.006307:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.006309:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.006311:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.006312:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.006314:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.006314:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.006315:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.006318:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.006320:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.006322:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.006323:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.006324:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.006326:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.006326:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.006327:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.006328:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.006341:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.006343:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.006344:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.006345:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.006346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.006348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.006716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.006720:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:4.2:1619677474.006720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677474.006723:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.006725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.006725:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.006728:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.006728:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:20.0:1619677474.006729:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:10.0:1619677474.006730:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.006731:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677474.006731:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.006733:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.006734:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.006735:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:10.0:1619677474.006735:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.006737:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.006739:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677474.006740:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677474.006741:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:10.0:1619677474.006741:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.006742:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.006743:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.006744:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.006745:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:10.0:1619677474.006745:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.006746:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.006747:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:10.0:1619677474.006748:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.006749:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:10.0:1619677474.006750:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.006751:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.006752:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.006752:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:10.0:1619677474.006753:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.006753:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.006754:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.006756:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.006756:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.006757:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677474.006757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:10.0:1619677474.006758:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677474.006763:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.006765:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:9.2:1619677474.006765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.006766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.006767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677474.006772:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.006775:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.006776:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.006776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.006778:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.006780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.006783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.2:1619677474.006789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677474.006794:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.006796:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:24.0:1619677474.006797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.006797:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.006797:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677474.006798:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:8.0:1619677474.006798:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677474.006799:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.006799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:21.0:1619677474.006801:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.006802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.006805:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.006811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.006819:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.006821:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.006822:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.006823:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.006823:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.006825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677474.006944:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.006947:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.006948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.006950:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.006952:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.006954:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.006955:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.006957:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.006957:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.006959:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.006962:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.006964:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.006967:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.006967:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.006968:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.006970:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.006970:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.006971:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.006971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677474.006986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.006992:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.006995:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.006996:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.006996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.006998:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.007001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.007408:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.007411:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.007413:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.007416:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.007416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.007417:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.007418:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.007419:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.007420:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.007421:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.007424:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.007426:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.007429:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.007430:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.007430:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.007432:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.007433:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.007434:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.007434:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.007448:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.007451:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.007451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:4.2:1619677474.007451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677474.007452:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.007454:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.007457:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677474.007459:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.007462:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.007463:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.007464:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.007467:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.007469:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.007470:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.007471:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.007472:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.007473:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.007476:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.007477:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.007480:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.007481:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.007482:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.007483:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.007484:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.007484:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.007485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.007497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.007499:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.007500:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.007501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.007503:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.007505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.007513:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.007516:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.007517:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.007517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.007518:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.007519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.007584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.007591:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.007594:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.007595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.007597:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.007600:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.007601:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.007603:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.007604:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.007605:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.007606:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.007609:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.007611:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.007614:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.007615:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.007615:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.007617:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.007617:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.007618:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.007619:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.007630:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.007632:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.007633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.007634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.007636:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.007639:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.007683:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.007686:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.007688:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.007690:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.007691:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.007692:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.007693:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.007694:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.007695:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.007696:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.007699:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.007700:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.007703:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.007704:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.007704:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.007706:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.007706:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.007707:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.007708:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.007711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.007718:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.007720:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.007721:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.007721:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.007723:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.007725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.007887:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.007890:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.007890:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.007893:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.007895:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.007896:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.007898:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.007899:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.007900:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.007901:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.007905:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.007906:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.007909:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.007910:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.007911:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.007912:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.007913:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.007914:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.007915:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.007916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.007926:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.007928:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.007928:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.007929:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.007930:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.007930:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677474.007931:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.007932:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.007933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.007934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.008083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.008090:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.008093:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.008094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.008096:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.008099:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.008100:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.008102:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.008103:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.008104:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.008105:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.008108:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.008109:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.008111:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.008112:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.008113:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.008115:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.008115:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.008116:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.008117:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.008121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.008128:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.008130:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.008130:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.008131:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.008132:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.008134:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677474.008274:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.008277:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.008277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.008280:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.008282:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.008284:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.008285:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.008287:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.008288:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.008289:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.008292:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.008300:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.008303:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.008303:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.008304:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.008306:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.008307:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.008308:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677474.008309:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.008309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.008313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.008320:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677474.008321:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677474.008323:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677474.008323:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000400:00000200:20.0:1619677474.008324:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.008324:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.008325:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677474.008326:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.008327:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.008329:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677474.008356:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.008359:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.008360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.008362:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.008364:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.008365:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677474.008367:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.008368:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.008369:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.008370:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.008373:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.008374:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.008377:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.008378:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677474.008378:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.008380:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677474.008380:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.008381:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.008381:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.008399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.008406:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.008408:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.008409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.008410:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.008412:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.008415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677474.008573:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.008576:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.008577:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.008579:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.008582:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.008583:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.008584:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.008585:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.008587:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.008587:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.008591:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.008592:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.008594:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.008595:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.008596:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.008597:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.008598:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.008598:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.008599:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677474.008614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.008620:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.008623:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.008624:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.008624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.008626:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.008629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.008748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.008756:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.008758:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.008759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.008761:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.008763:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.008765:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.008766:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.008767:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.008768:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.008769:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.008772:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.008774:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.008777:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.008778:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.008779:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.008781:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.008781:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.008782:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.008782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677474.008796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.008802:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.008804:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677474.008805:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677474.008807:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.008807:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677474.008808:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.008809:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.008809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:21.0:1619677474.008810:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.008812:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677474.008813:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.008814:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.008815:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.008816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:21.0:1619677474.008816:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.008817:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.008820:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.008822:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.008824:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.008825:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.008826:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.008827:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.008828:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.008829:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.008830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.008834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.008842:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.008845:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.008845:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.008846:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.008848:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.008851:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.008852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.008859:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.008861:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.008862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.008862:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.008863:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.008864:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.008982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:4.2:1619677474.008982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:18.0:1619677474.008990:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:31.0:1619677474.008991:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:18.0:1619677474.008993:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.008993:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:31.0:1619677474.008994:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:18.0:1619677474.008996:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.008998:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.008998:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.008999:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677474.009000:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.009000:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677474.009001:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.009002:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.009003:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.009003:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.009004:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.009005:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.009006:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.009007:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.009009:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:18.0:1619677474.009010:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.009013:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.009014:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.009016:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677474.009016:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.009017:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677474.009017:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.009019:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.009019:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.009020:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677474.009020:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.009020:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:31.0:1619677474.009021:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677474.009021:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:31.0:1619677474.009022:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.009023:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.009024:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677474.009026:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677474.009027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:4.2:1619677474.009032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:9.2:1619677474.009039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.009041:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677474.009042:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677474.009043:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.009044:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.009044:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:6.0:1619677474.009044:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.009045:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677474.009045:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:30.0:1619677474.009046:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.009047:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.009047:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.009048:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:22.0:1619677474.009048:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.009048:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.009049:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.009049:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.009049:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:6.0:1619677474.009050:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.009051:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.009052:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.009052:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.009053:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677474.009054:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.009055:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.009056:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.009059:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677474.009060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.009061:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.009063:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.009063:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:24.0:1619677474.009063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.009064:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.009064:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.009065:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.009066:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.009067:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.009067:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677474.009067:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.009067:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677474.009067:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.009067:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.009069:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.009069:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677474.009070:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.009071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.009072:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.009073:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.009073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.009077:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.009080:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.009081:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.009082:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.009084:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.009085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.009098:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.009101:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.009102:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.009103:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.009103:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.009105:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.009111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.009119:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.009121:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.009122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.009124:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.009127:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.009128:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.009129:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.009130:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.009131:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.009132:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.009135:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.009136:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.009138:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.009139:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.009140:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.009141:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.009141:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.009143:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.009144:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.009144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.009146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.009150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.009153:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.009155:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.009156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.009156:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.009157:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677474.009158:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.009159:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.009160:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.009161:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.009161:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.009165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.009171:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.009173:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.009174:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.009175:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.009175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.009176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677474.009570:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.009573:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.009574:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.009576:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.009578:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.009581:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677474.009582:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.009583:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.009584:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.009585:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.009589:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.009591:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.009594:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.009594:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677474.009595:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.009597:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677474.009597:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.009598:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.009598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.009614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.009621:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.009624:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.009625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.009625:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.009627:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.009630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677474.009658:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.009660:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.009661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.009663:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.009665:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.009667:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.009668:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.009670:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.009670:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.009672:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.009675:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.009677:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.009679:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.009680:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.009681:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.009682:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.009683:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.009684:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.009684:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.009696:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.009698:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.009699:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.009701:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.009702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.009703:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.009736:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.009739:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.009741:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.009743:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.009743:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.009745:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.009746:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.009747:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.009748:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.009749:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.009753:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.009755:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.009758:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.009759:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.009759:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.009761:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.009762:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.009763:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.009763:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677474.009775:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.009775:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.009778:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677474.009778:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.009779:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.009779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677474.009780:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.009782:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.009782:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677474.009782:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.009783:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.009784:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.009786:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.009787:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.009788:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.009791:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.009792:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.009793:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.009795:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.009795:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.009796:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.009797:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.009798:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.009799:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.009799:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.009810:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.009812:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.009813:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.009814:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.009816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.009817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.010055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.010063:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.010066:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.010067:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.010068:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.010071:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.010072:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.010073:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.010075:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.010075:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.010076:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.010080:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.010081:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.010084:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.010085:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.010085:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.010087:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.010087:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.010088:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.010088:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.010100:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:5.2:1619677474.010101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677474.010103:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.010104:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.010104:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.010106:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677474.010107:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.010109:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.010110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.010110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:1.0:1619677474.010112:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.010115:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.010116:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.010117:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.010119:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.010120:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.010121:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.010124:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.010125:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.010128:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.010129:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.010129:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.010131:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.010132:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.010132:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.010133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.010145:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.010147:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.010148:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.010148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.010150:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.010152:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.010865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.010875:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.010879:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.010883:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.010883:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.010886:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.010888:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677474.010890:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.010893:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.010894:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.010896:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677474.010902:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.010905:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677474.010908:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.010909:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677474.010910:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677474.010912:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677474.010913:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.010914:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.010915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.010919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.010926:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.010929:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.010930:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.010930:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.010932:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.010934:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677474.010945:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.010947:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.010948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.010950:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.010952:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.010954:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.010955:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.010957:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.010958:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.010959:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.010963:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.010965:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.010967:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.010968:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.010969:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.010970:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.010971:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.010972:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.010973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.010984:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.010986:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.010987:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.010989:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.010990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.010991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.011004:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.011006:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.011007:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.011008:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.011009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.011009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677474.011412:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.011415:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.011418:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.011420:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.011420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.011421:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.011423:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.011424:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.011425:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.011426:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.011430:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.011431:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.011434:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.011435:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.011436:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.011438:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.011438:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.011439:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.011440:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.011453:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.011456:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.011457:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.011457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.011460:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.011462:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.012288:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.012290:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677474.012294:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.012298:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.012301:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.012302:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.012304:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.012305:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.012306:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.012307:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.012310:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.012311:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.012314:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.012314:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.012315:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.012316:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.012317:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.012318:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.012320:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.012321:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.012322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.012322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.012330:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677474.012330:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.012332:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677474.012333:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:24.0:1619677474.012333:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.012334:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677474.012334:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.012335:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677474.012335:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.012335:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.012337:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.012338:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677474.012338:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.012338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.012339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677474.012339:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.012340:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.012342:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.012343:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.012343:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.012347:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.012348:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.012351:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.012352:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.012352:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.012354:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.012355:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.012355:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.012356:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.012369:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.012371:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.012372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.012373:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.012375:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.012378:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.012409:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.012412:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.012413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.012415:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.012417:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.012418:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.012419:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.012421:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.012422:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.012422:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.012426:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.012427:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.012429:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.012430:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.012431:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.012432:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.012433:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.012434:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.012434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.012446:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.012448:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.012449:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.012450:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.012452:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.012453:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.012787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677474.012792:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.012795:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.012797:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.012797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677474.012798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.012799:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.012801:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:21.0:1619677474.012802:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.012804:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.012804:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677474.012805:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.012806:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677474.012808:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.012808:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:21.0:1619677474.012810:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.012811:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.012812:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:8.0:1619677474.012814:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.012816:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677474.012816:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677474.012817:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677474.012818:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:8.0:1619677474.012818:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.012819:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.012820:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677474.012820:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000400:00000200:8.0:1619677474.012820:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.012821:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000400:00000200:8.0:1619677474.012821:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.012822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:8.0:1619677474.012824:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.012826:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.012829:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.012830:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.012831:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.012832:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:12.0:1619677474.012833:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:8.0:1619677474.012833:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.012834:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:12.0:1619677474.012835:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.012836:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.012836:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.012836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:12.0:1619677474.012838:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.012840:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.012841:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:4.2:1619677474.012842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.012846:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.012848:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.012849:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.012850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.012850:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.012851:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.012851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.012852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677474.012852:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.012855:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.012860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677474.012864:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.012867:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.012867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.012868:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677474.012868:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677474.012869:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.012869:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677474.012872:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.012874:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.012878:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.012881:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.012883:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.012885:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.012888:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.012889:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.012891:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.012897:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.012900:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.012904:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677474.012905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:2.0:1619677474.012905:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.012906:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.012909:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.012910:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.012911:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.012912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677474.012913:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.012916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.012916:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.012919:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677474.012921:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:10.0:1619677474.012921:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.012923:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677474.012924:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:10.0:1619677474.012924:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.012925:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.012925:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.012926:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.012927:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000400:00000200:10.0:1619677474.012927:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.012928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:10.0:1619677474.012931:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677474.012933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:10.0:1619677474.012933:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.012935:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.012936:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.012937:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.012939:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.012939:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.012940:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.012940:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.012945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.012954:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.012958:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.012960:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.012960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.012963:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.012968:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:30.0:1619677474.013153:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.013156:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.013158:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.013160:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.013160:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.013162:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.013163:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.013164:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.013165:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.013166:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.013169:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.013171:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.013173:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.013174:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.013175:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.013176:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.013177:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.013178:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.013178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.013183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.013191:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.013193:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.013194:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.013196:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.013197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.013199:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677474.013232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.013239:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.013242:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.013243:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.013245:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.109@o2ib7 : GET - for me
00000400:00000200:8.0:1619677474.013248:0:19938:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.109@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:8.0:1619677474.013250:0:19938:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.109@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:8.0:1619677474.013251:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.013252:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.013253:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677474.013256:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : REPLY try# 0
00000800:00000200:8.0:1619677474.013258:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.013259:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.013260:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.013260:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.013261:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.013262:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:9.2:1619677474.013279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.013285:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.013287:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.013288:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.013288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.013289:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: REPLY: OK
00000800:00000200:24.0:1619677474.013291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.013479:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.013481:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.013483:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.013486:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.013486:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.013487:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.013488:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.013490:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.013491:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.013491:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.013494:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.013496:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.013499:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.013500:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.013500:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.013502:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.013503:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.013504:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.013506:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.013506:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.013517:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.013520:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.013521:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.013521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.013521:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.013523:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.013524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.013524:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.013525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.013526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.013666:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.013669:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.013671:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.013673:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.013674:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.013675:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.013677:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.013678:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.013679:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.013680:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.013683:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.013685:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.013688:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.013688:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.013689:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.013691:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.013691:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.013692:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.013693:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.013696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.013703:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.013706:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.013707:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.013707:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.013709:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.013711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.013797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.013803:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.013806:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.013807:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.013809:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.013811:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.013812:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.013814:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.013815:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.013816:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.013817:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.013820:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.013822:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.013825:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.013825:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.013826:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.013828:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.013828:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.013829:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.013829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.013841:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.013844:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.013845:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.013845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.013847:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.013850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.013856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.013864:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.013867:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.013868:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.013870:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.013872:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.013874:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.013875:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.013876:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.013877:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.013878:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.013881:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.013882:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.013884:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.013885:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.013886:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.013887:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.013888:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.013888:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.013889:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.013893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.013899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.013901:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.013902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.013902:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.013904:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.013906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.014127:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.014130:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.014132:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.014134:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.014134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.014135:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677474.014135:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677474.014136:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.014138:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.014139:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.014139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.014139:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:31.0:1619677474.014142:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677474.014142:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.014144:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.014145:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.014146:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677474.014147:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677474.014148:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.014149:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.014150:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.014151:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.014151:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:6.0:1619677474.014151:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.014151:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.014153:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.014153:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.014154:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.014155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677474.014155:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.014157:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.014159:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.014160:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.014161:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.014163:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.014163:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677474.014165:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:6.0:1619677474.014165:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.014166:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.014166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677474.014167:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.014168:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.014170:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.014171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.014172:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.014177:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.014178:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.014179:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.014180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.014181:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.014181:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.014181:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.014183:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.014184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.014185:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677474.014857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.014863:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.014866:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.014866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.014869:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.014871:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.014872:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.014874:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.014875:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.014876:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.014877:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.014880:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.014882:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677474.014882:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.014885:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.014886:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677474.014886:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.014887:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:31.0:1619677474.014888:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.014889:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.014889:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677474.014889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677474.014890:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.014891:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.014891:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:31.0:1619677474.014892:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677474.014892:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.014892:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677474.014893:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.014894:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.014895:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.014896:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.014897:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677474.014897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677474.014899:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:31.0:1619677474.014900:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.014901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677474.014903:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.014903:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.014903:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.014904:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:31.0:1619677474.014905:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.014905:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.014905:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.014905:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677474.014906:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.014906:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677474.014906:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677474.014906:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.014906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.014907:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.014908:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.014908:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.014908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:10.0:1619677474.014908:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96506497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.014909:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.014910:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:6.0:1619677474.014910:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.014911:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.014911:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.014912:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.014912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677474.014914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:10.0:1619677474.014915:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.014917:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.014917:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677474.014920:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.014920:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677474.014921:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.014921:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.014922:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:10.0:1619677474.014923:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.014923:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677474.014923:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:30.0:1619677474.014924:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.014924:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.014924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:10.0:1619677474.014924:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.014925:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677474.014926:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:2.0:1619677474.014926:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.014927:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677474.014927:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.014928:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677474.014928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677474.014929:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:2.0:1619677474.014929:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.014930:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:2.0:1619677474.014930:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677474.014931:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.014931:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677474.014932:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.014933:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.014934:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:2.0:1619677474.014934:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.014936:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.014936:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677474.014937:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.014937:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:2.0:1619677474.014939:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.014940:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:20.0:1619677474.014940:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.014941:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.014941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:20.0:1619677474.014942:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677474.014942:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:20.0:1619677474.014943:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96506495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.014944:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677474.014945:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.014946:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.014946:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.014946:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:22.0:1619677474.014947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:20.0:1619677474.014947:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677474.014947:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677474.014949:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677474.014950:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.014950:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:20.0:1619677474.014951:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.014951:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.0:1619677474.014952:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.014955:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.014955:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:9.2:1619677474.014955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677474.014956:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.014958:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.014959:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.014960:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.014960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677474.014962:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.014964:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.014965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.014966:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.014968:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.014971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:8.2:1619677474.014976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.014985:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.014987:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.014988:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.014988:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.014990:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.014992:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677474.015404:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.015407:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.015410:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.015412:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.015412:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.015413:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.015415:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.015416:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.015417:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.015418:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.015422:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.015424:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.015427:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.015428:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.015429:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.015430:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.015431:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.015432:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.015432:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.015432:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.015435:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.015436:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.015438:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.015440:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.015442:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.015443:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.015445:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:6.0:1619677474.015445:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.015446:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.015447:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.015447:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.015448:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.015450:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.015451:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.015453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.015453:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.015454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.015455:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.015455:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.015456:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.015456:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.015457:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.015457:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.015458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.015459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.015459:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.015460:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.015461:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.015461:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.015473:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.015475:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.015476:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.015478:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.015479:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.015480:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677474.015621:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.015624:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.015624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.015627:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.015629:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.015631:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677474.015632:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.015633:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.015634:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.015635:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677474.015638:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.015640:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677474.015642:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.015643:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677474.015644:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677474.015646:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677474.015646:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.015647:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.015648:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.0:1619677474.015659:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.015661:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.015662:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.015664:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.015665:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.015666:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.015682:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.015684:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.015685:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.015686:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.015687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.015688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.015819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.015824:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.015826:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.015828:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.015829:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.015831:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.015833:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.015834:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.015835:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.015836:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.015837:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.015840:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.015842:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.015845:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.015846:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.015846:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.015848:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.015849:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.015850:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.015850:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.015854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.015861:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.015864:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.015865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.015866:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.015868:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.015870:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677474.016906:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.016909:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.016909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.016911:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.016914:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.016915:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.016916:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.016918:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.016919:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.016920:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.016923:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.016924:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.016927:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.016927:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.016928:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.016930:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.016930:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.016931:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.016932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.016944:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.016947:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.016947:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.016949:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.016951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.016952:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677474.017043:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.017046:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.017047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.017048:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.017051:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.017052:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:2.0:1619677474.017053:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.017055:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.017056:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.017056:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677474.017059:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.017061:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677474.017064:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.017065:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:2.0:1619677474.017065:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677474.017067:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:2.0:1619677474.017068:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.017069:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677474.017070:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.017070:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:8.2:1619677474.017075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677474.017081:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:21.0:1619677474.017083:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:18.0:1619677474.017083:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677474.017084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:21.0:1619677474.017085:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:18.0:1619677474.017085:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:21.0:1619677474.017086:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.017088:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.017089:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.017091:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.017098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.017106:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.017108:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.017109:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.017110:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.017110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.017112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677474.017430:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.017433:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.017435:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.017438:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.017438:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.017440:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.017441:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.017442:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.017443:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.017444:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.017448:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.017449:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.017452:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.017453:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.017454:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.017455:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.017456:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.017457:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.017458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.017471:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.017474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.017474:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.017475:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.017477:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.017480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.018094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.018100:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.018103:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.018104:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.018105:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.018108:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.018109:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.018110:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.018112:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.018113:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.018114:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.018117:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.018118:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677474.018119:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677474.018121:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.018122:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677474.018122:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.018123:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:31.0:1619677474.018124:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.018125:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.018125:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.018126:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677474.018126:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677474.018127:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.018127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677474.018129:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.018130:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.018132:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.018133:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.018134:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.018137:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.018138:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:31.0:1619677474.018139:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:13.0:1619677474.018141:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:31.0:1619677474.018142:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677474.018142:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.018143:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.018144:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:13.0:1619677474.018144:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.018146:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.018146:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677474.018146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677474.018147:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677474.018147:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677474.018148:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.018153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:12.0:1619677474.018154:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.018157:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.018158:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.018159:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.018159:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677474.018161:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:12.0:1619677474.018161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.0:1619677474.018163:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.018164:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.018166:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.018166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.018169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.018468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.018476:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.018478:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.018479:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.018481:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.018484:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.018485:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.018486:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.018487:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.018489:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.018489:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.018493:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.018494:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.018497:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.018498:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.018498:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.018500:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.018501:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.018502:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.018502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.018516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.018519:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.018519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.018520:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.018522:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.018525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.018580:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.018583:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.018585:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.018588:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.018588:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.018589:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.018590:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.018592:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.018593:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.018593:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.018596:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.018598:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.018599:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.018600:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.018600:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.018602:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.018603:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.018604:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.018604:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.018614:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.018617:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.018617:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.018619:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.018619:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677474.018621:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677474.018621:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:31.0:1619677474.018622:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.018622:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.018624:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.018626:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.018627:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.018629:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.018630:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.018631:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.018632:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.018634:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:6.0:1619677474.018635:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.018636:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.018637:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.018638:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.018639:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:6.0:1619677474.018639:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.018640:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677474.018640:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677474.018641:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677474.018641:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.018643:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.018643:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.018644:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.018645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:4.2:1619677474.018646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677474.018650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.018656:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677474.018657:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.018659:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677474.018659:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:20.0:1619677474.018660:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.018661:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:20.0:1619677474.018662:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.018663:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:21.0:1619677474.018663:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677474.018663:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.018665:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.018665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677474.018666:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.018667:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.018668:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.018670:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.018671:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.018672:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.018674:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.018675:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.018676:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.018677:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.018678:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.018678:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.018679:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.018680:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.018683:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.018684:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.018684:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.018684:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.018685:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.018686:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.018687:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.018688:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.018688:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (23)++
00000800:00000200:26.0:1619677474.018689:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.018690:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.018690:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (24)--
00000800:00000200:10.0:1619677474.018691:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.018692:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.018692:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.018692:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.018693:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.018693:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677474.018693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677474.018696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677474.018702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.018704:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.018706:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.018706:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677474.018707:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:24.0:1619677474.018708:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677474.018708:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677474.018710:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:13.0:1619677474.018710:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:24.0:1619677474.018711:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.018711:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.018712:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:13.0:1619677474.018714:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:24.0:1619677474.018715:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.018717:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.018718:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677474.018719:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.018724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.018730:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.018733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.018733:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.018734:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.018734:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.018735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.018895:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.018897:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.018900:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.018901:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.018901:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.018903:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.018904:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.018905:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.018906:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.018907:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.018910:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.018912:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.018915:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.018916:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.018916:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.018918:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.018919:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.018919:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.018920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.018932:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.018935:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.018935:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.018936:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.018938:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.018941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.018996:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.018999:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.019001:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.019002:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.019004:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.019005:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.019007:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.019008:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.019009:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.019009:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.019012:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.019014:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:4.2:1619677474.019016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:30.0:1619677474.019017:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.019018:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.019018:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.019020:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.019020:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.019021:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.019022:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677474.019023:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677474.019026:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677474.019027:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:8.2:1619677474.019027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:26.0:1619677474.019028:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.019030:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.019032:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677474.019033:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.019034:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.019035:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.019035:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:26.0:1619677474.019036:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.019037:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.019038:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.019039:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677474.019040:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.019040:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.019041:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.019043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677474.019044:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.019045:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677474.019045:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677474.019047:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677474.019048:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677474.019049:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (22)++
00000800:00000200:26.0:1619677474.019050:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:26.0:1619677474.019051:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:4.2:1619677474.019058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (21)++
00000800:00000200:20.0:1619677474.019063:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.019065:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677474.019065:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000400:00000200:20.0:1619677474.019066:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.019067:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000400:00000200:20.0:1619677474.019068:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.019068:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:21.0:1619677474.019069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677474.019069:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:20.0:1619677474.019071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.019192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677474.019200:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.019202:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677474.019203:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677474.019205:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.019208:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.019209:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.019210:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.019212:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.019212:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.019213:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.019216:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.019218:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.019220:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.019221:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.019221:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.019223:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.019224:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.019224:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677474.019225:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:9.2:1619677474.019229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.019238:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.019241:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.019242:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.019242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.019244:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.019247:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.019249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677474.019256:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.019259:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.019260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.019260:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.019261:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.019262:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:31.0:1619677474.019407:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.019410:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.019412:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.019414:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.019415:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.019416:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.019417:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.019419:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.019419:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.019420:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.019424:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.019425:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.019427:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.019428:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.019428:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.019430:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.019430:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.019432:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.019433:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.019434:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.019445:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.019446:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.019447:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.019448:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.019449:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.019450:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.019450:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.019451:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.019452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.019453:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.019518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677474.019523:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.019526:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.019528:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.019528:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.019531:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.019532:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677474.019534:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.019535:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.019536:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.019537:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677474.019541:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.019542:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677474.019545:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.019546:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677474.019547:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677474.019549:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677474.019549:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.019551:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677474.019552:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677474.019553:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:9.2:1619677474.019555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677474.019557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.019560:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677474.019562:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:10.0:1619677474.019562:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:20.0:1619677474.019563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677474.019564:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:10.0:1619677474.019565:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.019566:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.019566:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.019568:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.019571:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.019900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.019907:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.019909:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.019910:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.019912:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.019914:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.019915:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677474.019916:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.019917:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.019918:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.019918:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.019919:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677474.019920:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.019922:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677474.019922:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.019925:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.019927:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:6.0:1619677474.019928:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.019929:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677474.019930:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.019931:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.019931:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677474.019931:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.019932:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.019933:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.019933:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.019934:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677474.019934:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:26.0:1619677474.019935:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.019936:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.019937:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.019938:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.019940:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.019942:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677474.019942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.019944:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.019944:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.019946:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.019947:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.019948:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.019949:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677474.019949:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.019949:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:22.0:1619677474.019950:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.019950:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:10.0:1619677474.019951:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677474.019951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.019952:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.019952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677474.019952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.019953:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.019958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677474.019959:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.019962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.019962:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677474.019963:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677474.019963:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.019964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.019964:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.019964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.019965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.019965:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.019966:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.019967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677474.019967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677474.019967:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.019968:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.019968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677474.019983:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.019985:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:4.2:1619677474.019985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.0:1619677474.019986:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.019987:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.019987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.019989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677474.019993:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.019996:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.019997:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.019999:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.020001:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.020003:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.020004:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.020005:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.020006:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.020008:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.020011:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.020012:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.020015:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.020016:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.020016:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.020018:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.020019:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.020019:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.020020:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.020032:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.020035:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.020036:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.020036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.020038:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.020041:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.020052:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.020055:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.020056:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.020056:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.020057:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.020058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.020267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.020273:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.020276:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.020277:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.020279:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.020282:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.020283:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.020284:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.020286:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.020287:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.020288:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.020290:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.020292:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.020299:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.020299:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.020300:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.020301:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.020302:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.020303:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.020303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.020308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.020316:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.020318:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.020318:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.020319:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.020320:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.020321:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.020701:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.020704:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.020706:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.020708:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.020708:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.020709:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:4.2:1619677474.020710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:30.0:1619677474.020711:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.020712:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.020713:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.020714:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677474.020714:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.020716:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.020719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:30.0:1619677474.020720:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677474.020720:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.020722:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.020723:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:20.0:1619677474.020724:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.020725:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677474.020725:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.020726:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:20.0:1619677474.020726:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.020727:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.020728:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:20.0:1619677474.020728:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.020728:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.020729:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.020730:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.020731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677474.020732:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.020734:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677474.020734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677474.020736:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.020737:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.020738:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.020739:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.020740:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677474.020741:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.0:1619677474.020741:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.020741:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677474.020744:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.020745:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.020745:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.020747:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.020749:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.020753:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.020756:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.020757:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.020757:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.020759:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.020761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.020925:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.020928:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.020931:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.020932:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.020933:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.020934:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.020936:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.020937:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.020938:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.020939:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677474.020941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677474.020942:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.020944:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.020947:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.020947:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677474.020948:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.020948:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.020950:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.020950:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677474.020951:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677474.020951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677474.020952:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.020954:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.020954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:1.0:1619677474.020954:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.020956:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.020957:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.020958:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.020960:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.020961:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.020962:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.020964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677474.020965:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.020966:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.020967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.020968:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.020968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.020969:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.020969:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.020970:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:1.0:1619677474.020970:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:30.0:1619677474.020971:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.020971:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.020971:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677474.020972:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:1.0:1619677474.020972:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.020973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677474.020973:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.020974:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.020974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.020986:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.020988:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.020989:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.020990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.020991:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.020993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.021445:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.021447:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.021448:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.021450:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.021452:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.021453:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.021455:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.021456:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.021457:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.021458:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.021461:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.021463:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.021465:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.021466:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.021466:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.021468:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.021469:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.021469:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.021470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:18.0:1619677474.021481:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:21.0:1619677474.021483:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677474.021484:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.021484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:21.0:1619677474.021485:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.021486:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.021486:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.021488:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:18.0:1619677474.021488:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.021489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.021490:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:18.0:1619677474.021490:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677474.021491:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.021492:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.021493:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.021494:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677474.021498:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.021499:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677474.021502:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.021503:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677474.021503:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677474.021505:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677474.021505:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.021506:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.021507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:20.0:1619677474.021518:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.021520:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.021521:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.021523:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.021524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.021526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.021702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.021710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.021713:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.021715:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.021717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.021717:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.021719:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677474.021721:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.021722:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.021723:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.021724:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677474.021728:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.021729:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677474.021733:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.021733:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:12.0:1619677474.021734:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:12.0:1619677474.021736:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:12.0:1619677474.021736:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.021737:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.021738:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677474.021743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.021752:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.021754:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.021755:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.021755:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.2:1619677474.021756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677474.021758:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.021761:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677474.021761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677474.021764:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.021766:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.021766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.021766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000400:00000200:21.0:1619677474.021769:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.021770:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.021772:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.021773:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.021774:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677474.021774:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000400:00000200:21.0:1619677474.021775:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.021777:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.021777:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.021778:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000400:00000200:21.0:1619677474.021779:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.021779:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.021780:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677474.021781:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.021784:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.021785:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.021785:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.021787:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.021787:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.021789:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.021790:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.021791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.021793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.021796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.021799:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.021801:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.021801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677474.021801:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677474.021802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.021803:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.021804:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.021804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.021806:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.021808:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.022074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.022083:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.022085:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.022086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.022088:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.022091:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.022092:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677474.022093:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.022095:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.022096:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.022097:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677474.022100:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.022102:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.022103:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.022104:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.022105:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.022106:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.022106:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.022107:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.022108:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.022112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.022118:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.022119:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.022120:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.022121:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.022122:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.022123:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677474.022633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.022639:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.022642:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.022643:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.022644:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.022647:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.022649:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.022650:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.022652:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.022652:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.022654:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.022657:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.022659:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.022661:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.022662:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.022662:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.022664:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.022665:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.022666:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.022667:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.022667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.022671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.022673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677474.022679:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677474.022680:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677474.022681:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677474.022682:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.022682:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677474.022682:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.022683:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.022684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677474.022684:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.022687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677474.022794:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.022797:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.022799:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.022801:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.022802:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.022803:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.022805:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.022806:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.022807:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.022808:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.022812:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.022814:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.022817:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.022817:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.022818:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.022820:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.022820:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.022821:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.022822:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.022836:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.022838:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.022839:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.022840:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.022842:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.022845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677474.023009:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.023012:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.023013:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.023015:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.023017:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.023019:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677474.023020:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.023021:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.023022:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.023024:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677474.023027:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.023029:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677474.023032:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.023033:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677474.023033:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677474.023035:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677474.023036:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.023036:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.023037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.023041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.023049:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.023051:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.023052:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.023053:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.023055:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.023058:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677474.023060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.023066:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.023068:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.023069:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.023070:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.023071:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.023071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677474.023209:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.023212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.023212:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.023215:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.023217:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.023218:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.023220:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.023221:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.023222:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.023223:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.023226:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.023228:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.023231:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.023232:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.023233:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.023234:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.023235:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.023236:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.023236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.023248:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.023250:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.023251:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.023252:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.023253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.023255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.023438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.023446:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.023449:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.023450:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.023452:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.023454:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.023455:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.023457:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.023458:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.023459:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.023460:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.023463:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.023465:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.023467:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.023468:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.023468:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.023470:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.023470:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.023471:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.023472:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.023476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.023483:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.023486:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.023486:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.023487:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.023488:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.023491:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.023571:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.023574:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.023576:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.023579:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.023579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.023580:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.023582:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.023583:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.023584:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.023584:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.023587:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.023589:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.023591:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.023592:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.023592:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.023594:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.023594:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.023595:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.023596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.023599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.023605:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.023607:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.023608:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.023609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.023610:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.023612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677474.023713:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.023716:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.023717:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.023719:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.023722:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.023723:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.023725:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.023726:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.023727:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.023728:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.023731:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.023732:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.023735:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.023736:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.023736:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.023738:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.023738:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.023739:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.023739:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:22.0:1619677474.023752:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.023754:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.023755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.023756:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.023758:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.023761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.023769:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.023772:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.023772:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.023773:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.023774:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.023774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.023857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.023864:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.023867:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.023868:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.023870:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.023873:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.023874:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.023876:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.023877:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.023878:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.023879:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.023883:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.023884:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.023887:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.023888:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.023889:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.023889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:8.0:1619677474.023890:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.023891:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.023892:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.023892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677474.023894:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.023896:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.023899:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.023899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.023901:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.023902:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677474.023904:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677474.023904:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.023905:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.023906:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.023907:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:21.0:1619677474.023907:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.023908:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.023908:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.023911:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.023911:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.023914:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.023915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.023916:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.023917:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.023917:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.023919:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.023919:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.023920:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.023921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.023932:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.023935:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.023935:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.023936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.023937:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.023940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.024152:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.024155:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.024157:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.024159:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.024160:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.024161:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.024163:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.024164:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.024165:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.024166:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.024169:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.024171:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.024173:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.024174:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.024174:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.024176:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.024176:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.024177:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.024178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.024188:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.024191:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.024191:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.024193:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.024194:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.024195:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.024559:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.024562:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.024563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.024565:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.024567:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.024569:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.024570:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.024571:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.024572:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.024574:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.024576:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.024578:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.024580:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.024581:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.024582:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.024584:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.024584:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.024585:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.024586:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.024598:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.024601:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.024601:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.024603:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.024604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.024605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.024623:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.024625:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.024626:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.024627:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.024628:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.024628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.024662:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.024665:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.024668:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.024669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.024670:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.024671:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.024673:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.024674:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.024675:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.024676:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.024679:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.024681:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.024683:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.024683:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.024684:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.024685:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.024686:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.024687:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.024689:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.024689:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.024700:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.024701:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.024702:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.024703:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.024703:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.024704:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.024704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.024705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.024706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.024707:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.024738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.024746:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.024749:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.024749:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677474.024751:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.024754:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.024755:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.024756:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.024757:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.024758:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.024759:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.024762:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.024764:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.024767:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.024767:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.024768:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.024770:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.024770:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.024771:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.024771:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:17.0:1619677474.024783:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:6.0:1619677474.024784:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.024786:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:6.0:1619677474.024787:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.024787:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:22.0:1619677474.024788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.024788:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.024789:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.024790:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.024791:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.024792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677474.024793:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677474.024793:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:17.0:1619677474.024794:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.024796:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.024797:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.024798:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.024799:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677474.024801:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.024802:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677474.024803:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.024803:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.024805:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.024805:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.024806:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:26.0:1619677474.024807:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.024807:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677474.024809:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677474.024809:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.024809:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:26.0:1619677474.024810:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.024810:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677474.024811:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:26.0:1619677474.024812:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.024812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:26.0:1619677474.024813:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.024814:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.024817:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.024819:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.024821:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.024822:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.024822:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.024823:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:1.0:1619677474.024823:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:26.0:1619677474.024825:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (25)--
00000800:00000200:1.0:1619677474.024825:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:26.0:1619677474.024826:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.024826:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677474.024826:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:1.0:1619677474.024826:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:26.0:1619677474.024827:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.024827:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677474.024827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:22.0:1619677474.024828:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.024830:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.024834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.024835:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.024837:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.024838:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.024838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.024840:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.024842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.024856:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.024859:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.024860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.024860:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.024861:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.024861:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.025293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677474.025299:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.025301:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677474.025303:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.025304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677474.025306:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.025307:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677474.025309:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.025310:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.025311:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.025312:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677474.025315:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.025317:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.025319:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.025320:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.025321:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.025323:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.025323:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.025324:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677474.025324:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.025329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.025337:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.025339:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.025340:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.025342:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.025343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.025344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.025351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.025358:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.025361:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.025362:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.025363:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.025363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.025364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.025624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.025632:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.025634:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.025635:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.025637:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.025640:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.025641:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.025643:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.025644:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.025645:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.025646:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.025649:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.025651:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.025654:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.025655:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.025656:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.025657:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.025658:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.025659:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.025660:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.025660:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677474.025664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677474.025666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.025673:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677474.025673:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.025675:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:26.0:1619677474.025676:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.025676:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677474.025677:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.025677:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677474.025677:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.025679:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.025682:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:2.0:1619677474.025683:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:30.0:1619677474.025684:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.025687:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.025688:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.025689:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.025690:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.025692:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.025693:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.025694:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.025695:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.025698:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.025699:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.025702:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.025703:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.025704:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.025705:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.025706:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.025707:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.025707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.025720:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:5.2:1619677474.025722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677474.025723:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.025723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.025724:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.025726:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.025729:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677474.025729:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.025731:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.025733:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.025735:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.025737:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.025738:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.025740:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.025741:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.025742:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.025743:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.025747:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.025748:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.025751:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.025752:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.025752:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.025754:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.025754:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.025755:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.025755:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.025782:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.025785:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.025786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.025786:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.025788:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.025791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.025996:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.026000:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.026002:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.026003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.026005:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.026006:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.026008:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.026009:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.026010:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.026011:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.026015:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.026016:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.026018:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.026019:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.026020:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.026022:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.026022:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.026023:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.026024:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.026027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.026030:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.026033:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:24.0:1619677474.026034:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677474.026034:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677474.026036:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:22.0:1619677474.026036:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.026037:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.026038:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.026038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:22.0:1619677474.026040:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677474.026041:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.026042:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.026043:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677474.026043:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.026044:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.026045:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.026048:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.026050:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:9.2:1619677474.026051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677474.026053:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.026054:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.026054:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.026056:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.026057:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.0:1619677474.026057:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.026058:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.026059:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:8.0:1619677474.026059:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.026060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.026060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677474.026060:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.026061:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.026062:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677474.026071:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.026072:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.026073:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.026073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.026074:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.026075:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.026075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677474.026077:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.026077:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.026079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.026338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.026346:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.026349:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.026350:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.026351:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.026354:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.026356:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.026357:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.026358:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.026359:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.026360:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.026364:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.026366:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.026368:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.026369:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.026370:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.026372:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.026372:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.026373:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.026374:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.026375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677474.026378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.026380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677474.026388:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677474.026388:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.026390:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.026390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677474.026391:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677474.026391:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677474.026392:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.026392:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.026394:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.026398:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677474.026982:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.026985:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.026986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.026988:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.026990:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.026992:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.026993:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.026994:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.026995:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.026996:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.027000:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.027002:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.027004:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.027005:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.027006:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.027008:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.027008:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.027009:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677474.027010:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677474.027010:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:9.2:1619677474.027013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.027020:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:17.0:1619677474.027021:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:24.0:1619677474.027022:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:17.0:1619677474.027023:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:8.0:1619677474.027023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.027024:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.027024:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.027024:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000400:00000200:24.0:1619677474.027026:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.027029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.027426:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.027429:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.027431:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.027433:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.027433:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.027435:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.027436:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.027437:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.027438:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.027439:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.027443:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.027445:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.027448:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.027448:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.027449:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.027451:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.027451:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.027452:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.027453:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.027466:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.027469:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.027470:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.027470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.027472:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.027475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.027711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.027714:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.027717:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.027719:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.027720:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.027721:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.027722:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.027723:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.027724:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.027725:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.027726:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.027730:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.027731:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.027734:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.027734:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.027735:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677474.027735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677474.027737:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.027737:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677474.027737:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.027738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.027738:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.027739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677474.027740:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.027742:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.027743:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.2:1619677474.027744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:6.0:1619677474.027744:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.027746:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.027747:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.027748:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.027751:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.027752:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:6.0:1619677474.027753:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677474.027755:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:6.0:1619677474.027755:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677474.027756:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.027756:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:1.0:1619677474.027756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677474.027757:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:17.0:1619677474.027758:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.027758:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.027759:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.027760:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.027760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:17.0:1619677474.027761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677474.027772:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.027774:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.027775:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.027776:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.027777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.027778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.028251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.028259:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.028262:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.028263:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.028265:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.028268:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.028269:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.028271:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.028272:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.028273:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.028274:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.028278:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.028279:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.028282:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.028283:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.028284:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.028285:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.028286:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.028286:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.028288:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.028288:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.028291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.028294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.028298:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.028300:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.028301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.028301:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677474.028301:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.028302:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.028303:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.028304:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.028305:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.028305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.028487:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.028490:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.028492:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.028494:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.028494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.028496:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.028497:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.028498:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.028499:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.028500:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.028502:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.028504:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.028506:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.028507:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.028507:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.028509:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.028509:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.028510:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.028511:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.028515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.028521:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.028523:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.028524:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.028525:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.028526:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.028528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677474.028606:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.028609:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.028610:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.028611:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.028614:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.028616:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.028617:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.028618:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.028619:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.028620:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.028623:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.028624:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.028626:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.028627:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.028628:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.028630:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.028630:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.028631:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.028631:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:22.0:1619677474.028644:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.028646:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.028647:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.028648:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.028650:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.028652:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.028801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.028807:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.028810:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.028811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.028812:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.028814:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.028816:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.028817:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.028818:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.028819:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.028821:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.028824:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.028826:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.028828:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.028828:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.028829:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.028831:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.028831:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.028832:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.028833:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.028834:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.028837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.028840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.028846:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.028847:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.028847:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677474.028848:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.028848:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.028848:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.028850:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.028850:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.028851:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677474.028851:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.028854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.028861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.028863:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.028864:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.028864:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.028865:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.028865:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.029099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.029106:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.029109:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.029110:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.029112:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.029114:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.029116:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677474.029117:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000400:00000200:8.0:1619677474.029117:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.029119:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.029120:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000400:00000200:8.0:1619677474.029120:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677474.029120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:8.0:1619677474.029122:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.029123:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677474.029123:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677474.029125:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.029125:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.029125:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677474.029126:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677474.029127:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677474.029128:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.029128:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677474.029128:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.029129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677474.029130:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.029130:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.029131:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.029131:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677474.029132:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.029132:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:6.0:1619677474.029132:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.029133:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.029133:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.029133:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:30.0:1619677474.029134:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.0:1619677474.029135:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:6.0:1619677474.029135:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.029136:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677474.029137:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96506429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.029138:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:6.0:1619677474.029138:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96506429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.029139:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677474.029140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677474.029140:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.029141:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677474.029141:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.029141:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.029141:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677474.029141:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.029142:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677474.029142:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677474.029142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:6.0:1619677474.029143:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677474.029144:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.029145:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.029146:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:17.0:1619677474.029146:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677474.029148:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:17.0:1619677474.029148:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677474.029148:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000400:00000200:6.0:1619677474.029148:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.029149:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.029149:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:17.0:1619677474.029149:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677474.029150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:18.0:1619677474.029151:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:6.0:1619677474.029151:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.029152:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677474.029152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.029152:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.029152:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.029152:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:9.2:1619677474.029153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677474.029153:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.029154:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677474.029155:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677474.029155:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.029155:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.029155:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677474.029156:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:20.0:1619677474.029156:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.029157:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677474.029157:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.029157:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677474.029158:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:20.0:1619677474.029158:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96506427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677474.029158:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677474.029158:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677474.029159:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677474.029159:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.029159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677474.029160:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.029160:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:20.0:1619677474.029160:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.029161:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677474.029161:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.029162:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.029163:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.029163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677474.029165:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.029166:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.029168:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.029169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677474.029170:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.029170:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.029171:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.029171:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:21.0:1619677474.029172:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.029172:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:9.2:1619677474.029172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.029173:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:21.0:1619677474.029174:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.029174:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.029174:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.029175:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:31.0:1619677474.029176:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:21.0:1619677474.029176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677474.029177:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.029177:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.029177:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.029177:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:31.0:1619677474.029178:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.029178:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.029180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.029181:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.0:1619677474.029181:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.029182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.029182:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:4.2:1619677474.029182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677474.029183:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.029183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677474.029187:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.029188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:30.0:1619677474.029190:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.029190:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677474.029190:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:22.0:1619677474.029191:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.029191:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677474.029191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.029192:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:30.0:1619677474.029193:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.029194:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000400:00000200:22.0:1619677474.029194:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.029195:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.029196:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.029196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.029197:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677474.029206:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.029209:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.029210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.029210:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.029211:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.029211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.029257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677474.029264:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.029267:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677474.029268:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677474.029270:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.029273:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.029275:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.029276:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.029278:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.029279:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.029280:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.029284:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.029285:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.029288:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.029289:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.029290:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.029291:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.029292:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.029293:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.2:1619677474.029298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.029299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:2.0:1619677474.029307:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.029310:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.029311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.029311:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.029313:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.029317:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.029643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.029651:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.029655:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.029655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.029657:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.029659:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.029661:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.029662:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.029663:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.029664:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.029665:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.029668:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.029670:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.029672:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.029673:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.029674:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.029675:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.029676:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.029676:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.029677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677474.029683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677474.029690:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.029691:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.029693:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677474.029693:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.029694:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677474.029694:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.029694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677474.029695:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.029696:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.029698:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.029700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677474.029701:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.029702:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.029703:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.029704:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.029705:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.029708:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.029709:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.029712:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.029713:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.029713:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.029715:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.029715:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.029716:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.029717:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.029728:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.029731:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.029732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.029732:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.029734:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.029737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.029738:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.029741:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.029743:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.029744:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.029745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.029746:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.029747:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.029748:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:31.0:1619677474.029749:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.029750:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.029751:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.029751:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.029751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.029752:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.029753:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000400:00000200:31.0:1619677474.029754:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.029754:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.029756:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.029758:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.029759:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.029759:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.029761:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.029761:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.029763:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.029765:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.029765:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.029776:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.029777:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.029778:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.029779:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.029780:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677474.029780:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.029781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.029781:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.029783:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.029783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.029840:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.029843:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.029845:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.029847:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.029847:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.029849:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.029850:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.029851:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.029852:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.029853:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.029856:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.029857:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.029860:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.029861:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.029862:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.029864:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.029864:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.029865:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.029867:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.029867:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.029877:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.029880:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677474.029880:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.029881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.029881:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.029883:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.029884:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.029884:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.029885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.029886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.030042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.030048:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.030051:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.030052:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.030054:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.030056:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.030057:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.030059:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.030060:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.030061:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.030062:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.030065:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.030067:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.030070:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.030071:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.030072:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.030074:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.030075:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.030076:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.030076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677474.030087:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.030090:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.030090:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.030092:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.030093:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.030095:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.030542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.030548:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.030550:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:20.0:1619677474.030553:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.030556:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677474.030558:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.030559:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000400:00000200:21.0:1619677474.030560:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.030561:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.030562:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000400:00000200:21.0:1619677474.030563:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.030563:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:21.0:1619677474.030564:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.030565:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.030566:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.030568:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.030569:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677474.030570:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:21.0:1619677474.030571:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.030571:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.030573:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677474.030573:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.030574:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:1.0:1619677474.030574:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.030575:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677474.030575:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.030577:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.030577:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.030578:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000400:00000200:1.0:1619677474.030578:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.030579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:1.0:1619677474.030580:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.030583:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.030584:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.030585:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.030586:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.030587:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.030587:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.030588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:22.0:1619677474.030592:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:9.2:1619677474.030592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677474.030594:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.030595:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.030596:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.030598:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.030599:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677474.030600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.030602:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.030603:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.030603:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.030605:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.030608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.030613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.030620:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.030622:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.030623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.030625:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.030628:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.030629:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.030630:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.030631:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.030632:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.030633:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677474.030633:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.030637:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.030637:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:24.0:1619677474.030638:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.030640:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:6.0:1619677474.030640:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.030642:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.030643:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.030644:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.030644:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677474.030644:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.030645:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.030646:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.030646:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.030647:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677474.030647:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.030648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677474.030648:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.030649:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:9.2:1619677474.030652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677474.030652:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.030654:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.030656:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.030657:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.030658:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.030659:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.030660:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677474.030660:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:6.0:1619677474.030661:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.030661:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:2.0:1619677474.030663:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.030664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.030664:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.030667:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.030670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677474.030672:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.030675:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.030676:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.030677:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.030678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.030678:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.030680:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677474.030680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677474.030682:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.030685:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.030685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.030686:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.030688:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.030689:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.030690:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.030691:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:5.2:1619677474.030691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677474.030694:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.030694:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677474.030696:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.0:1619677474.030697:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677474.030697:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.0:1619677474.030698:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.030698:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677474.030699:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.030699:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:21.0:1619677474.030699:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677474.030700:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677474.030700:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.030700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.0:1619677474.030701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.030702:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.030702:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677474.030702:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.030703:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.030704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677474.030706:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.030707:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:9.2:1619677474.030707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677474.030708:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.030710:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.030711:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.030712:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677474.030714:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677474.030715:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.030716:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.030717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677474.030717:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677474.030717:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.030719:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.030720:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.030721:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.030721:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677474.030722:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677474.030723:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.030724:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.030724:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.030725:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.030737:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.030740:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.030741:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.030741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.030743:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.030746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.031623:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.031626:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.031628:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.031631:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.031631:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.031632:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.031633:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.031635:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.031636:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.031637:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.031640:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.031642:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.031645:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.031646:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.031646:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.031648:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.031648:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.031649:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.031650:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.031663:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.031666:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.031667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.031667:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.031669:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.031674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.031675:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.031677:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.031678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.031678:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.031679:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.031680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.032359:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.032361:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.032362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.032363:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.032365:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.032367:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.032368:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.032369:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.032370:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.032371:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.032374:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.032376:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.032378:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.032379:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.032380:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.032381:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.032382:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.032383:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.032384:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.032384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.032395:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.032396:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.032397:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.032397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.032398:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.032398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.032399:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.032401:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.032402:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.032403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.032639:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.032642:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.032645:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.032647:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.032648:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.032649:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.032650:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.032651:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.032653:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.032654:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.032657:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.032659:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.032661:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.032662:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.032663:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.032664:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.032665:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.032666:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.032666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.032670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.032677:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.032680:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.032680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.032681:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.032682:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.032684:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.032687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.032693:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.032695:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.032696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.032696:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.032697:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.032697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.033145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.033154:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.033156:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.033157:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.033159:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.033161:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.033163:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.033164:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.033165:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.033166:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.033167:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.033170:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.033171:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.033173:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.033174:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.033174:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.033176:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.033177:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.033177:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.033178:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.033181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.033188:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.033190:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.033191:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.033191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.033192:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.033195:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.033212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.033217:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.033219:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.033221:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.033222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.033224:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.033226:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.033227:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.033228:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.033229:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.033230:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.033234:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.033235:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.033238:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.033238:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.033239:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.033241:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.033241:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.033242:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.033242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677474.033265:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.033268:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.033269:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.033269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.033271:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.033273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.033576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.033579:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.033582:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.033584:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.033585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.033586:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.033588:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.033589:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.033590:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.033591:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.033592:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.033596:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.033598:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.033600:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.033601:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.033601:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.033603:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.033604:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.033605:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.033605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.033610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.033619:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.033621:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.033623:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.033623:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.033625:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.033627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.033631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.033638:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.033641:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.033642:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.033642:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.033643:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.033643:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677474.033671:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.033673:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.033674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.033676:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.033678:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.033679:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.033681:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.033682:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.033683:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.033684:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.033687:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.033689:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.033691:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.033692:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.033692:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.033694:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.033695:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.033696:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.033696:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.033708:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.033710:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.033711:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.033712:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.033714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.033715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.033899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.033906:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:30.0:1619677474.033908:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.033909:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.033910:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677474.033911:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677474.033913:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677474.033914:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.033916:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677474.033916:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.033917:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.033918:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677474.033918:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.033919:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.033920:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.033920:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.033921:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.033922:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.033922:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.033923:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.033923:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.033927:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.033927:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677474.033930:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.033930:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.033932:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.033933:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.033933:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.033933:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.033934:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.033934:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.033935:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.033935:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.033936:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677474.033936:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.033937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677474.033937:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.033938:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.033938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.033942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.033950:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677474.033951:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:20.0:1619677474.033953:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677474.033954:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.033954:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.033954:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.033956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.033956:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.033957:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677474.033958:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.033960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:31.0:1619677474.033961:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:1.0:1619677474.033966:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.033969:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.033970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.033972:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.033975:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.033976:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.033977:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.033979:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.033980:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.033981:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.033984:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.033986:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.033989:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.033990:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.033990:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.033992:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.033993:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.033994:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677474.033995:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677474.033996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:22.0:1619677474.034006:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677474.034006:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:17.0:1619677474.034008:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:22.0:1619677474.034009:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677474.034009:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:22.0:1619677474.034010:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.034010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:6.0:1619677474.034010:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.034012:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.034015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.034067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.034074:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:4.2:1619677474.034076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:8.0:1619677474.034077:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.034078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.034080:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.034083:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.034084:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:8.0:1619677474.034084:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.034085:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.034086:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.034087:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:8.0:1619677474.034087:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.034089:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.034090:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.034090:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677474.034092:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.034093:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.034094:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677474.034095:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.034095:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677474.034097:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.034097:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.034098:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.034098:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.034099:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.034100:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.034101:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:26.0:1619677474.034102:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.034102:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677474.034103:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.034103:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.034103:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677474.034106:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.034107:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677474.034107:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:9.2:1619677474.034108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677474.034109:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677474.034110:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.034110:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.034111:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.034116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.034117:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.034119:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.034120:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.034121:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.034123:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.034124:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677474.034126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677474.034126:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.034127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.034127:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.034130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000400:00000200:1.0:1619677474.034130:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.034133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:2.0:1619677474.034137:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.034140:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.034141:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.034141:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.034142:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.034143:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.034186:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.034189:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.034191:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.034193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.034193:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.034195:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.034196:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.034198:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.034199:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.034200:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.034203:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.034205:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.034208:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.034208:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.034209:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.034211:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.034211:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.034212:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.034213:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.034225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.034227:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.034228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.034229:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.034231:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.034233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.034553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.034561:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.034563:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.034564:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.034566:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.034568:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.034570:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.034571:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.034572:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677474.034572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677474.034573:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.034574:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.034578:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.034580:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.034580:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.034581:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677474.034584:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.034584:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.034585:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.034585:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:26.0:1619677474.034586:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.034587:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:26.0:1619677474.034588:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.034589:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.034590:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.034591:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.034592:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.034593:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.034594:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.034594:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:26.0:1619677474.034595:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.034595:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.034597:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.034599:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.034600:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.034600:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.034602:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.034602:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.034603:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.034603:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.034603:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.034605:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.034606:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.034606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.034609:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.034613:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.034615:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.034616:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.034616:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.034617:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.034619:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.034621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.034633:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.034636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.034636:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.034637:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.034638:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.034638:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.034881:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.034884:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.034886:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.034889:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.034889:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.034890:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.034891:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.034893:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.034894:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.034894:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.034897:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.034899:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.034901:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.034902:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.034902:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.034903:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.034904:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.034906:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.034907:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.034908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.034918:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.034919:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.034920:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.034922:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.034922:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.034923:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.034923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.034924:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.034925:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.034926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.035117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.035122:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.035125:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.035127:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.035127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.035129:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.035131:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.035132:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.035137:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.035138:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.035139:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677474.035139:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677474.035142:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:21.0:1619677474.035142:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677474.035143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.035145:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.035145:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677474.035148:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.035148:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.035148:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:22.0:1619677474.035149:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.0:1619677474.035149:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:22.0:1619677474.035151:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.035151:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:22.0:1619677474.035152:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.035152:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:22.0:1619677474.035153:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.035153:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.035153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:22.0:1619677474.035154:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.035157:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677474.035157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677474.035159:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.035162:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.035163:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.035163:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677474.035164:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677474.035165:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677474.035166:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:22.0:1619677474.035166:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.035166:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.035167:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677474.035167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.035168:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.035170:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.035173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677474.035179:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:4.2:1619677474.035180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677474.035183:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.035184:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.035185:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.035185:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.035187:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677474.035188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677474.035190:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.035191:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.035193:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.035196:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.035197:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.035199:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.035200:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.035201:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.035202:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.035205:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.035207:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.035210:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.035210:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.035211:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.035213:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.035213:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.035214:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.035215:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.035216:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.035222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.035228:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.035230:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.035231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.035232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.035236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.035245:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.035247:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.035248:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.035249:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.035251:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.035254:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.035366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.035372:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.035375:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.035375:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677474.035377:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.035379:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.2:1619677474.035381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000400:00000200:2.0:1619677474.035381:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.035382:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.035383:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.035384:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.035385:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000400:00000200:2.0:1619677474.035385:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.035387:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677474.035388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:2.0:1619677474.035388:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677474.035391:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.5.59@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.54@o2ib7 : PUT - routed
00000800:00000200:2.0:1619677474.035391:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.035393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:2.0:1619677474.035393:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.035394:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.035394:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.035396:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.035397:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.035397:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.035398:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.035403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.035411:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.035413:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.035414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.035414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.035416:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.035419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677474.035492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.035497:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.035500:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.035501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.035502:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.035504:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.035506:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.035507:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.035508:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.035509:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.035510:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.035513:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.035515:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.035518:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.035519:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.035519:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.035521:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.035521:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.035522:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.035523:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:4.2:1619677474.035524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a7e00] (20)++
00000800:00000200:1.0:1619677474.035524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.035529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677474.035532:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a7e00] (21)++
00000800:00000200:24.0:1619677474.035535:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.59@o2ib3
00000800:00000200:17.0:1619677474.035535:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:8.0:1619677474.035536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a7e00] (22)--
00000800:00000200:6.0:1619677474.035536:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:24.0:1619677474.035537:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.59@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.035537:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.035537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.035538:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.035539:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.035540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.035540:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.035541:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.035542:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.035543:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:24.0:1619677474.035544:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.035544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:24.0:1619677474.035546:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.035547:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.035548:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:24.0:1619677474.035551:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.59@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.035553:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:24.0:1619677474.035555:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.035556:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677474.035557:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.035558:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677474.035558:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a7e00] (21)++
00000800:00000200:24.0:1619677474.035560:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a7e00] (22)--
00000800:00000200:24.0:1619677474.035560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a7e00] (21)--
00000800:00000200:24.2:1619677474.035564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.035568:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.035569:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:8.0:1619677474.035570:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.035571:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.035572:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.035574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677474.035993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.036001:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.036004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.036004:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677474.036006:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.036009:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.036011:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677474.036012:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.036013:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.036014:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.036015:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.036018:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.036020:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.036022:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.036022:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677474.036023:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.036025:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.036025:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.036027:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.036028:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.036028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:4.2:1619677474.036034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677474.036039:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.036041:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:26.0:1619677474.036041:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:31.0:1619677474.036042:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.036043:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.036043:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.036043:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.036045:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677474.036046:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.036047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.036063:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.036066:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.036067:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.036067:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.036068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.036069:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.036218:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.036221:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.036223:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.036225:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.036225:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.036226:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.036227:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.036228:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.036229:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.036230:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.036233:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.036234:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.036238:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.036238:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.036239:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.036241:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.036241:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.036242:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.036243:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.036255:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.036257:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.036258:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.036259:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.036261:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.036263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.036853:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.036855:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.036856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.036858:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.036860:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.036861:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.036862:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.036864:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.036865:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.036866:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.036869:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.036870:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.036873:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.036874:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.036874:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.036876:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.036877:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.036878:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.036878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.036891:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.036893:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.036894:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.036896:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.036897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.036898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.036910:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.036913:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.036914:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.036915:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.036915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.036916:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677474.037617:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.037620:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.037621:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.037623:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.037626:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.037627:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677474.037629:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.037630:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.037631:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.037632:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677474.037636:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.037638:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677474.037641:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.037642:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677474.037642:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677474.037644:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677474.037645:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.037645:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.037646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.037651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.037659:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.037661:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.037662:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.037663:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.037665:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.037668:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677474.037672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677474.037678:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.037681:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677474.037681:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.037682:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.037683:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.037683:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677474.037934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.037941:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.037944:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.037945:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.037946:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.037948:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.037949:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677474.037950:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.037951:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.037952:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677474.037952:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.037954:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.037955:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.037955:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.037957:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.037958:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.037958:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.037959:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:10.0:1619677474.037960:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.037961:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.037962:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.037963:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.037963:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677474.037964:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.037964:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.037965:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.037966:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:30.0:1619677474.037967:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.037967:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.037968:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:30.0:1619677474.037969:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.037969:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.037970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677474.037971:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.037972:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:9.2:1619677474.037972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.037973:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:30.0:1619677474.037974:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (24)--
00000800:00000200:30.0:1619677474.037975:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677474.037975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677474.037976:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.037976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677474.037979:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:26.0:1619677474.037982:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677474.037982:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:24.0:1619677474.037983:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.037983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:8.0:1619677474.037983:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000800:00000200:26.0:1619677474.037984:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:24.0:1619677474.037984:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.037984:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.037985:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.037986:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677474.037986:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.037987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.037988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.037989:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.038109:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.038111:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.038112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.038114:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.038116:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.038118:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.038119:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.038121:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.038122:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.038123:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.038126:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.038128:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.038130:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.038131:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.038132:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.038133:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.038134:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.038135:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.038136:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.038137:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.038147:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.038147:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.038149:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.038150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.038150:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.038150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.038151:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.038152:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.038153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.038155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.038278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:10.0:1619677474.038285:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677474.038288:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677474.038289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:10.0:1619677474.038291:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.038298:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.038300:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677474.038301:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.038303:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.038304:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.038305:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:10.0:1619677474.038305:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677474.038308:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:10.0:1619677474.038308:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.038310:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.038311:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677474.038312:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.038312:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.038314:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677474.038314:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.038314:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:31.0:1619677474.038315:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.038315:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:31.0:1619677474.038316:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.038317:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.038317:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677474.038317:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:31.0:1619677474.038318:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.038318:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:10.0:1619677474.038320:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677474.038320:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:31.0:1619677474.038321:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.038322:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.2:1619677474.038324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677474.038325:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.038326:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:4.2:1619677474.038326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:31.0:1619677474.038327:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.038328:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.038329:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.038330:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.038332:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.038332:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:17.0:1619677474.038333:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677474.038334:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:17.0:1619677474.038335:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:26.0:1619677474.038336:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:1.0:1619677474.038336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:26.0:1619677474.038337:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:17.0:1619677474.038337:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.038338:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000400:00000200:17.0:1619677474.038339:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.038342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677474.038343:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.038346:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677474.038346:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.038347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.038347:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.038348:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677474.038349:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.038350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.038350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.038352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.038502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.038507:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.038510:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.038512:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.038512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.038514:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.038516:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.038517:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.038518:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.038519:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.038520:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.038523:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.038525:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.038527:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.038528:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.038529:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.038531:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.038531:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.038533:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.038534:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.038534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.038536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.038542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677474.038543:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.038545:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.038546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.038546:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:8.0:1619677474.038547:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.038549:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.038549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:8.0:1619677474.038549:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.038550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677474.038551:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.038613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.038621:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.038624:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.038624:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.038626:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.038629:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.038631:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.038632:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.038633:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.038634:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.038635:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.038638:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.038640:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.038643:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.038644:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.038644:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.038646:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.038646:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.038647:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.038648:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.038652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.038660:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.038663:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.038664:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.038664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.038667:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.038670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:30.0:1619677474.038895:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.038897:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.038899:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.038901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.038902:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.038904:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.038905:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.038906:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.038907:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.038908:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.038911:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.038913:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.038915:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.038915:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.038916:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.038917:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.038918:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677474.038918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677474.038919:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.038919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677474.038924:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.038926:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.038927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.038929:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.038930:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:2.0:1619677474.038931:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.038932:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.038933:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.038933:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.038934:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.038935:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:2.0:1619677474.038935:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.038936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:2.0:1619677474.038936:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.038937:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:2.0:1619677474.038938:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.038941:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.038943:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.038945:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.038945:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.038946:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.038948:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.038948:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.038949:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.038950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.038954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.038960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677474.038961:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.038963:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.038964:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.038964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.038966:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.038967:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677474.038967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677474.038970:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.038971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.038973:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.038975:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.038977:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.038978:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.038979:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.038980:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.038981:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.038985:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.038987:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.038990:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.038990:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.038991:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.038993:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.038993:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.038994:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.038994:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.039008:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.039010:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.039011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.039012:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.039014:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.039016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.039256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.039260:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.039262:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.039264:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.039265:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.039267:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.039269:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.039270:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.039271:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.039272:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.039273:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.039277:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.039279:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.039281:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.039282:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.039282:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.039284:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.039285:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.039285:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.039286:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.039300:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.039302:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.039303:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.039304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.039305:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.039307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.039320:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.039323:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.039323:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.039324:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.039325:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.039326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.039451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.039459:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.039462:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.039463:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.039465:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.039467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677474.039468:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.039469:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.039471:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.039473:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.039474:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.039475:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.039476:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.039477:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677474.039480:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.039480:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.039482:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.039482:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.039484:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.039486:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.039486:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.039487:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.039487:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:26.0:1619677474.039488:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.039488:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677474.039489:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.039490:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.039491:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677474.039492:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.039492:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.039493:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.039493:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677474.039494:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.039496:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.039497:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.039497:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.039499:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.039499:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.039500:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.039502:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.039502:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677474.039504:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.039506:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:4.2:1619677474.039507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677474.039508:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.039509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.039510:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.039511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.039512:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.039513:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.039513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.039514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.039515:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.039515:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.039517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677474.039517:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.039518:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.039519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.039531:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.039534:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.039535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.039535:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.039536:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.039537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.039607:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.039609:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.039610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.039612:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.039614:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.039615:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.039616:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.039619:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.039619:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.039620:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.039623:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.039625:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.039625:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.039627:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677474.039628:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.039628:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.039629:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677474.039630:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.039631:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.039631:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677474.039632:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.039632:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.039632:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.039632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:30.0:1619677474.039633:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.039634:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.039636:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.039637:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.039638:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.039642:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.039643:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677474.039644:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677474.039645:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.039646:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.0:1619677474.039646:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677474.039647:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677474.039647:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.039648:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.039649:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:21.0:1619677474.039649:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.039650:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.039650:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.039650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.039651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.039660:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.039663:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.039664:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.039666:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.039667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.039667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677474.039989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.039996:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.039999:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.040000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.040002:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.040005:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.040007:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.040008:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.040009:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.040010:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.040011:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.040014:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.040016:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.040019:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.040020:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.040020:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.040022:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.040022:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.040023:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.040023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.040036:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.040038:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.040039:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.040039:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.040042:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.040044:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.040139:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.040142:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.040144:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.040146:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.040146:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.040148:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.040149:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.040150:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.040151:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.040152:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.040155:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.040157:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.040160:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.040161:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.040161:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.040163:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.040163:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.040164:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.040165:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.040176:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.040179:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.040180:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.040180:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.040182:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.040185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.040307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.040313:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.040316:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.040317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.040319:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.040322:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.040323:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.040324:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.040326:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.040327:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.040328:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.040331:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.040333:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.040336:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.040336:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.040337:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.040339:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.040339:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.040340:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.040340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.040402:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.040404:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.040405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.040405:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.040408:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.040410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.041112:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.041115:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.041115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.041117:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.041119:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.041121:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677474.041122:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.041124:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.041125:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.041126:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677474.041129:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.041131:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.041134:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.041135:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677474.041136:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.041138:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.041138:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.041139:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.041139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677474.041143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.041150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.041153:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.041154:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.041154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.041156:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.041159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.041162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.041168:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.041171:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.041172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.041172:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.041173:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.041173:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677474.041351:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.041353:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.041354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.041356:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.041359:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.041360:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.041361:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.041362:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.041363:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.041364:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.041367:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.041368:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.041370:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.041371:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.041372:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.041373:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.041374:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.041375:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.041375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.041388:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.041390:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.041391:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.041393:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.041395:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.041395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.042221:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.042224:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.042226:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.042229:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.042229:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.042231:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.042232:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.042233:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.042234:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.042235:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.042238:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.042240:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.042243:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.042244:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.042244:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.042246:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.042247:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.042248:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.042248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.042260:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.042263:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.042264:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.042264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.042267:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.042269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.042280:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.042282:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.042283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.042284:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.042285:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.042285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.042526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677474.042534:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.042537:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677474.042538:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677474.042539:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.042542:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.042544:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677474.042545:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.042546:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.042547:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.042548:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677474.042551:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.042553:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.042555:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.042556:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677474.042557:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677474.042558:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677474.042559:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.042559:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677474.042560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677474.042564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.042573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.042576:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.042577:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.042577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.042580:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.042583:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677474.042641:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.042644:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.042644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.042646:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.042649:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.042650:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.042651:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.042653:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.042654:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.042655:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.042658:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.042660:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.042662:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.042663:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.042664:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.042665:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.042666:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.042667:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.042668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.042680:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.042682:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.042683:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.042685:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.042686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.042687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.042697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.0:1619677474.042702:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.042705:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677474.042705:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677474.042706:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.042707:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.042707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.042708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677474.042709:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.042711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.042714:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.042717:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.042719:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.042721:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.042724:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.042725:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.042727:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.042733:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.042736:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:4.2:1619677474.042738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677474.042740:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.042741:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.042743:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.042745:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.042746:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677474.042746:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.042747:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.042748:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677474.042749:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.042750:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.042752:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.042754:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.042755:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.042757:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677474.042758:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:10.0:1619677474.042758:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.042759:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.042760:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:13.0:1619677474.042761:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.042762:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.042763:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677474.042764:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.042765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.042765:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677474.042766:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677474.042768:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.042769:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.042769:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.042771:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.042771:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.042772:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.042772:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.042777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.042783:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.042785:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.042786:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.042786:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.042788:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.042791:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.043100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.043107:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.043109:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.043110:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.043112:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.043114:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.043115:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.043116:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.043117:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.043118:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.043119:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.043122:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.043123:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.043126:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.043127:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.043127:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.043129:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.043129:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.043130:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.043130:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.043135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.043144:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.043148:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.043150:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.043150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.043154:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.043157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677474.043251:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.043254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.043254:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677474.043257:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.043259:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.043260:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677474.043262:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.043263:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.043265:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.043266:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677474.043269:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.043271:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677474.043274:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.043275:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677474.043275:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677474.043277:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677474.043278:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.043279:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.043279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677474.043283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.043292:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.043300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.043302:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (22)--
00000800:00000200:9.2:1619677474.043304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (22)++
00000400:00000200:2.0:1619677474.043305:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.043309:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.043312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.043314:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.043317:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.043319:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.043319:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.043321:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.043322:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:30.0:1619677474.043338:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.043341:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.043343:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.043345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.043346:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.043348:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.043349:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.043350:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.043351:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.043352:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.043356:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.043358:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.043360:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.043361:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.043361:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.043363:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.043363:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.043365:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.043366:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.043367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.043378:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.043379:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.043380:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.043381:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.043381:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:31.0:1619677474.043382:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.043382:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.043383:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.043384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.043385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.043390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.043393:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.043396:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.043398:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.043399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.043400:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.043402:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.043403:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.043405:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.043406:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.043407:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.043410:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.043412:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.043415:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.043415:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.043416:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.043418:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.043418:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.043419:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.043419:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.043424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.043433:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.043435:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.043437:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.043437:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.043438:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.043441:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.043716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.043723:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.043726:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.043727:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.043729:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.043731:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.043733:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.043734:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.043735:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.043736:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.043737:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.043741:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.043743:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.043746:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.043746:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.043747:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.043749:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.043749:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.043750:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.043750:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.043763:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.043765:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.043766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.043767:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.043769:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.043771:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.043837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:9.2:1619677474.043839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677474.043844:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.043847:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000800:00000200:18.0:1619677474.043848:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000400:00000200:30.0:1619677474.043849:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.043850:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677474.043851:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677474.043851:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.043851:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:30.0:1619677474.043853:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677474.043853:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.102@o2ib7 : GET - for me
00000400:00000200:30.0:1619677474.043854:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.043856:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.043857:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.043857:0:19933:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.102@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:30.0:1619677474.043858:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:18.0:1619677474.043859:0:19933:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.102@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:18.0:1619677474.043861:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000400:00000200:30.0:1619677474.043862:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.043862:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677474.043864:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:18.0:1619677474.043864:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677474.043866:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.043867:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:30.0:1619677474.043868:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:18.0:1619677474.043868:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : REPLY try# 0
00000800:00000200:30.0:1619677474.043869:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:30.0:1619677474.043870:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:18.0:1619677474.043870:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.043871:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.043872:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:18.0:1619677474.043872:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.043873:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.043874:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.043875:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.043876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677474.043882:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.043885:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.043885:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.043887:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.043889:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.043890:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:9.2:1619677474.043898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.043906:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.043908:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677474.043909:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.043910:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.043911:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: REPLY: OK
00000800:00000200:2.0:1619677474.043912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:31.0:1619677474.043926:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.043929:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.043931:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.043933:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.043933:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.043934:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.043936:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.043937:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.043938:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.043939:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.043942:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.043944:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.043946:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.043947:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.043948:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.043949:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.043950:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.043951:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.043951:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.043963:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.043964:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.043965:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.043966:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.043967:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.043967:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.043968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.043969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:31.0:1619677474.043970:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.043971:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.043972:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.043974:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.043975:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.043976:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.043977:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.043978:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.043981:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.043981:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.043982:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.043983:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.043984:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.043984:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.043985:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.043985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.043985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.043986:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.043986:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.043988:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.043989:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.043989:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.043990:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.044023:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.044026:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.044027:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.044027:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.044029:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.044033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.044110:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.044113:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.044114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.044116:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.044117:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.044119:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.044120:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.044122:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.044123:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.044124:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.044127:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.044129:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.044131:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.044132:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.044132:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.044134:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.044135:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.044136:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.044137:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.044138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.044149:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.044150:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.044150:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.044151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.044152:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.044152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677474.044153:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.044155:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.044156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.044157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.044431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.044439:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.044442:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.044443:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.044445:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.044447:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.044449:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.044450:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.044451:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.044452:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.044453:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.044457:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.044458:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:10.0:1619677474.044459:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.044460:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677474.044461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:10.0:1619677474.044462:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677474.044463:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.044463:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.044464:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:17.0:1619677474.044466:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.044466:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.044466:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677474.044467:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:10.0:1619677474.044467:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.044467:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:17.0:1619677474.044469:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.044470:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.044471:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.044472:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677474.044476:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.044477:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677474.044480:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.044481:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677474.044481:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677474.044483:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677474.044484:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.044484:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677474.044485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677474.044490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.044499:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.044501:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.044502:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.044502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677474.044504:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677474.044504:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.044506:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677474.044507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:30.0:1619677474.044508:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.044511:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.044511:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.044513:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.044514:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.044515:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.044516:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.044517:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.044520:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.044522:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.044523:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.044524:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.044525:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677474.044525:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.044526:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.044526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.044527:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:22.0:1619677474.044527:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.044528:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.044529:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.044529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677474.044529:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.044532:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677474.044533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.044540:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.044542:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.044543:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.044543:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.044545:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.044548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.044693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.044701:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.044703:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.044704:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.044705:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.044708:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.044710:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.044711:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.044712:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.044713:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.044714:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.044717:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.044718:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.044721:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.044722:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.044722:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.044724:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.044725:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.044725:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.044726:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.044738:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.044741:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.044741:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.044742:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.044744:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.044747:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.045188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.045193:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.045196:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.045197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.045199:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.045201:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.045203:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.045204:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.045205:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.045206:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.045207:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.045211:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.045212:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.045215:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.045216:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.045216:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.045218:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.045218:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.045219:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.045219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.045232:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.045234:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.045235:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.045235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.045237:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.045240:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.045755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.045758:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.045761:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.045763:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.045764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.045765:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.045767:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.045768:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.045770:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.045771:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.045772:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.045775:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.045777:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.045780:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.045781:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.045781:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.045783:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.045784:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.045784:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.045785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.045790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.045797:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.045800:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.045801:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.045801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.045803:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.045806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.045813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.045818:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.045821:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.045822:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.045822:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.045823:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.045823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677474.045866:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.045869:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.045869:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.045871:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.045873:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.045874:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.045876:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.045877:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.045878:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.045879:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.045882:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.045883:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.045886:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.045886:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.045887:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.045889:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.045889:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.045890:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.045891:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.045902:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.045905:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.045905:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.045907:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.045908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.045909:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.046156:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.046159:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.046162:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.046164:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.046164:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.046165:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.046167:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.046168:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.046169:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.046170:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.046173:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.046175:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.046178:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.046179:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.046180:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.046182:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.046182:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.046183:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.046184:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.046188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.046196:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.046198:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.046200:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.046201:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.046201:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.046205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:30.0:1619677474.046356:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.046359:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.046361:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.046363:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.046364:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.046365:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.046367:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.046368:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.046369:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.046370:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.046373:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.046375:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.046378:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.046378:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.046379:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.046381:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.046381:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.046383:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.046384:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.046384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.046396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.046398:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.046398:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.046399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.046400:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.046401:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.046401:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.046402:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:22.0:1619677474.046402:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.046405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.046413:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.046416:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.046417:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.046417:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.046418:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.046419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.047165:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.047167:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.047168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.047170:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.047172:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.047173:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.047174:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.047176:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.047177:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.047178:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.047181:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.047182:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.047185:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.047186:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.047186:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.047188:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.047188:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.047189:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.047189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.047201:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.047203:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.047204:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.047206:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.047207:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.047208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.047299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.047307:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.047310:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.047311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.047312:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.047315:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.047317:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.047318:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.047319:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.047320:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.047322:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.047325:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.047327:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.047330:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.047330:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.047331:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.047332:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.047333:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.047334:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.047335:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.047336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677474.047339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.047342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677474.047348:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677474.047350:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.047351:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677474.047351:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.047352:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.047352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677474.047352:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.047354:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677474.047354:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.047358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677474.047587:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.047590:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.047590:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.047593:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.047595:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.047597:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.047598:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.047599:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.047600:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.047601:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.047605:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.047606:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.047609:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.047610:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.047610:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.047612:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.047613:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.047614:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.047614:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.047621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.047630:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.047632:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.047633:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.047634:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.047636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.047637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.047706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.047713:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.047716:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.047716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.047718:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.047721:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.047722:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.047723:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.047725:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.047726:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.047727:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.047731:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.047733:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.047736:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.047737:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.047737:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.047739:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.047740:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.047741:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.047742:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.047742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677474.047747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677474.047752:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:2.0:1619677474.047753:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677474.047755:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:2.0:1619677474.047755:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.047756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677474.047756:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677474.047757:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.047758:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.047759:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.047762:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677474.047774:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.047777:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.047777:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.047778:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.047779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.047779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.048011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.048019:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.048021:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.048022:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.048024:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.048027:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.048028:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.048030:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.048031:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.048032:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.048033:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.048037:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.048038:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.048041:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.048042:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.048043:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.048044:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.048045:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.048046:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.048047:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.048047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.048049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.048053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.048056:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.048058:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.048059:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.048060:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.048060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.048061:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.048062:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.048063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.048064:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677474.048064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.048445:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.048447:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.048450:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.048452:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.048452:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.048454:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.048455:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.048456:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.048457:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.048458:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.048462:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.048463:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.048466:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.048467:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.048468:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.048469:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.048470:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.048471:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.048471:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677474.048481:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.048484:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.048486:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.048488:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.048488:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.048490:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.048491:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.048492:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.048493:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.048493:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677474.048494:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.048496:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.048496:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.048497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.048497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.048498:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:6.0:1619677474.048499:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.048501:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.048501:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.048502:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.048502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.048503:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.048504:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.048506:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.048507:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.048508:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.048509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.048516:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.048519:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.048520:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.048520:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.048521:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.048522:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.048523:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.048524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677474.048524:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.048525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.048528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.048533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.048536:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.048537:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.048537:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.048537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.048538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.048671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.048674:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.048675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.048677:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.048679:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.048680:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.048682:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.048684:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.048685:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.048686:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.048689:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.048691:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.048694:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.048694:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.048695:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.048697:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.048697:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.048698:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.048699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.048711:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.048713:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.048714:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.048715:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.048717:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.048717:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.048728:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.048730:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.048731:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.048732:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.048733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.048734:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.048937:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.048940:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.048942:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.048944:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.048944:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.048946:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.048947:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.048948:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.048950:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.048950:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.048953:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.048955:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.048957:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.048958:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.048959:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.048960:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.048960:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.048961:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.048962:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.048972:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.048975:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.048976:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.048977:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.048979:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.048979:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677474.049205:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.049208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.049208:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677474.049211:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.049213:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.049215:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677474.049216:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.049217:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.049219:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.049219:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677474.049223:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.049224:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677474.049227:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.049228:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677474.049229:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677474.049231:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677474.049231:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.049233:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677474.049233:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677474.049234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677474.049237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677474.049244:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:2.0:1619677474.049244:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677474.049246:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677474.049247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:2.0:1619677474.049247:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:22.0:1619677474.049248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:2.0:1619677474.049248:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.049249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.049250:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.049253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.049259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.049267:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.049269:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.049270:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.049270:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.049271:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.049272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.049412:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.049415:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.049417:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.049419:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.049419:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.049421:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.049422:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.049423:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.049424:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.049425:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.049428:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.049430:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.049432:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.049433:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.049434:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.049435:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.049435:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.049436:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.049437:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.049442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.049448:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:10.0:1619677474.049450:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677474.049451:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.049451:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.049453:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.049453:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.049453:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677474.049455:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:10.0:1619677474.049455:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.049456:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:10.0:1619677474.049458:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.049459:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.049460:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.049461:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.049462:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.049463:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.049466:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.049467:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.049470:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.049471:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.049471:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.049473:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.049473:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.049474:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.049475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.049488:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.049491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.049491:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.049492:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.049494:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.049497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.049597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.049604:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.049607:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.049608:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.049610:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.049612:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.049613:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.049615:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.049616:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.049617:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.049617:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.049620:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.049622:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.049624:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.049625:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.049625:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.049627:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.049628:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.049628:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.049629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.049641:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.049643:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.049644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.049645:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.049647:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.049649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.049662:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.049665:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.049665:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.049666:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.049667:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.049667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.049716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.049724:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.049726:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.049727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.049728:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.049732:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.049733:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.049734:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.049735:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.049736:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.049737:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.049740:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.049742:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.049744:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.049745:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.049746:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.049747:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.049748:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.049749:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.049749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.049761:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.049763:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.049764:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.049765:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.049767:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.049770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.049999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.050005:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.050008:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.050009:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.050011:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.050013:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.050015:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.050016:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.050017:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.050018:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.050019:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.050023:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.050024:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.050027:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.050028:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.050028:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.050030:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.050030:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.050031:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.050032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.050044:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.050046:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.050047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.050047:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.050050:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.050052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.050113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.050119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.050122:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.050123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.050125:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.050127:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.050129:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.050130:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.050131:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.050132:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.050133:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.050136:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.050138:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.050140:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.050141:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.050141:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.050143:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.050144:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.050144:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.050145:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.050149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.050156:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.050158:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.050159:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.050159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.050161:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.050162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.050293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.050299:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.050301:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.050304:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.050304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.050306:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.050308:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.050309:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.050310:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.050312:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.050312:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.050316:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.050317:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.050320:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.050320:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.050321:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.050322:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.050323:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.050325:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.050326:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.050326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.050329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.050331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.050334:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.050336:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.050337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.050338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.050338:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.050340:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.050341:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.050341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.050343:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.050346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.050351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677474.050357:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:1.0:1619677474.050359:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.050360:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.050361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:1.0:1619677474.050362:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677474.050363:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.050363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.050363:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.050364:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.050364:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:22.0:1619677474.050365:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.050367:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.050368:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.050370:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.050371:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.050372:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.050375:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.050377:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.050379:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.050380:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.050381:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.050382:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.050383:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.050384:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.050385:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.050386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.050396:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.050396:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.050398:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.050398:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.050398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.050399:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.050399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.050401:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.050402:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.050403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.050532:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.050535:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.050537:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.050539:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.050540:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.050541:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.050543:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.050544:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.050545:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.050546:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.050549:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.050551:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.050554:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.050555:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.050555:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.050557:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.050558:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.050559:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.050559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.050563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.050571:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.050573:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.050574:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.050576:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.050576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.050579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:31.0:1619677474.050629:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.050632:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.050634:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.050636:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.050637:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.050639:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.050640:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.050641:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.050642:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.050643:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.050647:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.050648:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.050651:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.050652:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.050652:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.050654:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.050655:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.050656:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.050656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.050669:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.050672:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.050672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.050673:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.050675:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.050677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.050694:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.050696:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.050697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.050697:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.050699:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.050699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.051315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.051323:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.051325:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.051326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.051328:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.051331:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.051332:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.051333:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.051334:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.051335:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.051336:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.051339:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.051341:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.051343:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.051344:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.051344:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.051346:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.051346:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.051347:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.051348:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.051349:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677474.051355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677474.051360:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677474.051362:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:13.0:1619677474.051362:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.051363:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.051364:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.051364:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677474.051364:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677474.051366:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677474.051367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.051368:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677474.051659:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.051662:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.051663:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.051665:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.051667:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.051668:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.051669:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.051671:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.051672:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.051673:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.051676:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.051678:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.051680:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.051681:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.051682:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.051684:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.051684:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.051685:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.051685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.051697:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.051699:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.051700:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.051702:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.051703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.051704:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.052321:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.052324:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.052326:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.052328:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.052329:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.052330:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.052331:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.052332:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.052333:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.052334:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.052337:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.052339:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.052342:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.052343:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.052343:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.052345:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.052346:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.052347:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.052347:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.052361:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.052363:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.052364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.052365:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.052367:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.052370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.052372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.052378:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.052381:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.052382:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.052384:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.052386:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.052387:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.052389:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.052390:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.052391:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.052392:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.052396:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.052398:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.052400:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.052401:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.052402:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.052403:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.052404:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.052405:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.052405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.052410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.052419:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.052421:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.052422:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.052423:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.052425:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.052427:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.052431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.052438:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.052441:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.052442:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.052442:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.052443:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.052444:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.052833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.052841:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.052844:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.052845:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.052847:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.052849:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.052851:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.052852:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.052853:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.052854:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.052855:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.052858:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.052860:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.052862:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.052863:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.052864:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.052865:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.052866:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.052866:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.052867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.052874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.052881:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.052884:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.052885:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.052885:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.052886:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.052889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.052906:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.052908:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.052910:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.052912:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.052913:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.052915:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.052916:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.052917:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.052918:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.052918:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.052921:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.052923:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.052927:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.052927:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.052928:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.052930:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.052930:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.052932:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.052934:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.052934:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.052936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.052948:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.052948:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.052951:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.052951:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677474.052952:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677474.052952:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.052953:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677474.052954:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.052954:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.052955:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.052956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.052957:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:20.0:1619677474.052957:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.052958:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.052959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.052973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.052980:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.052983:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.052984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.052985:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.052988:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.052990:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.052991:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.052992:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.052993:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.052994:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.052997:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.052999:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.053002:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.053002:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.053003:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.053005:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.053005:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.053006:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.053006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.053011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.053019:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.053021:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.053022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.053023:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.053025:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.053028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677474.053133:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.053136:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.053136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.053139:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.053141:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.053142:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.053143:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.053145:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.053146:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.053147:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.053150:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.053152:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.053155:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.053155:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.053156:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.053158:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.053158:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.053159:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.053160:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.053171:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.053173:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.053174:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.053176:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.053177:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.053178:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.053233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.053240:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.053243:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.053243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.053245:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.053248:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.053250:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.053251:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.053252:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.053253:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.053254:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.053257:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.053259:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.053262:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.053263:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.053264:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.053265:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.053266:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.053267:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.053267:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.053275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.053283:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.053286:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.053287:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.053289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.053289:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.053292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:30.0:1619677474.053496:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.053499:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.053501:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.053503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.053504:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.053505:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.053506:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.053507:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.053508:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.053509:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.053512:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.053514:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.053516:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.053516:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.053517:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.053518:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.053519:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.053520:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.053520:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.053532:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.053534:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.053535:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.053536:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.053538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.053539:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.053809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.053814:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.053816:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.053819:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.053819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.053821:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.053823:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.053824:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.053826:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.053827:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.053827:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.053830:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.053832:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.053834:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.053835:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.053836:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.053838:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.053838:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.053839:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.053840:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.053844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.053852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.053854:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.053854:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.053856:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.053857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.053858:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.053926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.053934:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.053937:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.053937:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677474.053940:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.053942:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.053944:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.053945:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.053946:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.053948:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.053948:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.053952:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.053954:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.053956:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.053957:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.053958:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.053960:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.053960:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.053961:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.053961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.053974:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.053977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.053977:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.053978:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.053980:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.053983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.054076:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.054079:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.054081:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.054083:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.054083:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.054085:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.054086:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.054087:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.054088:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.054089:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.054092:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.054094:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.054095:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.054096:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.054097:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.054098:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.054099:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.054099:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.054100:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.054110:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.054113:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.054113:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.054115:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.054117:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.054117:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.054132:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.054134:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.054135:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.054136:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.054136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.054138:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.054263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.054270:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.054273:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.054274:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.054275:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.054278:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.054279:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.054280:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.054282:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.054282:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.054283:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.054286:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.054288:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.054290:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.054291:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.054292:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.054299:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.054300:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.054301:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.054301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.054308:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.054311:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.054312:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.054312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.054314:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.054317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.054450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677474.054456:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.054459:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677474.054460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677474.054462:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.054464:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.054466:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677474.054467:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.054468:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.054469:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.054470:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677474.054474:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.054475:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677474.054475:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677474.054478:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:17.0:1619677474.054478:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.054479:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677474.054480:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:30.0:1619677474.054481:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.054482:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677474.054482:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:31.0:1619677474.054483:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.054483:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.054483:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677474.054483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000400:00000200:30.0:1619677474.054484:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.054485:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.054487:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.054488:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677474.054488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:30.0:1619677474.054489:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.054492:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.054494:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.054494:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:30.0:1619677474.054497:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.054497:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:30.0:1619677474.054498:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.054498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.054498:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.054499:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.054500:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677474.054500:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.054501:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.054502:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:10.0:1619677474.054503:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:30.0:1619677474.054504:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.054504:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.054514:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.054518:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.054518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.054518:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.054519:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.054520:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.054521:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.054521:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.054522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.054524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.054672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.054680:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.054683:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.054684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.054686:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.054689:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.054690:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.054691:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.054693:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.054694:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.054695:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.054698:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.054700:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.054703:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.054703:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.054704:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.054706:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.054706:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.054707:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.054707:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.054719:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.054722:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.054723:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.054723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.054725:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.054728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.054737:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.054739:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.054740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.054741:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.054742:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.054742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.054886:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.054888:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.054889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.054891:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.054892:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.054894:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.054895:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.054897:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.054897:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.054898:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.054902:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.054904:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.054906:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.054907:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.054907:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.054909:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.054910:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.054910:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.054911:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.054923:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.054925:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:4.2:1619677474.054925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:21.0:1619677474.054926:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.054928:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.054930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.054931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677474.054933:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.054936:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.054937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.054939:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.054940:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:26.0:1619677474.054941:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.054942:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677474.054943:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:20.0:1619677474.054943:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000400:00000200:26.0:1619677474.054944:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.054944:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.054944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677474.054945:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.054945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677474.054946:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.054947:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677474.054950:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.054952:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.054954:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.054954:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.054955:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.054957:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.054957:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.054958:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.054958:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.054962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.054968:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.054970:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.054971:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.054972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.054973:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.054974:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:31.0:1619677474.055167:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.055169:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.055171:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.055173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.055174:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.055175:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.055176:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.055177:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.055178:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.055179:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.055182:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.055184:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.055186:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.055187:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.055187:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.055189:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.055190:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.055190:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.055191:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.055195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.055201:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.055204:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.055205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.055205:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.055207:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.055209:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.055373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.055379:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.055382:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.055383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.055384:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.055387:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.055389:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.055390:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.055391:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.055392:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.055393:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.055396:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.055398:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.055400:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.055401:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.055401:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.055403:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.055404:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.055405:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.055406:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.055407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.055412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.055418:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.055418:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.055421:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.055421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.055421:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.055422:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.055422:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.055423:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.055424:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.055427:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.055566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677474.055569:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.055572:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677474.055574:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.055575:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677474.055576:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.055578:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677474.055579:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.055581:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.055582:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.055583:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677474.055586:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.055588:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.055590:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.055591:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.055592:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.055594:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.055595:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.055595:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677474.055596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:21.0:1619677474.055607:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.055609:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.055610:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.055611:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.055612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.055613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677474.056274:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.056277:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.056277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.056279:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.056282:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.056283:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.056284:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.056286:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.056287:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.056288:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.056291:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.056299:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.056301:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.056302:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.056302:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.056304:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.056305:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.056306:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.056307:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.056307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.056318:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.056319:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.056321:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.056321:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.056322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.056322:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.056322:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.056323:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.056324:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.056326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.056390:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.056393:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.056395:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.056398:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.056398:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.056400:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.056401:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.056402:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.056403:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.056404:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.056408:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.056409:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.056412:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.056413:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.056414:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.056415:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.056416:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.056417:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.056417:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.056429:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.056432:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.056433:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.056433:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.056435:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.056438:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.056690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677474.056697:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.056700:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677474.056702:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.056704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677474.056705:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.056707:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.056708:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.056709:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.056710:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.056711:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.056714:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.056715:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.056717:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.056718:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.056719:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.056720:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.056721:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.056722:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.056722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.056726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.056733:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.056735:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.056736:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.056736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.056738:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.056740:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.056973:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.056976:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.056978:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.056979:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.056980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.056981:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.056982:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.056983:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.056984:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.056985:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.056987:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.056989:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.056991:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.056992:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.056992:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.056994:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.056994:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.056996:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.056997:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.056998:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.056999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.057006:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.057008:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.057009:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.057009:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.057011:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.057011:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.057013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.057014:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.057014:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.057015:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.057604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.057612:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.057615:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.057616:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.057617:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.057619:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.057619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677474.057621:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.057622:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.057623:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.057624:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.057624:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.057626:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677474.057627:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.057629:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.057629:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.057630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677474.057631:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.057632:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.057632:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:24.0:1619677474.057632:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.057634:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.057634:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677474.057634:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.057636:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677474.057636:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677474.057637:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.057637:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677474.057637:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.057638:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.057639:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.057640:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.2:1619677474.057640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:24.0:1619677474.057643:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.057643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.057645:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.057647:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.057648:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.057648:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.057649:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:8.0:1619677474.057649:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.057650:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677474.057650:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677474.057651:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.057652:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677474.057652:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.057652:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.057653:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677474.057653:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.057653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.057654:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.057654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677474.057655:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.057656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677474.057658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.057661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:9.2:1619677474.057664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677474.057668:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677474.057670:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.057670:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.057671:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.057671:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.057672:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.057672:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:2.0:1619677474.057672:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677474.057673:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.057674:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.057674:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677474.057674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:2.0:1619677474.057674:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.057675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.057677:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.057682:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677474.057826:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.057829:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.057830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.057832:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.057834:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.057836:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.057837:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.057839:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.057840:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.057841:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.057844:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.057846:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.057848:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.057849:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.057850:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.057851:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.057852:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.057853:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.057853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.057865:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.057870:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.057872:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677474.057873:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.057874:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.057876:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.057877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.057879:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.057881:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.057883:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.057884:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.057885:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.057886:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.057888:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.057889:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.057890:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.057893:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.057895:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.057898:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.057899:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.057899:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.057901:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.057902:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.057903:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.057903:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.057907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.057915:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.057918:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.057919:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.057921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.057921:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.057924:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677474.058087:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.058090:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677474.058091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677474.058093:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.058096:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.058097:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677474.058098:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.058100:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.058101:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.058102:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677474.058105:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.058107:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677474.058110:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.058111:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:6.0:1619677474.058112:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:6.0:1619677474.058114:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677474.058115:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.058116:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.058116:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:13.0:1619677474.058128:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.058130:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.058131:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.058133:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.058135:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.058136:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677474.058145:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.058148:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.058149:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.058150:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.058150:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.058152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677474.058430:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.058433:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.058435:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.058437:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.058438:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.058439:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.058440:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.058442:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.058443:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.058444:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.058447:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.058448:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.058450:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.058451:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.058452:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.058453:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.058454:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.058455:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.058455:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.058466:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.058468:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.058469:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.058471:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.058472:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.058473:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.058500:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.058502:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.058504:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.058506:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.058506:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.058508:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.058509:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.058510:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.058511:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.058512:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.058515:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.058516:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.058519:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.058520:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.058520:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.058522:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.058522:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.058523:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.058523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.058535:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.058538:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.058539:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.058539:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.058541:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.058544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.058775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.058783:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.058785:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.058786:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.058788:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.058790:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.058792:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.058793:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.058794:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.058795:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.058796:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.058799:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.058800:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.058803:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.058804:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.058805:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.058806:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.058807:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.058808:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.058809:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.058809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.058815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.058820:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.058822:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.058823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.058823:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677474.058824:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677474.058824:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.058825:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.058826:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.058826:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.058829:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.058841:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.058844:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.058845:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.058845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.058846:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.058847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.059083:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.059085:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.059087:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.059089:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.059090:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.059091:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.059093:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.059094:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.059095:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.059096:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.059099:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.059100:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.059102:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.059103:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.059104:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.059105:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.059106:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.059107:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.059107:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.059117:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.059119:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.059120:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.059122:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.059123:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.059124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.059143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.059150:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.059152:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.059153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.059155:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.059158:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.059159:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.059160:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.059161:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.059162:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.059163:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.059166:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.059167:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.059170:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.059171:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.059171:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.059173:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.059173:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.059174:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.059175:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.059187:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.059190:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.059190:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.059191:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.059193:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.059196:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.059217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.059224:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.059226:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.059227:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.059229:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.059232:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.059233:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.059234:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.059236:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.059236:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.059237:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.059240:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.059241:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.059243:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.059244:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.059245:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.059246:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.059247:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.059247:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.059248:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677474.059260:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.059262:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.059263:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.059264:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.059266:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.059267:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677474.059569:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.059571:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.059572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.059574:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.059577:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.059578:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.059580:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.059581:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.059582:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.059583:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.059586:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.059588:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.059591:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.059591:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.059592:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.059594:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.059594:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.059595:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.059596:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:4.2:1619677474.059600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.059607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677474.059608:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.0:1619677474.059610:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.059611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677474.059611:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.059611:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.059613:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.059613:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.059614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.059616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677474.059617:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.059618:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.059620:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.059621:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.059622:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.059623:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.059626:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.059627:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677474.059628:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.059629:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.059630:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.059631:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677474.059631:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.059632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677474.059632:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.059633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677474.059633:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.059634:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.059635:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.059636:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.059636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.059648:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.059651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.059651:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.059652:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.059654:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.059657:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.059666:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.059669:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.059670:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.059670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.059671:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.059672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.059786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.059793:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.059796:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.059796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.059799:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.059801:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.059802:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.059803:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.059805:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.059806:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.059807:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.059810:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.059812:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.059814:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.059814:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.059815:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.059817:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.059817:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.059818:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.059819:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.059823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.059830:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.059832:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.059833:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.059833:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.059834:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.059836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.059842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.059849:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.059851:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.059852:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.059852:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.059853:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.059854:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.060146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.060152:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.060155:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.060156:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.060157:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.060160:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.060162:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.060163:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.060164:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.060165:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.060166:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.060169:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.060171:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.060173:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.060174:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.060174:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.060176:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.060177:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.060177:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.060178:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.060190:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.060192:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.060193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.060193:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.060196:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.060198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.060255:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.060257:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.060260:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.060262:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.060262:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.060263:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.060265:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.060266:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.060267:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.060268:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.060271:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.060272:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.060276:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.060276:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.060277:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.060279:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.060279:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.060280:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.060281:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.060297:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.060299:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.060300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.060300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.060302:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.060305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.060667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.060673:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.060676:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.060677:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.060679:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.060681:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.060683:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.060684:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.060685:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.060686:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.060687:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.060690:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.060692:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.2:1619677474.060693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:2.0:1619677474.060694:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.060695:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:20.0:1619677474.060696:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:2.0:1619677474.060696:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.060697:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.0:1619677474.060698:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:2.0:1619677474.060698:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.060699:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.060699:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:20.0:1619677474.060701:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.060702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677474.060703:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.060704:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.2:1619677474.060704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:20.0:1619677474.060706:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.060707:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.060708:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.060709:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.060712:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.060713:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:20.0:1619677474.060714:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.060715:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:20.0:1619677474.060716:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.060717:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:17.0:1619677474.060717:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.060717:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.0:1619677474.060718:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:17.0:1619677474.060719:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.060720:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.060720:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.060721:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:17.0:1619677474.060721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677474.060722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677474.060735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.060737:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.060738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.060738:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.060740:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.060742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.060850:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.060852:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.060853:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.060855:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.060857:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.060858:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.060859:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.060861:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.060862:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.060863:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.060866:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.060867:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.060870:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.060870:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.060871:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.060873:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.060873:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.060874:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.060875:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.060886:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.060888:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.060889:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.060891:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.060892:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.060893:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.061006:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.061008:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.061011:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.061013:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.061013:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.061015:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.061016:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.061018:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.061018:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.061019:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.061022:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.061024:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.061026:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.061027:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.061028:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.061029:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.061030:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.061031:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.061033:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.061033:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.061035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.061041:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.061044:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.061045:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.061045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.061047:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.061047:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.061049:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677474.061049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.061050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.061051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.061127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.061134:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.061137:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.061138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.061139:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.061142:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.061144:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.061145:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.061146:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.061147:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.061147:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.061150:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.061152:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.061155:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.061156:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.061156:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.061158:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.061158:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.061159:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.061159:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.061164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.061172:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.061174:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.061176:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.061177:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.061178:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.061180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.061825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.061833:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.061835:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.061837:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.061840:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.061840:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.061842:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677474.061843:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.061844:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.061845:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.061846:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677474.061849:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.061851:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677474.061854:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.061854:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677474.061855:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677474.061857:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677474.061857:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.061859:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677474.061860:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677474.061861:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677474.061864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.061867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:4.2:1619677474.061871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677474.061872:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.061875:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:13.0:1619677474.061875:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:18.0:1619677474.061876:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.061876:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677474.061877:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:18.0:1619677474.061878:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.061878:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.061879:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.0:1619677474.061879:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.061882:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677474.061882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677474.061882:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.061884:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.061887:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.061889:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.061890:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.061891:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.061892:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.061893:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.061896:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.061898:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.061901:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.061902:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.061902:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.061904:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.061904:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.061905:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.061906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.061910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.061917:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.061920:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.061921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.061921:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.061923:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.061927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.062184:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.062187:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.062189:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.062191:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.062192:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.062193:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.062194:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.062196:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.062197:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.062198:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.062201:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.062203:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.062206:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.062207:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.062207:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.062209:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.062209:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.062211:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.062212:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.062213:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.062224:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.062226:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.062227:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.062227:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.062228:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.062229:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.062230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.062230:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.062231:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.062233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.062333:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677474.062333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677474.062336:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.062337:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.062340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.062340:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.062341:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677474.062342:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.062343:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.062343:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.062344:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677474.062345:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.062346:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.062346:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.062347:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677474.062347:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.062349:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:10.0:1619677474.062349:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.062350:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.062350:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:6.0:1619677474.062350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677474.062351:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.062353:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:22.0:1619677474.062353:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.062354:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.062355:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.062356:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677474.062356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.062356:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.062357:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.062357:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677474.062357:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:30.0:1619677474.062359:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:22.0:1619677474.062359:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96506273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.062360:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.062360:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.062361:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677474.062361:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.062361:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.062362:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.062363:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.062364:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.062366:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677474.062366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677474.062367:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.062367:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.062368:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.062369:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.062369:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.062370:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.062370:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.062371:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.062372:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.062373:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.062374:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.062374:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:22.0:1619677474.062375:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:9.2:1619677474.062375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.062376:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677474.062376:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:22.0:1619677474.062377:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:21.0:1619677474.062377:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.062378:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.062379:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677474.062379:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.062380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677474.062381:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.062384:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:21.0:1619677474.062384:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:24.0:1619677474.062385:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.062386:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.062386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677474.062389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677474.062389:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677474.062390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677474.062391:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.062392:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.062392:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.062392:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:6.0:1619677474.062392:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.062393:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.062393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677474.062394:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.062394:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677474.062394:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:21.0:1619677474.062395:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.062395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.062395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.062396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.062397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.062977:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.062980:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.062982:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.062984:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.062985:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.062986:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.062988:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.062989:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.062990:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.062991:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.062995:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.062997:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.062999:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.063000:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.063000:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.063002:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.063002:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.063004:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.063004:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.063014:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.063016:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.063017:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.063019:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.063020:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.063021:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.063193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.063200:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.063203:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.063204:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.063206:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.063207:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.063209:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.063210:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.063212:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.063213:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.063213:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.063217:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.063219:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.063222:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.063222:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.063223:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.063225:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.063225:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.063226:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.063226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.063239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.063241:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.063242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.063243:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.063245:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.063248:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.063509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677474.063517:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.063519:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677474.063521:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677474.063522:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.063525:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.063527:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677474.063528:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.063530:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.063530:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.063531:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677474.063534:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.063536:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.063538:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.063539:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677474.063539:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677474.063541:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677474.063541:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.063542:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677474.063542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:31.0:1619677474.063554:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.063557:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.063557:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.063559:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.063561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.063562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677474.063741:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.063744:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.063746:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.063747:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.063748:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.063749:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.063750:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.063751:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.063752:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.063753:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.063756:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.063757:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.063760:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.063760:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.063761:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.063762:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.063763:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.063764:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.063764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.063768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.063775:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.063777:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.063778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.063778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.063780:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.063782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.063906:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.063909:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.063911:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.063913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.063914:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.063915:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.063916:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.063918:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.063919:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.063919:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.063923:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.063924:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.063927:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.063928:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.063928:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.063930:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.063931:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.063931:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.063932:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.063944:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.063947:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.063948:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.063948:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.063951:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.063953:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.063963:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.063965:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.063966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.063966:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.063967:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.063968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.064006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.064014:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.064017:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.064018:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.064020:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.064022:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.064023:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.064025:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.064026:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.064027:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.064028:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.064030:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.064032:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.064035:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.064035:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.064036:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.064038:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.064038:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.064039:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.064039:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.064052:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.064053:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.064054:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.064054:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677474.064056:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.064056:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.064058:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.064059:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.064061:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.064062:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.064064:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.064065:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.064066:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.064067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.064068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.064070:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:6.0:1619677474.064070:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.064073:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.064075:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.064076:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.064077:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.064079:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.064079:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.064080:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.064081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.064092:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.064094:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.064095:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.064097:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.064098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.064099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.064806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.064813:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.064816:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.064817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.064818:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.064821:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.064823:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.064824:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.064825:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.064826:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.064827:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.064830:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.064832:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.064835:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.064836:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.064836:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.064838:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.064838:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.064839:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.064840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.064853:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.064856:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.064857:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.064857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.064860:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.064862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.064865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.064871:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.064874:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.064875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.064877:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.064879:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.064881:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.064882:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.064884:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.064885:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.064886:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.064889:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.064891:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.064893:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.064894:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.064895:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.064896:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.064897:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.064898:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.064898:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.064910:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.064913:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.064914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.064914:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.064916:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.064918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.065260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.065267:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.065270:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.065271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.065273:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.065275:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.065277:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.065278:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.065280:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.065281:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.065282:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.065286:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.065288:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.065291:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.065291:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.065292:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.065298:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.065299:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.065300:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.065301:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677474.065308:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.065311:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.065312:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.065314:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.065315:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.065317:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:22.0:1619677474.065347:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.065349:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.065350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.065352:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.065354:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.065356:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.065357:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.065359:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.065360:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.065361:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.065364:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.065366:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.065368:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.065369:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.065370:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.065372:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.065372:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.065373:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.065374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.065385:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.065387:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.065388:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.065390:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.065391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.065392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.065409:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.065412:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.065413:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.065413:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.065414:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.065415:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677474.065472:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677474.065475:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.29@o2ib3
00000800:00000200:1.0:1619677474.065475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:17.0:1619677474.065478:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.29@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.065480:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.065481:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677474.065483:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.065484:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.065485:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.065486:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677474.065489:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.29@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.065490:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677474.065493:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.065494:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677474.065494:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677474.065496:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677474.065496:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1200] (21)++
00000800:00000200:17.0:1619677474.065497:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:17.0:1619677474.065499:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:17.0:1619677474.065499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000800:00000200:20.0:1619677474.065509:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:1.0:1619677474.065509:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)++
00000800:00000200:20.0:1619677474.065512:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:1.0:1619677474.065512:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1200] (23)--
00000800:00000200:1.0:1619677474.065512:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (22)--
00000400:00000200:20.0:1619677474.065513:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.065513:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1200] (21)--
00000400:00000200:20.0:1619677474.065514:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.065516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.065517:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677474.065543:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.065545:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.065547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.065548:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.065551:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.065552:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677474.065553:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.065555:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.065556:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.065557:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677474.065560:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.065562:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.065564:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.065565:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677474.065565:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.065567:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677474.065568:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.065569:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.065569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:6.0:1619677474.065581:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.065583:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.065584:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.065584:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.065587:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.065589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.065601:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.065604:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.065606:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.065607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.065609:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.065610:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.065611:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.065613:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.065614:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.065615:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.065618:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.065620:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.065624:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.065624:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.065625:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.065627:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.065627:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.065628:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.065629:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.065633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:8.2:1619677474.065634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.065641:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.065643:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000800:00000200:21.0:1619677474.065643:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.065645:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:31.0:1619677474.065646:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.065646:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.065648:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.065648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:21.0:1619677474.065648:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.065648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:31.0:1619677474.065650:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677474.065651:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.065652:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.065654:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.065654:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.065656:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:31.0:1619677474.065658:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.065660:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.065663:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.065664:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677474.065665:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677474.065667:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677474.065667:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.065668:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677474.065669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677474.065673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.065682:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.065684:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.065685:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.065686:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.065688:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.065691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.065912:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.065915:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.065917:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.065918:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.065920:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.065921:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.065922:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.065924:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.065925:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.065925:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.065929:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.065931:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.065933:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.065934:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.065935:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.065936:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.065937:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.065938:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.065940:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.065940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.065942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.065951:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.065953:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677474.065954:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677474.065954:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.065956:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:20.0:1619677474.065956:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.065957:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.065957:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.065958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.065959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677474.066206:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.066208:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677474.066209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677474.066211:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.066213:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.066215:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677474.066216:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.066218:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.066218:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.066219:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677474.066222:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.066224:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677474.066227:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.066227:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677474.066228:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.066230:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677474.066230:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.066231:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677474.066232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677474.066236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.066244:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.066247:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.066248:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.066248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.066250:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.066252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.066618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.066626:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.066629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.066629:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.066631:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.066635:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.066636:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.066638:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.066639:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.066640:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.066641:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.066644:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.066646:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.066649:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.066650:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.066650:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.066652:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.066653:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.066654:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.066655:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.066656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677474.066659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.066661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.066668:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677474.066668:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677474.066670:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.066671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677474.066671:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677474.066672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677474.066672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.066672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.066674:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.066678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.066680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.066687:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.066689:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.066690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.066691:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.066692:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.066692:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.066866:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.066869:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:6.0:1619677474.066869:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677474.066872:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.066872:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.066873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677474.066874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.066875:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.066876:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.066878:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677474.066878:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.066879:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.066879:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:31.0:1619677474.066881:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.066881:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.066882:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.066883:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677474.066883:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.066884:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.066885:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.066890:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.066891:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.066893:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.066894:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.066896:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.066897:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:31.0:1619677474.066898:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.066898:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.066898:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677474.066899:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.066899:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.066900:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677474.066901:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.066901:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.066901:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.066902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677474.066903:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.066903:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677474.066913:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.066915:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.066916:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.066917:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677474.066917:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.066918:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.066919:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677474.066920:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.066921:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.066921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.066923:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.066928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.066930:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.066932:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.066933:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.066933:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.066934:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.066935:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.066998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677474.067004:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677474.067004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.067007:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:30.0:1619677474.067008:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677474.067008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677474.067011:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.067011:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.067011:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.067013:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.067015:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.067015:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.067016:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677474.067016:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.067018:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.067019:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.067020:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.067021:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.067021:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.067022:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.067023:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677474.067026:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.067028:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.067028:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.067030:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.067031:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677474.067032:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.067032:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.067034:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.067034:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677474.067034:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.067034:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677474.067035:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.067036:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677474.067036:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:30.0:1619677474.067037:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677474.067037:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:30.0:1619677474.067038:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.067038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677474.067038:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.067040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.067043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.067049:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:31.0:1619677474.067050:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:18.0:1619677474.067050:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.067050:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677474.067051:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.067051:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.067052:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677474.067053:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:18.0:1619677474.067053:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.067053:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:31.0:1619677474.067054:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.067054:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677474.067055:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:31.0:1619677474.067056:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.067057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.067058:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.067070:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.067073:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.067074:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.067074:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.067075:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.067076:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.067338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.067345:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.067348:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.067349:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.067351:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.067353:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.067355:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.067356:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.067357:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.067358:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.067359:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.067363:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.067364:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.067367:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.067368:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.067369:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.067370:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.067370:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.067371:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.067372:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677474.067386:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.067389:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.067390:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.067392:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.067393:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.067395:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.067535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.067542:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.067545:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.067546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.067548:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.067550:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.067552:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.067553:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.067554:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.067555:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.067556:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.067560:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.067561:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.067563:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.067564:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.067564:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.067565:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.067566:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.067567:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.067568:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.067568:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.067571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.067574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.067578:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.067580:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677474.067581:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677474.067581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.067581:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.067583:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677474.067583:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.067584:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.067585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.067585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.067590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.067596:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.067598:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.067599:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.067599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.067600:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.067600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.067744:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.067746:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.067748:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.067750:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.067751:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.067752:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.067753:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.067754:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.067755:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.067756:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.067759:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.067760:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.067762:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.067763:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.067763:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.067765:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.067765:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.067766:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.067767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.067777:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.067780:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.067780:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.067782:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.067783:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.067784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.067917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.067924:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.067926:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.067927:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.067929:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.067931:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.067933:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677474.067934:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.067935:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.067936:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.067937:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677474.067940:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.067942:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677474.067945:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.067945:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677474.067946:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677474.067948:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677474.067948:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.067949:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.067950:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.067951:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677474.067954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677474.067956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:4.2:1619677474.067960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:1.0:1619677474.067962:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:8.0:1619677474.067963:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.067965:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:1.0:1619677474.067965:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.067966:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.067966:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.067967:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:24.0:1619677474.067967:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:26.0:1619677474.067969:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:8.0:1619677474.067969:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.067969:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.067970:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.067971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:8.0:1619677474.067972:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.067972:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:8.0:1619677474.067973:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.067975:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.067976:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.067977:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.067978:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.067981:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.067982:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.067985:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.067986:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.067986:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.067988:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.067988:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.067989:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677474.067990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677474.068002:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.068004:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.068005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.068006:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.068008:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.068011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.068206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.068213:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.068216:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.068217:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.068219:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.068221:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.068223:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.068224:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.068225:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.068226:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.068227:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.068230:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.068232:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.068235:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.068235:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.068236:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.068238:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.068238:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.068239:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.068239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.068251:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.068254:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.068254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.068255:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.068257:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.068260:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.068269:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.068272:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.068272:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.068273:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.068274:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.068274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677474.068411:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.068414:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.068415:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.068416:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.068418:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.068419:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.068421:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.068422:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.068423:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.068424:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.068427:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.068429:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.068431:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.068431:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.068432:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.068434:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.068434:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.068435:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.068436:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.068440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.068448:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.068451:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.068451:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.068452:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.068453:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.068454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.068521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.068524:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.068526:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.068528:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.068528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.068529:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.068531:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.068532:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.068533:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.068534:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.068538:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.068539:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.068542:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.068543:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.068543:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.068543:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677474.068545:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.068545:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.068545:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:31.0:1619677474.068546:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677474.068546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677474.068547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677474.068548:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.068550:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.068550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677474.068551:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.068553:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.068554:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.068555:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.068556:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677474.068557:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.068560:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:22.0:1619677474.068560:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.068561:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.068561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.068562:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:24.0:1619677474.068563:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.068565:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.068565:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677474.068566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.068566:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.068568:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.068568:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.068570:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.068571:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.068571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.068581:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.068582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.068583:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.068583:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.068584:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.068584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.068585:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.068587:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.068588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.068589:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.068819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.068827:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.068830:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.068830:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.068832:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.068835:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.068837:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.068838:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.068839:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.068840:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.068841:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.068844:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.068846:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.068848:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.068849:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.068850:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.068851:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.068852:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.068853:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.068853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.068865:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.068868:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.068868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.068869:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.068871:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.068874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.068940:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.068943:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.068945:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.068947:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.068948:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.068950:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.068951:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.068952:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.068953:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.068954:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.068958:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.068959:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.068962:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.068963:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.068963:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.068965:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.068966:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.068967:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.068969:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.068969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.068979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.068981:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.068982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.068982:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.068983:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.068984:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.068985:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.068986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.068987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.068988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677474.069434:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.069437:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.069437:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.069439:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.069441:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.069443:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.069444:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.069446:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.069446:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.069448:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.069451:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.069453:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.069455:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.069456:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.069457:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.069459:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.069459:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.069460:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.069460:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.069465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.069474:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.069477:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.069477:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.069479:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.069480:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.069482:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.069494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.069501:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.069504:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.069505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.069506:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.069509:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.069511:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.069512:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.069514:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.069515:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.069516:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.069520:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.069521:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.069524:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.069525:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.069526:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.069527:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.069528:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.069528:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.069529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.069543:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.069546:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.069546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.069547:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.069549:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.069555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.069557:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.069559:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.069560:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.069560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.069561:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.069562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.069840:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.069842:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.069843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.069845:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.069847:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.069848:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.069849:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.069851:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.069852:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.069853:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.069856:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.069858:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.069861:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.069861:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.069862:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.069864:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.069864:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.069865:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.069866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.069877:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.069880:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.069880:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.069882:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.069883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.069884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.069902:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.069905:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.069906:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.069906:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.069907:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.069908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.069935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.069941:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.069944:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.069945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.069946:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:4.2:1619677474.069949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:2.0:1619677474.069949:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.069950:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.069951:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.069953:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.069954:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677474.069954:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.069955:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.0:1619677474.069956:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677474.069959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:2.0:1619677474.069959:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:21.0:1619677474.069960:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677474.069961:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:21.0:1619677474.069962:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.069964:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677474.069964:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677474.069965:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.069965:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.069965:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:21.0:1619677474.069966:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.069967:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.069967:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:21.0:1619677474.069968:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677474.069968:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.069969:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.069970:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.069970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:21.0:1619677474.069971:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.069973:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.2:1619677474.069973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677474.069975:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677474.069975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677474.069976:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.069977:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.069978:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.069979:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.069980:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.069980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:1.0:1619677474.069981:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677474.069982:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.069984:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:8.2:1619677474.069984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.069984:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677474.069985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677474.069986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.069986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677474.069986:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.069987:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.069991:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677474.069991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677474.069994:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.069994:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.069996:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.069997:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.069998:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.070004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.070012:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.070014:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.070015:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.070016:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.070016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.070017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677474.070122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.070128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.070131:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.070132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.070134:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.070136:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.070137:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.070139:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.070140:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.070141:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.070142:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.070146:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.070147:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.070150:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.070151:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.070151:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.070153:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.070154:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.070154:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.070155:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.070167:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.070169:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.070170:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.070170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.070173:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.070175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.070474:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.070477:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.070479:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.070481:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.070482:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.070483:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.070485:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.070486:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.070487:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.070488:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.070491:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.070493:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.070495:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.070496:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.070497:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.070498:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.070499:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.070500:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.070502:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.070502:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.070505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.070512:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.070514:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.070515:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.070515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.070517:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.070517:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.070519:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677474.070519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.070520:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.070521:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.070761:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.070764:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.070766:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.070768:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.070769:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.070770:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.070771:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.070772:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.070773:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.070774:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.070777:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.070779:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.070782:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.070782:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.070783:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.070785:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.070785:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.070786:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.070786:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.070799:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.070802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.070802:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.070803:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.070805:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.070808:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.070884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.070892:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.070894:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.070895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.070897:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.070900:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.070901:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.070903:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.070904:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.070905:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.070906:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.070909:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.070910:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.070913:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.070914:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.070914:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.070916:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.070917:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.070918:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.070918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.070923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.070931:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.070933:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.070934:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.070934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.070937:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.070940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677474.071295:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.071299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.071302:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677474.071304:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.071306:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.071308:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.071309:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.071310:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.071311:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.071312:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.071316:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.071317:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.071320:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.071320:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.071321:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.071323:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.071323:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.071324:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.071325:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.071337:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.071339:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.071340:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.071342:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.071343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.071344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677474.071852:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.071855:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.071856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.071858:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.071861:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.071862:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.071864:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.071865:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.071866:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.071867:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:4.2:1619677474.071868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:1.0:1619677474.071870:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.071871:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.071874:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.071875:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:1.0:1619677474.071875:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.071876:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.071878:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:1.0:1619677474.071878:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.071878:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:8.0:1619677474.071879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:1.0:1619677474.071879:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.071880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000400:00000200:24.0:1619677474.071881:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.071884:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.071884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:24.0:1619677474.071885:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.071887:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.071889:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.071890:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.071891:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.071891:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.071893:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.071894:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.071896:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677474.071896:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.071898:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.071900:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.071901:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.071901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677474.071902:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.071903:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.071905:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.071906:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.071906:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.071907:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677474.071919:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.071922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.071922:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.071923:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.071925:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.071928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.072151:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.072154:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.072156:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.072158:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.072159:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.072160:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.072162:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.072163:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.072164:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.072165:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.072168:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.072170:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.072172:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.072173:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.072173:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.072175:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.072176:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.072176:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.072177:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.072187:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.072190:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.072190:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.072192:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.072193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.072194:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.072319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.072327:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.072329:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.072331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.072332:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.072335:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.072336:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.072337:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.072338:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.072340:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.072340:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.072343:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.072345:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.072347:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.072348:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.072348:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.072350:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.072350:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.072351:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.072351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.072356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.072363:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.072365:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.072366:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.072366:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.072368:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.072371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.072736:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.072738:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677474.072739:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677474.072740:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.072740:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.072742:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.072742:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.072744:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.072745:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.072746:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.072747:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.072748:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.072750:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.072751:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.072751:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.072752:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.072753:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.072754:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.072755:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.072755:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.072759:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.072760:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.072762:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.072763:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.072763:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.072763:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.072764:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.072764:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.072764:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.072765:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.072766:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.072766:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.072766:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.072767:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.072768:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.072768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.072769:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677474.072770:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.072777:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.072779:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.072780:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.072781:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.072783:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.072783:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.072784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.072785:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.072786:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.072787:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.072789:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.072792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.072918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.072924:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.072927:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.072928:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.072930:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.072932:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.072934:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677474.072935:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.072936:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.072938:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.072939:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677474.072942:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.072944:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677474.072947:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.072948:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677474.072949:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677474.072951:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677474.072952:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.072953:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677474.072954:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677474.072954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677474.072958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.2:1619677474.072959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:24.0:1619677474.072965:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677474.072965:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:24.0:1619677474.072967:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:18.0:1619677474.072967:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677474.072967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.0:1619677474.072968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.072969:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.072969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:24.0:1619677474.072971:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.072974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.072976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677474.072982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.072983:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.072986:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.072986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677474.072988:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:26.0:1619677474.072989:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.072991:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.072991:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.072992:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.072992:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:26.0:1619677474.072993:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677474.072993:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:26.0:1619677474.072994:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.072994:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:26.0:1619677474.072995:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.072996:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.072997:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.073000:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.073002:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.073004:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.073005:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.073006:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.073008:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.073008:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.073009:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.073009:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.073023:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.073023:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.073024:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.073025:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.073026:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.073027:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.073028:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.073029:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.073032:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.073033:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.073034:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.073035:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.073036:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.073038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.073039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.073040:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.073040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.073042:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.073045:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.073045:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.073046:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.073048:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.073049:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.073049:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.073050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.073061:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.073064:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.073064:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.073066:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.073067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.073068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.073536:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.073539:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.073541:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.073543:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.073544:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.073545:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.073547:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.073548:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.073549:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.073550:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.073553:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.073554:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.073558:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.073558:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.073559:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.073561:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.073561:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.073562:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.073563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.073568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.073576:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.073578:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.073579:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.073581:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.073582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.073584:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677474.073947:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.073949:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.073950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.073952:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.073954:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.073956:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.073957:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.073959:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.073959:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.073961:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.073964:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.073965:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.073968:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.073969:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.073969:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.073971:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.073972:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.073973:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.073974:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.073974:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.073984:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.073985:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.073986:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.073987:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.073987:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677474.073988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.073988:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.073990:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.073991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.073992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.074166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.074174:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.074177:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.074178:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.074180:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.074183:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.074183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677474.074184:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.074186:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.074187:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.074188:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.074189:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677474.074189:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.074192:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677474.074192:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.074193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677474.074194:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.074195:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.074197:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.074197:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.074198:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:24.0:1619677474.074199:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677474.074199:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.074199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666800] (20)++
00000400:00000200:24.0:1619677474.074200:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.074200:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.074201:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.074201:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677474.074202:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.074202:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677474.074203:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.074204:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.074207:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.074207:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (21)++
00000800:00000200:8.0:1619677474.074208:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.4.27@o2ib3
00000800:00000200:24.0:1619677474.074209:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.074211:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.074212:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.074213:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000400:00000200:8.0:1619677474.074213:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.074214:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000400:00000200:8.0:1619677474.074215:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:24.0:1619677474.074216:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000400:00000200:8.0:1619677474.074216:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.074217:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.074218:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.074219:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:8.0:1619677474.074221:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.074222:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.074223:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.074223:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:24.0:1619677474.074224:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677474.074225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (22)--
00000800:00000200:8.0:1619677474.074225:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.074225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:8.0:1619677474.074226:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:8.0:1619677474.074226:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:22.0:1619677474.074228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.074228:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:8.0:1619677474.074228:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (21)++
00000800:00000200:6.0:1619677474.074228:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:8.0:1619677474.074229:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (22)--
00000400:00000200:6.0:1619677474.074229:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.074230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (21)--
00000400:00000200:6.0:1619677474.074231:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.2:1619677474.074233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:4.2:1619677474.074235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666800] (20)++
00000800:00000200:6.0:1619677474.074236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.074238:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.074239:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.074240:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:6.0:1619677474.074240:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.074241:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:22.0:1619677474.074242:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677474.074242:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.074243:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677474.074243:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.074243:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677474.074244:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677474.074245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.074245:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (21)++
00000800:00000200:6.0:1619677474.074245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:24.0:1619677474.074246:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.074246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.074247:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000800:00000200:24.0:1619677474.074249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677474.074250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000400:00000200:8.0:1619677474.074250:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.074251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (22)--
00000400:00000200:8.0:1619677474.074251:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.074253:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677474.074254:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.074255:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.074256:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.074257:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677474.074259:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.074261:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677474.074263:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.074264:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:8.0:1619677474.074264:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:8.0:1619677474.074266:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:8.0:1619677474.074266:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (21)++
00000800:00000200:8.0:1619677474.074267:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (22)--
00000800:00000200:4.2:1619677474.074268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666800] (22)++
00000800:00000200:8.0:1619677474.074271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (22)--
00000800:00000200:8.0:1619677474.074272:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (21)++
00000800:00000200:8.0:1619677474.074274:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000800:00000200:24.0:1619677474.074275:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (22)++
00000400:00000200:8.0:1619677474.074276:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.074277:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000400:00000200:8.0:1619677474.074278:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.074279:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.074279:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677474.074280:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677474.074281:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:8.0:1619677474.074282:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.074282:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.074283:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677474.074284:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.074284:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.074285:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.074287:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677474.074288:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.074288:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.074288:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.074289:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:24.0:1619677474.074290:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.074290:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:24.0:1619677474.074291:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.074292:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677474.074300:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:24.0:1619677474.074302:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.074302:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677474.074303:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)++
00000800:00000200:24.0:1619677474.074304:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677474.074304:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (24)--
00000800:00000200:12.0:1619677474.074304:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (24)--
00000800:00000200:13.0:1619677474.074305:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:8.0:1619677474.074305:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.074306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677474.074306:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677474.074306:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677474.074307:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.074308:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677474.074308:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677474.074309:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677474.074309:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (23)++
00000800:00000200:8.0:1619677474.074310:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (24)--
00000800:00000200:24.0:1619677474.074311:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677474.074311:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (23)++
00000800:00000200:8.0:1619677474.074311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000800:00000200:24.0:1619677474.074313:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8666800] (23)++
00000800:00000200:8.0:1619677474.074313:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (23)++
00000800:00000200:9.2:1619677474.074314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677474.074314:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000800:00000200:24.0:1619677474.074315:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (25)--
00000800:00000200:24.0:1619677474.074316:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (24)--
00000800:00000200:8.2:1619677474.074316:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677474.074318:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000800:00000200:4.2:1619677474.074320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666800] (22)++
00000400:00000200:8.0:1619677474.074321:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.074324:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:8.0:1619677474.074324:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.074325:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:17.0:1619677474.074326:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:8.0:1619677474.074326:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.074327:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.074328:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.074328:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.074329:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:10.0:1619677474.074329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:8.0:1619677474.074329:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.074330:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.074330:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (23)++
00000400:00000200:17.0:1619677474.074330:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.074330:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677474.074330:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677474.074331:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8666800] (24)--
00000800:00000200:24.0:1619677474.074332:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000800:00000200:24.0:1619677474.074333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (22)--
00000400:00000200:17.0:1619677474.074333:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.074334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:8.0:1619677474.074335:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.074337:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677474.074338:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:4.2:1619677474.074338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666800] (21)++
00000800:00000200:8.0:1619677474.074341:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.074342:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.074343:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.074345:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.074346:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (23)++
00000800:00000200:8.0:1619677474.074346:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (22)++
00000800:00000200:8.0:1619677474.074347:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (23)--
00000800:00000200:8.0:1619677474.074347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000800:00000200:24.0:1619677474.074348:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000800:00000200:8.0:1619677474.074348:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (22)--
00000400:00000200:24.0:1619677474.074350:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677474.074352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:24.0:1619677474.074354:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.074355:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.074356:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.074357:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677474.074357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666800] (21)++
00000400:00000200:24.0:1619677474.074358:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.074360:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.074360:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:24.0:1619677474.074363:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.074363:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.074364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.074364:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (22)++
00000400:00000200:2.0:1619677474.074364:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.074365:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.074367:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000400:00000200:2.0:1619677474.074367:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.074368:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.074369:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.074369:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (22)++
00000400:00000200:8.0:1619677474.074369:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.074371:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677474.074372:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677474.074373:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (23)++
00000800:00000200:9.2:1619677474.074373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:8.0:1619677474.074373:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.074374:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (24)--
00000800:00000200:24.0:1619677474.074375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000400:00000200:8.0:1619677474.074375:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677474.074376:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (22)++
00000800:00000200:24.0:1619677474.074377:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000400:00000200:8.0:1619677474.074377:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.074379:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677474.074380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:8.0:1619677474.074380:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.074381:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677474.074383:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.074383:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.074383:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000800:00000200:18.0:1619677474.074384:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (23)++
00000800:00000200:2.0:1619677474.074384:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000400:00000200:24.0:1619677474.074385:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.074385:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (24)--
00000400:00000200:2.0:1619677474.074385:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.074386:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:8.0:1619677474.074386:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.074387:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:24.0:1619677474.074388:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677474.074388:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.074388:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.074389:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.074391:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.074392:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.074392:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677474.074394:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677474.074395:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677474.074396:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.074396:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:8.0:1619677474.074397:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677474.074398:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677474.074399:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.0:1619677474.074400:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (23)++
00000800:00000200:24.0:1619677474.074401:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.074401:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (24)--
00000800:00000200:8.0:1619677474.074401:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000800:00000200:24.0:1619677474.074402:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:8.0:1619677474.074402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (22)++
00000800:00000200:24.0:1619677474.074403:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677474.074404:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000800:00000200:24.0:1619677474.074405:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677474.074405:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (23)++
00000800:00000200:24.0:1619677474.074406:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8666800] (24)++
00000400:00000200:8.0:1619677474.074406:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.074408:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (25)--
00000800:00000200:24.0:1619677474.074408:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (24)--
00000400:00000200:8.0:1619677474.074408:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.074409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000400:00000200:8.0:1619677474.074410:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:13.0:1619677474.074411:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:8.0:1619677474.074411:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.074413:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.074413:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677474.074414:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.074414:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.074414:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:4.2:1619677474.074414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666800] (22)++
00000400:00000200:13.0:1619677474.074416:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.074418:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:8.0:1619677474.074418:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.074419:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:13.0:1619677474.074420:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677474.074421:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (23)++
00000800:00000200:8.0:1619677474.074421:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677474.074422:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:24.0:1619677474.074423:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8666800] (24)--
00000800:00000200:22.0:1619677474.074423:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:8.0:1619677474.074423:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.074424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000800:00000200:8.0:1619677474.074424:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:6.0:1619677474.074424:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.074425:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (22)++
00000800:00000200:8.0:1619677474.074425:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677474.074426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:24.0:1619677474.074427:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000800:00000200:8.0:1619677474.074427:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:8.0:1619677474.074427:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (23)++
00000400:00000200:6.0:1619677474.074427:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.074428:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (24)--
00000800:00000200:8.0:1619677474.074429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000400:00000200:24.0:1619677474.074430:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.074430:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (22)++
00000800:00000200:8.0:1619677474.074431:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000800:00000200:6.0:1619677474.074431:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677474.074432:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:9.2:1619677474.074432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:8.0:1619677474.074433:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.074434:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.074434:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.074435:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.074435:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:24.0:1619677474.074437:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677474.074438:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.074439:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.074439:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:24.0:1619677474.074440:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.074441:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.074441:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.074442:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677474.074442:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:24.0:1619677474.074443:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677474.074443:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.074443:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.074444:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:18.0:1619677474.074445:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.074446:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.074446:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677474.074447:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.074448:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.074448:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.074450:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.074450:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.074451:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.074453:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677474.074454:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.074454:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.074456:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:18.0:1619677474.074457:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.074459:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677474.074459:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.074460:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.074460:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.074461:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677474.074461:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.074461:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677474.074462:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677474.074462:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.074463:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677474.074464:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677474.074464:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.074464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677474.074464:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.074464:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:24.0:1619677474.074465:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (23)++
00000800:00000200:8.0:1619677474.074465:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.074466:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (24)--
00000800:00000200:24.0:1619677474.074467:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000800:00000200:8.0:1619677474.074467:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677474.074467:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (22)++
00000800:00000200:24.0:1619677474.074468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000800:00000200:9.2:1619677474.074468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.074468:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (23)--
00000800:00000200:8.2:1619677474.074470:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.074475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666800] (21)++
00000800:00000200:2.0:1619677474.074475:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.074477:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677474.074478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:2.0:1619677474.074478:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.074479:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.074479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (22)--
00000800:00000200:8.0:1619677474.074480:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (21)++
00000400:00000200:2.0:1619677474.074480:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.074481:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.074482:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.074482:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000400:00000200:21.0:1619677474.074483:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.074483:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677474.074484:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (22)--
00000800:00000200:20.0:1619677474.074484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:8.0:1619677474.074485:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.074487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:8.0:1619677474.074488:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677474.074490:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.074493:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:8.0:1619677474.074494:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677474.074495:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.074496:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.074496:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.074496:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.074498:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000400:00000200:8.0:1619677474.074498:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.074499:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.074499:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.074500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677474.074501:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.074502:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.074502:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.074503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.074504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:8.0:1619677474.074504:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.074506:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.074507:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.074508:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677474.074508:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.074508:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)++
00000800:00000200:8.0:1619677474.074509:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (23)++
00000800:00000200:30.0:1619677474.074510:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.074511:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (24)--
00000800:00000200:8.0:1619677474.074511:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (24)--
00000800:00000200:4.2:1619677474.074511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666800] (21)++
00000800:00000200:30.0:1619677474.074512:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:8.0:1619677474.074512:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (21)++
00000800:00000200:30.0:1619677474.074513:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.074514:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677474.074514:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8666800] (23)++
00000800:00000200:8.0:1619677474.074515:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (24)--
00000800:00000200:8.0:1619677474.074516:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000800:00000200:8.0:1619677474.074517:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (22)++
00000800:00000200:24.0:1619677474.074519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (23)++
00000800:00000200:8.0:1619677474.074519:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000800:00000200:24.0:1619677474.074520:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8666800] (24)--
00000800:00000200:24.0:1619677474.074521:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000400:00000200:8.0:1619677474.074521:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.074523:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (22)++
00000800:00000200:24.0:1619677474.074524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (23)--
00000400:00000200:8.0:1619677474.074524:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.074525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (22)--
00000400:00000200:8.0:1619677474.074525:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677474.074526:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.074528:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.074529:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.074530:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677474.074533:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:8.0:1619677474.074533:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.074535:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:8.0:1619677474.074535:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:31.0:1619677474.074536:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.074536:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.074537:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:8.0:1619677474.074537:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:31.0:1619677474.074538:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.074539:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677474.074540:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.074540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677474.074540:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (21)++
00000800:00000200:8.0:1619677474.074541:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (22)--
00000800:00000200:8.0:1619677474.074541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (21)--
00000800:00000200:9.2:1619677474.074544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.074551:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.074553:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.074554:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.074554:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.074556:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.074558:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677474.074567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.074572:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.074575:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.074576:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.074576:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.074577:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.074577:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.074591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666800] (20)++
00000800:00000200:24.0:1619677474.074598:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (21)++
00000800:00000200:24.0:1619677474.074600:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000800:00000200:8.0:1619677474.074601:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (22)--
00000400:00000200:24.0:1619677474.074603:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.074606:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.074607:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677474.074608:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.074609:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.074610:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.074611:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677474.074613:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.074615:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.074616:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.074617:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.074617:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.074618:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.074619:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (21)++
00000800:00000200:24.0:1619677474.074620:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (22)--
00000800:00000200:24.0:1619677474.074620:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (21)--
00000800:00000200:9.2:1619677474.074624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.074630:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.074632:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.074632:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.074633:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.074634:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.074635:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.074643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666800] (20)++
00000800:00000200:24.0:1619677474.074649:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666800] (21)++
00000800:00000200:24.0:1619677474.074652:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.27@o2ib3
00000800:00000200:8.0:1619677474.074653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (22)--
00000400:00000200:24.0:1619677474.074654:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.074656:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.074657:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677474.074658:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.074659:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.074660:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.074661:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677474.074664:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.074665:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.074668:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.074669:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677474.074669:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677474.074670:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677474.074671:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666800] (21)++
00000800:00000200:24.0:1619677474.074672:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666800] (22)--
00000800:00000200:24.0:1619677474.074672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666800] (21)--
00000800:00000200:8.2:1619677474.074676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.074684:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.074687:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.074688:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.074689:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.074691:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.074693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.074696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.074697:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.074700:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677474.074700:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.074701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.074701:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.074702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.074709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.074715:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.074718:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.074719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.074721:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.074723:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.074725:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677474.074726:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.074727:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.074729:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.074730:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677474.074733:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.074735:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677474.074738:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.074739:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677474.074739:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677474.074741:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677474.074742:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.074743:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.074743:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:30.0:1619677474.074746:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:9.2:1619677474.074747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.074749:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.074751:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.074753:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677474.074753:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:30.0:1619677474.074754:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.074755:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.074756:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.074756:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:30.0:1619677474.074757:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.074757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.074757:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.074758:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.074760:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.074760:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.074763:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.074763:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.074765:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.074769:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.074769:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.074770:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.074772:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.074772:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.074773:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.074774:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.074796:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.074799:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.074800:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.074800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.074802:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.074805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.074833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.074839:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.074842:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.074842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.074845:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.074847:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.074849:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.074850:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.074851:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.074852:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.074853:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.074856:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.074858:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.074860:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.074861:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.074862:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.074863:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.074864:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.074865:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.074866:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.074867:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.074872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677474.074877:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.074878:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.074880:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.074880:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677474.074881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.074881:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.074881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.074882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.074883:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.074886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.074897:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.074899:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.074900:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.074900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.074901:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.074902:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.075025:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.075027:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.075028:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.075030:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.075032:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.075033:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.075034:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.075036:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.075037:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.075038:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.075041:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.075043:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.075045:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.075046:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.075047:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.075048:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.075049:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.075049:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.075050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.075062:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.075064:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.075065:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.075066:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.075068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.075069:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.075079:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.075082:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.075083:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.075083:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.075084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.075085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677474.075376:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.075378:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677474.075379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677474.075381:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677474.075383:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.075385:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.075386:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.075387:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.075388:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.075389:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.075393:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677474.075395:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.075397:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.075398:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.075398:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.075400:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.075401:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.075401:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677474.075402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:22.0:1619677474.075426:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.075429:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.075429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.075430:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.075432:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.075435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.075452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.075460:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.075463:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.075464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.075466:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.075469:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.075470:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.075472:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.075473:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.075474:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.075475:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.075478:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.075480:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.075483:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.075484:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:4.2:1619677474.075484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:24.0:1619677474.075485:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.075487:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.075487:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.075488:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.075490:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.075490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677474.075491:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:9.2:1619677474.075493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677474.075494:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.075495:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677474.075496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677474.075497:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.075499:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.075500:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.075501:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.075501:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:26.0:1619677474.075502:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.075503:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.075503:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677474.075503:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:26.0:1619677474.075504:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.075505:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.075505:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:2.0:1619677474.075505:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.075506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677474.075506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677474.075507:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.075508:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.075511:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.075511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677474.075513:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.075514:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.075515:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.075517:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.075517:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.075518:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.075518:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677474.075530:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.075532:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.075533:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.075535:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.075537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.075538:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677474.075548:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.075550:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.075551:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.075551:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.075552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.075553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677474.075944:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.075947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.075947:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.075949:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.075951:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.075953:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.075954:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.075955:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.075956:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.075958:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.075961:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.075962:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.075965:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.075966:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.075966:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.075968:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.075969:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.075970:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.075970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.075982:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.075984:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.075985:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.075986:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.075987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.075989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.076641:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.076644:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.076646:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.076648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.076649:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.076650:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.076652:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.076653:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.076654:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.076655:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.076658:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.076660:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.076662:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.076663:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.076664:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.076665:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.076666:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.076667:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.076669:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.076669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.076680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.076681:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.076682:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.076683:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.076683:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.076684:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.076684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.076685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.076686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.076687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.076783:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.076786:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.076787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.076789:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.076791:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.076793:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.076794:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.076796:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.076796:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.076798:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.076801:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.076802:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.076805:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.076805:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.076806:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.076808:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.076808:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.076809:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.076810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.076821:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.076823:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.076824:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.076826:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.076827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.076828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.077115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.077123:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.077125:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.077126:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.077128:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.077131:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.077132:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.077134:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.077135:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.077136:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.077137:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.077141:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.077142:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.077145:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.077146:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.077146:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.077148:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.077148:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.077149:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.077150:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.077151:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.077153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.077156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.077160:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.077162:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.077163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.077163:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.077164:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.077165:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677474.077165:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.077166:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.077167:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677474.077168:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.077168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.077171:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.077173:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.077175:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.077176:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.077178:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.077179:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.077180:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.077181:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.077182:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.077185:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.077187:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.077190:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.077191:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.077192:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.077194:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.077194:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.077195:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.077197:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.077197:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.077207:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.077210:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.077211:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.077211:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.077211:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.077213:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677474.077213:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.077214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.077215:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.077217:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.077230:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.077233:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.077234:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.077234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.077235:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.077235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.077260:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.077263:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.077265:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.077267:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.077268:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.077269:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.077271:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.077272:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.077273:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.077274:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.077277:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.077278:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.077280:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.077280:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.077281:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.077282:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.077283:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.077284:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.077284:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.077296:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.077303:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.077304:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.077304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.077305:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.077307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.077759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.077767:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:4.2:1619677474.077769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:10.0:1619677474.077770:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.077771:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.077772:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.077774:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.077776:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:10.0:1619677474.077776:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.077777:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.077778:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677474.077779:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:10.0:1619677474.077779:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.077781:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:13.0:1619677474.077782:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.077784:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:10.0:1619677474.077784:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677474.077785:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.077786:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677474.077786:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:13.0:1619677474.077787:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.077789:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.077789:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677474.077790:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.077790:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:13.0:1619677474.077791:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677474.077791:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.077792:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.077793:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:13.0:1619677474.077794:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.077794:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.077795:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:13.0:1619677474.077796:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677474.077796:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677474.077799:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.077799:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:13.0:1619677474.077800:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677474.077801:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677474.077801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:13.0:1619677474.077802:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677474.077802:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.077803:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:30.0:1619677474.077804:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677474.077804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:30.0:1619677474.077806:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.077807:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:31.0:1619677474.077808:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.077808:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.077808:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:8.2:1619677474.077808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:30.0:1619677474.077809:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.077810:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96506185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.077810:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677474.077810:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677474.077811:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677474.077811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.077811:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.077812:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.077812:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677474.077813:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.077814:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.077815:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677474.077817:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:30.0:1619677474.077818:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.077818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.077820:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677474.077820:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.077821:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.077821:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.077822:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.077823:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.077823:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:1.0:1619677474.077823:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.077825:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.077825:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.077826:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677474.077826:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677474.077827:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.077837:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.077840:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.077841:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.077843:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.077844:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.077845:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.077855:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.077857:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.077858:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.077858:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.077859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.077860:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.077872:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.077875:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.077876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.077878:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.077880:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.077881:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.077883:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.077884:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.077885:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.077887:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.077890:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.077892:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.077894:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.077895:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.077895:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.077897:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.077898:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.077899:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.077900:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.077900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.077911:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.077913:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.077913:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.077914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.077915:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.077915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677474.077916:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.077918:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.077919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.077920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.078271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.078278:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.078280:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.078282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.078283:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.078286:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.078287:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677474.078289:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.078290:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.078291:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.078292:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677474.078299:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.078300:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.078302:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.078302:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.078303:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.078304:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.078305:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.078306:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677474.078307:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677474.078307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:4.2:1619677474.078314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:4.2:1619677474.078319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:8.0:1619677474.078320:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:9.2:1619677474.078322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.078322:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:20.0:1619677474.078323:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677474.078323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677474.078324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:24.0:1619677474.078325:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.078326:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677474.078326:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:24.0:1619677474.078327:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.078328:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.078329:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.078329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.0:1619677474.078329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677474.078330:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.078331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:20.0:1619677474.078332:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.078333:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.078334:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.078335:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.078336:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.078339:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.078341:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.078344:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.078345:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.078345:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.078347:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.078348:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.078349:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.078350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677474.078361:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.078364:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.078365:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.078367:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.078368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.078370:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.078417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.078424:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.078427:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.078428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.078429:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.078432:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.078434:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.078435:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.078437:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.078437:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.078439:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.078442:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.078444:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.078447:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.078448:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.078448:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.078450:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.078451:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.078452:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.078452:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:4.2:1619677474.078452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677474.078461:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.078463:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000800:00000200:30.0:1619677474.078464:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:31.0:1619677474.078465:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.078465:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.078466:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.078467:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.078468:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.078469:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.078470:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.078471:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.078472:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.078473:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.078476:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.078476:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.078478:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.078478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.078478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677474.078481:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.078481:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.078482:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.078484:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.078484:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.078485:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677474.078486:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:31.0:1619677474.078487:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:9.2:1619677474.078489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.078490:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.078491:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.078491:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.078492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.078493:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:8.0:1619677474.078495:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.078498:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.078499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.078499:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.078501:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.078504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.078509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.078514:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.078517:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.078518:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.078518:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.078519:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.078519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.078616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.078621:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.078624:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.078626:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.078626:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.078629:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.078630:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.078632:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.078633:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.078634:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.078635:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.078638:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.078640:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.078643:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.078643:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.078644:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.078646:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.078646:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.078647:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.078648:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.078652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.078660:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.078663:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.078664:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.078664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.078666:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.078668:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677474.078705:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.078708:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.078709:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.078711:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.078714:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.078715:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.078716:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.078718:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.078719:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.078720:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.078723:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.078725:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.078727:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.078728:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.078728:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.078730:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.078731:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.078732:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.078732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.078743:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.078745:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.078746:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.078748:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.078749:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.078750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.078767:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.078770:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.078771:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.078771:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.078772:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.078773:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677474.079132:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.079135:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.079135:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000400:00000200:1.0:1619677474.079137:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.079140:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.079141:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.079142:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.079143:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.079144:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.079145:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.079148:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.079150:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.079153:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.079154:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.079154:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.079156:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.079157:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.079157:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.079158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677474.079162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.079169:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.079171:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.079172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.079172:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.079174:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.079177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.079181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.079187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.079189:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.079190:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.079190:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.079192:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.079192:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677474.079199:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.079202:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.079203:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.079204:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.079207:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.079208:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.079210:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.079211:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.079212:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.079212:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.079215:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.079216:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.079219:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.079219:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.079220:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.079221:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.079222:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.079223:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.079223:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.079228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.079235:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.079238:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.079238:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.079240:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.079241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.079242:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677474.079442:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.079444:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.079447:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.079449:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.079449:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.079451:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.079452:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.079453:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.079454:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.079455:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.079459:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.079460:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.079463:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.079464:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.079465:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.079466:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.079467:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.079468:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.079468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.079473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.079480:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.079482:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.079483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.079483:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.079485:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.079487:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.079518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.079525:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.079528:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.079529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.079530:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.079532:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.079534:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.079535:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.079536:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.079537:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.079538:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.079541:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.079543:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.079545:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.079546:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.079547:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.079548:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.079549:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.079549:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.079550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.079554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.079562:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.079565:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.079566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.079566:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.079568:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.079572:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.079574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.079582:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.079585:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.079585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.079586:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.079587:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.079587:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677474.079612:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.079615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.079615:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.079617:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.079620:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.079621:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.079622:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.079624:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.079625:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.079626:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.079629:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.079631:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.079633:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.079634:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.079635:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.079637:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.079637:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.079638:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.079639:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.079650:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.079652:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.079653:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.079655:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.079655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.079657:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.079774:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.079778:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.079780:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.079781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.079783:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.079784:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.079786:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.079787:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.079788:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.079789:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.079792:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.079794:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.079797:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.079798:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.079799:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.079800:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.079801:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.079802:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.079802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.079814:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.079817:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.079818:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.079818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.079821:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.079823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.080011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.080019:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.080022:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.080023:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.080025:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.080028:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.080029:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.080030:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.080031:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.080032:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.080033:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.080036:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.080038:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.080040:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.080041:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.080042:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.080043:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.080044:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.080044:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.080045:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.080057:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.080060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.080060:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.080061:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.080063:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.080066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.080140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.080147:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.080150:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.080151:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.080152:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.080155:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.080156:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.080157:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.080158:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.080159:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.080160:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.080163:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.080165:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.080167:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.080168:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.080169:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.080170:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.080171:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.080171:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.080172:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.080184:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.080187:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.080188:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.080188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.080190:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.080192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.080630:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.080632:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.080633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.080635:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.080637:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.080639:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.080640:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.080641:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.080643:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.080643:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.080646:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.080648:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.080650:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.080651:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.080652:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.080654:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.080654:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.080655:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.080655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.080669:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.080671:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.080672:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.080673:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.080675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.080676:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.081445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.081453:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.081456:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.081456:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.081458:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.081461:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.081463:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.081464:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.081465:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.081466:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.081467:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.081470:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.081472:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.081475:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.081476:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.081476:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.081478:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.081479:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.081479:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.081480:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.081495:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.081497:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.081497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:22.0:1619677474.081499:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.081499:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677474.081502:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.081503:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.081506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.081506:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.081507:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677474.081508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.081509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.081509:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.081511:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.081512:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.081513:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.081517:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.081519:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.081521:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.081522:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.081523:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.081525:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.081526:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.081527:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.081528:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.081529:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.081539:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.081541:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.081541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.081541:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.081542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.081543:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.081544:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.081546:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.081546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.081548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677474.081687:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.081690:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.081691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.081693:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.081696:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.081697:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.081698:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.081699:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.081700:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.081701:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.081704:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.081706:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.081708:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.081709:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.081710:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.081712:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.081712:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.081713:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.081713:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677474.081725:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.081728:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.081729:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.081729:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.081731:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.081734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.081742:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.081745:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.081746:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.081746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.081747:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.081748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.081877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.081885:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.081888:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.081889:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.081891:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.081893:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677474.081894:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.081895:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677474.081895:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.081897:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.081898:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.081898:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.081899:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.081900:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.081900:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677474.081900:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.081902:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.081903:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.081904:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.081904:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.081905:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.081906:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.081906:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.081908:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.081909:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:31.0:1619677474.081910:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.081910:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.081911:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.081912:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.081912:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.081912:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.081913:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677474.081914:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.081914:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.081915:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.081916:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.081917:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677474.081917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.081918:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.081920:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.081920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.081923:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.081926:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.081927:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.081927:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.081929:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.081931:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:24.0:1619677474.081931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.081932:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.081933:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.081934:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.081934:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.081935:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.081936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.081937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.081938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.081939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.082019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.082026:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.082029:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.082030:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.082031:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.082034:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.082035:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.082036:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.082038:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.082038:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.082039:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.082043:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.082044:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.082047:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.082048:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.082048:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.082050:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.082050:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.082051:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.082053:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.082053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677474.082056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677474.082059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677474.082065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677474.082066:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.082068:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:2.0:1619677474.082068:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.082069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.082069:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677474.082069:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.082070:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677474.082071:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.082075:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.082271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677474.082278:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.082281:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677474.082282:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677474.082283:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.082286:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.082287:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677474.082288:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.082289:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.082290:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.082291:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677474.082300:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.082301:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677474.082303:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.082304:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:26.0:1619677474.082305:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677474.082306:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677474.082307:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.082307:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677474.082308:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:12.0:1619677474.082321:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.082323:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.082324:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.082325:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.082328:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.082329:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:6.0:1619677474.082336:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.082339:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.082340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.082341:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.082344:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.082345:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.082346:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.082348:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.082349:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.082350:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.082353:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.082355:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.082357:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.082358:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.082359:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.082360:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.082361:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.082361:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.082362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.082374:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.082376:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.082377:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.082379:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.082380:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.082381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.082392:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.082394:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.082395:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.082396:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.082397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.082397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.082449:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.082452:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.082454:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.082456:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.082456:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.082457:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.082459:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.082460:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.082461:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.082462:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.082466:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.082468:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.082470:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.082471:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.082471:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.082473:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.082473:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.082474:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.082475:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.082485:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.082487:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.082488:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.082490:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.082491:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.082492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.082655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.082663:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.082666:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.082667:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.082668:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.082670:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.082672:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.082673:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.082674:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.082675:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.082676:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.082679:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.082680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:10.0:1619677474.082681:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677474.082682:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.082684:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.082684:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.082685:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.082685:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.082686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.082687:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.082687:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.082687:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677474.082688:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677474.082688:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677474.082689:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.082689:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677474.082691:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.082691:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.082692:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.082695:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.082697:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.082700:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.082700:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.082701:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.082701:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:30.0:1619677474.082703:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.082703:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.082703:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:30.0:1619677474.082704:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677474.082704:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.082704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:30.0:1619677474.082705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:22.0:1619677474.082707:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.082712:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.082714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.082716:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.082717:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.082717:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.082719:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.082721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.082845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.082852:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.082855:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.082856:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.082858:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.082860:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.082862:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.082863:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.082865:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.082866:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.082867:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.082870:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.082872:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.082876:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.082877:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.082877:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.082879:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.082880:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.082881:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.082881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677474.082892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.082894:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.082895:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.082897:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.082898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.082900:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677474.083329:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.083331:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.083332:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.083334:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.083336:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.083337:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.083339:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.083340:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.083341:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.083342:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.083345:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.083347:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.083349:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.083350:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.083350:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.083352:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.083353:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.083353:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.083354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.083366:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.083369:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.083369:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.083371:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.083372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.083373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.083500:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.083503:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.083506:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.083508:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.083508:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.083510:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.083511:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.083512:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.083514:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.083514:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.083518:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.083519:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.083522:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.083523:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.083523:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.083525:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.083526:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.083527:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.083527:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.083531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.083538:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.083540:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.083541:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.083541:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.083541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.083543:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:8.0:1619677474.083543:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.083544:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.083546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:24.0:1619677474.083547:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.083547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.083550:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.083551:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.083552:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.083554:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.083554:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.083555:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.083558:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.083560:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.083563:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.083564:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.083564:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.083566:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.083566:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.083567:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.083567:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.083572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.083579:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.083582:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.083583:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.083583:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.083586:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.083589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.083930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.083935:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.083938:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.083939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.083941:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.083943:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.083944:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.083945:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.083947:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.083948:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.083949:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.083951:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.083953:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.083955:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.083955:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.083956:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.083958:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.083958:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.083959:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.083960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.083964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.083971:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.083973:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.083973:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.083974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.083975:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.083976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677474.084189:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.084191:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.084192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.084194:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.084196:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.084198:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.084199:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.084200:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.084201:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.084202:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.084206:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.084207:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.084210:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.084211:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.084211:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.084213:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.084214:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.084214:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.084215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.084227:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.084229:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.084230:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.084232:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.084233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.084234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.084381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:30.0:1619677474.084384:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.084387:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.084389:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.084389:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:31.0:1619677474.084391:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677474.084391:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:30.0:1619677474.084392:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.084392:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:30.0:1619677474.084393:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.084395:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.084395:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677474.084396:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.084397:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.084398:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.084398:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.084399:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:8.0:1619677474.084400:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.084402:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.084403:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.084403:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.084405:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.084405:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677474.084408:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677474.084409:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.084409:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.084410:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.084410:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677474.084412:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.084412:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.084412:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.084413:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677474.084413:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:8.0:1619677474.084413:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:30.0:1619677474.084414:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.0:1619677474.084415:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:8.0:1619677474.084415:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.084416:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.084417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677474.084422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677474.084427:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.084429:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:6.0:1619677474.084430:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.084431:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:22.0:1619677474.084431:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.084431:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.084432:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.084432:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:6.0:1619677474.084434:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.084436:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.084437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677474.084439:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:5.2:1619677474.084992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.085000:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.085004:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.085006:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.085009:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.085013:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.085015:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.085017:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.085019:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.085021:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.085023:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.085028:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.085031:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.085034:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.085036:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.085037:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.085039:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.085040:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.085041:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.085042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.085054:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.085054:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.085055:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.085056:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.085056:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677474.085059:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.085059:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.085062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.085062:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.085063:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677474.085063:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.085064:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.085065:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.085066:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000400:00000200:6.0:1619677474.085066:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.085067:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.085067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.085067:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.085068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.085068:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.085069:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:6.0:1619677474.085072:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.085074:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.085076:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.085077:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.085078:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.085080:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.085080:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.085081:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.085082:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.085082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.085093:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.085094:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.085095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.085095:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.085096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.085097:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.085098:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.085099:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.085101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.085102:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.085426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.085434:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.085437:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.085438:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.085440:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.085442:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.085443:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.085444:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.085446:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.085447:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.085447:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.085451:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.085452:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.085455:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.085456:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.085456:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.085458:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.085459:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.085459:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.085460:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.085474:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.085477:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.085478:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.085478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.085480:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.085483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.085537:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.085541:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.085542:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.085545:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.085545:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.085547:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.085548:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.085549:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.085550:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.085551:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.085554:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.085556:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.085558:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.085559:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.085559:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.085561:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.085561:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.085562:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.085563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.085567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.085574:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.085577:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.085578:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.085578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.085580:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.085582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.085584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.085590:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.085592:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.085593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.085593:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.085594:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.085594:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.085661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.085669:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.085672:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.085674:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677474.085676:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.085677:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.085678:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:12.0:1619677474.085679:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.085680:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.085682:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.085682:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:12.0:1619677474.085686:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.085687:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677474.085690:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.085691:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:12.0:1619677474.085691:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:12.0:1619677474.085693:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:12.0:1619677474.085693:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.085694:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.085695:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.085699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.085705:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.085707:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.085708:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.085708:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.085710:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.085713:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677474.086051:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.086053:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.086054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.086056:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.086058:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.086059:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.086061:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.086062:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.086063:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.086064:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.086067:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.086067:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:22.0:1619677474.086068:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677474.086070:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:22.0:1619677474.086071:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.086071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:22.0:1619677474.086072:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.086072:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:17.0:1619677474.086073:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.086074:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.086074:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.086075:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:17.0:1619677474.086075:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.086076:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:17.0:1619677474.086076:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677474.086078:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.086079:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.086080:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.086081:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677474.086084:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.086086:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.086087:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:17.0:1619677474.086088:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.086088:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.086089:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.086090:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.086091:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.086091:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.086091:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:20.0:1619677474.086092:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.086093:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:21.0:1619677474.086094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677474.086094:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:20.0:1619677474.086095:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677474.086095:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677474.086098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.086107:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.086108:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677474.086109:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677474.086109:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000400:00000200:1.0:1619677474.086109:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.086110:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:21.0:1619677474.086111:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677474.086111:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677474.086111:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.086112:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:17.0:1619677474.086112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.086113:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:17.0:1619677474.086113:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677474.086113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677474.086114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.086115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.086149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.086154:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.086157:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.086159:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.086159:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.086161:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.086163:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.086164:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.086165:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.086166:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.086167:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.086171:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.086172:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.086175:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.086176:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.086177:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.086179:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.086179:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.086180:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.086181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.086185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.086187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.086194:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.086196:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.086197:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.086197:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.086198:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677474.086199:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.086201:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.086202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.086203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:8.0:1619677474.086203:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.086204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.086205:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.086206:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.086208:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.086209:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.086210:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.086213:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.086215:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.086218:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.086219:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.086219:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.086221:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.086222:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.086223:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.086224:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.086225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677474.086228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.086230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677474.086236:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677474.086237:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.086239:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:18.0:1619677474.086239:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677474.086240:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677474.086240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677474.086241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677474.086241:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.086243:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.086246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.086481:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.086484:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.086486:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677474.086488:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.086488:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.086489:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.086491:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.086492:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.086493:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.086494:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.086497:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677474.086499:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.086501:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.086502:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.086503:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.086504:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.086505:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.086506:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.086508:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.086508:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.086519:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.086522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.086522:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.086523:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.086523:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.086525:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677474.086525:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.086526:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.086527:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.086528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.086539:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.086542:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.086542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.086543:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.086544:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.086544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.086653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.086661:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.086664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.086664:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.086666:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.086668:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.086670:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.086671:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.086672:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.086673:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.086674:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.086677:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.086679:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.086681:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.086682:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.086683:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.086685:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.086685:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.086686:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.086686:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.086691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.086699:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.086702:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.086703:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.086705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.086705:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.086708:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677474.086873:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.086876:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.086877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.086878:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.086880:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.086881:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.086883:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.086884:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.086885:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.086886:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.086889:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.086890:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.086893:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.086894:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.086894:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.086896:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.086896:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.086897:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.086897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.086914:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677474.086916:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.086916:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.086917:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.086919:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:21.0:1619677474.086919:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.086919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677474.086922:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.086922:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.086923:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.086925:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.086926:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.086927:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.086929:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.086930:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.086931:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.086934:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.086936:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.086938:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.086939:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.086940:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.086941:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.086942:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.086943:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.086945:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.086945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.086960:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.086962:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.086963:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.086964:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.086965:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.086965:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.086966:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677474.086966:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.086967:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.086969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.087144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:4.2:1619677474.087148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.087151:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.087154:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677474.087156:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.087156:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677474.087158:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.087158:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:26.0:1619677474.087158:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:31.0:1619677474.087159:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677474.087159:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677474.087161:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.087161:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.087162:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.087163:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.087164:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677474.087164:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.087166:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.087167:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.087168:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.087169:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.087169:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.087170:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677474.087170:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.087173:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.087173:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.087174:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677474.087174:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677474.087175:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.087176:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677474.087177:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.087177:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.087178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:26.0:1619677474.087178:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.087178:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.087179:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.087180:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.087181:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.087182:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:9.2:1619677474.087182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677474.087183:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.087184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.087186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.087189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677474.087191:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677474.087192:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:2.0:1619677474.087193:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677474.087195:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:2.0:1619677474.087195:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.087196:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:8.0:1619677474.087196:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.087197:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.087197:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:2.0:1619677474.087197:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.087199:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.087199:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677474.087199:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.087200:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677474.087201:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677474.087202:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.087204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.087212:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.087215:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.087216:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.087216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.087217:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.087218:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.087401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.087407:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.087409:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.087410:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.087412:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.087414:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.087416:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.087416:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.087418:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.087419:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.087420:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.087424:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.087425:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.087428:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.087429:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.087430:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.087432:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.087432:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.087433:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.087433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677474.087446:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.087449:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.087450:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.087451:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.087453:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.087454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.087479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.087487:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.087489:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.087490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.087492:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.087494:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.087496:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.087497:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.087499:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.087500:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.087501:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.087504:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.087506:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.087509:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.087510:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.087511:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.087513:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.087513:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.087514:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.087514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.087528:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.087530:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.087531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.087532:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.087534:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.087537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.087701:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.087704:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.087705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.087707:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.087708:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.087710:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.087711:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.087712:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.087713:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.087714:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.087717:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.087719:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.087721:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.087722:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.087722:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.087724:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.087724:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.087725:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.087725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.087737:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.087740:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.087741:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.087741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:20.0:1619677474.087743:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.087744:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.087745:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.087746:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677474.087746:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.087747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.087748:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677474.087751:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.087752:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.087754:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.087755:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.087756:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.087757:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.087761:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.087763:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.087765:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.087766:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.087767:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.087769:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.087769:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.087770:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.087771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677474.087782:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.087785:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.087786:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.087788:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.087789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.087790:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677474.087854:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.087857:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.087859:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.087861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.087862:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.087863:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.087865:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.087866:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.087867:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.087868:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.087871:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.087873:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.087876:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.087876:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.087877:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.087878:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.087879:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.087880:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.087881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.087891:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.087893:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.087894:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.087895:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.087897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.087898:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.087899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.087907:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677474.087908:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.087909:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.087910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677474.087911:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.087911:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.087912:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.087912:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:24.0:1619677474.087912:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.087913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:24.0:1619677474.087914:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.087916:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.087917:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.087918:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.087919:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.087920:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.087923:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.087925:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.087928:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.087928:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.087929:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.087931:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.087931:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.087932:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.087932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.087937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.087945:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.087948:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.087949:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.087949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.087951:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.087954:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.088096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.088101:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.088104:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.088105:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.088107:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.088109:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.088110:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677474.088111:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.088112:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.088113:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.088114:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677474.088117:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.088119:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677474.088121:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.088122:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677474.088123:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677474.088124:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677474.088125:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.088126:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.088126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677474.088131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.088139:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.088142:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.088143:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.088144:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.088145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.088147:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677474.088726:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.088729:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.088730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.088732:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.088734:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.088735:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.088736:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.088737:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.088738:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.088739:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.088742:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.088744:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.088746:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.088747:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.088748:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.088749:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.088750:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.088751:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.088752:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.088752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.088763:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.088765:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.088765:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.088765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.088766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.088767:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.088768:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.088770:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.088771:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.088772:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.089580:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.089583:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.089585:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.089587:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.089587:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.089588:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.089590:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.089591:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.089592:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.089593:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.089596:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.089598:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.089600:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.089601:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677474.089601:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677474.089602:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.089603:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.089604:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.089604:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:30.0:1619677474.089605:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.089605:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677474.089605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.089606:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.089609:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.089610:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:9.2:1619677474.089610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677474.089611:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.089612:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.089613:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.089614:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.089617:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.089617:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677474.089619:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677474.089619:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.089620:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.089620:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.089621:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.089622:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.089623:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677474.089623:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.089625:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.089625:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.089626:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.089626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677474.089626:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677474.089639:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.089641:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.089642:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.089644:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.089645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.089646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.089656:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.089659:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.089660:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.089660:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.089661:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.089662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677474.089810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.089816:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.089819:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.089820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.089821:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.089823:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.089825:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.089826:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.089827:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.089828:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.089830:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.089832:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.089834:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.089836:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.089837:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.089838:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.089840:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.089840:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.089841:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.089841:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.089853:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.089856:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.089857:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.089857:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.089859:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.089861:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.090235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.090243:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.090246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.090246:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.090249:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.090251:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.090253:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.090254:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.090255:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.090256:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.090258:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.090261:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.090263:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.090266:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.090266:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.090267:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.090269:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.090269:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.090270:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.090270:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.090282:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.090284:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.090285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.090286:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.090288:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.090291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.090447:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.090449:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.090450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.090452:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.090454:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.090455:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.090456:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.090458:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.090458:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.090460:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.090463:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.090465:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.090467:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.090468:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.090468:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.090470:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.090470:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.090471:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.090471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.090485:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.090487:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.090488:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.090490:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.090491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.090492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.090613:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.090616:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.090618:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.090620:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.090620:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.090622:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.090623:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.090625:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.090625:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.090627:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.090630:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.090632:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.090634:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.090635:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.090635:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.090637:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.090637:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.090638:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.090639:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.090649:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.090651:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.090652:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.090653:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.090655:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.090656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.090715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.090723:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.090725:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.090726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.090727:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.090731:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.090733:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.090734:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.090735:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.090736:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.090737:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.090740:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.090742:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.090745:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.090746:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.090746:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.090748:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.090748:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.090749:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.090749:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.090761:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.090764:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.090765:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.090765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.090767:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.090770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.090938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.090945:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.090948:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.090948:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.090951:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.090953:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.090954:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.090955:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.090957:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.090958:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.090959:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.090962:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.090964:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.090966:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.090967:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.090967:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.090969:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.090970:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.090971:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.090971:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.090975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.090982:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.090984:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.090985:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.090986:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.090987:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.090988:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.091348:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.091351:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.091353:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.091355:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.091356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.091356:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.091358:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.091359:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.091360:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.091361:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.091364:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.091365:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.091367:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.091368:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.091369:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.091370:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.091371:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.091372:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.091374:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.091374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.091385:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.091387:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.091388:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.091388:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677474.091389:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.091389:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.091389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.091391:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.091392:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.091393:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677474.091446:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.091449:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.091450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.091452:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.091454:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.091455:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.091456:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.091458:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.091459:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.091460:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.091463:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.091464:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.091467:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.091468:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.091468:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.091470:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.091470:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.091471:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.091471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.091483:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.091485:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.091486:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.091488:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.091489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.091490:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.091579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677474.091586:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.091589:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677474.091590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677474.091592:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.091594:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.091596:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.091597:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.091599:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.091600:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.091600:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.091604:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.091606:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.091609:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.091610:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.091611:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.091612:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.091613:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.091613:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677474.091614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677474.091619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.091627:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.091629:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.091630:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.091631:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.091633:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.091634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.091804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.091811:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.091814:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.091815:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.091816:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.091819:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.091821:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.091822:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.091823:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.091824:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.091825:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.091829:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.091830:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.091833:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.091833:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.091834:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.091836:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.091836:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.091837:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.091837:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:4.2:1619677474.091841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.091848:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.091850:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.091851:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.091852:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.091853:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.091853:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.091853:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.091854:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.091855:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.091856:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.091857:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.091858:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.091859:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.091860:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.091861:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.091863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.091863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:26.0:1619677474.091864:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.091867:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.091869:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.091870:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.091871:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.091871:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.091873:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.091873:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.091873:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.091874:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.091875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:10.0:1619677474.091875:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677474.091877:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677474.091878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677474.091879:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.091881:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.091882:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.091883:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.091884:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.091884:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677474.091885:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:10.0:1619677474.091887:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.091887:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.091888:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.091888:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677474.091888:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.091890:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.091891:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.091891:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.091892:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677474.091893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.091894:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.091894:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.091895:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.091895:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677474.091907:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.091910:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.091910:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.091912:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.091914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:30.0:1619677474.091915:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677474.091915:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677474.091918:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.091920:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.091921:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.091922:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.091923:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.091924:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.091925:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.091926:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.091927:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.091930:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.091932:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.091933:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.091934:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.091935:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.091936:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.091936:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.091937:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.091938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.091948:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.091951:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.091951:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.091953:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.091954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.091955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.091967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.091975:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.091977:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.091978:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.091980:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.091982:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.091984:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.091985:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.091986:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.091987:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.091988:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.091991:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.091992:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.091995:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.091996:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.091996:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.091998:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.091998:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.091999:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.091999:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.092004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.092013:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.092016:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.092017:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.092017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.092019:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.092022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677474.092319:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.092322:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.092323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.092325:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.092327:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.092328:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.092329:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.092331:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.092332:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.092333:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.092336:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.092338:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.092341:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.092341:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.092342:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.092344:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.092344:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.092345:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.092346:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.092346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.092357:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.092358:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.092359:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.092359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.092360:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677474.092361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.092361:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.092363:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.092364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.092365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677474.093170:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.093173:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.093174:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.093175:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.093177:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.093178:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.093180:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.093181:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.093182:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.093183:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.093186:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.093187:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.093189:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.093190:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.093190:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.093192:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.093192:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.093192:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.093193:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.093205:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.093207:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.093208:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.093210:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.093211:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.093212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.093222:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.093225:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.093226:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.093227:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.093227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.093228:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677474.093512:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.093515:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.093516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.093518:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.093521:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.093522:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.093524:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.093525:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.093526:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.093527:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.093530:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.093532:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.093535:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.093535:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.093536:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.093538:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.093538:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.093539:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677474.093540:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.093541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.093544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.093552:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677474.093553:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677474.093554:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677474.093555:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677474.093555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677474.093556:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.093556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677474.093557:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.093559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.093560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.093618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677474.093626:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.093629:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677474.093631:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.093633:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.093633:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677474.093635:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677474.093636:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.093637:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.093639:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.093639:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677474.093643:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.093645:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677474.093648:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.093649:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677474.093650:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.093651:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677474.093652:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.093654:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:31.0:1619677474.093655:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:31.0:1619677474.093656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:9.2:1619677474.093658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.2:1619677474.093661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:2.0:1619677474.093667:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:30.0:1619677474.093669:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:2.0:1619677474.093670:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:30.0:1619677474.093672:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:18.0:1619677474.093672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.093672:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.093673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.093673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000400:00000200:2.0:1619677474.093674:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.093675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.093678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.093680:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.093683:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.093684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.093684:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.093685:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.093686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:30.0:1619677474.093842:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.093845:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.093847:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.093849:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.093850:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.093852:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.093853:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.093854:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.093855:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.093856:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.093859:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.093861:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.093863:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.093864:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.093864:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.093866:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.093867:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.093868:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.093868:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.093872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.093878:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.093880:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.093881:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.093882:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.093883:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.093886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.093889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.093895:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.093898:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.093899:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.093899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.093900:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.093900:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.094198:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.094201:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.094201:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.094203:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.094205:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.094207:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.094208:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.094209:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.094210:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.094211:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.094214:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.094215:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.094218:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.094219:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.094219:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.094221:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.094221:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.094222:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.094222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.094235:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.094237:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.094238:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.094239:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.094241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.094242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.094288:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.094291:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677474.094295:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.094298:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677474.094301:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.094303:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.094304:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.094305:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.094307:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.094308:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.094311:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677474.094312:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.094315:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.094316:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.094316:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.094318:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.094319:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.094320:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.094320:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.094324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.094331:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.094334:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.094335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.094335:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.094337:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.094340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677474.095026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.095033:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677474.095034:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677474.095035:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.095036:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677474.095037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.095037:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:17.0:1619677474.095040:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.095041:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.095042:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.095043:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.095044:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.095045:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.095045:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.095047:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.095048:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.095049:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.095049:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.095050:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.095051:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677474.095052:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677474.095058:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.095058:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.095060:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677474.095061:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.095063:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.095064:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.095064:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677474.095065:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.095065:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677474.095066:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.095066:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677474.095067:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.095067:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677474.095068:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.095068:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.095068:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.095068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:17.0:1619677474.095069:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.095070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.095075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.0:1619677474.095080:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:1.0:1619677474.095080:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:20.0:1619677474.095082:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:1.0:1619677474.095083:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.095083:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677474.095084:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:20.0:1619677474.095084:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.095084:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677474.095085:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.095086:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:21.0:1619677474.095086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:22.0:1619677474.095087:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.095087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677474.095088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.095090:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.095092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.095097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677474.095102:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.095104:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.095105:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.095105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.095105:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.095106:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.095107:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:6.0:1619677474.095107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677474.095108:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.095110:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.095113:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.095114:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.095115:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.095117:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.095118:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.095119:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.095123:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.095124:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.095127:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.095128:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.095128:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.095130:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.095130:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.095131:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.095132:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.095143:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:4.2:1619677474.095145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677474.095146:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.095147:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.095147:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.095149:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.095152:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677474.095153:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.095155:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.095156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.095158:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.095161:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.095162:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.095163:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.095165:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.095166:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.095167:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.095170:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.095172:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.095175:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.095175:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.095176:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.095177:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.095178:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.095179:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.095180:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.095180:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.095186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.095190:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.095193:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.095193:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.095194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.095194:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.095195:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.095196:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677474.095196:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.095197:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.095200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.095209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.095212:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.095213:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.095213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.095214:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.095214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.095438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.095445:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.095448:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.095449:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.095451:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.095452:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.095454:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.095455:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.095456:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.095457:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.095458:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.095462:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.095463:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.095466:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.095467:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.095467:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.095469:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.095470:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.095471:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.095472:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.095472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.095476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.095478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.095484:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677474.095485:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677474.095486:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.095486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677474.095487:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677474.095487:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:17.0:1619677474.095488:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.095489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.095491:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.095493:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677474.095628:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.095631:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.095633:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.095635:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.095636:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.095637:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.095638:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.095639:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.095640:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.095641:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.095644:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.095646:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.095649:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.095649:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.095650:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.095651:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.095652:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.095653:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.095653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.095658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.095664:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.095667:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.095668:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.095668:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.095670:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.095672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.095775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.095780:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.095783:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.095785:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.095786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.095788:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.095789:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.095791:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.095792:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.095793:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.095794:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.095797:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.095799:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.095802:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.095803:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.095803:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.095806:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.095806:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.095808:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.095809:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.095809:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.095811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.095814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.095818:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677474.095818:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677474.095820:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:8.0:1619677474.095820:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.095821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677474.095822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:8.0:1619677474.095822:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.095823:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677474.095824:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.095826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677474.095883:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.095886:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.095887:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.095889:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.095891:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.095892:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.095894:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.095895:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.095896:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.095897:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.095900:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.095902:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.095904:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.095905:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.095906:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.095907:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.095908:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.095909:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.095910:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.095910:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.095920:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.095921:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.095923:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677474.095923:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677474.095924:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.095924:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.095924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.095926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.095926:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.095928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.096092:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.096095:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.096097:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.096099:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.096099:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.096101:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.096102:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.096103:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.096104:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.096104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677474.096105:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.096108:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.096110:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.096111:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677474.096113:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.096113:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.096114:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.096114:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677474.096115:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.096115:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677474.096116:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677474.096117:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.096118:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.096119:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:24.0:1619677474.096119:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.096120:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677474.096120:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:9.2:1619677474.096121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:24.0:1619677474.096122:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.096124:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.096125:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.096126:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.096128:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.096129:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.096130:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.096130:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.096132:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.096132:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.096133:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677474.096134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.096135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.096136:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.096136:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.096136:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.096137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677474.096137:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.096138:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.096139:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.096140:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.096141:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.096141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.096153:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.096156:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.096157:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.096157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.096159:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.096162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.096591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.096599:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677474.096600:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.096601:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.096602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677474.096603:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.096606:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.096606:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.096607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.096608:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.096609:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.096610:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:4.2:1619677474.096610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677474.096611:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677474.096612:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.096613:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.096613:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.096614:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.096615:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.096615:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.096616:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.096617:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.096618:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677474.096619:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.096619:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677474.096621:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677474.096622:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.096622:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.096623:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.096624:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.096624:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.096624:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.096625:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.096626:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.096626:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677474.096626:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677474.096627:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.096627:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677474.096628:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96506091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.096628:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.096629:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.096629:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:26.0:1619677474.096630:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.096630:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:26.0:1619677474.096631:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.096632:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.096632:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.096634:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:26.0:1619677474.096635:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.096635:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.096636:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.096637:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.096637:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677474.096639:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:26.0:1619677474.096639:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.096640:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.096640:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.096641:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:4.2:1619677474.096641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677474.096642:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.096642:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.096642:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677474.096643:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.096643:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:30.0:1619677474.096645:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677474.096645:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677474.096645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.096646:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677474.096647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:26.0:1619677474.096647:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:9.2:1619677474.096647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.096648:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677474.096649:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677474.096649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.096649:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.096650:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677474.096650:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (23)--
00000800:00000200:4.2:1619677474.096651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677474.096652:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.096655:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677474.096655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.096656:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677474.096657:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.096658:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.096659:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677474.096659:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.096659:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:30.0:1619677474.096660:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.096660:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000400:00000200:8.0:1619677474.096660:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.096661:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.096661:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.096662:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677474.096663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:10.0:1619677474.096663:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.096663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.096867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.096874:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.096877:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.096878:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.096880:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.096882:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.096883:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.096884:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.096886:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.096887:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.096887:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.096891:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.096892:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.096894:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.096895:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.096896:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.096898:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.096898:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.096899:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.096899:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677474.096915:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.096917:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.096917:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:22.0:1619677474.096918:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.096918:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677474.096920:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.096921:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.096924:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.096924:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.096925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:6.0:1619677474.096925:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677474.096926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.096927:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.096929:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.096930:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.096931:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.096934:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.096935:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.096937:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.096938:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.096939:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.096941:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.096941:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.096942:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.096942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.096954:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.096956:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.096957:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.096959:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.096960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.096961:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.096972:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.096974:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.096975:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.096976:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.096976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.096977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677474.097016:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.097018:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.097019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.097021:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.097024:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.097025:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677474.097027:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.097028:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.097029:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.097029:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677474.097033:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.097034:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677474.097037:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.097037:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677474.097038:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677474.097040:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677474.097041:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.097042:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677474.097042:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677474.097043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:31.0:1619677474.097052:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:6.0:1619677474.097053:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:31.0:1619677474.097055:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:6.0:1619677474.097055:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677474.097056:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:31.0:1619677474.097057:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.097057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:31.0:1619677474.097058:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.097059:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.097061:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677474.097140:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.097143:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.097145:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.097147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.097147:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.097149:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.097150:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.097151:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.097152:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.097153:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.097156:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.097158:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.097161:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.097162:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.097162:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.097164:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.097165:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.097166:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.097166:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.097170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.097178:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.097181:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.097182:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.097184:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.097184:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.097187:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.097188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.097196:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.097199:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.097200:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.097200:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.097201:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.097202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677474.097759:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.097762:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.097762:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.097764:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.097767:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.097769:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.097770:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.097771:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.097772:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.097773:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.097777:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.097778:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.097781:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.097782:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.097782:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.097784:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.097784:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.097785:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.097786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.097799:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.097801:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.097802:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.097803:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.097805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.097806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677474.098612:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.098614:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.098615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.098617:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.098619:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.098620:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.098621:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.098623:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.098623:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.098624:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.098627:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.098629:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.098631:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.098631:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.098632:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.098633:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.098634:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.098634:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.098635:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.098648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.098650:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.098651:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.098653:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.098654:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.098655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.099358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.099366:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.099368:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.099370:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.099371:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.099374:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.099376:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.099377:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.099378:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.099379:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.099380:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.099383:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.099385:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.099388:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.099389:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.099389:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.099391:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.099392:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.099392:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.099393:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.099398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.099407:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.099410:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.099411:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.099411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.099413:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.099417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.099419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.099426:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.099429:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.099430:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.099430:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.099431:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.099431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.099458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.099465:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.099468:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.099469:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.099471:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.099474:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.099475:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.099476:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.099478:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.099479:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.099480:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.099483:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.099484:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.099487:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.099488:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.099489:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.099491:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.099491:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.099492:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.099492:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.099497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.099506:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.099508:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.099509:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.099511:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.099512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.099514:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.099544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.099553:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.099558:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.099560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.099564:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.099568:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.099570:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.099573:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.099576:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.099578:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.099579:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.099586:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.099589:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.099592:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.099594:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.099595:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.099597:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.099598:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.099600:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.099602:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.099603:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.099604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.099608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.099615:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.099618:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.099618:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.099619:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677474.099619:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.099621:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.099623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.099623:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.099625:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.099626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677474.099636:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.099639:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.099640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.099643:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.099645:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.099647:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.099649:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.099651:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.099652:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.099653:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.099657:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.099659:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.099662:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.099663:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.099664:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.099665:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.099666:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.099667:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.099668:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.099669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.099679:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.099679:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.099682:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.099682:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677474.099683:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.099683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.099684:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.099685:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.099686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.099687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.099716:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.099719:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.099721:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.099724:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.099725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.099726:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.099727:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.099728:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.099729:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.099730:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.099734:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.099735:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.099738:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.099738:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.099739:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.099741:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.099741:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.099742:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.099743:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.099754:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.099757:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.099758:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.099761:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.099762:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.099763:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677474.099805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.099813:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.099818:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.099819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.099823:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.099826:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.099828:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.099830:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.099832:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.099834:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.099836:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.099841:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.099844:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.099848:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.099849:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.099850:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.099852:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.099853:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.099854:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.099855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.099866:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.099868:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.099869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.099870:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.099872:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.099874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.099903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.099911:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.099914:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.099914:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.099917:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.099919:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.099921:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.099922:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.099923:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.099925:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.099925:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.099929:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.099931:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.099934:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.099934:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.099935:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.099937:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.099937:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.099938:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.099938:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.099950:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.099953:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.099954:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.099954:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.099956:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.099959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.100327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.100336:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.100341:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.100342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.100345:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.100349:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.100351:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.100353:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.100357:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.100358:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.100360:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.100365:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.100368:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.100372:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.100374:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.100375:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.100377:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.100378:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.100380:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.100381:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677474.100393:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.100395:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.100396:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.100398:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.100401:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.100402:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.100403:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.100403:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.100404:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.100405:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.100406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677474.100498:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.100501:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.100501:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.100504:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.100506:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.100508:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.100510:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.100512:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.100513:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.100514:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.100518:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.100520:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.100523:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.100524:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.100525:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.100527:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.100527:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.100528:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.100529:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.100541:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.100543:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.100544:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.100546:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.100547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.100548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.100563:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.100565:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.100567:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.100567:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.100568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.100569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.100903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677474.100911:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.100913:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677474.100916:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.100918:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.100918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677474.100919:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677474.100921:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.100922:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.100923:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.100924:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677474.100928:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.100930:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.100933:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.100934:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677474.100934:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677474.100936:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677474.100937:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.100938:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677474.100938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677474.100942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.100951:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.100953:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.100954:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.100955:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.100957:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.100960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.101019:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.101022:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.101024:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.101026:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.101027:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.101028:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.101030:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.101031:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.101032:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.101033:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.101036:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.101038:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:4.2:1619677474.101038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:30.0:1619677474.101040:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.101041:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.101041:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.101043:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.101043:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.101044:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.101046:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.101046:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677474.101046:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.101048:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:9.2:1619677474.101048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.101049:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.101051:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.101054:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.101055:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.101056:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.101056:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:10.0:1619677474.101057:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.101058:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.101058:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.101059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677474.101059:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677474.101059:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.101060:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.101061:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.101063:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.101063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677474.101063:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.101064:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677474.101064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.101065:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.101067:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677474.101067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.101068:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.101068:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.101070:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.101070:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.101071:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.101071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:24.0:1619677474.101073:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.101076:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.101077:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.101077:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.101077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.101078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677474.101082:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.101085:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.101085:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.101087:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.101089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.101090:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.101298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.101305:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.101308:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.101309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.101309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:8.0:1619677474.101310:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.101313:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:8.0:1619677474.101313:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.101314:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.101315:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.101316:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:8.0:1619677474.101317:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.101318:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.101319:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.101319:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.101320:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.101322:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.101323:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677474.101324:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.101325:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.101325:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.101326:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.101327:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.101328:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.101328:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.101329:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.101329:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.101331:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.101331:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677474.101332:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.101332:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.101333:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677474.101334:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.101337:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.101338:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.101339:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:20.0:1619677474.101341:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:20.0:1619677474.101342:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.101342:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.101343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.101346:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:6.0:1619677474.101348:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677474.101349:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:22.0:1619677474.101349:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677474.101349:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.101350:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.101352:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.101352:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.101355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.101356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.101357:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.101359:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.101360:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.101361:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.101361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.101361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:4.2:1619677474.101364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.101372:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.101375:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.101375:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.101377:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.101380:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.101381:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.101383:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.101384:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677474.101384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:26.0:1619677474.101385:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.101386:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.101390:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.101392:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.101392:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.101393:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.101395:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.101395:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.101396:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.101397:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677474.101398:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.101399:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677474.101399:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.101401:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.101402:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.101403:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.101404:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677474.101404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.101405:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.101406:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.101406:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677474.101407:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.101408:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.101410:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677474.101410:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677474.101412:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.101413:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.101413:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.101414:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.101414:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677474.101414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.101415:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.101416:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.101417:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677474.101418:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.101418:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.101419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.101428:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.101430:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.101432:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:6.0:1619677474.101433:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.101434:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.101435:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.101435:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.101437:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.101437:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.101438:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.101440:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.101441:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.101442:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.101445:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.101446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.101446:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.101447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.101448:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.101450:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.101451:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.101452:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.101453:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.101454:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.101454:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.101455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.101467:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.101469:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.101470:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.101472:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.101472:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.101474:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.101642:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.101645:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.101647:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.101649:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.101649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.101651:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.101652:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.101654:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.101655:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.101656:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.101659:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.101661:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.101664:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.101665:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.101665:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.101667:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.101668:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.101669:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.101670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.101674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.101682:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.101684:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.101685:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.101687:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.101688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.101690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:30.0:1619677474.102137:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.102140:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.102142:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.102144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.102145:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.102146:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.102148:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.102149:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.102150:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.102151:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.102154:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.102156:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.102159:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.102160:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.102160:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.102162:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.102162:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.102163:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.102164:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.102168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.102176:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.102179:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.102180:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.102181:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.102182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.102184:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677474.102463:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.102466:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.102466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.102469:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.102471:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.102472:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.102473:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.102475:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.102476:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.102477:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.102480:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.102482:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.102484:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.102485:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.102485:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.102487:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.102487:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.102488:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.102489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.102501:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.102504:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.102504:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.102506:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.102507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.102509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.102818:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.102821:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.102823:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.102825:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.102826:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.102828:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.102829:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.102830:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.102831:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.102832:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.102835:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.102837:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.102840:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.102840:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.102841:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.102842:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.102843:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.102844:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.102845:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.102848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.102855:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.102858:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.102859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.102859:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.102861:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.102863:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.102937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.102944:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.102947:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.102948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.102949:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.102951:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.102953:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.102954:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.102955:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.102956:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.102957:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.102961:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.102962:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.102965:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.102966:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.102967:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.102968:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.102969:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.102970:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.102971:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.102972:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.102974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:9.2:1619677474.102975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.102977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677474.102979:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.102981:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.102983:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.102983:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677474.102984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.0:1619677474.102984:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:21.0:1619677474.102986:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.102986:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.102987:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.102987:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677474.102987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:21.0:1619677474.102988:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:8.0:1619677474.102988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.102989:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:21.0:1619677474.102990:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.102990:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.102991:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.102992:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.102993:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.102994:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:21.0:1619677474.102997:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.102999:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.103002:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.103003:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.103004:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.103006:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.103007:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.103008:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.103009:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.103010:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:4.2:1619677474.103015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677474.103019:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677474.103019:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.103021:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:30.0:1619677474.103022:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677474.103022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.103023:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677474.103024:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.103025:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.103026:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.103028:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:6.0:1619677474.103307:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.103310:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.103311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.103313:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.103315:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.103316:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.103318:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.103319:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.103320:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.103321:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.103325:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.103326:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.103329:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.103330:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.103330:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.103332:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.103332:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.103333:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.103334:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.103334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.103345:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.103346:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.103347:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.103347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.103348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.103348:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.103349:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.103351:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.103352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.103353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.103368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91946f784800] (20)++
00000800:00000200:21.0:1619677474.103374:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f784800] (21)++
00000800:00000200:21.0:1619677474.103377:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.12.14@o2ib3
00000400:00000200:21.0:1619677474.103379:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.14@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.103379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f784800] (22)--
00000400:00000200:21.0:1619677474.103383:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.103384:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:21.0:1619677474.103386:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.103387:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.103388:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.103389:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:21.0:1619677474.103393:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.103394:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 3584 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:21.0:1619677474.103398:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.103399:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:21.0:1619677474.103400:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:21.0:1619677474.103402:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:21.0:1619677474.103402:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946f784800] (21)++
00000800:00000200:21.0:1619677474.103403:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946f784800] (22)--
00000800:00000200:21.0:1619677474.103404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f784800] (21)--
00000800:00000200:24.2:1619677474.103409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.103413:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.103415:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677474.103416:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.103416:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677474.103419:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.103422:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677474.103854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.103862:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.103865:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.103866:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.103868:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.103870:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.103872:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.103873:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.103874:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.103875:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.103876:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.103880:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.103881:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.103884:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.103885:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.103886:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.103888:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.103888:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.103889:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.103891:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.103891:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677474.103894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677474.103897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677474.103902:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677474.103904:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677474.103905:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.103906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.103906:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:2.0:1619677474.103906:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.103907:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.103908:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677474.103908:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.103911:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677474.104157:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.104160:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.104161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.104163:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.104165:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.104166:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.104167:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.104169:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.104170:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.104171:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.104174:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.104176:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.104178:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.104179:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.104180:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.104182:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.104182:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.104182:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.104183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.104195:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.104197:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.104198:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.104200:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.104201:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.104202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.104215:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.104218:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.104219:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.104219:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.104220:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.104221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.104382:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.104385:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.104387:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.104389:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.104389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.104390:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.104392:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.104393:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.104394:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.104395:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.104398:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.104400:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.104402:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.104403:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.104404:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.104405:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.104406:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.104407:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.104409:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.104409:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.104420:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.104422:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.104422:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.104424:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.104424:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.104425:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.104426:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.104427:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.104428:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.104429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677474.104647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.104654:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.104657:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.104658:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.104661:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.104663:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.104665:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.104666:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.104668:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.104669:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.104670:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.104673:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.104675:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.104677:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.104678:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.104679:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.104680:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.104681:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.104681:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.104682:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.104694:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.104696:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.104697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.104698:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.104700:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.104702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.104715:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.104718:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.104719:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.104719:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.104720:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.104721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.104725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.104733:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.104736:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.104737:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.104739:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.104742:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.104744:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.104745:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.104746:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.104747:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.104748:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.104752:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.104753:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.104756:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.104757:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.104758:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.104760:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.104760:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.104761:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.104761:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.104773:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.104776:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.104776:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.104777:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.104779:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.104782:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.105117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.105125:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.105127:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.105130:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.105132:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.105132:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.105133:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677474.105135:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.105136:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.105137:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.105138:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677474.105141:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.105143:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677474.105145:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.105146:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677474.105146:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677474.105147:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677474.105148:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.105149:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.105149:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.105153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.105160:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.105162:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.105163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.105163:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.105165:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.105167:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677474.105187:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.105190:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.105191:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.105193:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.105195:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.105197:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.105198:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.105199:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.105200:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.105201:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.105205:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.105206:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.105209:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.105210:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.105210:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.105212:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.105213:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.105213:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.105214:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.105226:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.105228:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.105229:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.105231:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.105231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.105233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.105301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.105308:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.105311:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.105312:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.105314:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.105316:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.105318:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.105319:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.105320:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.105321:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.105322:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.105326:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.105328:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.105330:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.105331:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.105332:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.105333:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.105334:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.105335:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.105335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:13.0:1619677474.105347:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.105350:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.105350:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.105352:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.105354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.105355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677474.105386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.105394:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.105396:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.105397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.105399:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.105401:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.105403:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.105404:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.105406:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.105406:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.105407:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.105411:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.105412:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.105415:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.105416:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.105417:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.105419:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.105420:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.105421:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.105421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677474.105434:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.105438:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.105439:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.105442:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677474.105443:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.105446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.106001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.106009:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.106012:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.106013:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.106014:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.106017:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.106018:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677474.106018:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.106020:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.106021:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:8.0:1619677474.106021:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.106022:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.106022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.106024:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.106024:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.106027:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.106029:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.106030:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.106031:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.0:1619677474.106032:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677474.106033:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.106033:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:22.0:1619677474.106034:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.106034:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677474.106035:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.106035:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.106036:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677474.106036:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.106037:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.106037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.106039:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.106041:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.106044:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.106044:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.106045:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.106047:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.106047:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.106048:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.106048:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677474.106060:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.106062:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.106063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.106063:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.106064:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:6.0:1619677474.106065:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.106066:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.106067:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.106068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:20.0:1619677474.106069:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.106070:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.106071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.106123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:4.2:1619677474.106127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.0:1619677474.106128:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.106131:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.106133:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.106133:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:10.0:1619677474.106134:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:21.0:1619677474.106136:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.106136:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.106137:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:21.0:1619677474.106138:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.106139:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.106141:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.106141:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.106143:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.106143:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.106144:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.106145:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:21.0:1619677474.106147:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677474.106147:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.106149:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.106150:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.106151:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.106152:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:21.0:1619677474.106153:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.106154:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.106154:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.106155:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.106156:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.106157:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.106157:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:10.0:1619677474.106157:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.0:1619677474.106158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:10.0:1619677474.106160:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.106161:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.106161:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.106162:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.106163:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.106164:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.106165:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.106166:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.106168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.106169:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:4.2:1619677474.106171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.106172:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.106173:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.106173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.106174:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:22.0:1619677474.106175:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.106177:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:22.0:1619677474.106177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677474.106178:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677474.106178:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.106178:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.106180:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.106180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.106181:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.106182:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.106183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.106184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677474.106184:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.106186:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.106187:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677474.106188:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.106189:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.106191:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677474.106191:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.106191:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.106192:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677474.106193:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677474.106193:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677474.106194:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.106194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677474.106195:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.106196:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.106197:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.106200:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.106202:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.106205:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.106206:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.106206:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.106208:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.106209:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.106209:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.106210:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.106221:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.106224:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.106225:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.106225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.106227:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.106230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.106606:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.106609:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.106611:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.106613:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.106613:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.106614:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.106615:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.106617:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.106617:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.106618:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.106621:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.106623:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.106626:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.106627:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.106627:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.106629:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.106630:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.106631:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.106631:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.106636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.106644:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.106647:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.106648:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.106649:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.106651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.106653:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.106653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.106661:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.106664:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.106665:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.106666:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.106666:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.106667:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.106690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.106696:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.106699:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.106700:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.106703:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.106705:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.106707:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.106708:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.106710:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.106711:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.106712:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.106715:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.106717:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.106719:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.106720:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.106721:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.106722:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.106723:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.106724:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.106724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.106729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.106737:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.106739:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.106740:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.106740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.106742:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.106743:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.106747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.106755:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.106757:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.106758:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.106758:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.106758:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.106759:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677474.106868:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.106871:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.106872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.106874:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.106876:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.106877:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.106878:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.106880:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.106881:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.106882:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.106885:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.106887:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.106890:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.106890:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.106891:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.106893:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.106893:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.106895:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.106896:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.106896:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.106907:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.106908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.106909:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.106909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.106910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.106911:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.106911:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.106913:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.106914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.106915:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677474.107101:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.107104:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.107104:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677474.107107:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.107109:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.107110:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.107112:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.107113:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.107114:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.107115:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.107119:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.107121:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.107123:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.107124:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.107125:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.107126:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.107127:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.107127:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.107128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.107132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.107140:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.107142:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.107143:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.107144:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.107145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.107147:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677474.107194:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.107197:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.107199:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.107201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.107202:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.107203:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.107205:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.107206:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.107207:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.107208:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.107212:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.107214:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.107216:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.107217:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.107217:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.107219:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.107219:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.107220:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.107221:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.107231:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.107233:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.107234:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.107235:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.107237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.107238:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.107472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.107480:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.107482:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.107483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.107485:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.107488:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.107489:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.107490:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.107491:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.107492:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.107493:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.107496:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.107498:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.107501:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.107501:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.107502:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.107503:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.107504:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.107505:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.107505:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.107510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.107519:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.107522:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.107523:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.107524:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.107526:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.107529:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.107800:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.107803:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.107805:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.107807:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.107808:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.107808:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.107810:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.107811:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.107812:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.107813:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.107816:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.107817:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.107819:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.107820:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.107821:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.107822:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.107823:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.107824:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.107824:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.107828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.107834:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.107837:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.107838:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.107838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.107839:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.107842:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.107845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.107851:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.107854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.107854:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.107855:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.107855:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.107856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677474.107942:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.107945:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.107946:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.107948:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.107950:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.107951:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.107952:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.107954:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.107955:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.107956:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.107959:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.107960:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.107963:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.107963:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.107964:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.107966:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.107966:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.107967:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.107967:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.107979:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.107981:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.107982:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.107984:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.107984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.107986:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.107997:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.107999:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.108000:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.108001:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.108002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.108003:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.108614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.108621:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.108624:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.108625:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.108627:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.108630:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.108632:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.108633:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.108634:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.108635:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.108636:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.108640:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.108641:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.108644:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.108645:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.108645:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.108647:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.108648:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.108648:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.108649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.108654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.108663:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.108665:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.108666:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.108668:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.108671:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.108672:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677474.108674:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.108677:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.108677:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.108680:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.108682:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.108684:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677474.108685:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.108687:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.108687:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.108689:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.108692:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.108694:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.108697:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.108698:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.108699:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.108701:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.108701:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.108702:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.108702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:31.0:1619677474.108716:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.108719:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.108720:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.108721:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.108723:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.108724:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.108732:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.108734:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.108735:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.108736:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.108736:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.108738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.108793:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.108796:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.108798:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.108800:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.108800:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.108801:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:6.0:1619677474.108801:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677474.108802:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.108803:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.108804:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.108804:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.108805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677474.108807:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677474.108808:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.108810:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.108811:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:6.0:1619677474.108811:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.108812:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.108813:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.108814:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.108815:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.108815:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:6.0:1619677474.108815:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.108816:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.108817:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.108817:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677474.108817:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.108819:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.108820:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677474.108820:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.108821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677474.108822:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:9.2:1619677474.108823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.108825:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.108826:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.108826:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.108828:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.108828:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677474.108829:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677474.108829:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.108829:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677474.108832:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.108833:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.108833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.108834:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.108835:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.108836:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.108837:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.108838:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677474.108838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677474.108841:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.108844:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.108844:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.108846:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.108847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.108848:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677474.109429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.109435:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.109438:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.109439:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.109441:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.109443:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.109445:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.109446:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.109447:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.109448:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.109449:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.109453:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.109455:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.109457:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.109458:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:4.2:1619677474.109458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:17.0:1619677474.109459:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.109460:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.109461:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.109461:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.109462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.0:1619677474.109463:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.109466:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.109468:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.109469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.109471:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.109473:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.109474:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96506032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.109475:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:21.0:1619677474.109475:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.109476:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.109477:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:21.0:1619677474.109477:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677474.109478:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.109478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.109481:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.109482:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.109484:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677474.109484:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.109487:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.109488:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.109489:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.109491:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.109491:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.109492:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.109493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.109497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.109503:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.109506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.109506:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.109507:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.109509:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.109512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.109528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.109533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.109536:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.109536:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.109537:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.109537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.109538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.109571:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.109574:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.109576:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.109578:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.109579:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.109580:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.109581:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.109583:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.109584:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.109584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a5200] (20)++
00000400:00000200:30.0:1619677474.109585:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.109588:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.109589:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.109592:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.109592:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677474.109592:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (21)++
00000800:00000200:30.0:1619677474.109593:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.109594:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.109595:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.109595:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (22)++
00000800:00000200:8.0:1619677474.109595:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000800:00000200:30.0:1619677474.109596:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.109596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677474.109596:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000400:00000200:8.0:1619677474.109598:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.109599:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.109600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:8.0:1619677474.109601:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.109603:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677474.109604:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.109605:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.109606:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.109607:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:8.0:1619677474.109607:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:24.0:1619677474.109608:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.109608:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677474.109610:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.109610:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.109610:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:26.0:1619677474.109611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.109611:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:8.0:1619677474.109611:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.109612:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.109612:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.109614:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.109614:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:24.0:1619677474.109614:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.109614:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.109614:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:24.0:1619677474.109615:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.109616:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.109616:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.109618:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.109619:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96506028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.109620:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677474.109621:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.109621:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.109621:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:10.0:1619677474.109622:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.109622:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.109622:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677474.109623:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677474.109624:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677474.109625:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.109625:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (23)++
00000800:00000200:24.0:1619677474.109626:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.109626:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (24)--
00000800:00000200:24.0:1619677474.109627:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.109627:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.109627:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (23)--
00000800:00000200:24.0:1619677474.109628:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:8.0:1619677474.109628:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (22)++
00000800:00000200:24.0:1619677474.109630:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.109630:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.109630:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000800:00000200:24.0:1619677474.109631:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (23)++
00000800:00000200:10.0:1619677474.109631:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.109631:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.109632:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (24)--
00000800:00000200:24.0:1619677474.109632:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (23)--
00000400:00000200:8.0:1619677474.109632:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.109633:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (22)++
00000800:00000200:10.0:1619677474.109633:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.109634:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000800:00000200:10.0:1619677474.109634:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.109634:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.109635:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677474.109635:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.109636:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677474.109636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.0:1619677474.109637:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677474.109637:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677474.109638:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96506027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.109639:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:8.0:1619677474.109639:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.109640:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.109640:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.109642:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.109642:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677474.109644:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677474.109645:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.109645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677474.109645:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:20.0:1619677474.109646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677474.109646:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.109647:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677474.109647:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.109648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:2.0:1619677474.109648:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:24.0:1619677474.109649:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96506026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677474.109649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:8.0:1619677474.109649:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:2.0:1619677474.109649:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.109651:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.109651:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.109652:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.109653:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.109654:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677474.109654:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.109655:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677474.109656:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677474.109657:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.109658:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.109658:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000400:00000200:6.0:1619677474.109658:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96506025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.109659:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:6.0:1619677474.109659:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.109660:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.109660:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:6.0:1619677474.109660:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.109661:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.109661:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (23)++
00000400:00000200:6.0:1619677474.109662:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.109663:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.109663:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c773a5200] (24)++
00000800:00000200:8.0:1619677474.109665:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (25)--
00000800:00000200:8.0:1619677474.109665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (24)--
00000800:00000200:8.0:1619677474.109666:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (23)++
00000400:00000200:6.0:1619677474.109667:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.109668:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000400:00000200:24.0:1619677474.109669:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.109670:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677474.109671:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.109672:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.109674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:13.0:1619677474.109674:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:22.0:1619677474.109675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:8.0:1619677474.109675:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.109675:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.109676:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.109676:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677474.109677:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:8.0:1619677474.109677:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677474.109677:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677474.109678:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:13.0:1619677474.109678:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:8.0:1619677474.109678:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96506024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.109679:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.109679:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677474.109680:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.109680:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677474.109681:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.109681:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.109681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:13.0:1619677474.109682:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.109682:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.109684:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (24)++
00000800:00000200:12.0:1619677474.109684:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:8.0:1619677474.109684:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.109685:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (25)--
00000800:00000200:24.0:1619677474.109686:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (24)--
00000800:00000200:13.0:1619677474.109686:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:24.0:1619677474.109687:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (23)++
00000400:00000200:8.0:1619677474.109688:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.2:1619677474.109689:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:8.0:1619677474.109690:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.109691:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000800:00000200:21.0:1619677474.109692:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:24.0:1619677474.109693:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.109693:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.109694:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:24.0:1619677474.109695:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.109695:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.109695:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.109695:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:24.0:1619677474.109697:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:8.0:1619677474.109697:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:21.0:1619677474.109698:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.109698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:24.0:1619677474.109699:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96506023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.109700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:24.0:1619677474.109701:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.109702:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.109703:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:22.0:1619677474.109703:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.2:1619677474.109703:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.109706:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:24.0:1619677474.109707:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.109707:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.109707:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (24)++
00000800:00000200:6.0:1619677474.109707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.109708:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (25)--
00000800:00000200:24.0:1619677474.109709:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:22.0:1619677474.109709:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.109709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (24)--
00000800:00000200:8.0:1619677474.109709:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (23)++
00000800:00000200:8.0:1619677474.109710:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c773a5200] (24)--
00000800:00000200:8.0:1619677474.109711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (23)--
00000800:00000200:1.0:1619677474.109711:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:24.0:1619677474.109712:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.109712:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.109712:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.109713:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677474.109714:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677474.109714:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677474.109715:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677474.109715:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:24.0:1619677474.109716:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (22)++
00000800:00000200:17.0:1619677474.109716:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:8.0:1619677474.109716:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.54@o2ib7 : PUT - routed
00000400:00000200:1.0:1619677474.109716:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.109717:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (23)--
00000800:00000200:24.0:1619677474.109717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (22)--
00000800:00000200:24.0:1619677474.109718:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (21)++
00000800:00000200:8.0:1619677474.109719:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:1.0:1619677474.109719:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.109720:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000800:00000200:8.0:1619677474.109720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:9.2:1619677474.109721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677474.109721:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (22)++
00000400:00000200:24.0:1619677474.109722:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.109722:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677474.109723:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000400:00000200:8.0:1619677474.109724:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.109725:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.109727:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:24.0:1619677474.109727:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677474.109728:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.109729:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.109730:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:24.0:1619677474.109730:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.109731:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.109731:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677474.109731:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:8.0:1619677474.109732:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.109733:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.109734:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677474.109735:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.109736:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.109737:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677474.109737:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.109738:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.109739:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677474.109741:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.2:1619677474.109742:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677474.109744:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.109744:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677474.109745:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:24.0:1619677474.109747:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:24.0:1619677474.109748:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (23)++
00000400:00000200:8.0:1619677474.109748:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.109749:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c773a5200] (24)++
00000800:00000200:24.0:1619677474.109750:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (25)--
00000800:00000200:17.0:1619677474.109750:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:8.0:1619677474.109750:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677474.109751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (24)--
00000800:00000200:24.0:1619677474.109752:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (23)++
00000800:00000200:17.0:1619677474.109753:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677474.109753:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.109753:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:24.0:1619677474.109754:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000800:00000200:17.0:1619677474.109754:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (24)++
00000800:00000200:8.0:1619677474.109754:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677474.109755:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.109755:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (24)--
00000400:00000200:24.0:1619677474.109756:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.109756:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (24)--
00000800:00000200:17.0:1619677474.109756:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.109756:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.109757:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677474.109757:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:24.0:1619677474.109758:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.109758:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (24)++
00000400:00000200:1.0:1619677474.109758:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.109759:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (25)--
00000400:00000200:24.0:1619677474.109760:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.0:1619677474.109760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (24)--
00000800:00000200:8.0:1619677474.109760:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (23)++
00000400:00000200:24.0:1619677474.109762:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677474.109762:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677474.109764:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.109765:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.109766:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.109767:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000400:00000200:24.0:1619677474.109769:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.109771:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677474.109771:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677474.109772:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677474.109773:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.109773:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:8.0:1619677474.109773:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.109774:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.109774:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:20.0:1619677474.109774:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.109776:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.109776:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (24)++
00000800:00000200:21.0:1619677474.109776:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.109776:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.109776:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.109777:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (25)--
00000800:00000200:24.0:1619677474.109778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (24)--
00000800:00000200:24.0:1619677474.109778:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (23)++
00000400:00000200:8.0:1619677474.109778:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:24.0:1619677474.109780:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000400:00000200:8.0:1619677474.109780:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677474.109781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:8.0:1619677474.109781:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.109782:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.109783:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677474.109785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:24.0:1619677474.109787:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677474.109788:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:24.0:1619677474.109790:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.109792:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.109793:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.109793:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.109795:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.109795:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.109796:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.109797:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.109797:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.109799:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.109800:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677474.109800:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677474.109801:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.109801:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:8.0:1619677474.109801:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.109801:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677474.109802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.109802:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677474.109803:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.109803:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.109804:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (24)++
00000800:00000200:24.0:1619677474.109805:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.109805:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (25)--
00000800:00000200:24.0:1619677474.109806:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.109806:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677474.109806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (24)--
00000800:00000200:24.0:1619677474.109807:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677474.109807:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (23)++
00000800:00000200:24.0:1619677474.109808:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (24)++
00000800:00000200:8.0:1619677474.109808:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c773a5200] (24)--
00000800:00000200:24.0:1619677474.109809:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c773a5200] (24)++
00000800:00000200:9.2:1619677474.109809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677474.109809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (24)--
00000800:00000200:24.0:1619677474.109810:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (24)--
00000800:00000200:8.0:1619677474.109810:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.109810:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.109811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (23)--
00000800:00000200:24.0:1619677474.109811:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (22)++
00000400:00000200:8.0:1619677474.109811:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.109813:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000400:00000200:8.0:1619677474.109813:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.109815:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.109815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.109817:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.109817:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000800:00000200:8.0:1619677474.109817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677474.109817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (23)++
00000400:00000200:24.0:1619677474.109818:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:18.0:1619677474.109819:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000800:00000200:8.0:1619677474.109819:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000400:00000200:24.0:1619677474.109820:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.109820:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (24)++
00000400:00000200:24.0:1619677474.109821:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.109821:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.109821:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000400:00000200:2.0:1619677474.109822:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.109823:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.109823:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677474.109824:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.109825:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677474.109827:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.109828:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.109829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677474.109829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:24.0:1619677474.109830:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.109831:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.109832:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677474.109833:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677474.109834:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.109835:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.109836:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.109836:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677474.109839:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.109840:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:24.0:1619677474.109841:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:8.0:1619677474.109841:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.109843:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:8.0:1619677474.109843:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677474.109844:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (24)++
00000800:00000200:24.0:1619677474.109845:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (25)--
00000800:00000200:8.0:1619677474.109845:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.109846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (24)--
00000800:00000200:8.0:1619677474.109846:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:24.0:1619677474.109847:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (23)++
00000800:00000200:8.0:1619677474.109847:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677474.109848:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000800:00000200:8.0:1619677474.109849:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:24.0:1619677474.109850:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.109850:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (24)++
00000800:00000200:8.0:1619677474.109850:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (25)--
00000800:00000200:8.0:1619677474.109851:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (24)--
00000800:00000200:8.0:1619677474.109851:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (23)++
00000400:00000200:24.0:1619677474.109852:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.109853:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000400:00000200:24.0:1619677474.109854:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677474.109855:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.109856:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.109857:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.109857:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677474.109857:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:8.0:1619677474.109859:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.109859:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.109860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:24.0:1619677474.109861:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.109861:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.109862:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.109863:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677474.109863:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:6.0:1619677474.109863:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.109864:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:12.0:1619677474.109866:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:8.0:1619677474.109866:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.109867:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.109867:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.109868:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.109870:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.109870:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:13.0:1619677474.109871:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:24.0:1619677474.109872:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.109873:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677474.109873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677474.109874:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:13.0:1619677474.109874:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.109874:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.109875:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.109875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:8.0:1619677474.109875:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.109876:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:24.0:1619677474.109876:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (24)++
00000800:00000200:13.0:1619677474.109876:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:8.0:1619677474.109876:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.109877:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (25)--
00000800:00000200:13.0:1619677474.109877:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:24.0:1619677474.109878:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (24)--
00000800:00000200:24.0:1619677474.109879:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (23)++
00000800:00000200:8.0:1619677474.109879:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.109880:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c773a5200] (24)--
00000800:00000200:8.0:1619677474.109880:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677474.109881:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (23)--
00000800:00000200:24.0:1619677474.109881:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (22)++
00000800:00000200:9.2:1619677474.109881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.0:1619677474.109881:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677474.109883:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000800:00000200:8.0:1619677474.109883:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677474.109883:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (23)++
00000800:00000200:8.0:1619677474.109884:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c773a5200] (24)++
00000400:00000200:24.0:1619677474.109885:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.109885:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (25)--
00000800:00000200:8.0:1619677474.109886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (24)--
00000800:00000200:10.0:1619677474.109887:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:8.0:1619677474.109887:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (23)--
00000400:00000200:24.0:1619677474.109888:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.109890:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677474.109890:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.109891:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:24.0:1619677474.109891:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96506013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.109891:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.109891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a5200] (22)++
00000400:00000200:24.0:1619677474.109893:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.109893:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.109893:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.109895:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:24.0:1619677474.109895:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.109897:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:10.0:1619677474.109897:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:30.0:1619677474.109898:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.109899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (23)++
00000400:00000200:30.0:1619677474.109900:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.109900:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.109900:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c773a5200] (24)--
00000800:00000200:31.0:1619677474.109901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677474.109901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (23)--
00000800:00000200:30.0:1619677474.109902:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:24.0:1619677474.109902:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677474.109903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (22)--
00000800:00000200:24.0:1619677474.109906:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.109907:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677474.109907:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677474.109909:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677474.109910:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (21)++
00000800:00000200:24.0:1619677474.109910:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (22)--
00000800:00000200:24.0:1619677474.109911:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (21)--
00000800:00000200:8.2:1619677474.109915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.109918:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.109921:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.109922:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.109922:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.109923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.109923:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677474.109924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.0:1619677474.109925:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.109926:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.109928:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.109929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.109931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.110023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a5200] (20)++
00000800:00000200:8.0:1619677474.110030:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (21)++
00000800:00000200:8.0:1619677474.110033:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000800:00000200:24.0:1619677474.110034:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (22)--
00000400:00000200:8.0:1619677474.110035:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.110038:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.110040:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.110041:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.110042:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.110043:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.110044:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.110047:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.110048:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.110051:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.110052:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.110052:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.110054:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.110054:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (21)++
00000800:00000200:8.0:1619677474.110055:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (22)--
00000800:00000200:8.0:1619677474.110055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (21)--
00000800:00000200:9.2:1619677474.110059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.110067:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.110070:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.110071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.110071:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.110073:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.110077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.110079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.110086:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.110089:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.110090:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.110090:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.110091:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.110091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.110452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.110459:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.110462:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677474.110464:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.110464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.110468:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.110468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a5200] (20)++
00000400:00000200:24.0:1619677474.110469:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.110470:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96506011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.110472:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.110473:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.110474:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.110476:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a5200] (21)++
00000800:00000200:8.0:1619677474.110477:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.6@o2ib3
00000400:00000200:24.0:1619677474.110478:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.110479:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.6@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.110480:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.110482:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677474.110482:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.110483:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:8.0:1619677474.110483:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677474.110484:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:8.0:1619677474.110484:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96506010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.110486:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:8.0:1619677474.110486:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.110487:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.110487:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677474.110488:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.110489:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.110490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677474.110491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (22)--
00000400:00000200:8.0:1619677474.110491:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.6@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.110493:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:4.2:1619677474.110494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677474.110496:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.110496:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677474.110497:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677474.110498:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677474.110499:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a5200] (21)++
00000800:00000200:8.0:1619677474.110499:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a5200] (22)--
00000800:00000200:8.0:1619677474.110500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a5200] (21)--
00000800:00000200:26.0:1619677474.110501:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.110501:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.110504:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677474.110504:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.2:1619677474.110504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.110505:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677474.110505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677474.110506:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.110506:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.110508:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.110509:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.110510:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677474.110511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677474.110512:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96506009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.110512:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:26.0:1619677474.110513:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.110514:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.110514:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:26.0:1619677474.110515:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677474.110515:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.110518:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.110519:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:26.0:1619677474.110520:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.110521:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677474.110522:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.110525:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.110525:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.110526:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.110528:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.110528:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.110529:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.110529:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.110541:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.110544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.110544:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.110545:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.110547:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.110550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.110559:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.110561:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.110562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.110563:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.110564:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.110564:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.110706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677474.110707:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f2800] (21)++
00000800:00000200:31.0:1619677474.110710:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.4.60@o2ib3
00000800:00000200:6.0:1619677474.110711:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677474.110713:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.60@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.110714:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.110714:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:30.0:1619677474.110715:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f2800] (22)--
00000400:00000200:31.0:1619677474.110716:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.110717:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677474.110717:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.110718:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.110718:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.110720:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:31.0:1619677474.110721:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.110722:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.110723:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.110724:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677474.110724:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.110725:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.110726:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677474.110729:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.60@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.110731:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.110732:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677474.110733:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.110734:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.110735:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:31.0:1619677474.110736:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677474.110736:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.110737:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677474.110737:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:31.0:1619677474.110738:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f2800] (21)++
00000800:00000200:31.0:1619677474.110738:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f2800] (22)--
00000800:00000200:6.0:1619677474.110738:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677474.110739:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f2800] (21)--
00000800:00000200:6.0:1619677474.110739:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.110740:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.110741:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:9.2:1619677474.110743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677474.110743:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.110743:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.110748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:10.0:1619677474.110750:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.110752:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.110753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.110753:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.110754:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.110755:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:10.0:1619677474.110755:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.110756:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.110757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.110757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.110757:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.110758:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.110758:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:21.0:1619677474.110760:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.110761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.110762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677474.110765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.110771:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677474.110772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677474.110774:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.110775:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:20.0:1619677474.110775:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.110775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.110776:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677474.110776:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677474.110777:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.0:1619677474.110777:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.110777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.110779:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.110849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.110852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.110855:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.110857:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.110858:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.110860:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.110861:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.110863:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.110864:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.110865:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.110866:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.110870:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.110872:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.110874:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.110875:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.110876:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.110877:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.110878:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.110879:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.110879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.110884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.110892:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.110895:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.110896:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.110896:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.110898:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.110900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.110950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f2800] (20)++
00000800:00000200:30.0:1619677474.110958:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f2800] (21)++
00000800:00000200:30.0:1619677474.110961:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.60@o2ib3
00000400:00000200:30.0:1619677474.110963:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.60@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.110965:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f2800] (22)--
00000400:00000200:30.0:1619677474.110966:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.110967:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677474.110969:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.110970:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.110971:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.110972:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677474.110976:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.60@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.110977:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677474.110981:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.110981:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:30.0:1619677474.110982:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677474.110984:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:30.0:1619677474.110984:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f2800] (21)++
00000800:00000200:30.0:1619677474.110985:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f2800] (22)--
00000800:00000200:30.0:1619677474.110985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f2800] (21)--
00000800:00000200:6.0:1619677474.110997:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.111000:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.111001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.111001:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.111004:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.111007:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.111085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.111093:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.111095:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.111096:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.111098:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.111101:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.111102:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.111103:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.111105:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.111106:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.111106:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.111109:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.111111:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.111113:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.111113:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.111114:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.111115:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.111116:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.111116:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.111117:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.111121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.111127:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.111130:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.111131:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.111132:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.111133:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.111135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.111148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677474.111153:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.111156:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677474.111158:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.111158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677474.111161:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.111162:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.111163:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.111165:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.111166:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.111167:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.111170:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.111172:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.111174:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.111175:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.111175:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.111177:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.111177:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.111178:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677474.111179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.111183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.111190:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.111192:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.111193:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.111195:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.111196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.111197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.111282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.111290:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.111299:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:13.0:1619677474.111300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.111301:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677474.111303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.111305:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677474.111306:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.111307:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.111308:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.111309:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677474.111313:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.111315:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677474.111318:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.111318:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677474.111319:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677474.111320:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677474.111321:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.111323:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677474.111324:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677474.111325:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:4.2:1619677474.111331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.111339:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677474.111341:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677474.111342:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.111343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:31.0:1619677474.111345:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.111348:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.111349:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.111350:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.111352:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.111353:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.111459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.111466:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.111469:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.111471:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.111473:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.111473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.111474:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.111475:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96506001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.111477:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.111477:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.111478:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.111481:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.111482:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.111484:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.111484:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.111485:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.111486:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.111487:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.111488:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.111488:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.111499:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.111501:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.111502:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.111503:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.111505:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.111506:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.111597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.111603:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.111606:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.111606:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.111608:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.111610:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.111612:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.111613:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96506000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.111614:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.111615:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.111616:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.111619:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.111621:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.111623:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.111624:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.111625:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.111626:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.111627:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.111628:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.111628:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.111640:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.111642:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.111643:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.111645:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.111645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.111647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.111872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677474.111879:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.111882:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677474.111883:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677474.111885:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.111887:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.111889:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677474.111890:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.111891:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.111892:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.111893:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677474.111896:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.111898:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677474.111900:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.111901:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677474.111901:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677474.111903:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677474.111904:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.111904:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677474.111905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:21.2:1619677474.111907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:9.2:1619677474.111909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677474.111913:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:26.0:1619677474.111915:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:17.0:1619677474.111916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.111916:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:26.0:1619677474.111918:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.111918:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.111918:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677474.111919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677474.111920:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.111921:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.111922:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677474.111923:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:1.0:1619677474.111923:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.111925:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.111926:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.111927:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.111930:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:9.2:1619677474.111931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677474.111932:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.111934:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.111935:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.111936:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.111937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677474.111937:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.111938:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677474.111938:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.111939:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677474.111940:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:26.0:1619677474.111941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.111941:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677474.111941:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:10.0:1619677474.111942:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.111942:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.111943:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:24.0:1619677474.111944:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677474.111946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:24.0:1619677474.111947:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.111948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.111950:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.111951:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:24.0:1619677474.111952:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.111952:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000400:00000200:24.0:1619677474.111953:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677474.111953:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677474.111954:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000400:00000200:6.0:1619677474.111954:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.111955:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96505997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.111955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.111955:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677474.111956:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000400:00000200:24.0:1619677474.111957:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.111957:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:24.0:1619677474.111958:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.111959:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.111963:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.111965:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.111965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.111966:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.111967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.111968:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.111969:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.111970:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.111970:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.111970:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.111971:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.111971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.111973:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.111973:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.111975:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.111976:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.111977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.111982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.111986:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.111988:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.111989:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.111989:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.111989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.111991:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.111991:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.111992:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.111993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.111994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.112253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.112257:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.112260:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.112262:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.112263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.112264:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.112266:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.112267:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.112268:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.112269:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.112270:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.112274:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.112276:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.112278:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.112279:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.112280:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.112282:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.112282:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.112284:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.112285:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.112285:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677474.112292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:9.2:1619677474.112299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.112300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.112302:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.112303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.112303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677474.112305:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.112308:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.112309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.112309:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.112311:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.112313:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.112449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.112455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.112458:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.112459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.112461:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677474.112462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:6.0:1619677474.112463:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.112464:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.112465:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.112467:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.112468:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.112468:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677474.112469:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677474.112471:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.112472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677474.112472:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.112474:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677474.112475:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.112477:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677474.112477:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.112478:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.112478:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:2.0:1619677474.112478:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.112479:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.112480:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.112481:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.112481:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:2.0:1619677474.112481:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.112482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:2.0:1619677474.112482:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.112483:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.112486:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.112488:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.112490:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.112491:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.112492:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677474.112494:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677474.112494:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.112494:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.112495:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677474.112496:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:2.0:1619677474.112496:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:21.0:1619677474.112497:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.112498:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.112499:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.2:1619677474.112500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677474.112501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677474.112508:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.112510:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.112512:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.112512:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.112514:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.112516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.112521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.112528:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.112531:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.112531:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.112532:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.112533:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.112533:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.113415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677474.113420:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.113423:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.113424:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.113426:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.113428:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.113429:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.113430:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.113431:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.113433:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.113433:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.113437:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.113438:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.113441:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.113442:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.113442:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.113444:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.113445:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.113445:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.113446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677474.113463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.113470:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.113473:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.113474:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.113474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.113476:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.113479:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.113542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:21.2:1619677474.113549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677474.113549:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.113552:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.113552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.113554:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.113557:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677474.113557:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.113558:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.113559:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.113560:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677474.113561:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.113562:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.113564:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.113564:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.113565:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.113566:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.113568:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677474.113568:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.113569:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677474.113570:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.113571:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.113571:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677474.113573:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677474.113574:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677474.113574:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.113574:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.113576:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:30.0:1619677474.113577:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.113577:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.113578:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.113578:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677474.113579:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.113581:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.113582:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:9.2:1619677474.113582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.113583:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.113584:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.113585:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.113586:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.113586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677474.113589:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.113591:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.113592:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.113593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.2:1619677474.113594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:2.0:1619677474.113594:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.113597:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:2.0:1619677474.113597:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.113599:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.113600:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.113600:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677474.113602:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.113603:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.113603:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:31.0:1619677474.113604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677474.113604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.113606:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.113608:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.113609:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.113611:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.113612:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.113613:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.113614:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.113618:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.113619:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.113622:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.113622:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.113623:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.113625:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.113625:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.113626:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.113627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.113638:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.113640:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.113641:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.113643:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.113644:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.113645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.113770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.113775:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.113778:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.113780:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.113780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.113783:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.113784:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.113785:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.113787:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.113788:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.113789:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.113792:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.113793:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.113795:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.113796:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.113797:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.113798:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.113799:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.113800:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.113800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.113812:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.113814:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.113815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.113815:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.113817:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.113819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.114175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.114181:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.114183:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.114184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.114186:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.114189:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.114191:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.114192:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.114193:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.114194:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.114195:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.114198:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.114199:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.114202:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.114203:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.114203:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.114205:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.114205:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.114207:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.114208:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.114208:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.114213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677474.114215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:22.0:1619677474.114218:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.114219:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.114221:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.114221:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.114221:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677474.114222:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.114222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:30.0:1619677474.114223:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:1.0:1619677474.114223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.114225:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.114226:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:22.0:1619677474.114227:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:30.0:1619677474.114228:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.114230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.114231:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.114233:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.114234:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.114235:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.114237:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.114237:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.114241:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.114242:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.114245:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.114246:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.114247:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.114248:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.114249:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.114250:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.114250:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.114265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.114271:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.114274:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.114274:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.114275:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.114276:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.114279:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.114438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.114444:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.114447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.114447:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.114450:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.114452:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.114453:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.114454:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.114455:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.114457:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.114458:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.114461:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.114462:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.114465:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.114466:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.114466:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.114468:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.114468:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.114469:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.114470:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.114471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.114476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.114481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.114482:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677474.114482:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.114483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.114484:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.114485:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.114485:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.114487:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.114488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.114489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.114499:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.114501:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.114502:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.114503:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.114504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.2:1619677474.114505:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.0:1619677474.114508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.114513:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.114516:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.114518:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.114520:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.114520:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.114522:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.114523:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.114525:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.114525:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.114527:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.114530:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.114532:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.114534:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.114535:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.114535:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.114537:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.114537:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.114539:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.114540:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.114541:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.114547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.114551:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.114552:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.114553:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.114554:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.114555:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.114556:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.114556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.114557:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.114559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.114559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.114567:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.114570:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.114570:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.114571:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.114571:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.114572:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.114979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677474.114987:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.114989:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677474.114990:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677474.114992:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.114994:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.114995:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677474.114997:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.114998:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.114999:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.115000:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677474.115003:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.115005:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677474.115008:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.115009:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677474.115009:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.115011:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.115011:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.115012:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677474.115012:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677474.115018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.115026:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.115029:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.115030:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.115030:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.115033:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.115036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.115096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.115104:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.115106:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.115107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.115109:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.115112:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.115114:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.115115:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.115116:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.115117:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.115118:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.115122:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.115123:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.115125:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.115126:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.115127:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.115128:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.115129:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.115129:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.115130:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.115142:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.115145:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.115146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.115146:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.115149:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.115151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.115282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.115287:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.115289:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.115290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.115292:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.115300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.115301:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.115302:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.115304:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.115305:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.115306:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.115309:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.115311:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.115313:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.115314:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.115314:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.115316:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.115316:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.115317:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.115317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.115331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.115333:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.115334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.115336:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.115338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.115339:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.115341:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.115349:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.115351:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.115354:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.115356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.115356:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.115357:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.115358:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.115360:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.115361:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.115362:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:4.2:1619677474.115364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677474.115365:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.115366:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.115369:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.115369:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.115370:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.115372:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.115372:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677474.115372:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677474.115373:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.115374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677474.115374:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.115375:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.115377:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677474.115377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:26.0:1619677474.115380:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.115382:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.115383:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.115383:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:26.0:1619677474.115384:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.115385:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.115386:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677474.115386:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.115387:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.115388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:26.0:1619677474.115389:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.115389:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.115391:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677474.115391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.115394:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.115395:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.115395:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:9.2:1619677474.115395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.115397:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.115397:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.115399:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.115400:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.115400:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.0:1619677474.115401:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.115403:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.115404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.115404:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.115405:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.115406:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.115406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.115410:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.115413:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.115413:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.115414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.115414:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.115415:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.115415:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.115416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.115417:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.115419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.115645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.115651:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.115654:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.115655:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.115656:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.115660:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.115661:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.115662:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.115664:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.115665:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.115665:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.115669:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.115670:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.115673:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.115674:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.115674:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.115676:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.115676:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.115677:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.115678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.115682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.115691:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.115693:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.115694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.115695:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.115697:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.115700:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.115802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.115808:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.115810:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.115811:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.115813:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.115815:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.115816:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.115817:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.115819:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.115819:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.115820:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.115824:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.115826:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.115828:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.115829:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.115829:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.115831:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.115831:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.115832:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.115833:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677474.115837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.115845:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.115847:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.115848:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.115849:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.115851:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.115853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.116183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.116190:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.116193:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.116193:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.116196:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.116198:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.116200:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.116201:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.116202:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.116203:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.116204:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.116208:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.116210:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.116212:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.116213:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.116214:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.116215:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.116215:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.116217:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.116218:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.116218:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.116221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.116224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.116227:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:4.2:1619677474.116228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677474.116229:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.116230:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.116231:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677474.116232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.116233:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:10.0:1619677474.116233:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.116233:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677474.116233:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.116234:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677474.116235:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677474.116236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.116236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:24.0:1619677474.116237:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.116237:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677474.116240:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.116241:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677474.116242:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.116243:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:24.0:1619677474.116244:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.116244:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.116245:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677474.116246:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.116246:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.116249:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.116249:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.116251:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.116251:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677474.116252:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.116253:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.116254:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.116255:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.116255:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677474.116255:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:26.0:1619677474.116256:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.116257:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677474.116257:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677474.116258:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.116259:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000400:00000200:26.0:1619677474.116260:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.116260:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:26.0:1619677474.116261:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677474.116261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:26.0:1619677474.116263:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677474.116263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.116264:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.116265:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.116266:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:4.2:1619677474.116266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:26.0:1619677474.116267:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.116267:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.116268:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:17.0:1619677474.116271:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.116273:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:8.0:1619677474.116273:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:1.0:1619677474.116274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.116275:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.116275:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677474.116276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677474.116277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:17.0:1619677474.116277:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.116279:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:17.0:1619677474.116280:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:13.0:1619677474.116281:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.116282:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677474.116282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:13.0:1619677474.116283:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.116285:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.116287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:1.0:1619677474.116289:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.116292:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.116293:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.116299:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.116300:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.116301:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.116337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.116343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.116347:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.116347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.116349:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.116351:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.116353:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.116354:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.116355:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.116356:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.116357:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.116360:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.116362:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.116365:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.116366:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.116366:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.116368:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.116368:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.116369:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.116369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.116382:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.116384:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.116385:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.116387:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.116388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.116389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.116429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.116437:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.116440:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.116443:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.116445:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.116445:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.116447:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.116448:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.116449:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.116450:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.116451:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.116455:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.116456:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.116459:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.116460:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.116460:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.116462:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.116462:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.116463:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.116464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.116467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.116474:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.116476:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.116477:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.116477:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.116479:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.116482:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.116687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.116695:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.116697:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.116698:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.116700:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.116702:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.116704:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.116705:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.116706:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.116707:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.116708:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.116711:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.116713:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.116715:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.116716:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.116717:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.116719:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.116719:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.116720:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.116720:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677474.116734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.116739:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677474.116741:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.116742:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.116744:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.116744:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.116745:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.116745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677474.116745:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.116747:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.116748:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.116749:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.116750:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.116750:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:21.0:1619677474.116751:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.116752:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.116753:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.116757:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.116758:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.116761:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.116761:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.116762:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.116763:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.116764:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.116764:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.116765:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677474.116766:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.116766:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677474.116767:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.116767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677474.116767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.116768:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.116769:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677474.116770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.116772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.116775:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.116777:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.116778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.116778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.116778:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.116781:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.116782:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.116782:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.116784:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.116786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.117229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.117233:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.117235:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.117237:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.2:1619677474.117238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:20.0:1619677474.117239:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.117241:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677474.117242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.117242:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.117243:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:20.0:1619677474.117244:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.117244:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.117246:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.117246:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.117247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.117249:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.117251:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:6.0:1619677474.117252:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.117254:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.117254:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:6.0:1619677474.117254:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.117255:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677474.117255:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.0:1619677474.117257:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.117257:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.117258:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.117258:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000400:00000200:6.0:1619677474.117258:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.117259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677474.117259:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.117260:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.117263:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.117265:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.117268:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.117269:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.117269:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.117271:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.117271:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.117272:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.117272:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:22.0:1619677474.117273:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.117273:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.117274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:22.0:1619677474.117275:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.117275:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.117278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.117284:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.117286:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.117287:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.117289:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.117290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.117291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.117953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.117960:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.117962:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.117963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.117965:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.117968:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.117969:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.117970:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.117972:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.117973:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.117974:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.117978:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.117980:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.117983:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.117984:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.117984:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.117986:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.117987:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.117989:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.117990:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.117990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677474.117996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.118001:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:2.0:1619677474.118002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677474.118003:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:2.0:1619677474.118004:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:30.0:1619677474.118005:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.118005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677474.118006:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.118006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677474.118008:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.118010:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.118105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.118111:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.118114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.118114:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.118116:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.118119:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.118120:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.118121:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.118123:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.118124:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.118125:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.118128:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.118130:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.118132:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.118133:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.118133:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.118135:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.118136:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.118137:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.118138:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.118138:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.118143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.118148:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.118148:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.118150:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.118151:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.118151:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.118151:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.118152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.118153:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.118154:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.118155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.118168:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.118170:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.118171:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.118172:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.118173:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.118173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.118268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.118276:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.118279:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.118281:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.118283:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.118283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.118285:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.118286:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.118288:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.118289:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.118290:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.118299:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.118301:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.118303:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.118304:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.118305:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.118306:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.118307:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.118308:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.118308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.118319:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.118322:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.118322:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.118324:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.118326:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.118327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.118341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.118348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.118350:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.118351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.118353:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.118356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.118357:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677474.118359:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.118360:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.118361:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.118362:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677474.118365:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.118367:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677474.118371:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.118371:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677474.118372:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677474.118374:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677474.118374:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.118375:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.118375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:12.0:1619677474.118388:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.118390:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.118391:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.118393:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.118394:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.118396:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.118824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677474.118832:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.118835:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677474.118837:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.118838:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677474.118838:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.118840:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677474.118841:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.118843:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.118844:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.118845:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677474.118848:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.118850:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677474.118853:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.118854:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:30.0:1619677474.118854:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677474.118856:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677474.118857:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.118858:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.118858:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677474.118863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.118871:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.118874:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.118875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.118876:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.118878:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.118881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677474.118943:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.118946:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.118947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.118949:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.112@o2ib7 : GET - for me
00000400:00000200:6.0:1619677474.118951:0:19937:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.112@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:6.0:1619677474.118954:0:19937:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.112@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:6.0:1619677474.118955:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.118956:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.118958:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677474.118961:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : REPLY try# 0
00000800:00000200:6.0:1619677474.118963:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.118965:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.118965:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.118966:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.118967:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.118968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.118980:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.118982:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.118983:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.118983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.118984:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: REPLY: OK
00000800:00000200:22.0:1619677474.118986:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.119168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.119173:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.119175:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.119176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.119177:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.119180:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.119181:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.119182:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.119183:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.119184:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.119185:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.119188:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.119189:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.119192:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.119192:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.119193:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.119194:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.119195:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.119195:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.119196:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.119208:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.119210:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.119211:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.119212:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.119213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.119215:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.119504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.119513:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.119516:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.119517:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.119518:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677474.119519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:8.0:1619677474.119521:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.119523:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.119524:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.119525:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.119526:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677474.119526:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.119527:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677474.119528:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.119529:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.119532:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.119532:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.119534:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.119535:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:17.0:1619677474.119536:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.119537:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.119538:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677474.119539:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.119539:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.119539:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:17.0:1619677474.119540:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.119541:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677474.119541:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.119542:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.119542:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.119543:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677474.119544:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.119546:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.119548:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677474.119548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677474.119549:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.119550:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.119551:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.119552:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.119553:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.119553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:18.0:1619677474.119557:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.119559:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.119560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.119561:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.119563:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.119566:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:18.0:1619677474.119566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677474.119568:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.119570:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.119570:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.119572:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.119575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.119624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.119632:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.119636:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.119638:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.119640:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.119641:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.119642:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.119644:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.119645:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.119646:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.119647:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.119651:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.119652:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.119656:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.119657:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.119657:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.119659:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.119660:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.119661:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.119663:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.119663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.119665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.119669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.119673:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.119676:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677474.119677:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677474.119677:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.119678:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.119679:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.119679:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.119680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.119681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.119682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.119683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.119690:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.119693:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.119694:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.119695:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.119695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.119697:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.119946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.119951:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.119953:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.119955:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.119956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.119957:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.119959:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.119960:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.119961:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.119962:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.119963:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.119966:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.119968:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.119970:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.119971:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.119972:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.119974:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.119974:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.119975:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.119975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.119979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.119986:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.119989:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.119989:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.119991:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.119992:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.119994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.120009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.120017:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.120020:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.120021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.120023:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.120025:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.120027:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.120028:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.120029:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.120031:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.120031:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.120035:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.120037:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.120040:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.120041:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.120041:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.120043:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.120044:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.120044:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.120044:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.120045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.120054:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.120056:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.120058:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.120059:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.120059:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.120061:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.120062:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.120063:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.120063:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.120065:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.120066:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.120068:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.120069:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.120072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.120073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.120073:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.120074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.120075:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.120078:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.120079:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.120080:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.120081:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.120082:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.120082:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.120083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.120095:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.120097:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.120098:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.120100:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.120101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.120102:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.120297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.120305:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.120308:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.120309:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.120311:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.120313:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.120315:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.120316:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.120318:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.120319:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.120320:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.120323:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.120324:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.120327:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.120328:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.120329:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.120330:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.120331:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.120331:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.120331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677474.120343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:22.0:1619677474.120344:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.120347:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.120348:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.120348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.120349:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:22.0:1619677474.120351:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.120352:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.14.21@o2ib3
00000800:00000200:22.0:1619677474.120353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677474.120353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.120355:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : REPLY - routed
00000800:00000200:1.0:1619677474.120358:0:19931:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 951898
00000800:00000200:1.0:1619677474.120362:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.120363:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677474.120364:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677474.120365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:21.2:1619677474.120372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.120377:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677474.120379:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677474.120380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:21.2:1619677474.120519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.120524:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677474.120527:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.120528:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677474.120529:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.120530:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.120532:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.120533:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.120534:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.120535:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.120536:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.120540:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : REPLY try# 0
00000800:00000200:17.0:1619677474.120541:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 951898 bytes in 256 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.120543:0:19930:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 951898
00000800:00000200:17.0:1619677474.120552:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.120553:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.120553:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.120555:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.120555:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.120556:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.120556:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677474.120569:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.120571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.120572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.120606:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.120608:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[3] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.120609:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.120613:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.120625:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.120626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:4.2:1619677474.120639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.120643:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.120645:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.120648:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.120649:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.120649:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.120651:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.120652:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.120653:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.120655:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.120656:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.120659:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.120661:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.120664:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.120665:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.120666:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.120667:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.120668:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.120669:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.120669:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.120699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.120706:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.120708:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.120709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.120710:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.120712:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.120715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.120722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.120729:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.120731:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.120732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.120732:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.120733:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.120734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677474.120800:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.120803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.120804:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.120805:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.120806:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: REPLY: OK
00000800:00000200:6.0:1619677474.120809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.120864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.120872:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.120874:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.120877:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.120878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.120879:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.120881:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.120882:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.120884:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.120884:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.120885:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.120888:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.120890:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.120892:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.120893:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.120894:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.120895:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.120896:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.120897:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.120897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.120901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.120907:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.120910:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.120910:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.120911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.120912:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.120915:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.120952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.120957:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.120960:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.120961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.120963:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.120965:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.120967:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.120968:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.120969:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.120970:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.120971:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.120974:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.120976:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.120978:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.120979:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.120979:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.120981:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.120981:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.120982:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.120982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.120995:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.120997:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.120998:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.120999:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.121000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.121002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.121305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.121313:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.121316:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.121316:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.121318:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.121321:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.121323:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.121324:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.121325:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.121326:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.121327:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.121331:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.121332:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.121335:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.121336:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.121337:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.121339:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.121339:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.121340:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.121340:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.121354:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.121356:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.121357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.121358:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.121360:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.121362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.121470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.121479:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.121482:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.121484:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.121486:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.121487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.121488:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.121489:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.121490:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.121491:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.121492:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.121496:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.121498:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.121500:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.121500:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.121501:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.121502:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.121503:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.121504:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.121505:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.121515:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.121517:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.121518:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.121519:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.121521:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.121522:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.121647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.121654:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.121657:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.121657:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.121660:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.121662:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.121664:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.121665:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.121666:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.121667:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.121668:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.121671:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.121673:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.121675:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.121676:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.121677:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.121678:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.121679:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.121679:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.121680:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.121684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.121693:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.121696:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.121697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.121697:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.121699:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.121703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.121728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.121736:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.121738:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.121739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.121740:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.121743:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.121745:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.121746:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.121747:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.121748:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.121749:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.121752:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.121754:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.121757:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.121757:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.121758:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.121760:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.121760:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.121761:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.121763:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.121763:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.121769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.121774:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.121776:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.121776:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.121777:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.121778:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677474.121778:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.121779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.121780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.121780:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.121786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.121788:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.121790:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.121791:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.121791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.121792:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.121793:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.121898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.121905:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.121908:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.121909:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.121911:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.121913:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.121914:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.121916:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.121917:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.121918:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.121919:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.121922:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.121923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:18.0:1619677474.121924:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.121926:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.121927:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:18.0:1619677474.121927:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.121928:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.121929:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677474.121930:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:18.0:1619677474.121930:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.121931:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.121931:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.0:1619677474.121932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.121932:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.121934:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.121935:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:9.2:1619677474.121935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:20.0:1619677474.121936:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.121938:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.121938:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.121939:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.121942:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.121942:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677474.121944:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677474.121944:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.121945:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.121946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677474.121947:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.121948:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:2.0:1619677474.121948:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.121949:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.121950:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677474.121950:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677474.121951:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.121952:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.121953:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.121954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.121955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:9.2:1619677474.121959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.121961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:18.0:1619677474.121963:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.121965:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.121966:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677474.121966:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:2.0:1619677474.121966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677474.121967:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:18.0:1619677474.121967:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.121968:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677474.121968:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.121969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.121969:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.121970:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.121970:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677474.121970:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.121971:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.121974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.122003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.122009:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.122012:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.122012:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.122015:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.122016:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.122018:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.122019:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.122020:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.122021:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.122022:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.122025:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.122027:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.122029:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.122030:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.122031:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.122033:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.122033:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.122034:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.122035:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.122036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.122041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.122046:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.122048:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677474.122048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.122049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.122050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.122051:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.122051:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.122053:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.122054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.122055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.122068:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.122071:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.122071:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.122072:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.122073:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.122074:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.122224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677474.122228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.122233:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.122235:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677474.122237:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:31.0:1619677474.122238:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.122238:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.122239:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.122241:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.122241:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.122243:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.122244:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.122245:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.122246:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.122249:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.122251:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.122252:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.122254:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677474.122254:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677474.122255:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.122255:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677474.122255:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.122256:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.122257:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677474.122257:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.122258:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.122259:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.122260:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.122261:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.122261:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.122262:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.122263:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.122264:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.122265:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.122265:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.122266:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.122266:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.122266:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.122267:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677474.122268:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.122269:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.122270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.122270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.122270:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.122271:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.122271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.122282:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.122285:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.122286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.122286:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.122288:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.122291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.122835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.122841:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.122844:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.122845:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.122846:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.122849:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.122851:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.122852:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677474.122853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:2.0:1619677474.122853:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.122855:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.122856:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677474.122859:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:2.0:1619677474.122859:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.122861:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677474.122862:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.122863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:2.0:1619677474.122864:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677474.122865:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.122865:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.122866:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:22.0:1619677474.122867:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.122868:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677474.122868:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.122868:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:22.0:1619677474.122869:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.122869:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.122870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677474.122871:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.122872:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.122873:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.122875:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.122877:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.122879:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.122880:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.122881:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.122882:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:13.0:1619677474.122882:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:22.0:1619677474.122883:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.122883:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.122884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:13.0:1619677474.122884:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.122885:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.122887:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.122889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.122890:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.0:1619677474.122895:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.122898:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.122899:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.122900:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.122901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.122903:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.123286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.123294:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.123302:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.123303:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.123305:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.123308:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.123309:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.123311:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.123312:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.123313:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.123314:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.123316:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.123318:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.123320:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.123321:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.123322:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.123323:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.123324:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.123325:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.123327:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.123327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.123334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.123338:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.123340:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.123340:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.123342:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.123342:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.123343:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.123343:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.123344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.123346:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.123346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.123355:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.123357:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.123358:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.123359:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.123359:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.123360:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.123408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.123417:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.123420:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.123421:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.123422:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.123426:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.123428:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.123429:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.123431:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.123432:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.123433:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.123436:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.123438:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.123441:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.123442:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.123442:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.123444:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.123445:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.123445:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.123446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.123450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.123459:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.123462:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.123463:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.123463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.123465:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.123468:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.123513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.123518:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.123521:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.123523:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.123523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.123525:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.123527:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.123528:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.123530:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.123531:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.123532:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.123535:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.123537:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.123539:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.123540:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.123541:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.123542:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.123543:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.123544:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.123545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.123549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.123557:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.123560:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.123561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.123561:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.123563:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.123565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.123678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.123684:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.123687:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.123687:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.123689:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.123691:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.123693:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.123694:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.123695:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.123696:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.123697:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.123701:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.123703:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.123705:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.123706:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.123706:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.123708:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.123708:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.123709:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.123709:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.123721:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.123723:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.123724:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.123726:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.123727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.123728:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.124024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.124031:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.124034:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.124036:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.124038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.124038:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.124039:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.124041:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.124042:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.124043:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.124044:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.124048:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.124050:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.124052:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.124053:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.124054:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.124055:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.124056:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.124057:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.124058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.124070:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.124073:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.124074:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.124074:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.124077:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.124079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.124244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.124250:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.124253:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.124254:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.124255:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.124257:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.124259:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.124260:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.124261:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.124262:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.124263:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.124266:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.124267:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.124270:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.124271:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.124271:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.124273:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.124273:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.124274:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.124274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.124286:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.124289:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.124290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.124290:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.124292:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.124300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.124514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.124520:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.124523:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.124524:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.124526:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.124528:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.124530:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.124531:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.124532:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.124533:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.124534:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.124538:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.124540:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.124543:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.124544:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.124545:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.124547:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.124547:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.124548:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677474.124549:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.124550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:21.2:1619677474.124555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.124560:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:31.0:1619677474.124561:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677474.124562:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677474.124563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:31.0:1619677474.124564:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:18.0:1619677474.124564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:31.0:1619677474.124565:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.124567:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.124568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.124570:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.124713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.124720:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.124723:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.124723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.124725:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.124728:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.124729:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.124730:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.124732:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.124733:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.124734:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.124737:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.124739:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.124741:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.124742:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.124743:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.124744:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.124745:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.124745:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.124746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.124758:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.124760:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.124761:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.124763:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.124764:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.124765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.125006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.125014:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.125017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.125017:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.125019:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.125022:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.125024:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.125025:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.125026:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.125027:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.125028:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.125032:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.125033:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.125036:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.125037:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.125038:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.125040:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.125040:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.125041:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.125041:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.125053:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.125056:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.125057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.125057:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.125060:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.125062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.125092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.125099:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.125102:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.125103:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.125105:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.125108:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.125109:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.125110:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677474.125110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000400:00000200:26.0:1619677474.125111:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.125112:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.125113:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.125116:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.125117:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677474.125118:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.125120:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000800:00000200:26.0:1619677474.125121:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.125121:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:30.0:1619677474.125122:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.125122:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.125123:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:31.0:1619677474.125124:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:26.0:1619677474.125124:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677474.125125:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.125125:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.125125:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677474.125126:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677474.125127:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.125129:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.125129:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.125130:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677474.125134:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.125135:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.125137:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:30.0:1619677474.125138:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.125139:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (23)++
00000800:00000200:22.0:1619677474.125140:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000800:00000200:30.0:1619677474.125141:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (24)++
00000800:00000200:6.0:1619677474.125141:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (24)--
00000400:00000200:22.0:1619677474.125142:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.125144:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (23)--
00000800:00000200:30.0:1619677474.125144:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000400:00000200:22.0:1619677474.125144:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.125146:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:30.0:1619677474.125147:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:22.0:1619677474.125147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:30.0:1619677474.125148:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:4.2:1619677474.125154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:6.0:1619677474.125156:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.125159:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.125160:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.125160:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.125161:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.125162:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000800:00000200:31.0:1619677474.125164:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677474.125165:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677474.125166:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:6.0:1619677474.125167:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.125168:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.125170:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.125171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.125171:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.125172:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.125173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.125199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.125207:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.125210:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.125212:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.125214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.125215:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.125217:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.125218:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.125219:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.125220:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.125221:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.125224:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.125225:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.125228:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.125229:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.125230:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.125231:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.125232:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.125233:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.125233:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.125238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.125247:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.125249:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.125250:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.125252:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.125252:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.125255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677474.125421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677474.125427:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677474.125430:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677474.125431:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677474.125434:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.125436:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.125437:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677474.125439:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.125440:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.125441:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.125442:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677474.125446:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.125448:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677474.125450:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.125451:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677474.125452:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677474.125454:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:1.0:1619677474.125454:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677474.125455:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677474.125456:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:30.0:1619677474.125468:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.125471:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.125472:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.125474:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.125476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.125478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677474.125489:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.125492:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.125493:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.125494:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.125494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.125496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.125619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.125626:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.125628:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.125629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.125631:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.125633:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.125635:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.125636:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.125637:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.125638:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.125639:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.125643:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.125644:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.125647:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.125648:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.125648:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.125650:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.125650:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.125652:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.125652:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.125653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.125658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.125664:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.125666:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.125666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.125667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.125668:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.125670:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.125671:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.125672:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.125674:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.125674:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.125675:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.125676:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.125677:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (23)++
00000800:00000200:20.0:1619677474.125677:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.125678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (24)--
00000800:00000200:20.0:1619677474.125679:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.125680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.126136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677474.126144:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.126147:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677474.126148:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677474.126150:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677474.126153:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.126155:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677474.126156:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.126157:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.126158:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.126159:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677474.126163:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.126164:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677474.126167:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.126168:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:8.0:1619677474.126168:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677474.126170:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.0:1619677474.126170:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.126171:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.126172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:4.2:1619677474.126173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.126176:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.126179:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.126181:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.126182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.126183:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.126183:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:20.0:1619677474.126185:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.126186:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677474.126186:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.126186:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.126187:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.126188:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:20.0:1619677474.126189:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.126189:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677474.126190:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.126191:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.126193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:20.0:1619677474.126193:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.126195:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677474.126197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677474.126199:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.126199:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.126200:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677474.126200:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.126202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677474.126202:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:13.0:1619677474.126202:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:30.0:1619677474.126203:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:20.0:1619677474.126203:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.126204:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.126204:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:30.0:1619677474.126205:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.126205:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.126205:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.126206:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677474.126206:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677474.126207:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:13.0:1619677474.126207:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:30.0:1619677474.126208:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.126208:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677474.126208:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677474.126209:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677474.126209:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677474.126210:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.126210:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.126211:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.126212:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.126213:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.126214:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.126214:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.126215:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.126215:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:22.0:1619677474.126216:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677474.126217:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.126218:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.126218:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:30.0:1619677474.126219:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.126219:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.126219:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.126219:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677474.126220:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.126220:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.126222:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677474.126222:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.126222:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:10.0:1619677474.126222:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.126223:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96505922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.126225:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.126225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677474.126226:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.126226:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.126227:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.126228:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.126230:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.126231:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.126231:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.126231:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.126232:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.126232:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.126233:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.126233:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.126234:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677474.126234:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.126235:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:10.0:1619677474.126235:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677474.126236:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.126236:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:10.0:1619677474.126236:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.126236:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.126237:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.126237:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.126237:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677474.126238:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:10.0:1619677474.126238:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677474.126239:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.126239:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.126239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.126240:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.126240:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677474.126240:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:6.0:1619677474.126240:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.126241:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.126241:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677474.126241:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677474.126242:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677474.126242:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (25)--
00000400:00000200:22.0:1619677474.126243:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.126243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:4.2:1619677474.126243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.126244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.126245:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.126245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.2:1619677474.126246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677474.126247:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677474.126248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677474.126249:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.126250:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.126251:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:30.0:1619677474.126252:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.126252:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.126252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.126253:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.126254:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677474.126255:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:26.0:1619677474.126255:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.0:1619677474.126255:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.126256:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677474.126256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.126257:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677474.126257:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.126258:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.126260:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.126260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:9.2:1619677474.126260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:21.0:1619677474.126263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677474.126266:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.126269:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.126270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.126270:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.126271:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.126272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.126505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.126511:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.126513:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.126514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.126516:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.126518:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.126520:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.126521:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.126522:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.126523:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.126525:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.126528:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.126530:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.126532:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.126533:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.126534:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.126535:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.126536:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.126537:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.126537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.126550:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.126552:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.126553:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.126555:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.126556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.126557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.126828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.126835:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.126838:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.126839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.126841:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.126843:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.126845:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.126846:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.126848:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.126848:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677474.126849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677474.126850:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.126853:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.126855:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.126857:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.126857:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.126858:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.126859:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.126860:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677474.126861:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.126861:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677474.126862:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.126862:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.126862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677474.126864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.126864:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.126866:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.126867:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.126868:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.126869:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.126870:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.126873:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.126874:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.126875:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.126877:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.126877:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.126878:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.126878:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:22.0:1619677474.126878:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.126878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.126880:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.126880:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.126881:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.126881:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677474.126881:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.126884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.126892:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.126895:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.126896:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.126897:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.126898:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.126899:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.126972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.126980:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.126983:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.126984:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.126986:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.126988:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.126990:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.126991:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.126992:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.126993:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.126994:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.126997:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.126999:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.127002:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.127003:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.127003:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.127005:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.127006:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.127006:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.127008:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.127008:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677474.127014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.127014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.127021:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.127023:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:21.0:1619677474.127023:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677474.127024:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.127025:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677474.127025:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.127026:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.127028:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.127029:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.127031:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.127546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.127552:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.127554:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.127555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.127557:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.127559:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.127560:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.127561:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.127563:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.127564:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.127565:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.127568:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.127569:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.127572:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.127573:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.127573:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.127575:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.127575:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.127576:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.127576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.127589:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.127592:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.127592:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.127594:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.127595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.127596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.127795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.127802:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.127805:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.127807:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.127810:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.127810:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.127811:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.127812:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.127814:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.127815:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.127816:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.127818:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.127820:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.127822:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.127823:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.127824:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.127825:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.127825:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.127827:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.127828:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.127829:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.127834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.127839:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.127841:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.127841:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.127842:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677474.127843:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.127843:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.127843:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.127844:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.127846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.127846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.127981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.127989:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.127991:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.127993:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.127995:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.127995:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.127997:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.127998:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.127999:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.128000:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.128001:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.128003:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.128005:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.128008:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.128009:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.128009:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.128011:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.128011:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.128012:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.128012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.128024:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.128027:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.128028:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.128028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.128030:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.128033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.128415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.128421:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.128423:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.128424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.128425:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.128427:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.128428:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.128429:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.128430:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.128431:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.128432:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.128435:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677474.128437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677474.128437:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.128439:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.128440:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.128441:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.128442:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.128443:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.2:1619677474.128444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:6.0:1619677474.128444:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.128444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677474.128445:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.128448:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.128450:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.128450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.128452:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:31.0:1619677474.128453:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.128454:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677474.128454:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677474.128454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677474.128456:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.128457:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.128458:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.128458:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:31.0:1619677474.128459:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.128459:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.128460:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:21.0:1619677474.128460:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.128461:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.128461:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.128462:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96505912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.128463:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.128463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:2.0:1619677474.128464:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.128465:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.128465:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.128466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:2.0:1619677474.128466:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.128469:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:2.0:1619677474.128470:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.128471:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.128472:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677474.128472:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.128473:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.128474:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677474.128474:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.128475:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677474.128475:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.128475:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677474.128476:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.128477:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677474.128477:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.128478:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.128479:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:9.2:1619677474.128480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677474.128481:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.128481:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.128483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.128486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677474.128487:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.128490:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.128491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.128491:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.128492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677474.128493:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:8.0:1619677474.128493:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.128494:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.128494:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677474.128495:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.128495:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677474.128495:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.128495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677474.128496:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:18.0:1619677474.128497:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.128497:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.128498:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677474.128498:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677474.128499:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.128501:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.128502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.128503:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.128504:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.128506:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.128507:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.128508:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.128509:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.128512:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.128514:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.128516:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.128517:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.128517:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.128519:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.128519:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.128520:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.128521:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.128525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.128533:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.128535:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.128536:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.128537:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.128539:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.128540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677474.129023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.129031:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.129036:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.129037:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.129041:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.129044:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.129046:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.129048:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.129051:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.129052:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.129054:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.129060:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.129062:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.129066:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.129067:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.129068:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.129070:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.129071:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.129072:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.129073:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.129084:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.129087:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.129088:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.129088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.129090:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.129092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.129111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.129118:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.129121:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.129122:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.129124:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.129127:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.129128:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.129129:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.129131:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.129132:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.129133:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.129137:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.129139:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.129142:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.129142:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.129143:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.129145:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.129145:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.129146:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.129147:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.129151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.129161:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.129165:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.129167:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.129167:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.129170:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.129175:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.129383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.129391:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.129395:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.129395:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677474.129397:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.129400:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.129402:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.129403:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.129404:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.129405:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.129406:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.129409:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.129411:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.129414:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.129414:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.129415:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.129417:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.129417:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.129418:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.129419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.129432:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.129434:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.129435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.129436:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677474.129437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:6.0:1619677474.129438:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.129441:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677474.129445:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.129450:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.129451:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.129454:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.129457:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.129459:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.129462:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.129464:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.129466:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.129467:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.129473:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.129476:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.129479:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.129481:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.129482:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.129484:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.129485:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.129486:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.129487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.129490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.129501:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.129505:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.129506:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.129507:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.129509:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.129513:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.129765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.129771:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.129774:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.129774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.129776:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.129779:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.129780:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.129781:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.129783:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.129784:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.129785:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.129788:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.129790:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.129792:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.129793:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.129794:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.129795:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.129796:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.129797:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.129798:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.129798:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.129804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677474.129808:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.129809:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.129811:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.129811:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677474.129812:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.129812:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.129813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.129814:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.129815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677474.129816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.129818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.129819:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.129822:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.129823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.129824:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677474.129824:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.129825:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.129826:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677474.129826:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677474.129827:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.129828:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.129828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.129828:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.129829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:20.0:1619677474.129829:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.129830:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.129831:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.129835:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.129837:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.129839:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.129840:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.129840:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.129842:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.129842:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.129843:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.129843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.129855:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.129858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.129858:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.129859:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.129861:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.129863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.129876:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.129879:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.129880:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.129880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.129881:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.129882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.129909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.129916:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.129919:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.129920:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.129922:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.129924:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.129926:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.129927:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.129929:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.129930:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.129931:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.129934:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.129935:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.129938:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.129939:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.129940:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.129941:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.129942:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.129943:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.129943:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.129955:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.129957:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.129958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.129959:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.129961:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.129963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.130589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.130597:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677474.130599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.130600:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.130602:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.130604:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.130605:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.130605:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677474.130606:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677474.130607:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.130608:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.130608:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.130609:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.130610:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.130613:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.130615:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.130616:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.130616:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.130617:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.130617:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677474.130618:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.130619:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677474.130620:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.130621:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.130623:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.130623:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.130624:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.130625:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.130626:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.130626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677474.130628:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.130628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.130629:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.130629:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.130630:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.130631:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.130632:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677474.130632:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.130633:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.130634:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.130634:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.130635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.130636:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.130636:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.130637:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.130638:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.130650:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.130652:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.130653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.130654:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.130656:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.130659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.130988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.130995:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.130998:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.130999:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.131001:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.131004:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.131005:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.131006:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.131008:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.131009:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.131010:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.131012:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.131014:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.131016:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.131017:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.131017:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.131019:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.131019:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.131020:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.131020:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.131024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.131032:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.131034:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.131035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.131035:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.131037:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.131039:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.131451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.131458:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677474.131458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:24.0:1619677474.131460:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.131462:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.131463:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.131463:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677474.131465:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.131466:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:24.0:1619677474.131467:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677474.131467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677474.131469:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.131470:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.131471:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.131472:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.131472:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.131473:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677474.131473:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.131475:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.131476:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.131477:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.131478:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.131483:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.131484:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.131486:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677474.131487:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.131488:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.131489:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677474.131490:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.131490:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677474.131491:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.131492:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.131492:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.131492:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677474.131493:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.131493:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677474.131494:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.131494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677474.131495:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.131496:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.131497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.131503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677474.131506:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.131508:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.131509:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.131511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677474.131511:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.131511:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:20.0:1619677474.131512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.131513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677474.131513:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677474.131514:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.131514:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677474.131515:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.131515:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.131516:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.131517:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.131520:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.131889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.131896:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.131899:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.131901:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.131904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.131904:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.131905:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.131906:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.131908:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.131909:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.131910:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.131913:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.131915:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.131917:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.131917:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.131918:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.131919:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.131919:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.131920:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.131921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.131931:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.131934:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.131934:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.131936:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.131938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.131939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.131955:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.131958:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.131959:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.131959:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.131960:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.131961:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.132904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.132912:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.132915:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.132917:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.132919:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.132920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.132921:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.132922:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.132923:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.132924:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.132925:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.132928:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.132929:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.132932:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.132933:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.132933:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.132935:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.132935:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.132936:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.132936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.132950:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.132952:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.132953:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.132954:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.132956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677474.132956:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.132959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.132963:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.132966:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.132968:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.132968:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.132970:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677474.132970:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.132971:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.132971:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.132972:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.132972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677474.132973:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677474.132973:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677474.132974:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.132975:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.132976:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.132977:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.132981:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.132982:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.132984:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.132985:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.132986:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.132987:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.132988:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.132989:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.132991:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.132992:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.132997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.133002:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.133003:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.133004:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.133005:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.133005:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.133006:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.133007:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.133007:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.133008:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.133009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.133288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.133294:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.133299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.133305:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.133307:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.133309:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.133311:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.133312:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.133313:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.133314:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.133315:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.133319:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.133321:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.133323:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.133324:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.133324:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.133326:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.133326:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.133327:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.133327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.133341:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.133343:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.133344:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.133345:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.133347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.133347:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.133506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.133514:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.133516:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.133517:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.133519:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.133522:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.133523:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.133524:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.133526:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.133527:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.133528:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:4.2:1619677474.133528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:8.0:1619677474.133531:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.133532:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.133533:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.133535:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:8.0:1619677474.133535:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.133536:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:21.0:1619677474.133537:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677474.133537:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677474.133538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677474.133539:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.133539:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677474.133540:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.133540:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.133540:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:21.0:1619677474.133541:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.133542:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.133544:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.133545:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677474.133545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:21.0:1619677474.133546:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.133550:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.133552:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:2.0:1619677474.133552:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677474.133555:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.133555:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677474.133555:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677474.133556:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677474.133556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.133556:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.133558:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.133558:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:2.0:1619677474.133558:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.133559:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.133560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:2.0:1619677474.133562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.133564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:9.2:1619677474.133568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677474.133571:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.133574:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:18.0:1619677474.133574:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.133575:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.133575:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.133576:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:18.0:1619677474.133576:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.133577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677474.133577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.133578:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.133580:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.133583:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.133616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.133624:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.133627:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.133629:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.133632:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.133632:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.133633:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.133635:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.133636:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.133637:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.133638:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.133641:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.133643:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.133646:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.133647:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.133647:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.133649:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.133650:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.133651:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.133651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.133656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.133664:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.133666:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.133667:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.133669:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.133670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.133672:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.133673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.133681:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.133683:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.133684:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.133685:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.133686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.133687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.133855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:5.2:1619677474.133859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677474.133861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.133864:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.133865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:17.0:1619677474.133866:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677474.133866:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.133869:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:2.0:1619677474.133869:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.133870:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:1.0:1619677474.133870:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.133872:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.133872:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.133874:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.133875:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.133875:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.133877:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677474.133877:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677474.133879:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.133880:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.133881:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.133881:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.133882:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677474.133883:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:17.0:1619677474.133886:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.133887:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.133888:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677474.133888:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677474.133888:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677474.133890:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.133890:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677474.133891:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:2.0:1619677474.133891:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:17.0:1619677474.133892:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677474.133892:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:17.0:1619677474.133893:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677474.133893:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:17.0:1619677474.133894:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.133895:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.133896:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:9.2:1619677474.133896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677474.133897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.133901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677474.133906:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:1.0:1619677474.133907:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.133908:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.133910:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:1.0:1619677474.133910:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.133910:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677474.133911:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.133911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677474.133912:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.133912:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.133912:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:6.0:1619677474.133912:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.133914:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.133917:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.133917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677474.133921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677474.134171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.134180:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.134184:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.134185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.134188:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.134191:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.134194:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.134196:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.134198:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.134200:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.134201:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.134207:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.134210:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.134214:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.134215:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.134216:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.134218:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.134219:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.134221:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.134223:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.134223:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.134229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.134232:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.134235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.134235:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.134236:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.134237:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677474.134238:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.134241:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677474.134241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677474.134242:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.134244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.134249:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.134252:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.134253:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.134253:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.134254:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.134255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.134310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.134320:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.134325:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.134326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.134329:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.134333:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.134335:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.134337:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.134339:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.134341:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.134342:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.134348:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.134350:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.134354:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.134355:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.134356:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.134359:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.134360:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.134361:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.134361:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.134365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.134374:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.134376:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.134377:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.134379:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.134379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.134382:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.134489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.134491:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.134494:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.134496:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.134497:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.134499:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.134500:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.134502:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.134503:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.134504:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.134505:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.134509:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.134510:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.134513:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.134514:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.134514:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.134516:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.134516:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.134517:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.134518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.134522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.134530:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.134533:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.134534:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.134534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.134536:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.134538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677474.134538:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677474.134546:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677474.134546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:30.0:1619677474.134549:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.134552:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.134553:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677474.134554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.134555:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.134555:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677474.134556:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:17.0:1619677474.134556:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.134556:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:30.0:1619677474.134557:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.134557:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.134558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:30.0:1619677474.134559:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.134560:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.134561:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.134564:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.134566:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.134568:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.134569:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.134570:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.134571:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.134572:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.134573:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.134573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.134578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.134584:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.134586:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.134587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.134588:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.134590:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.134592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.134651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677474.134659:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.134661:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677474.134663:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.134665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677474.134666:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.134667:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677474.134669:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.134670:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.134671:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.134672:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677474.134675:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.134676:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677474.134679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677474.134680:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.134681:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677474.134681:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677474.134683:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677474.134683:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.134684:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677474.134685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:6.0:1619677474.134686:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.134688:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.134689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.2:1619677474.134689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:6.0:1619677474.134691:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.134693:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.134695:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.134696:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.134697:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:6.0:1619677474.134697:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.134698:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677474.134699:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:6.0:1619677474.134699:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677474.134700:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.134700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.134703:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.134704:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.134706:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677474.134706:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.134709:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.134710:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.134711:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.134712:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.134713:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.134713:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.134714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.134725:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.134728:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.134728:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.134730:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.134731:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.134733:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.134766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.134773:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.134776:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.134777:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.134779:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.134781:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.134783:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.134784:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.134785:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.134786:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.134787:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.134791:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.134793:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.134796:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.134796:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.134797:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.134799:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.134799:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.134800:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.134800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.134812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.134815:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.134816:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.134816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.134818:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.134821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.134835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.134842:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.134845:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.134846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.134847:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.134850:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.134852:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677474.134853:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.134854:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.134855:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.134856:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677474.134858:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.134860:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677474.134862:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.134862:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677474.134862:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.134864:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.134864:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.134866:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.134867:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.134867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677474.134869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677474.134873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.134878:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.134879:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.134879:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.134881:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.134881:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.134882:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.134882:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.134883:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.134884:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677474.134884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.2:1619677474.134973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.134981:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.134984:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.134986:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.134988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.134988:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.134990:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.134991:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.134993:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.134993:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.134994:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.134997:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.134999:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.135002:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.135002:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.135003:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.135005:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.135005:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.135007:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.135008:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.135008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.135014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.135018:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.135021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.135021:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.135022:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.135023:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.135024:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.135025:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.135026:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.135027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.135027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.135539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.135546:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.135549:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.135550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.135552:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.135555:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.135556:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.135557:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.135559:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.135560:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.135561:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.135564:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.135566:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.135569:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.135569:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.135570:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.135572:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.135573:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.135574:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.135574:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677474.135587:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.135589:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.135590:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.135592:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.135594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.135595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.135655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.135663:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.135666:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.135668:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.135670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.135671:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.135672:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.135673:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.135674:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.135675:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.135676:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.135679:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.135681:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.135683:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.135684:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.135684:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.135686:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.135686:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.135687:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.135688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.135698:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.135701:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.135701:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.135703:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.135705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.135705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.135896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.135901:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.135904:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.135905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.135907:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.135910:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.135911:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.135912:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.135914:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.135915:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.135916:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.135918:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.135920:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.135922:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.135923:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.135923:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.135925:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.135925:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.135926:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.135927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.135931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.135938:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.135940:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.135941:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.135941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.135942:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.135944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.136013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.136021:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.136023:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.136024:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.136026:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.136028:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.136029:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.136031:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.136032:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.136033:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.136034:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.136036:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.136038:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.136040:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.136041:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.136042:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.136043:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.136044:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.136045:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.136046:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.136046:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.136049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.136052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.136055:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.136057:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.136058:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.136059:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.136059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.136060:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.136061:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.136062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.136063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.136063:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.136072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:4.2:1619677474.136075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.136080:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.136082:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.136083:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.136083:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.136084:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.136084:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.136085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.136086:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.136086:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677474.136087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.136089:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.136091:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.136092:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.136093:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.136094:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.136095:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.136098:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.136100:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.136102:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.136103:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.136104:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.136105:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.136106:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.136107:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.136107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.136120:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.136122:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.136123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.136124:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.136126:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.136129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.136339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.136345:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.136347:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.136348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.136350:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.136352:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.136353:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.136354:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.136356:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.136357:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.136358:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.136361:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.136363:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.136365:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.136366:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.136367:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.136368:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.136368:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.136369:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.136370:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.136371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.136376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.136381:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.136382:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.136383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.136383:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.136384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.136385:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.136386:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.136388:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.136389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.136390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.136401:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.136403:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.136404:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.136405:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.136405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.136407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.137202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.137211:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.137214:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.137216:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.137217:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.137218:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.137219:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.137220:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.137222:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.137222:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.137224:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.137227:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.137229:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.137231:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.137231:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.137232:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.137234:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.137234:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.137236:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.137237:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.137238:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.137243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.137248:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.2:1619677474.137249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.137250:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.137251:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.137252:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.137252:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.137254:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.137254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.137254:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.137256:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.137256:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.137257:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.137258:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677474.137259:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.137260:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.137262:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.137263:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.137264:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.137266:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.137266:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.137267:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.137270:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.137272:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.137275:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.137275:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.137276:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.137278:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.137278:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.137279:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.137279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.137298:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.137301:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.137302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.137302:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.137304:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.137307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.137335:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.137337:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.137339:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.137339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.137340:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.137340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.137826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.137834:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.137837:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.137838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.137840:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.137843:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.137844:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.137846:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.137847:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.137848:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.137849:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.137852:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.137854:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.137857:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.137857:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.137858:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.137859:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.137860:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.137861:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.137861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.137866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.137874:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.137877:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.137878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.137878:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.137880:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.137884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.138180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.138185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.138188:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.138189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.138191:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.138193:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.138194:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.138195:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.138196:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.138197:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.138198:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.138201:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.138203:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.138205:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.138206:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.138207:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.138208:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.138209:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.138209:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.138210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.138222:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.138224:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.138225:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.138227:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.138228:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.138230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.138305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.138313:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.138316:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.138319:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.138321:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.138321:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.138323:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.138324:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.138325:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.138326:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.138327:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.138330:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.138332:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.138334:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.138335:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.138336:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.138337:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.138337:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.138338:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.138339:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.138350:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.138353:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.138353:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.138355:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.138357:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.138357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.138372:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.138374:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.138375:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.138376:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.138376:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.138377:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677474.138493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.138499:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.138502:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.138503:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.138505:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.138508:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.138510:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.138511:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.138512:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.138513:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.138514:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.138517:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.138519:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.138521:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.138522:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.138523:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.138525:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.138525:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.138526:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.138526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.138538:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.138541:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.138542:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.138542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.138544:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.138546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.138861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.138869:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.138872:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.138874:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.138876:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.138876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.138878:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.138879:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.138881:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.138882:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.138883:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.138886:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.138888:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.138891:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.138892:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.138893:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.138894:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.138895:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.138896:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.138896:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.138908:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.138910:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.138911:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.138911:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.138914:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.138916:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.139007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.139014:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.139017:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.139018:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.139020:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.139022:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.139024:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.139025:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.139026:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.139027:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.139028:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.139032:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.139033:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:4.2:1619677474.139035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.139036:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.139037:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.139037:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.139040:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.139041:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.139041:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.139042:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677474.139043:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.139045:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.139045:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677474.139046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:26.0:1619677474.139047:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.139049:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.139051:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.139052:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.139053:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.139054:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.139054:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677474.139054:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:26.0:1619677474.139057:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.139057:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.139058:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.139058:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.139059:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677474.139061:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:26.0:1619677474.139062:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.139063:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.139063:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:18.0:1619677474.139064:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677474.139065:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.139065:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.139066:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.139067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677474.139068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677474.139074:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.139077:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.139078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.139078:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:22.0:1619677474.139079:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:2.0:1619677474.139079:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.139080:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677474.139082:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.139083:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.139083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.139086:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.139089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.139553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.139559:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.139561:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.139563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.139565:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.139567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.139568:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.139569:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.139571:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.139572:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.139573:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.139577:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.139579:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.139582:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.139583:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.139583:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.139585:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.139586:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.139586:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.139587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.2:1619677474.139595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:21.0:1619677474.139602:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:17.0:1619677474.139604:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:21.0:1619677474.139605:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.139606:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.139607:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.139609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677474.139609:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:21.0:1619677474.139610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677474.139611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677474.139614:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.139618:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.139621:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:21.2:1619677474.139623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677474.139623:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.139626:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.139628:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.139630:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.139631:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.139634:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.139637:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.139637:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.139638:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.139640:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.139640:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:30.0:1619677474.139642:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.139643:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.139645:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.139645:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677474.139646:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.139647:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677474.139647:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677474.139648:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677474.139650:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:30.0:1619677474.139651:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.139651:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.139652:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:30.0:1619677474.139653:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677474.139654:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:30.0:1619677474.139656:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677474.139656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677474.139657:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.139658:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.139659:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.139660:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.139661:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.139662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677474.139665:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:8.2:1619677474.139666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677474.139668:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.139669:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.139670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.139672:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.139675:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:18.0:1619677474.139675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.0:1619677474.139678:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.139679:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.139681:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.139681:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.139684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.139876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677474.139884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.139887:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677474.139890:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.139891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677474.139892:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.139894:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.139896:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.139897:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.139898:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.139899:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.139902:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.139904:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.139906:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.139907:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.139908:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.139909:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.139910:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.139911:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677474.139913:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677474.139913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677474.139915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.139919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:24.0:1619677474.139922:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.139924:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.139925:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.139925:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.139927:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000400:00000200:24.0:1619677474.139927:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.139929:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677474.139930:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:24.0:1619677474.139930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.139931:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:20.2:1619677474.140282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.140288:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.140290:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.140292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.140298:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.140301:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.140302:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.140304:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.140305:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.140306:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.140307:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.140310:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.140312:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.140316:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.140317:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.140317:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.140319:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.140320:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.140321:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.140323:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.140323:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677474.140329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677474.140334:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677474.140335:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677474.140336:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677474.140337:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:12.0:1619677474.140337:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.140338:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.140338:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677474.140339:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.140341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.140342:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.140345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.140353:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.140356:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.140358:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.140360:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.140361:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.140362:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.140363:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.140364:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.140365:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.140366:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.140369:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.140371:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.140373:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.140374:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.140375:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.140376:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.140377:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.140378:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.140378:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.140382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.140389:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.140391:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.140392:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.140392:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.140394:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.140397:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.140570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.140578:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.140580:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.140582:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.140584:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.140585:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.140586:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.140587:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.140588:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.140589:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.140590:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.140593:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.140595:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.140598:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.140598:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.140599:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.140601:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.140601:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.140602:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.140603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.140615:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.140618:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.140619:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.140619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.140621:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.140624:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.140736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.140741:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.140744:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.140746:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.140746:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.140749:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.140750:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.140751:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.140753:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.140754:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.140755:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.140758:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.140760:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.140762:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.140763:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.140764:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.140766:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.140766:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.140767:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.140767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.140772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677474.140776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677474.140780:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.140783:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.140784:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677474.140784:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677474.140784:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.140787:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.140787:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:1.0:1619677474.140787:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.140790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:10.0:1619677474.140791:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.140793:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.140795:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.140796:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.140797:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.140798:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.140799:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.140803:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.140805:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.140807:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.140808:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.140809:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.140810:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.140810:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.140811:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.140812:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.140815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.140822:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.140824:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.140825:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.140825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.140827:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.140829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.141048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.141056:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.141058:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.141061:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.141062:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.141063:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.141065:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.141066:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.141067:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.141068:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.141069:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.141072:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.141073:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.141076:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.141077:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.141077:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.141079:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.141079:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.141081:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.141082:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.141083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.141084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.141088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.141090:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.141092:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.141093:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.141093:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.141095:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.141096:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677474.141097:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.141098:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.141099:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.141100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.141193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.141199:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.141201:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.141202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.141204:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.141206:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.141207:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.141209:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.141210:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.141211:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.141212:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.141215:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.141217:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.141219:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.141220:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.141221:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.141222:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.141223:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.141223:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.141223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.141236:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.141238:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.141239:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.141240:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.141241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.141243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.141515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.141523:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.141526:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.141526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.141528:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.141531:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.141533:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.141534:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.141535:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.141537:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.141537:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.141541:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.141543:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.141545:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.141546:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.141547:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.141549:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.141549:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.141550:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.141551:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.141552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.141558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.141563:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.141565:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.141566:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.141567:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.141567:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677474.141567:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.141568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.141569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.141569:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.141572:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.141653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.141661:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.141664:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.141664:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.141666:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.141668:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.141669:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.141671:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.141672:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.141673:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.141674:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.141676:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.141678:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.141680:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.141680:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.141681:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.141683:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.141683:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.141684:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.141684:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.141688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.141695:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.141697:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.141698:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.141698:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.141700:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.141702:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.141711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.141716:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.141719:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.141720:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.141720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.141721:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.141721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.142050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.142056:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.142058:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.142060:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.142061:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.142062:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.142064:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.142066:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.142067:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.142068:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.142069:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.142072:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.142073:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.142076:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.142077:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.142077:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.142079:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.142080:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.142080:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.142081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677474.142093:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.142096:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.142097:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.142099:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.142100:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.142102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.142668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.142675:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.142678:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.142679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.142680:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.142683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:8.0:1619677474.142683:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.142685:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.142686:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.142687:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.142688:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.142689:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677474.142691:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:8.0:1619677474.142692:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.142694:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:8.0:1619677474.142694:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677474.142696:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.142697:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.142698:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.142698:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677474.142699:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.142699:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.142700:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677474.142700:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:30.0:1619677474.142701:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.142701:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.142701:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677474.142702:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.142702:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:30.0:1619677474.142703:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.142704:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677474.142706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:30.0:1619677474.142707:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.142709:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.142712:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.142712:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.142713:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.142714:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.142715:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677474.142715:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:30.0:1619677474.142717:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:17.0:1619677474.142717:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:30.0:1619677474.142718:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:1.0:1619677474.142718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:30.0:1619677474.142719:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:17.0:1619677474.142719:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.142721:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.142724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677474.142724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677474.142730:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.142732:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.142733:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.142733:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.142734:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.142735:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.142736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.142736:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.142737:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.142739:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.142747:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.142749:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.142750:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.142750:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.142751:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.142752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.143033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677474.143040:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.143043:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677474.143046:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.143047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677474.143048:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.143050:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677474.143051:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.143052:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.143054:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.143054:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677474.143058:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.143060:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.143063:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.143063:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677474.143064:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677474.143066:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677474.143066:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.143067:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677474.143068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677474.143072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.143079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:18.0:1619677474.143080:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.143083:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.143084:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.143084:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677474.143085:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:18.0:1619677474.143087:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.143088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.143088:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:18.0:1619677474.143091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:6.0:1619677474.143091:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.143092:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.143094:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.143095:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.143096:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.143098:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.143099:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.143102:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.143104:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.143106:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.143107:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.143108:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.143109:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.143110:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.143111:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.143112:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.143112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.143123:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.143125:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.143126:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.143127:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.143128:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.143130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.143145:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.143147:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.143148:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.143149:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.143149:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.143151:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.143185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.143191:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.143193:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.143194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.143194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:5.2:1619677474.143288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.143294:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.143298:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.143301:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677474.143303:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.143305:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.143307:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.143308:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.143309:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.143310:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.143312:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.143314:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.143316:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.143319:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.143319:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.143320:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.143321:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.143322:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.143323:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.143323:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.143336:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.143339:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.143340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.143340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.143342:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.143344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.143451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.143459:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.143463:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.143464:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.143467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.143467:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.143468:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.143469:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.143471:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.143472:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.143473:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.143475:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.143477:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.143479:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.143480:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.143481:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.143482:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.143483:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.143484:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.143484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.143494:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.143497:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.143497:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.143499:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.143501:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.143502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.143827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.143834:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.143837:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.143838:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.143840:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.143842:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.143843:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.143845:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.143846:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.143847:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.143847:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.143850:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.143852:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.143855:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.143856:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.143856:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.143858:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.143858:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.143859:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.143859:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.143871:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.143874:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.143875:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.143875:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.143877:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.143880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.143892:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.143894:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.143895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.143896:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.143897:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.143897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.144270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.144277:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.144280:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.144281:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.144283:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.144286:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.144287:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.144288:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.144290:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.144291:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.144292:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.144300:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.144301:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.144304:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.144305:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.144305:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.144307:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.144308:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.144308:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.144309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.144313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.144322:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.144324:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.144325:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.144326:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.144328:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.144330:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.144332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.144340:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.144342:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.144343:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.144344:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.144344:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.144345:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.144541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.144549:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.144551:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.144552:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.2:1619677474.144553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907b780e00] (20)++
00000400:00000200:26.0:1619677474.144554:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.144557:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.144559:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677474.144559:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907b780e00] (21)++
00000400:00000200:26.0:1619677474.144560:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.144562:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.144562:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.52@o2ib3
00000800:00000200:18.0:1619677474.144563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b780e00] (22)--
00000400:00000200:26.0:1619677474.144564:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.144565:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677474.144565:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.52@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.144568:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.144569:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.144570:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677474.144571:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677474.144571:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.144572:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.144573:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.144574:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.144575:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677474.144575:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:2.0:1619677474.144575:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.144577:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677474.144577:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.144578:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000400:00000200:2.0:1619677474.144578:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.52@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.144579:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:2.0:1619677474.144580:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677474.144582:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.144583:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677474.144584:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677474.144585:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677474.144586:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907b780e00] (21)++
00000800:00000200:2.0:1619677474.144587:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907b780e00] (22)--
00000800:00000200:2.0:1619677474.144587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907b780e00] (21)--
00000800:00000200:31.0:1619677474.144591:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:8.2:1619677474.144592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.144594:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.144594:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.144596:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.144598:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.2:1619677474.144598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677474.144599:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.0:1619677474.144600:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.144602:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.144603:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.144604:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.144604:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677474.144606:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.144607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677474.144607:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.144608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.144609:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.144611:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.144612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677474.144612:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.144614:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.144615:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.144616:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.144617:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.144620:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677474.144621:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.144623:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677474.144623:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:31.0:1619677474.144626:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.144626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677474.144626:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.144627:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:31.0:1619677474.144628:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.144628:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.144629:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677474.144630:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677474.144630:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.144630:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677474.144631:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.144631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:31.0:1619677474.144632:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677474.144632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:31.0:1619677474.144633:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.144634:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.144638:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.144639:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677474.144639:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.144641:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:31.0:1619677474.144642:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.144642:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677474.144642:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677474.144643:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677474.144643:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:18.0:1619677474.144644:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.144645:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.144645:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.144645:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677474.144646:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677474.144646:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.144646:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.144647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677474.144648:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.144648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:18.0:1619677474.144648:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.144649:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96505839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.144651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:18.0:1619677474.144651:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.144652:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.144653:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.144656:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.144657:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:18.0:1619677474.144658:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.144659:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.144660:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.144660:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.144660:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.144661:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:30.0:1619677474.144662:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.144663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677474.144663:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.144664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:18.0:1619677474.144664:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.144665:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.144665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.144669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.144676:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.144678:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.144679:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.144679:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.144681:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.144682:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.144687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.144694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.144697:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.144698:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.144698:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.144698:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.144699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.144793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.144800:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.144803:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.144805:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.144807:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.144807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.144809:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.144810:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.144811:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.144813:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.144813:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.144817:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.144818:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.144821:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.144822:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.144823:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.144825:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.144825:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.144826:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.144827:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.144838:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.144841:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.144842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.144843:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.144845:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.144847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.145529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.145537:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.145540:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.145542:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.145544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.145545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.145545:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.145546:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.145547:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.145549:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.145550:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.145551:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.145551:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677474.145554:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.145554:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677474.145555:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.145555:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.145557:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.145558:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.145558:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.145559:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677474.145559:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.145560:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677474.145560:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677474.145561:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677474.145561:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.145562:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.145562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677474.145562:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.145563:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.145564:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.145567:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677474.145567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.145569:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.145571:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.145571:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677474.145572:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:10.0:1619677474.145573:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (24)--
00000800:00000200:10.0:1619677474.145574:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.145574:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (24)++
00000800:00000200:10.0:1619677474.145575:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.145575:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.145576:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:8.0:1619677474.145577:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.145578:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:24.0:1619677474.145579:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:8.0:1619677474.145579:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.145580:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.145581:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.145582:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.145583:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.145584:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.146049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.146056:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.146058:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.146059:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.146061:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.146063:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.146065:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677474.146066:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.146067:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.146068:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.146069:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677474.146071:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.146073:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677474.146074:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.146075:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677474.146075:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677474.146077:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677474.146077:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.146078:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.146078:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677474.146089:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.146092:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.146093:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.146094:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.146096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.146097:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.146106:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.146109:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.146110:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.146110:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.146111:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.146112:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.146128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.146135:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.146137:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.146138:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.146140:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.146142:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.146144:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.146145:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.146146:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.146147:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.146148:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.146151:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.146152:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.146155:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.146156:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.146156:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.146158:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.146158:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.146159:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.146160:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.146172:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.146175:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.146176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.146176:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.146179:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.146181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.146230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.146238:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677474.146239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677474.146241:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.146243:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.146245:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.146245:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.146245:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677474.146247:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.146248:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.146248:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.146248:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:30.0:1619677474.146250:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.146251:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.146252:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.146253:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.146253:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.146255:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.146256:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.146258:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.146258:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.146260:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.146261:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.146262:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.146264:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.146265:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.146265:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:22.0:1619677474.146265:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.146267:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.146267:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.146267:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.146269:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677474.146270:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.146270:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677474.146271:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:22.0:1619677474.146271:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677474.146272:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677474.146273:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.146273:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.146274:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.146275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.2:1619677474.146277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.146281:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.146283:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.146284:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.146284:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677474.146285:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.146285:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.146286:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677474.146286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677474.146287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.146287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.146288:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.146288:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.146289:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.146291:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.146292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.146299:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.146435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.146442:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.146445:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.146446:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.146447:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.146450:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.146451:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.146452:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.146453:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.146454:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.146455:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.146459:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.146461:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.146463:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.146464:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.146464:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.146465:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.146466:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.146467:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.146468:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.146469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.146472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.146475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.146478:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.146480:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.146481:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.146482:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677474.146482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.146483:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677474.146483:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.146484:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.146485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.146486:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.146488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.146496:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.146499:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.146501:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.146503:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.146503:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.146504:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.146505:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.146506:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.146507:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.146508:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.146511:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.146512:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.146515:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.146516:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.146517:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.146518:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.146519:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.146520:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.146520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.146532:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.146535:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.146536:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.146536:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.146538:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.146541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.146640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.146646:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.146649:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.146651:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.146651:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.146654:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.146655:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.146657:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.146658:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.146659:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.146660:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.146664:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.146666:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.146668:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.146669:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.146669:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.146671:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.146672:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.146673:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.146674:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.146675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.146678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.146680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.2:1619677474.146682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.0:1619677474.146686:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677474.146686:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:2.0:1619677474.146688:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677474.146689:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677474.146689:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:20.0:1619677474.146690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677474.146690:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.146690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.146691:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677474.146691:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.146692:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677474.146693:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677474.146694:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677474.146695:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:2.0:1619677474.146696:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.146698:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.146699:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.146700:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.146701:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.146702:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.146706:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.146708:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.146710:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.146711:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.146711:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.146713:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.146713:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.146714:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.146715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.146719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.146726:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.146729:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.146730:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.146730:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.146732:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.146734:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.147408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.147415:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.147418:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.147419:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.147421:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.147423:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.147425:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.147426:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.147427:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.147428:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.147429:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.147432:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.147434:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.147437:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.147437:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.147438:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.147440:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.147441:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.147441:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.147442:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.147447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.147455:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.147458:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.147458:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.147460:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.147462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.147463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.147465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.147472:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.147475:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.147476:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.147476:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.147477:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.147478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.148026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.148034:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.148037:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677474.148037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:30.0:1619677474.148039:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.148042:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.148042:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.148043:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677474.148043:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677474.148044:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.148046:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.148046:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:30.0:1619677474.148047:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.148047:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:30.0:1619677474.148049:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.148049:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.148051:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.148053:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677474.148054:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.148054:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.148056:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.148057:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.148057:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.148058:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.148059:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.148060:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.148060:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.148062:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:22.0:1619677474.148062:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.148063:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.148064:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.148064:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.148064:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.148066:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.148067:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.148068:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.148069:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.148070:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.148071:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.148071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677474.148075:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.148078:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.148078:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.148080:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.148082:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.148082:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.0:1619677474.148083:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.148085:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.148086:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.148088:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.148088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.148090:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.148115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.148122:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.148125:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.148126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.148128:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.148131:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.148132:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.148133:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.148134:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.148135:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.148136:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.148140:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.148141:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.148144:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.148145:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.148145:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.148147:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.148148:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.148148:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.148149:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.148161:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.148163:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.148164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.148165:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.148167:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.148169:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.148230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.148238:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.148240:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.148241:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.148243:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.148246:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.148247:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.148249:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.148250:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.148251:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.148252:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.148255:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.148257:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.148260:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.148261:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.148261:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.148263:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.148264:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.148265:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.148266:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.148267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677474.148269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.148272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677474.148277:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677474.148279:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677474.148280:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.148281:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.148281:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:2.0:1619677474.148281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.148282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.148283:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677474.148284:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.148287:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.148578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.148584:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.148587:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.148588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.148590:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.148592:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.148594:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.148595:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.148597:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.148598:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677474.148599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:17.0:1619677474.148599:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.148602:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.148604:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.148606:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.148607:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677474.148607:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.148608:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677474.148609:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:17.0:1619677474.148610:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.148610:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.148611:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.148611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677474.148612:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.148614:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.148614:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.148615:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.148617:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.148618:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.148619:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.148620:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.148623:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.148624:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:30.0:1619677474.148625:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.148627:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:30.0:1619677474.148628:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677474.148628:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.148628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:30.0:1619677474.148629:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.148630:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677474.148630:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.148632:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.148632:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.148633:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.148633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.148634:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.148645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.148647:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.148648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.148648:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.148650:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.148653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.148667:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.148670:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.148670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.148671:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.148672:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.148672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.149030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677474.149036:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.149039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.149039:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:1.0:1619677474.149041:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.149044:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.149045:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677474.149046:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.149048:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.149049:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.149050:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677474.149053:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.149054:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.149056:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.149057:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.149057:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.149059:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.149059:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.149060:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.149060:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677474.149065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.149073:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.149075:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.149076:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.149077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.149078:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.149079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.149114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677474.149121:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.149124:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677474.149126:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.149128:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677474.149130:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.149131:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677474.149132:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.149133:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.149134:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.149135:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677474.149138:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.149140:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.149142:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.149143:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.149143:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.149145:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.149145:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.149146:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.149147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:30.0:1619677474.149157:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.149159:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.149160:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.149162:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.149163:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.149164:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.149461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.149467:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.149469:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.149470:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.149472:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.149474:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.149476:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.149477:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.149478:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.149479:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.149480:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.149483:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.149485:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.149487:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.149488:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.149488:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.149490:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.149491:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.149492:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.149493:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.149493:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.149499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.149504:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.149505:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.149506:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.149506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.149507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.149507:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.149508:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.149509:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.149511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.149512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.149531:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.149533:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.149534:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.149535:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.149536:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.149536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.149712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.149718:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.149721:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.149722:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.149724:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.149725:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.149727:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677474.149728:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.149729:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.149731:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.149732:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677474.149735:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.149737:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677474.149740:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.149741:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677474.149741:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677474.149743:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677474.149744:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.149744:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.149745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.149749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.149757:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.149759:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.149760:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.149761:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.149763:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.149766:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.149779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.149787:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.149789:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.149791:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.149793:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.149794:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.149794:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.149795:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.149796:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.149797:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.149798:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.149801:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.149803:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.149805:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.149805:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.149806:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.149807:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.149808:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677474.149808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677474.149809:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.149811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.149813:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.149814:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000800:00000200:31.0:1619677474.149816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677474.149816:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.149818:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.149819:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.149820:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.149821:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.149822:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.149822:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.149825:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.149827:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.149830:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.149831:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.149831:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.149833:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.149833:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.149834:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.149835:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677474.149840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.149847:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.0:1619677474.149848:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677474.149849:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.149850:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.149850:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.149851:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.149852:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.149853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677474.149853:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.149853:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.149855:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.0:1619677474.149856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.150281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.150288:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.150290:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.150293:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.150294:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.150302:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.150303:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.150305:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.150306:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.150307:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.150307:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.150311:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.150312:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.150315:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.150315:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.150316:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.150317:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.150318:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.150319:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.150320:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.150321:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.150327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.150332:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.150335:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.150335:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.150335:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.150336:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.150338:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.150338:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.150338:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.150339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.150341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.150804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.150812:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.150814:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.150815:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.150817:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.150819:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.150820:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.150822:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.150823:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.150824:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.150825:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.150828:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.150829:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.150832:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.150832:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.150833:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.150835:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.150835:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.150836:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.150836:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.150840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.150847:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.150849:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.150850:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.150851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.150852:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.150854:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.150861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.150867:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.150869:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.150870:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.150870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.150871:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.150872:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.151032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.151038:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.151041:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.151042:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.151044:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.151046:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.151047:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.151048:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.151050:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.151051:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.151052:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.151055:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.151057:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.151058:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.151059:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.151060:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.151062:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.151063:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.151064:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.151064:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.151068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.151075:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.151077:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.151078:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.151079:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.151080:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.151081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677474.151081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.151089:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.151092:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.151094:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.151096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.151097:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.151098:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.151099:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.151101:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.151102:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.151103:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.151106:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.151108:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.151110:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.151111:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.151111:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.151113:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.151113:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.151114:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.151115:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.151125:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.151128:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.151129:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.151130:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.151132:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.151133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.151140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677474.151145:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.151145:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677474.151148:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.151149:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.151149:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:22.0:1619677474.151149:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.151149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677474.151150:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.151151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:22.0:1619677474.151152:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.151154:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.151155:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.151156:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.151157:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.151159:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.151160:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.151163:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.151165:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.151167:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.151168:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.151169:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.151170:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.151171:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.151171:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.151172:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.151184:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.151186:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.151187:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.151188:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.151189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.151191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.151286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.151293:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.151300:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.151302:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.151304:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.151307:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.151308:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.151310:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.151311:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.151312:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.151313:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.151316:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.151318:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.151321:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.151321:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.151322:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.151324:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.151324:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.151325:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.151326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.151339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.151341:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.151342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.151343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.151345:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.151346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:6.0:1619677474.151347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677474.151353:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.151356:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.151357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.151358:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.151361:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.151362:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677474.151363:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.151365:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.151366:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.151366:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.151370:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.151371:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.151373:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.151374:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677474.151374:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.151376:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.151376:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.151377:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.151377:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:30.0:1619677474.151389:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.151392:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.151392:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.151394:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.151396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.151397:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.151505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.151512:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.151514:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.151515:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.151517:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.151519:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.151520:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.151521:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.151522:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.151523:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.151524:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.151527:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.151528:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.151530:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.151531:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.151531:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.151533:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.151533:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.151534:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.151534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.151538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.151545:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.151547:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.151548:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.151549:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.151550:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.151552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.151945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.151952:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.151955:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.151955:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.151957:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.151959:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.151961:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.151962:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.151963:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.151964:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.151965:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.151968:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.151969:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.151972:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.151972:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.151973:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.151975:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.151975:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.151976:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.151978:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.151978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677474.151981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677474.151984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677474.151989:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677474.151991:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677474.151992:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:26.0:1619677474.151993:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.151993:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677474.151993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.151993:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.151995:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677474.151995:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.151998:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.152335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677474.152338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.152343:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:4.2:1619677474.152344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677474.152345:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677474.152346:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677474.152348:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.152348:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.152350:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.152351:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.152352:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.152352:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677474.152353:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.152355:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.152355:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.152356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677474.152357:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.152358:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.152359:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.152359:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.152361:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677474.152362:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.152363:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.152363:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677474.152364:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.152364:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.152364:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96505804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.152365:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.152366:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.152366:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.152367:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677474.152368:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.152368:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.152370:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.152371:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.152372:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.152373:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677474.152373:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.152373:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.152374:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.152375:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.152375:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.152376:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677474.152376:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.152377:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677474.152377:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.152378:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.152378:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.152379:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.152379:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.152379:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.152380:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.152380:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.152381:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.152382:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.152382:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677474.152383:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677474.152383:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677474.152384:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.152384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677474.152385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.152386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.152389:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:9.2:1619677474.152389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.152390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.152391:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.152392:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677474.152392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.152393:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.152395:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:22.0:1619677474.152395:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.152396:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.152396:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.152398:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.152398:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.152398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677474.152398:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:31.0:1619677474.152401:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677474.152401:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677474.152401:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:31.0:1619677474.152402:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677474.152402:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.152402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:31.0:1619677474.152403:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677474.152404:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.152408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.152956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.152964:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.152967:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.152968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.152970:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.152972:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.152974:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.152975:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.152977:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.152978:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.152978:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.152982:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.152983:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.152987:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.152987:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.152988:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.152990:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.152990:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.152991:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.152992:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.152993:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.152998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.153004:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.153005:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.153006:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677474.153007:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677474.153007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677474.153008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.153008:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.153009:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.153010:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.153013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.153028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.153032:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.153035:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.153036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.153038:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.153040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.153041:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.153042:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.153043:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.153045:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.153046:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.153049:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.153050:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.153053:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.153053:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.153054:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.153056:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.153056:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.153057:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.153057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.2:1619677474.153064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.0:1619677474.153069:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677474.153070:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677474.153072:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.153073:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.153073:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.153074:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:21.0:1619677474.153075:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.153075:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:2.0:1619677474.153077:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.153079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:2.0:1619677474.153079:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.153080:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.153082:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.153083:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.153084:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.153085:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.153089:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.153091:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.153094:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.153095:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.153096:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.153097:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.153098:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.153099:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.153100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677474.153110:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.153113:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.153114:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.153116:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.153117:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.153119:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677474.153131:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.153134:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.153135:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.153136:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.153136:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.153137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.153145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.153148:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.153151:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.153153:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.153154:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.153156:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.153157:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.153159:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.153160:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.153161:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.153162:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.153166:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.153167:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.153170:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.153171:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.153171:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.153173:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.153174:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.153175:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.153175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.153180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.153188:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.153191:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.153192:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.153192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.153194:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.153196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.153201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.153208:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.153211:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.153212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.153212:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.153213:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.153213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677474.153279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.153284:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.153287:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.153288:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.153290:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.153292:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.153300:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.153301:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.153302:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.153303:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.153304:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.153307:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.153309:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.153312:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.153312:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.153313:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.153315:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.153315:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.153316:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.153317:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.153317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.153323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677474.153329:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.153329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.153331:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677474.153332:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.153332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677474.153333:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.153333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.153334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.153335:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.153337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.153346:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.153348:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.153349:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.153349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.153350:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.153351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.154019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.154028:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.154030:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.154032:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.154035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.154035:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.154036:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.154037:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.154039:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.154040:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.154040:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.154043:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.154045:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.154047:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.154048:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.154048:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.154050:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.154050:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.154051:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.154052:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.154062:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.154064:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.154065:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.154066:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.154068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.154069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.154118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.154126:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.154128:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.154130:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.154132:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.154133:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.154133:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.154134:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.154136:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.154136:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.154137:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.154140:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.154142:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.154145:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.154146:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.154146:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.154148:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.154148:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.154149:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.154150:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.154161:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.154164:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.154165:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.154165:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.154167:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.154170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.154181:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.154184:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.154185:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.154185:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.154186:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.154187:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.154373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677474.154379:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.154381:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677474.154382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677474.154384:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.154385:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.154387:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677474.154388:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.154390:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.154391:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.154392:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677474.154395:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.154397:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677474.154399:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.154400:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677474.154401:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677474.154403:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677474.154403:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.154403:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.154404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677474.154409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.154419:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.154421:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.154422:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.154422:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.154425:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.154428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.154445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.154450:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.154453:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.154454:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.154456:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.154458:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.154459:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.154460:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.154461:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.154462:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.154463:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.154466:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.154467:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.154469:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.154470:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.154470:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.154472:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.154472:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.154473:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.154473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.154485:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.154488:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.154488:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.154490:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.154491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.154492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.154721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.154728:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.154731:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.154731:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.154734:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.154736:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.154738:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.154739:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.154740:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.154742:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.154743:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.154746:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.154748:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.154750:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.154751:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.154752:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.154753:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.154754:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.154754:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.154755:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677474.154767:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.154769:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.154770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.154770:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.154773:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.154775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.155014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.155022:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.155025:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.155027:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.155030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.155030:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.155031:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.155033:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.155034:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.155035:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.155036:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.155040:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.155041:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.155044:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.155045:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.155045:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.155047:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.155047:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.155048:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.155049:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.155059:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.155061:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.155062:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.155063:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.155065:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.155066:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.155505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.155512:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.155515:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.155517:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.155519:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.155520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.155521:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.155522:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.155523:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.155524:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.155525:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.155528:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.155529:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.155532:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.155533:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.155533:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.155535:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.155536:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.155536:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.155537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.155550:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.155553:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.155554:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.155554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.155556:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.155559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.155573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.155581:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.155584:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.155585:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.155586:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.155589:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.155590:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.155591:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.155593:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.155594:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.155594:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.155598:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.155599:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.155601:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.155602:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.155602:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.155604:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.155604:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.155605:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.155605:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.155609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.155616:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.155618:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.155619:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.155620:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.155621:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.155624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.156142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.156148:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.156151:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.156152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.156153:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.156155:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.156157:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.156158:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.156159:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.156160:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.156161:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.156164:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.156166:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.156168:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.156169:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.156170:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.156172:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.156172:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.156173:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.156173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.156185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.156188:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.156189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.156189:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.156191:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.156194:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.156257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.156262:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.156265:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.156265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677474.156266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:22.0:1619677474.156267:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.156269:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.156270:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.156271:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.156273:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677474.156273:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.156274:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.156275:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:26.0:1619677474.156277:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.156277:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677474.156279:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.156281:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:26.0:1619677474.156282:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.156284:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.156284:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.156284:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:26.0:1619677474.156285:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:22.0:1619677474.156285:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:26.0:1619677474.156287:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.156287:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.156287:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677474.156288:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.156288:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.2:1619677474.156288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000400:00000200:26.0:1619677474.156289:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.156289:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:26.0:1619677474.156290:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.156290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.156295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677474.156295:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000400:00000200:26.0:1619677474.156298:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.156299:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:26.0:1619677474.156300:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677474.156300:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677474.156301:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:26.0:1619677474.156302:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.156302:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.156302:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:26.0:1619677474.156303:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:20.0:1619677474.156303:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.156304:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.156304:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:2.0:1619677474.156304:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.156305:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.156305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.156305:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.156305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677474.156306:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677474.156306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:2.0:1619677474.156306:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.156307:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.156308:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:20.0:1619677474.156308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:2.0:1619677474.156308:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:26.0:1619677474.156309:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677474.156309:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.156310:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677474.156311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:2.0:1619677474.156311:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.156313:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:4.2:1619677474.156314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677474.156316:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.156318:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:2.0:1619677474.156318:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.156320:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.156321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.156321:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677474.156321:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.156321:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.156322:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677474.156323:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677474.156323:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.156323:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677474.156324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.156325:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:2.0:1619677474.156325:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.156325:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:8.0:1619677474.156326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677474.156326:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.156327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:21.0:1619677474.156329:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:9.2:1619677474.156329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:21.0:1619677474.156332:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.156333:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.156333:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.156334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.156335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677474.156336:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.156338:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.156339:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:8.0:1619677474.156339:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.156340:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:31.0:1619677474.156341:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:24.0:1619677474.156341:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.156341:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:31.0:1619677474.156342:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.156344:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.156345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.156347:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.156394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.156401:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.156403:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.156406:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.156408:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.156408:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.156410:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.156411:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.156412:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.156413:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.156414:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.156418:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.156420:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.156422:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.156423:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.156423:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.156425:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.156426:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.156426:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.156427:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.156430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.156437:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.156439:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.156440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.156440:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.156442:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.156444:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.156624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.156632:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.156634:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.156636:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.156638:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.156639:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.156640:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.156641:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.156643:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.156643:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.156644:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.156647:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.156649:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.156652:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.156653:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.156653:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.156655:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.156655:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.156657:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.156658:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.156659:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.156665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.156669:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.156671:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.156672:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.156672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.156673:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.156675:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677474.156675:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.156676:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.156677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.156678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.156688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.156690:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.156691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.156692:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.156693:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.156693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.157111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.157118:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.157121:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.157122:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.157124:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.157126:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.157128:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.157129:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.157130:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.157131:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.157132:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.157135:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.157137:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.157139:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.157140:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.157140:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.157142:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.157142:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.157143:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.157143:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677474.157155:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.157158:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.157159:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.157161:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.157163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.157164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.157670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.157678:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.157681:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.157683:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.157685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.157685:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.157687:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677474.157688:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.157690:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.157690:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.157691:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677474.157695:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.157697:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.157700:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.157700:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677474.157701:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677474.157703:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677474.157703:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.157704:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.157705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.157710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.157719:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.157721:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.157722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.157723:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.157725:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.157728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.157731:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.157732:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.157735:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.157736:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.157736:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.157737:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.157738:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.157740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.157748:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.157751:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.157752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.157753:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.157756:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.157757:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.157759:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.157760:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.157761:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.157762:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.157765:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.157766:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.157769:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.157770:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:21.2:1619677474.157771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:10.0:1619677474.157771:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.157772:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.157773:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.157774:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.157774:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677474.157777:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.157780:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.157781:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.157783:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.157786:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:1.0:1619677474.157786:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.157787:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677474.157788:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.157789:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.157790:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.157790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.157790:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.157791:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.157792:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677474.157793:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.157796:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.157798:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677474.157798:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677474.157801:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.157802:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677474.157802:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677474.157804:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677474.157804:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.157805:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677474.157806:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677474.157807:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677474.157810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.157816:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:21.2:1619677474.157817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:26.0:1619677474.157819:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.157820:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.157820:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.157823:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.157823:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:26.0:1619677474.157825:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:17.0:1619677474.157825:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677474.157826:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.157827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:5.2:1619677474.157944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.157949:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.157952:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.157952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.157955:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.157957:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.157958:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.157960:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.157961:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.157962:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.157963:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.157966:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.157968:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.157970:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.157971:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.157971:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.157973:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.157973:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.157973:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.157974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.157986:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.157988:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.157989:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.157989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.157991:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.157993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.158145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.158150:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.158152:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.158153:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.158154:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.158156:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.158157:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.158158:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.158160:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.158161:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.158162:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.158165:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.158167:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.158169:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677474.158170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:6.0:1619677474.158170:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.158170:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.158172:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.158173:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.158173:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.158174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.2:1619677474.158177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:31.0:1619677474.158178:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.158181:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:2.0:1619677474.158182:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677474.158183:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.158185:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:2.0:1619677474.158185:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:31.0:1619677474.158186:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.158186:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677474.158186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677474.158187:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677474.158188:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:2.0:1619677474.158188:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677474.158189:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.158189:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.158191:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.158191:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.158191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:2.0:1619677474.158191:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.158192:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.158193:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:2.0:1619677474.158193:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677474.158194:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677474.158194:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.158196:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.158198:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.158199:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.158199:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677474.158202:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677474.158203:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677474.158206:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.158206:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677474.158207:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.158208:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.158209:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.158210:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.158210:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.2:1619677474.158210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677474.158210:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:31.0:1619677474.158211:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:2.0:1619677474.158211:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:31.0:1619677474.158213:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:2.0:1619677474.158213:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677474.158214:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677474.158214:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.158215:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.158215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677474.158217:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677474.158218:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.158219:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.158221:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.158222:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:22.0:1619677474.158223:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677474.158224:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96505775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.158225:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.158225:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:31.0:1619677474.158226:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.158226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677474.158227:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677474.158227:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.158227:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.158229:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:22.0:1619677474.158230:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:13.0:1619677474.158230:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.158232:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677474.158232:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.158233:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:12.0:1619677474.158234:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677474.158235:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:13.0:1619677474.158236:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677474.158238:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.158239:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.158239:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.158241:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.158241:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.158243:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.158245:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.158245:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.158246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.158251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.158253:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.158256:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.158257:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.158258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.158259:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.158259:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.158261:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.158262:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.158262:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.158263:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.158769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.158777:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.158779:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.158781:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.158784:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.158784:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.158785:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.158786:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.158788:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.158788:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.158789:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.158792:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.158794:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.158796:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.158797:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.158797:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.158798:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.158799:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.158800:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.158800:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.158805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.158812:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.158814:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.158815:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.158815:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.158817:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.158819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.158914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.158920:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.158923:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.158924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.158926:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.158929:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.158930:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.158931:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.158933:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.158934:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.158934:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.158937:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.158939:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.158941:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.158942:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.158942:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.158944:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.158945:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.158946:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.158947:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.158948:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.158952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.158954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.158960:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.158961:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.158962:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.158962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.158962:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.158962:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.158964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.158964:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.158965:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.158965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.158972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.158980:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.158982:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.158983:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.158983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.158984:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.158984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.159042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.159046:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.159048:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.159050:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.159052:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.159053:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.159054:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.159055:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.159057:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.159058:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.159059:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.159062:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.159063:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.159066:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.159066:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.159067:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.159069:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.159069:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.159070:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.159071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.159075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.159083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.159086:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.159087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.159087:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.159089:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.159091:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.159629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.159634:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.159637:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.159638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.159640:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.159642:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.159643:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.159644:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.159646:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.159646:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.159648:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.159650:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.159652:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.159654:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.159655:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.159656:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.159657:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.159658:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.159658:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.159659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.159672:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.159674:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.159675:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.159677:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.159678:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.159679:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.159761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.159770:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.159773:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.159775:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.159777:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.159777:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.159778:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.159780:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.159781:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.159782:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.159783:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.159786:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.159788:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.159790:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.159791:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.159792:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.159793:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.159794:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.159795:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.159795:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.159805:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.159808:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.159808:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.159810:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.159811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.159813:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.159824:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.159826:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.159827:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.159828:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.159828:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.159829:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.2:1619677474.159916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.159924:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.159927:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.159929:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.107@o2ib7 : GET - for me
00000800:00000200:21.0:1619677474.159930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.159932:0:19934:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.107@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:20.0:1619677474.159934:0:19934:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.107@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:20.0:1619677474.159935:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.159936:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.159938:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.159942:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : REPLY try# 0
00000800:00000200:20.0:1619677474.159943:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.159946:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.159946:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.159947:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.159948:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.159949:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677474.159954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.159962:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.159964:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.159965:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.159967:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: REPLY: OK
00000800:00000200:20.0:1619677474.159968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.159969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.160080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.160087:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.160090:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.160091:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.160092:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.160095:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.160097:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.160098:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.160099:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.160100:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.160101:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.160104:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.160106:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.160109:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.160109:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.160110:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.160112:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.160112:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.160113:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.160114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677474.160125:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.160128:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.160129:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.160129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.160131:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.160134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.160248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.160255:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.160258:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.160260:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.160262:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.160262:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.160263:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.160264:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.160266:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.160267:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.160268:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.160271:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.160273:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.160275:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.160276:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.160277:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.160278:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.160279:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.160280:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.160280:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.160291:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.160296:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.160299:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.160300:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.160302:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.160304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.160630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.160637:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.160639:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.160640:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.160642:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.160644:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.160646:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677474.160647:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.160648:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.160649:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.160651:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677474.160654:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.160656:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677474.160657:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.160658:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677474.160659:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.160660:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.160661:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.160662:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677474.160663:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677474.160663:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677474.160667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.2:1619677474.160670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.160676:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.160677:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.160677:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000400:00000200:18.0:1619677474.160678:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.160678:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677474.160679:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.160680:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.160680:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.160681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677474.160681:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.2:1619677474.160685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.160690:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.160693:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.160694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.160695:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.160696:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.160698:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677474.160699:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.160700:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.160701:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.160702:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677474.160705:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.160706:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677474.160709:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.160710:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677474.160710:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677474.160712:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677474.160712:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.160713:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.160714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677474.160718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.160725:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.160728:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.160729:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.160729:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.160731:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.160734:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677474.160740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677474.160746:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.160749:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677474.160750:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.160750:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.160751:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.160752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677474.160805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.160812:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.160814:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.160815:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.160817:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.160819:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.160821:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.160822:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.160823:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.160824:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.160825:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.160828:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.160830:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.160832:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.160833:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.160833:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.160835:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.160835:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.160836:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.160836:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.160840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.160848:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.160850:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.160851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.160851:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.160852:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.160855:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.160857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.160863:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.160865:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.160866:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.160866:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.160867:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.160868:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.160942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.160949:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.160951:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.160952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.160953:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.160956:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.160957:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.160958:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.160959:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.160960:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.160961:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.160964:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.160966:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.160968:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.160969:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.160970:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.160971:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.160972:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.160973:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.160974:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.160975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.160980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.160985:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.160987:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.160988:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.160988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.160989:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677474.160989:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.160990:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.160991:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.160991:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.160993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.161016:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.161018:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.161019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.161019:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.161020:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.161021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.161254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.161259:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.161261:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.161262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.161264:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.161265:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.161267:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.161268:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.161269:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.161270:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.161271:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.161274:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.161276:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.161279:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.161279:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.161280:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.161282:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.161282:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.161283:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.161283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.161295:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.161301:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.161302:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.161303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.161304:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.161307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.161543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.161550:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.161553:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.161554:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.161556:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.161558:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.161560:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.161561:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.161562:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.161563:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.161564:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.161568:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.161569:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.161572:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.161572:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.161573:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.161574:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.161575:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.161576:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.161576:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.161581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.161588:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.161591:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.161591:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.161592:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.161593:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.161596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.161869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.161876:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.161879:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.161880:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.161882:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.161886:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.161887:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.161888:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.161890:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.161891:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.161892:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.161895:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.161897:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.161901:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.161902:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.161903:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.161904:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.161905:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.161906:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.161906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:12.0:1619677474.161918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.161921:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.161922:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.161924:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.161925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.161926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:13.0:1619677474.161945:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.161948:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.161948:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.161949:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.161950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.161951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.162128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.162137:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.162139:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.162142:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.162144:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.162144:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.162146:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.162147:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.162148:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.162149:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.162150:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.162153:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.162155:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.162158:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.162159:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.162159:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.162161:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.162162:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.162163:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.162163:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.162175:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.162178:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.162179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.162179:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.162181:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.162184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.162317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.162325:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.162328:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.162330:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.162333:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.162333:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.162334:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.162335:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.162337:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.162338:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.162339:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.162342:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.162343:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.162345:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.162346:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.162346:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.162348:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.162348:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.162349:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.162350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.162354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.162361:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.162364:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.162365:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.162365:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.162367:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.162369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.162552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.162559:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.162562:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.162563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.162564:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.162566:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.162568:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.162569:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.162570:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.162571:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.162572:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.162575:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.162576:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.162579:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.162580:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.162580:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.162582:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.162583:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.162583:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.162584:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.162596:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.162599:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.162600:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.162600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.162602:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.162605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.162689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.162695:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.162698:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.162698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.162700:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.162702:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.162704:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.162705:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.162707:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.162707:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.162708:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.162711:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.162713:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.162715:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.162716:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.162717:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.162718:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.162719:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.162719:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.162720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.162731:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.162734:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.162735:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.162735:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.162737:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.162739:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.162750:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.162753:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.162753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.162754:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.162755:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.162755:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.163042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.163047:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.163049:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.163050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.163052:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.163054:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.163055:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.163056:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.163058:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.163059:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.163060:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.163063:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.163065:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.163068:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.163068:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.163069:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.163071:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.163071:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.163072:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.163073:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.163074:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.163079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.163084:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.163086:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.163086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.163086:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.163087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.163088:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.163089:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.163091:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.163091:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.163093:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.163103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.163106:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.163107:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.163107:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.163108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.163109:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.163416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.163424:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.163427:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.163429:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.163432:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.163432:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.163433:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.163434:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.163436:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.163437:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677474.163437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000400:00000200:31.0:1619677474.163438:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677474.163440:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000400:00000200:31.0:1619677474.163442:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.163442:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:31.0:1619677474.163443:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677474.163445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677474.163445:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.163446:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.163446:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.163447:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:20.0:1619677474.163447:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.163449:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.163449:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677474.163449:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677474.163450:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.163451:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677474.163451:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.163452:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:20.0:1619677474.163452:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.163453:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677474.163453:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677474.163457:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677474.163458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.163458:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.163460:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.163461:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.163462:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.163463:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.163464:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:20.0:1619677474.163464:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.163464:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000400:00000200:30.0:1619677474.163465:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.163465:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677474.163465:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:31.0:1619677474.163467:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.163467:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.163468:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.163468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.163469:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.163470:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.2:1619677474.163470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.163471:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677474.163478:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.163481:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.163481:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.163483:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.163484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.163485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.163890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.163898:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.163901:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.163903:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.163905:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.163905:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.163907:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677474.163908:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.163909:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.163910:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.163911:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677474.163915:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.163918:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677474.163921:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.163921:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677474.163922:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677474.163924:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677474.163924:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.163925:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.163926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.163930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.163938:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.163941:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.163942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.163942:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.163944:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.163948:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677474.164003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.164009:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.164011:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.164012:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.164014:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.164016:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.164017:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.164018:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.164020:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.164021:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.164022:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.164025:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.164026:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.164029:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.164029:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.164030:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.164032:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.164032:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.164034:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.164035:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.164035:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.164038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.164041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.164046:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677474.164046:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677474.164048:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677474.164048:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677474.164049:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677474.164050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.164050:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677474.164050:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.164052:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.164054:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.164063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.164071:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.164074:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.164076:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.164078:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.164078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.164080:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.164081:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.164083:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.164084:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.164085:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.164087:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.164089:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.164091:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.164092:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.164093:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.164094:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.164095:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.164096:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.164096:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.164100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.164106:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.164109:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.164110:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.164110:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.164112:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.164114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.164234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.164241:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.164244:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.164246:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.164248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.164248:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.164249:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.164250:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.164252:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.164253:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.164253:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.164257:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.164258:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.164261:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.164262:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.164262:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.164264:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.164265:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.164266:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.164267:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.164268:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.164274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.164278:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.164281:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.164282:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.164282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.164282:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.164284:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677474.164284:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.164285:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.164286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.164287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.164368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.164373:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.164375:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.164376:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.164378:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.164379:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.164381:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.164382:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.164383:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.164384:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.164385:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.164388:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.164389:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.164391:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.164392:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.164393:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.164395:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.164396:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.164396:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.164396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.164409:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.164411:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.164412:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.164414:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.164415:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.164416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.164578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.164586:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.164589:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.164590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.164592:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.164595:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.164596:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.164597:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.164599:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.164600:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.164601:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.164604:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.164606:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.164609:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.164610:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.164611:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.164612:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.164613:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.164614:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.164614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.164619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.164627:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.164629:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.164630:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.164632:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.164632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.164635:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.164764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.164771:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.164774:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.164775:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.164777:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.164779:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.164781:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677474.164782:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.164783:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.164785:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.164786:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677474.164789:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.164791:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677474.164794:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.164795:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677474.164795:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677474.164797:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677474.164798:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.164799:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.164799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677474.164803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.164810:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.164812:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.164813:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.164814:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.164816:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.164819:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.164948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.164956:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.164959:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.164961:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.164963:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.164964:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.164966:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.164967:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.164968:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.164969:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.164970:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.164974:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.164976:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.164978:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.164979:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.164980:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.164981:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.164982:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.164983:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.164983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.164987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.164993:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.164995:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.164996:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.164997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.164998:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.165000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.165004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.165010:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.165012:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.165013:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.165014:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.165014:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.165015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.165588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.165596:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.165598:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.165599:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.165601:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.165603:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.165604:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.165605:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.165606:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.165607:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.165608:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.165611:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.165613:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.165615:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.165615:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.165616:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.165617:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.165618:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.165618:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.165619:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.165624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.165630:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.165633:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.165634:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.165634:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.165635:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.165637:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.165645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677474.165652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.165655:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677474.165656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677474.165657:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.165660:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.165661:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.165662:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.165663:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.165664:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.165665:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.165668:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.165669:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.165672:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.165673:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.165673:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.165675:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.165676:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.165676:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677474.165677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677474.165689:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.165692:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.165693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.165693:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.165695:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.165698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.165802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.165809:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.165812:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.165813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.165815:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.165818:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.165819:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.165820:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.165821:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.165822:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.165823:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.165826:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.165828:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.165830:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.165831:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.165831:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.165833:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.165834:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.165834:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.165835:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.165846:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.165849:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.165850:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.165850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.165852:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.165855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.165863:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.165866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.165866:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.165867:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.165868:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.165868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.165890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.165893:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.165896:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.165898:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.165899:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.165900:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.165902:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.165903:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.165904:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.165905:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.165906:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.165910:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.165912:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.165914:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.165915:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.165915:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.165917:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.165918:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.165918:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.165919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677474.165923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.2:1619677474.165923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.165929:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.165931:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:17.0:1619677474.165932:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:6.0:1619677474.165932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.165934:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.165935:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:22.0:1619677474.165936:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.165936:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.165936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:22.0:1619677474.165938:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677474.165939:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.165940:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.165941:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.165941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:22.0:1619677474.165942:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.165943:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.165946:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.165948:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.165950:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.165951:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.165951:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.165953:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.165954:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.165954:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.165955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.165967:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.165969:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.165970:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.165971:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.165973:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.165974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.166260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.166268:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.166271:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.166273:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.166275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.166275:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.166277:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.166278:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.166280:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.166281:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.166282:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.166285:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.166287:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.166290:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.166291:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.166292:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.166297:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.166298:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.166299:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.166299:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.166303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677474.166307:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.166310:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.166311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.166311:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677474.166311:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.166314:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.166314:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:6.0:1619677474.166314:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.166316:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.166316:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677474.166319:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.166320:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.166321:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.166322:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.166323:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.166324:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.166327:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.166329:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.166331:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.166331:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.166332:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.166333:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.166333:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.166335:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.166336:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.166337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.166339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.166342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.166345:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.166347:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.166348:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.166349:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677474.166349:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.166350:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.166351:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.166352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.166352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.166353:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677474.166851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.166858:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.166861:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.166861:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677474.166863:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.166865:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.166866:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.166867:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.166869:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.166869:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.166870:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.166873:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.166874:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.166877:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.166878:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.166878:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.166880:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.166880:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.166881:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.166882:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.166894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.166896:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.166897:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.166897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.166900:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.166902:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.167304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.167310:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.167312:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.167313:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.167315:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.167317:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.167319:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.167320:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677474.167321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198613d7600] (20)++
00000400:00000200:1.0:1619677474.167322:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.167322:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.167324:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:6.0:1619677474.167327:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198613d7600] (21)++
00000400:00000200:1.0:1619677474.167327:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.167329:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677474.167330:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.0.72@o2ib3
00000800:00000200:22.0:1619677474.167331:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198613d7600] (22)--
00000800:00000200:1.0:1619677474.167332:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677474.167333:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.72@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.167333:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.167333:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677474.167335:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.167335:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.167335:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.167336:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:6.0:1619677474.167337:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677474.167337:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:6.0:1619677474.167338:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.167339:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.167340:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.167341:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.2:1619677474.167342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677474.167345:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.167346:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.167349:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:6.0:1619677474.167349:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.167350:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677474.167350:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.167351:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.167352:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.167352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677474.167352:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677474.167353:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198613d7600] (21)++
00000800:00000200:6.0:1619677474.167353:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198613d7600] (22)--
00000400:00000200:24.0:1619677474.167354:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.167354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198613d7600] (21)--
00000800:00000200:5.2:1619677474.167356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677474.167358:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.167358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677474.167362:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677474.167364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677474.167365:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:2.0:1619677474.167366:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:1.0:1619677474.167366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.167368:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.167368:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.167369:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.167370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:17.0:1619677474.167370:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.167370:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:2.0:1619677474.167371:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.167372:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.167373:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.167373:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.167374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.167374:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:2.0:1619677474.167374:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:17.0:1619677474.167375:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.167375:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:17.0:1619677474.167376:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.167376:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:17.0:1619677474.167377:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.167380:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.167381:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.167384:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677474.167385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677474.167385:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.167386:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.167387:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.167388:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.167388:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677474.167389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198613d7600] (20)++
00000800:00000200:17.0:1619677474.167389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677474.167393:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.167396:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677474.167398:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198613d7600] (21)++
00000400:00000200:31.0:1619677474.167399:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.167400:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.72@o2ib3
00000800:00000200:30.0:1619677474.167401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.167402:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.167402:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:22.0:1619677474.167403:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.72@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.167404:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677474.167404:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:31.0:1619677474.167405:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.167405:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.167407:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.167407:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.167408:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.167409:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.167409:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.167411:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.167411:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.167413:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.167415:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.167416:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.167417:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.167420:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.167421:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.167422:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.167423:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.167423:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.167423:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.167424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:31.0:1619677474.167425:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.167425:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.167425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198613d7600] (22)--
00000800:00000200:31.0:1619677474.167426:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.167426:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.167426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.167427:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.167427:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.167427:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.167429:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.167430:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198613d7600] (21)++
00000800:00000200:22.0:1619677474.167431:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198613d7600] (22)--
00000800:00000200:22.0:1619677474.167432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198613d7600] (21)--
00000800:00000200:30.0:1619677474.167438:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.167440:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.167441:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.167442:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677474.167443:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.167444:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.167445:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677474.167445:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.167445:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.167446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.167448:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.167449:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.167506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.167513:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.167516:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.167517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.167519:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.167521:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.167523:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677474.167524:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.167525:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.167527:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.167527:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677474.167531:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.167533:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677474.167536:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.167537:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677474.167537:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677474.167539:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677474.167539:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.167541:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.167542:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.167542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677474.167545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.167548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:18.0:1619677474.167553:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:10.0:1619677474.167555:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:18.0:1619677474.167556:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.167557:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.167557:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:2.0:1619677474.167557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677474.167558:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.167559:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:18.0:1619677474.167559:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.167563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.167651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.167656:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.167659:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.167659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.167661:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.167663:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.167664:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.167666:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.167667:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.167668:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.167669:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.167673:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.167674:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.167677:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.167678:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.167679:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.167680:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.167681:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.167682:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.167682:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.167694:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.167696:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.167697:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.167699:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.167700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.167701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.168197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.168202:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.168205:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.168206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.168209:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.168211:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.168212:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.168214:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.168215:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.168216:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.168217:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.168220:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.168222:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.168224:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.168225:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.168226:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.168227:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.168228:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.168229:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.168229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.168234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.168242:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.168245:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.168245:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.168247:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.168248:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.168250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.168300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.168308:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.168311:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.168313:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.168316:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.168316:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.168317:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.168318:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.168320:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.168321:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.168322:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.168325:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677474.168325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677474.168326:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.168330:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.168330:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.168331:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.168331:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677474.168333:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.168333:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.168334:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:2.0:1619677474.168334:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:31.0:1619677474.168335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:18.0:1619677474.168336:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.168337:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.168340:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.168342:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.168343:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.168344:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.168345:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.168346:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677474.168347:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:2.0:1619677474.168349:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.168350:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.168351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.168351:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.168351:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:6.0:1619677474.168354:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.168354:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.168354:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.168355:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677474.168357:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677474.168357:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.168358:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.168359:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.168359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.168363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.168371:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.168373:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.168374:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.168374:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.168376:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.168378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.168522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.168529:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.168532:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.168535:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.168536:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.168538:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.168539:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.168541:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.168542:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.168543:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.168544:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.168547:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.168549:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.168551:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.168552:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.168553:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.168554:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.168555:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.168557:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.168558:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.168559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.168561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.168564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.168568:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.168571:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.168572:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.168572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.168573:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.168574:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.168575:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.168576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.168576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.168577:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.168918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198613d7600] (20)++
00000800:00000200:22.0:1619677474.168923:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198613d7600] (21)++
00000800:00000200:22.0:1619677474.168925:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.72@o2ib3
00000800:00000200:6.0:1619677474.168926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198613d7600] (22)--
00000400:00000200:22.0:1619677474.168928:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.72@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.168930:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.168932:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677474.168933:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.168934:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.168935:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.168936:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.168940:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.168941:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.168945:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.168945:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.168946:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.168948:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.168948:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198613d7600] (21)++
00000800:00000200:22.0:1619677474.168949:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198613d7600] (22)--
00000800:00000200:22.0:1619677474.168950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198613d7600] (21)--
00000800:00000200:31.0:1619677474.168961:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.168964:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.168965:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.168967:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.168968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.168970:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.168975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.168980:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.168983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.168983:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.168985:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.168988:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.168989:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.168991:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.168992:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.168993:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.168994:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.168996:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.168998:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.169000:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.169000:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.169001:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.169002:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.169003:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.169004:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.169005:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.169005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.169010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.169015:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.169016:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.169017:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.169018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.169018:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677474.169019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.169019:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.169021:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.169022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.169023:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.169037:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.169039:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.169041:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.169041:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.169042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.169043:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.169361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677474.169369:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677474.169372:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677474.169372:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000400:00000200:10.0:1619677474.169375:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.169377:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.169379:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677474.169380:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.169382:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.169382:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.169383:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677474.169387:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.169388:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.169391:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.169392:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677474.169393:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677474.169395:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677474.169395:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677474.169396:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677474.169396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:21.0:1619677474.169409:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.169411:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.169412:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.169414:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.169415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.169416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.169498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.169506:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.169508:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.169509:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.169511:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.169514:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.169516:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.169517:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.169518:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.169519:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.169520:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.169523:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.169525:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.169527:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.169528:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.169528:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.169530:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.169531:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.169531:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.169531:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.169537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.169545:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.169548:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.169549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.169549:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.169551:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.169555:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.170014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.170023:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.170026:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.170027:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.170030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.170030:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.170031:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.170032:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.170034:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.170035:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.170036:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.170039:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.170040:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.170043:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.170044:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.170044:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.170046:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.170047:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.170048:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.170048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.170061:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.170063:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.170064:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:4.2:1619677474.170064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:6.0:1619677474.170065:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.170066:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.170069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.170072:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.170075:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677474.170076:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677474.170077:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677474.170079:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.170080:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:8.0:1619677474.170081:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:22.0:1619677474.170082:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.170083:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.170083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.170084:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.170084:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000400:00000200:8.0:1619677474.170084:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.170085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:8.0:1619677474.170085:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.170086:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677474.170089:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.170090:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.170091:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.170092:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.170093:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.170094:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.170095:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.170096:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677474.170096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677474.170100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.170106:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.170108:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.170108:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.170109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.170110:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.170112:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.170116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.170123:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.170126:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.170128:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.170130:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.170131:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.170132:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.170133:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.170134:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.170135:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.170136:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.170139:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.170140:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.170143:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.170143:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.170144:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.170145:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.170146:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.170147:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.170147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.170151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.170158:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.170160:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.170161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.170161:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.170163:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.170165:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.170171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.170176:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.170179:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.170180:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.170180:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.170181:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.170181:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.170449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.170457:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.170459:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.170460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.170462:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.170464:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.170466:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.170467:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.170468:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.170469:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.170470:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.170473:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.170474:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.170477:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.170478:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.170478:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.170480:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.170480:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.170481:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.170482:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.170496:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.2:1619677474.170497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.170498:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.170499:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.170500:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.170501:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:6.0:1619677474.170501:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.170503:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.170504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:22.0:1619677474.170505:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.170506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.170507:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.170509:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.170510:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.170511:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.170512:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.170513:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.170517:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.170518:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.170521:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.170521:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.170522:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.170524:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.170524:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.170525:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.170525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.170537:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.170540:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.170541:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.170542:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.170543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.170544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.170678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.170685:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.170688:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.170689:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.170691:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.170693:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.170695:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.170696:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.170697:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.170698:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.170699:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.170703:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.170704:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.170706:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.170707:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.170708:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.170709:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.170710:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.170710:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.170711:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.170715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.170721:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.170723:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.170724:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.170725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.170726:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.170728:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.170729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.170736:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.170739:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.170740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.170742:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.170744:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.170746:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.170747:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677474.170749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677474.170749:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.170749:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.170751:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.170754:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.170756:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.170757:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677474.170759:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.170760:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677474.170760:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.170761:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:30.0:1619677474.170762:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.170762:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.170763:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:30.0:1619677474.170764:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.170764:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.170764:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677474.170765:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.170766:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.170767:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.170768:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.170769:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.170770:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.170773:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.170775:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.170775:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.170776:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.170777:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.170777:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.170778:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:31.0:1619677474.170779:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.170779:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.170780:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.170781:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.170782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.170782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677474.170782:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.170783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.170793:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.170795:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.170796:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.170797:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.170799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.170800:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.170809:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.170812:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.170813:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.170814:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.170814:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.170815:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.171408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.171415:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.171418:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.171419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.171420:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.171423:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.171424:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.171425:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.171426:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.171427:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.171428:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.171431:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.171432:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.171434:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.171435:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.171435:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.171437:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.171437:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.171438:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.171438:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.171443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.171450:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.171452:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.171453:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.171454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.171455:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.171457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.171703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.171710:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.171713:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.171714:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.171715:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.171718:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.171719:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.171720:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.171721:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.171722:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.171723:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.171726:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.171728:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.171730:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.171731:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.171732:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.171733:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.171734:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.171735:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.171736:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.171737:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.171742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.171747:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.171749:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.171750:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677474.171751:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677474.171751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.171751:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.171752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.171753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.171753:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.171756:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.172012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.172020:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.172023:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.172025:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.172027:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.172028:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.172029:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.172030:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.172031:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.172032:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.172033:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.172036:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.172037:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.172040:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.172041:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.172041:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.172043:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.172044:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.172045:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.172047:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.172047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.172053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.172056:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.172059:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.172060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.172061:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.172061:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.172063:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.172064:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.172065:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.172065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.172066:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677474.172068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.172073:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.172076:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.172078:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.172079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.172080:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.172082:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.172083:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.172085:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.172086:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.172087:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.172090:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.172092:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.172094:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.172095:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.172095:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.172097:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.172098:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.172099:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.172100:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.172101:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.172103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.172106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.172109:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.172111:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.172112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.172112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.172112:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.172115:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.172116:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.172116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:5.2:1619677474.172117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:17.0:1619677474.172118:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.172121:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677474.172122:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.172124:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.172124:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.172127:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.172128:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.172130:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.172131:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.172132:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.172133:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.172134:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.172138:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.172139:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.172142:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.172142:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.172143:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.172145:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.172145:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.172146:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.172147:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.172148:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.172153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677474.172158:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.172158:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.172160:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.172160:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.172161:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677474.172162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.172162:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.172162:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.172164:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.172166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.172180:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.172182:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.172183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.172184:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.172185:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.172185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.172253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.172259:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.172261:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.172262:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.172264:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.172266:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.172267:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.172268:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.172269:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.172270:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.172271:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.172275:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.172277:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.172279:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.172280:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.172280:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.172282:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.172283:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.172283:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.172283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.172296:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.172298:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.172299:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.172301:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.172302:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677474.172303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.172304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.172310:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.172313:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.172315:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.172317:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.172318:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.172320:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.172321:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.172322:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.172323:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.172325:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.172328:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.172330:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.172332:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.172333:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.172333:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.172335:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.172336:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.172337:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.172337:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.172348:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.172350:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.172351:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.172353:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.172354:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.172355:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.173307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677474.173313:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.173316:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677474.173317:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677474.173319:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.173321:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.173323:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677474.173324:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.173325:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.173326:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.173327:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677474.173331:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.173333:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677474.173335:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.173336:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677474.173336:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.173338:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677474.173339:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.173339:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677474.173340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677474.173345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.173353:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.173356:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.173357:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.173357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.173359:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.173361:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.173364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.173371:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.173374:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.173375:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.173375:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.173376:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.173377:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.173581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.173586:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.173589:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.173590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.173591:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.173594:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.173595:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.173596:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.173597:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.173598:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.173599:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.173602:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.173603:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.173605:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.173606:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.173606:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.173608:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.173609:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.173609:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.173610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.173623:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.173625:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.173626:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.173627:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.173629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.173629:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.173639:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.173641:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.173642:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.173643:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.173643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.173644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.174031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.174039:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.174042:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.174042:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.174045:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.174048:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.174049:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.174050:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.174052:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.174053:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.174054:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.174057:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.174059:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.174062:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.174063:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.174063:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.174065:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.174066:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.174067:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.174068:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.174068:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677474.174072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677474.174074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.174081:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677474.174081:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677474.174083:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.174084:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.174085:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677474.174085:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677474.174087:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.174088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:9.2:1619677474.174090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:2.0:1619677474.174090:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.174096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.174098:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.174101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.174102:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677474.174103:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.174105:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.174106:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.174308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.174316:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.174319:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.174321:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.174324:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.174324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.174325:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:31.0:1619677474.174326:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.174328:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.174329:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.174330:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677474.174333:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.174334:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677474.174337:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.174338:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:31.0:1619677474.174338:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:31.0:1619677474.174340:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:31.0:1619677474.174340:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.174342:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.174342:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.174346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.174353:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.174356:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.174357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.174357:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.174359:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.174362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677474.174365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.174373:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:4.2:1619677474.174375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:24.0:1619677474.174376:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.174377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.174379:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.174381:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.174381:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:24.0:1619677474.174383:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.174384:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.174384:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.174386:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.174386:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:24.0:1619677474.174387:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.174388:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.174389:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:21.0:1619677474.174389:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.174391:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.174392:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.174393:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677474.174394:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.174395:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.174396:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:21.0:1619677474.174396:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.174399:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.174400:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.174400:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:21.0:1619677474.174400:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.174402:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.0:1619677474.174402:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.174403:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.174403:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.174404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677474.174405:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.174406:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.174407:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.2:1619677474.174408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677474.174409:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.174409:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.174410:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.174411:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.174412:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:17.0:1619677474.174417:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:4.2:1619677474.174417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:17.0:1619677474.174420:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:30.0:1619677474.174421:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677474.174421:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:17.0:1619677474.174421:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.174421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.0:1619677474.174423:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:17.0:1619677474.174423:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.174424:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677474.174424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677474.174425:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.174425:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677474.174426:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:30.0:1619677474.174427:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.174428:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.174430:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:8.2:1619677474.174430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.174437:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.174440:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.174441:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.174441:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.174443:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.174443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.174502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.174510:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.174513:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.174515:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.174517:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.174517:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.174519:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.174520:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.174521:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.174522:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.174523:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.174526:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.174528:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.174530:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.174531:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.174532:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.174533:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.174534:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.174535:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.174537:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.174537:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.174543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.174547:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.174549:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.174549:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.174551:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.174551:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.174552:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.174553:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.174554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.174555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.174556:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677474.175101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677474.175106:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.175109:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677474.175110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677474.175112:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677474.175114:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.175116:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677474.175117:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.175118:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.175119:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.175120:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677474.175124:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677474.175126:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.2:1619677474.175127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:17.0:1619677474.175128:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.175129:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:17.0:1619677474.175129:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677474.175131:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677474.175131:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.175132:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:22.0:1619677474.175133:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677474.175133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:22.0:1619677474.175136:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:9.2:1619677474.175137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677474.175137:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.175138:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.175141:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.175142:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.175143:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.175145:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677474.175145:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:22.0:1619677474.175146:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.175147:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677474.175148:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.175149:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.175149:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:22.0:1619677474.175151:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677474.175152:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.175153:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677474.175154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677474.175155:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.175156:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.175157:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.175158:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:9.2:1619677474.175158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:22.0:1619677474.175159:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.175160:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.175161:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.175161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:2.0:1619677474.175165:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.175166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677474.175168:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.175169:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.175170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.175170:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.175171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677474.175172:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.175172:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.175174:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.175174:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677474.175175:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.175175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.175176:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.175177:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.175178:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.175179:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.175355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.175363:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.175365:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.175368:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.175370:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.175370:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.175371:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.175372:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.175374:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.175375:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.175376:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.175379:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.175381:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.175383:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.175384:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.175385:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.175386:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.175387:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.175388:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.175389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.175400:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.175403:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.175403:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.175405:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.175406:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.175408:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.175428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.175434:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.175437:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.175438:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.175440:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.175442:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.175444:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.175445:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.175446:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.175448:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.175449:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.175452:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.175454:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.175457:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.175458:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.175459:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.175460:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.175461:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.175462:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:4.2:1619677474.175462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677474.175463:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:26.0:1619677474.175471:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.175473:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:13.0:1619677474.175474:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677474.175474:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.175476:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.175477:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.175478:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.175479:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.175479:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:26.0:1619677474.175480:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.175481:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677474.175481:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.175482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:26.0:1619677474.175483:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.175484:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.175485:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.175488:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.175490:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.175492:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.175493:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.175494:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.175495:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.175496:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.175496:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.175497:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.175501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.175508:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.175510:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.175511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.175511:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.175513:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.175515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.175959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.175966:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.175969:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.175971:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.175973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.175973:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.175974:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.175975:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.175976:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.175977:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.175978:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.175981:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.175983:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.175986:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.175986:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.175987:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.175989:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.175989:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.175990:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.175990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.176002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.176005:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.176006:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.176006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.176009:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.176012:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.176084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677474.176087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:24.0:1619677474.176091:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677474.176093:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677474.176094:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.176095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677474.176096:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:24.0:1619677474.176097:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.176097:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.176099:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.176101:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.176101:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.176102:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677474.176103:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.176104:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.176105:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.176106:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.176107:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.176108:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.176108:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.176109:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677474.176109:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.176114:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.176115:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.176117:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.176118:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.176119:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.176120:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.176121:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.176121:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.176122:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677474.176122:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.176123:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.176123:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:24.0:1619677474.176124:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677474.176124:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.176124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677474.176125:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.176126:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.176127:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.176128:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677474.176129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.176134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677474.176137:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.176139:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.176139:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.176139:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677474.176140:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.176141:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.176141:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.176142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.176142:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:18.0:1619677474.176143:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:6.0:1619677474.176143:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.176144:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.176144:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.176145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677474.176145:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.176149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.176157:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.176159:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.176160:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.176160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.176161:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.176162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.176182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.176189:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.176192:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.176192:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.176194:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.176196:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.176198:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.176199:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677474.176200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (20)++
00000400:00000200:10.0:1619677474.176201:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.176202:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.176203:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.176206:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.176207:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)++
00000800:00000200:10.0:1619677474.176208:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.0:1619677474.176210:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:10.0:1619677474.176210:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.176211:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:21.0:1619677474.176212:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : GET - for me
00000800:00000200:10.0:1619677474.176212:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677474.176213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:10.0:1619677474.176213:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.176213:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.176214:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:21.0:1619677474.176216:0:19935:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.2.63@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:10.0:1619677474.176216:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.176216:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:21.0:1619677474.176217:0:19935:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.2.63@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:9.2:1619677474.176218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677474.176219:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (21)++
00000800:00000200:21.0:1619677474.176221:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e7f44800] (22)++
00000800:00000200:4.2:1619677474.176222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677474.176223:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (23)--
00000400:00000200:21.0:1619677474.176225:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.2.63@o2ib3 local destination
00000800:00000200:8.0:1619677474.176225:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.176227:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.176228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.2:1619677474.176228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (22)++
00000400:00000200:8.0:1619677474.176228:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.176229:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:21.0:1619677474.176229:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.2.63@o2ib3(10.51.2.63@o2ib3:10.51.2.63@o2ib3) : REPLY try# 0
00000400:00000200:8.0:1619677474.176230:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.176231:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:21.0:1619677474.176231:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.2.63@o2ib3
00000800:00000200:20.0:1619677474.176231:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)++
00000800:00000200:26.0:1619677474.176232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677474.176232:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e7f44800] (24)--
00000800:00000200:8.0:1619677474.176232:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.176233:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677474.176233:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)--
00000800:00000200:21.0:1619677474.176234:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9197c81a8700] -> 10.51.2.63@o2ib3 (2) version: 12
00000800:00000200:20.0:1619677474.176234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:21.0:1619677474.176235:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197e7f44800] (21)++
00000800:00000200:21.0:1619677474.176235:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e7f44800] (22)++
00000800:00000200:21.0:1619677474.176236:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197e7f44800] (23)--
00000800:00000200:21.0:1619677474.176237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:9.2:1619677474.176238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.2:1619677474.176242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (21)++
00000800:00000200:24.0:1619677474.176244:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677474.176244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)++
00000800:00000200:20.0:1619677474.176246:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e7f44800] (23)--
00000800:00000200:24.0:1619677474.176247:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677474.176247:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.176248:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.176248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.176249:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:20.0:1619677474.176249:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.2.63@o2ib3: REPLY: OK
00000800:00000200:24.0:1619677474.176250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677474.176250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:20.0:1619677474.176251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)--
00000800:00000200:21.2:1619677474.176490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.176498:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.176501:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.176503:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.176505:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.176506:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.176507:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.176508:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.176510:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.176511:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.176512:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.176514:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.176516:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.176518:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.176519:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.176520:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.176521:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.176522:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.176523:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.176523:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.176527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.176534:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.176537:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.176538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.176538:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.176539:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.176542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.176550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.176558:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.176560:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.176561:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.176563:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.176565:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.176567:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.176568:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.176569:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.176570:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.176571:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.176574:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.176576:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.176578:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.176579:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.176580:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.176581:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.176582:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.176582:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.176583:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.176596:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.176598:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.176599:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.176600:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.176602:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.176605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.176913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.176918:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.176921:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.176922:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.176924:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.176926:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.176928:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.176929:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.176930:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.176931:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.176932:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.176935:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.176937:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.176939:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.176940:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.176940:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.176942:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.176942:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:5.2:1619677474.176942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677474.176943:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.176943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:1.0:1619677474.176947:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.176950:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.176951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.176953:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.176955:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:1.0:1619677474.176955:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.176957:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:21.0:1619677474.176958:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.176958:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.176958:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.176960:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.176961:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.176961:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:1.0:1619677474.176961:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.176962:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677474.176963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677474.176966:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.176968:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.176971:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.176971:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.176972:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.176974:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.176974:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.176975:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.176976:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.176987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.176990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.176991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.176991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.176993:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.176995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.177442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.177450:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.177453:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.177455:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.177457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.177458:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.177460:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.177461:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.177463:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.177464:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.177465:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.177468:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.177470:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.177473:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.177474:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.177474:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.177476:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.177476:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.177477:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.177478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.177490:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.177493:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.177494:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.177494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.177496:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.177499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.177507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.177515:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.177519:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.177521:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.177522:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.177524:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.177526:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.177527:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.177529:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.177530:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.177531:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.177535:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.177537:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.177539:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.177540:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.177541:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.177542:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.177543:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.177543:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.177544:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.177554:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.177556:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.177557:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.177559:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.177560:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.177561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.177571:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.177573:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.177574:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.177574:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.177575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.177576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.177745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.177751:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.177754:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.177755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.177757:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.177759:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.177761:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677474.177762:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.177764:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.177765:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.177766:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677474.177769:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.177771:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677474.177774:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.177775:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677474.177776:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677474.177777:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677474.177778:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.177780:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677474.177781:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677474.177781:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677474.177783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.2:1619677474.177786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:24.0:1619677474.177790:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.177793:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:18.0:1619677474.177793:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000400:00000200:24.0:1619677474.177794:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.177794:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:8.0:1619677474.177794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677474.177795:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.177796:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:24.0:1619677474.177797:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.177799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.178039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.178046:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.178049:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.178050:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.178052:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.178054:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.178056:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677474.178057:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.178058:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.178059:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.178060:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677474.178062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:26.0:1619677474.178063:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.178065:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677474.178068:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.178068:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677474.178068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:26.0:1619677474.178069:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677474.178071:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677474.178071:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:17.0:1619677474.178071:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:26.0:1619677474.178072:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.178072:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:1.0:1619677474.178072:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.178074:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677474.178076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677474.178076:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.178078:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677474.178079:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677474.178079:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.178080:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.178082:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:17.0:1619677474.178082:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.178083:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677474.178084:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:21.0:1619677474.178085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.178085:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.178086:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:17.0:1619677474.178087:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677474.178087:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.178089:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.178090:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.178091:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.178091:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677474.178092:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.178092:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:20.0:1619677474.178093:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:17.0:1619677474.178093:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:20.0:1619677474.178094:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.178094:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.178095:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.178096:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.178096:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:20.0:1619677474.178097:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677474.178097:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.178097:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677474.178098:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.178101:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677474.178102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.178103:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.178106:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.178107:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.178107:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.178109:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.178110:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.178110:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.178110:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.178111:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.178111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:21.0:1619677474.178112:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.178112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677474.178114:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.178115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.178116:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677474.178123:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.178125:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.178127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.178127:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.178129:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.178131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.178254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.178260:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.178262:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.178263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.178265:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.178267:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.178269:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.178270:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.178271:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.178272:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.178273:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.178277:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.178279:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.178281:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.178282:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.178282:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.178284:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.178285:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.178285:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.178286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.178298:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.178300:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.178301:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.178302:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.178303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.178305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.179253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.179261:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.179263:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.179266:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.179267:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.179268:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.179269:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.179270:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.179271:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.179272:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.179273:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.179277:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.179278:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.179280:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.179281:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.179282:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.179283:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.179284:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.179285:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.179287:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.179287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.179293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.179299:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.179301:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.179301:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.179302:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.2:1619677474.179302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000400:00000200:31.0:1619677474.179303:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.179303:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.179303:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.179305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.179306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.179307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677474.179307:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.179309:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.179312:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.179312:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.179314:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.179316:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.179317:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.179319:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.179320:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.179321:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.179324:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.179326:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.179328:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.179329:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.179329:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.179331:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.179332:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.179333:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677474.179333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:22.0:1619677474.179346:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.179349:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.179350:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.179350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.179352:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.179354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.179505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.179512:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.179515:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.179518:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.179519:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.179520:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.179521:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.179523:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.179524:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.179525:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.179526:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.179528:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.179530:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.179533:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.179534:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.179535:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.179537:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.179537:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.179539:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.179540:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.179541:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.179547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.179550:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.179552:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.179553:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.179554:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.179554:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.179556:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.179557:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.179558:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.179558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.179559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.179695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677474.179703:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677474.179706:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677474.179706:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000400:00000200:10.0:1619677474.179709:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.179711:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.179712:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.179714:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.179715:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.179716:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.179717:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.179720:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.179722:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.179724:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.179725:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.179726:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.179727:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.179728:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677474.179728:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677474.179729:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:9.2:1619677474.179733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.179742:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.179745:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.179746:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.179746:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.179748:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.179752:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.179817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.179822:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.179825:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.179826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.179828:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.179830:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.179831:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.179833:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.179834:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.179835:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.179836:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.179839:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.179841:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.179843:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.179844:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.179845:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.179846:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.179847:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.179848:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.179848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.179860:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.179862:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.179863:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.179864:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.179866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.179867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.179878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.179881:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.179881:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.179882:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.179883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.179884:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.180198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677474.180203:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.180205:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:21.0:1619677474.180208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677474.180208:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.180209:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.180211:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677474.180212:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.180214:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.180214:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.180216:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677474.180219:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.180221:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677474.180224:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.180224:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:20.0:1619677474.180225:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677474.180227:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677474.180227:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.180229:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677474.180230:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677474.180230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:9.2:1619677474.180233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677474.180235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:26.0:1619677474.180239:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:21.0:1619677474.180241:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:26.0:1619677474.180242:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.180243:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.180243:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:10.0:1619677474.180243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677474.180244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677474.180244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:26.0:1619677474.180246:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.180249:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.180457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.180464:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.180467:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.180468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.180469:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.180472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.180473:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.180474:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.180475:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.180476:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.180478:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.180481:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.180483:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.180485:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.180486:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.180487:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.180488:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.180489:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.180490:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.180490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.180505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.180513:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677474.180514:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.180516:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677474.180517:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:30.0:1619677474.180518:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.180518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.180518:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.180521:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.180521:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.180522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.180522:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.180524:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.180524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677474.180525:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.180526:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.180527:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.180530:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.180532:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.180533:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.180534:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.180534:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.180535:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.180536:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.180536:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.180536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.180537:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.180537:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.180538:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.180539:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.180539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.180539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.180549:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.180552:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.180553:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.180554:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.180556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.180557:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.180619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.180627:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.180629:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.180630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.180632:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.180635:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.180636:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.180638:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.180639:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.180640:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.180641:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.180644:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.180645:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.180648:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.180649:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.180650:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.180651:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.180652:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.180653:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.180653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.180665:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.180667:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.180668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.180668:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.180671:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.180673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.181222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.181229:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.181231:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.181232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.181234:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.181237:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.181238:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.181239:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.181240:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.181241:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.181242:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.2:1619677474.181246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:18.0:1619677474.181246:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.181247:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.181250:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.181251:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.181251:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.181253:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:30.0:1619677474.181254:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:18.0:1619677474.181254:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.181255:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.181255:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677474.181257:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.181259:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677474.181259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677474.181261:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.181261:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.181263:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.181264:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.181265:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.181266:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.181267:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.181268:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:30.0:1619677474.181270:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.181270:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.181271:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.181271:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:30.0:1619677474.181272:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.181273:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.181275:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.181276:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.181276:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.181276:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677474.181278:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.181279:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.181279:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.181280:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.181291:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.181295:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.181299:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.181300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.181302:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.181305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.181313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.181321:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.181324:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.181324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.181326:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.181329:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.181330:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.181332:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.181333:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.181334:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.181335:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.181339:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.181340:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.181343:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.181344:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.181344:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.181346:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.181346:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.181347:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.181347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.181360:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.181362:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.181363:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.181364:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.181366:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.181368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.181578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.181584:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.181586:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.181587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.181589:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.181591:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.181592:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.181593:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.181595:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.181596:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.181597:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.181600:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.181602:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.181605:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.181605:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.181606:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.181608:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.181608:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.181609:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.181610:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.181611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.181616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.181621:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.181623:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.181623:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.181624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.181625:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.181625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677474.181626:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.181628:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.181628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.181630:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.181778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.181786:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.181789:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.181791:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.181793:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.181794:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.181795:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.181796:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.181797:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.181799:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.181800:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.181803:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.181805:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.181807:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.181808:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.181808:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.181810:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.181810:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.181811:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.181812:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.181822:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.181824:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.181825:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.181827:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.181828:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.181829:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.181969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.181975:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.181978:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.181979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.181981:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.181984:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.181985:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677474.181987:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.181988:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.181989:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.181990:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677474.181994:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.181995:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677474.181999:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.182000:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677474.182000:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677474.182002:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677474.182003:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.182004:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.182004:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677474.182008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.182015:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.182017:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.182018:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.182019:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.182021:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.182024:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.182453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.182460:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.182463:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.182463:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.182465:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.182467:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.182468:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.182469:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.182471:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.182471:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.182472:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.182475:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.182477:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.182480:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.182481:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.182481:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.182483:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.182483:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.182484:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.182484:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.182490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.2:1619677474.182494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677474.182498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:20.2:1619677474.182499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.182501:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:30.0:1619677474.182502:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677474.182502:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.182503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.182504:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677474.182505:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677474.182505:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.182506:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:30.0:1619677474.182509:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.182510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677474.182510:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:30.0:1619677474.182511:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.182511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.182511:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.182512:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.182514:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.182514:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.182515:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.182515:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.182516:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.182517:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677474.182517:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.182519:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.182520:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.182521:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.182524:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.182526:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.182527:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.182529:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.182529:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.182530:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.182531:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677474.182531:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.182531:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.182532:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.182532:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677474.182532:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.182533:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.182534:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677474.182534:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.182534:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.182535:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.182536:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.182540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677474.182545:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:18.0:1619677474.182547:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:31.0:1619677474.182548:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.182549:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.182550:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:31.0:1619677474.182551:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:18.0:1619677474.182551:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.182551:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.182552:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.182553:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:18.0:1619677474.182553:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.182555:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.182562:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.182565:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.182566:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.182566:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.182567:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.182568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.183153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.183159:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.183162:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.183163:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.183165:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.183167:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.183169:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.183170:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.183171:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.183172:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.183173:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.183177:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.183179:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.183181:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.183181:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.183182:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.183184:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.183184:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.183185:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.183185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.183197:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.183199:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.183200:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.183202:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.183203:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.183204:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.183319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.183327:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.183330:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.183332:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.183334:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.183334:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.183336:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.183337:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.183338:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.183339:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.183340:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.183344:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.183346:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.183349:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.183350:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.183350:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.183352:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.183353:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.183354:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.183354:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.183367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.183369:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.183370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.183371:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.183373:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.183376:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.183384:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.183387:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.183387:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.183388:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.183389:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.183389:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.183467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.183473:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.183475:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.183478:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.183478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.183480:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.183481:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.183483:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.183484:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.183485:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.183486:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.183489:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.183491:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.183493:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.183494:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.183495:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.183497:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.183497:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.183498:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.183500:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.183500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677474.183505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.183509:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677474.183510:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.183511:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.183512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677474.183512:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.183513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.183513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677474.183513:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.183515:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.183518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.183718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.183725:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.183728:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.183730:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.183732:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.183733:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.183734:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.183736:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.183737:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.183738:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.183739:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.183742:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.183744:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.183747:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.183748:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.183748:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.183750:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.183751:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.183752:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.183753:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.183754:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.183757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.183759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.183765:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.183767:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677474.183768:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677474.183768:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.183770:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:20.0:1619677474.183770:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.183771:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.183771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.183772:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.183773:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.183781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.183788:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.183791:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.183792:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.183793:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.183793:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.183794:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.184002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.184007:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677474.184007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.0:1619677474.184009:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.184011:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.184013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.184014:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.184015:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:21.0:1619677474.184016:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.184017:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.184017:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.184018:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:21.0:1619677474.184018:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.184020:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.184021:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677474.184023:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.184025:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.184026:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.184028:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677474.184028:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.184029:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.184030:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.184031:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.184031:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:10.0:1619677474.184031:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.184032:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.184033:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:10.0:1619677474.184033:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677474.184034:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.184034:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.184035:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:10.0:1619677474.184036:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.184038:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.2:1619677474.184039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677474.184041:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.184042:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.184043:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.184044:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.184045:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.184046:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.184046:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677474.184048:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.184050:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.184051:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.184052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.184053:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.184056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677474.184058:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.184061:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.184062:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.184062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.184064:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.184067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.184220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.184226:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.184229:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.184230:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.184232:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.184234:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.184236:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.184237:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.184238:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.184239:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.184240:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.184244:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.184245:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.184248:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.184248:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.184249:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.184251:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.184251:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.184252:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.184252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.184264:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.184267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.184267:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.184268:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.184270:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.184272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.184286:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.184288:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.184289:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.184289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.184290:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.184291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.184494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.184499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.184501:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.184503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.184504:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.184506:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.184507:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.184509:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.184510:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.184511:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.184512:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.184516:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.184517:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.184520:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.184520:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.184521:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.184523:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.184523:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.184524:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.184524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.184537:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.184539:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.184540:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.184542:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.184543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.184544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.184755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.184764:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.184766:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.184769:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.184771:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.184771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.184773:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.184774:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.184776:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.184777:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.184778:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.184781:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.184783:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.184785:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.184786:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.184787:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.184788:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.184789:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.184790:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.184790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.184800:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.184803:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.184804:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.184805:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.184807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.184808:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.185078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.185086:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.185089:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.185090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.185092:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.185095:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.185097:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.185098:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.185099:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.185100:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.185101:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.185104:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.185105:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.185108:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.185109:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.185110:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.185112:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.185112:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.185113:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.185113:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.185125:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.185128:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.185128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.185129:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.185131:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.185134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.185399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.185407:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.185410:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.185412:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.185413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.185415:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.185415:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.185416:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.185417:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.185419:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.185420:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.185421:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.185421:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677474.185424:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.185424:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.185425:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677474.185426:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.185427:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.185429:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.185430:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:26.0:1619677474.185430:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.185431:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677474.185431:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.185432:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677474.185432:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.185433:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.185433:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.185434:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:26.0:1619677474.185434:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.185434:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.185435:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.185439:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.185440:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.185443:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.185444:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.185444:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:26.0:1619677474.185447:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677474.185447:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:26.0:1619677474.185448:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.185448:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.185449:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.185450:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677474.185451:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:22.0:1619677474.185452:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677474.185452:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.185453:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.185454:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.185454:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.185457:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.185457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.185458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.185476:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.185479:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.185480:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.185480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.185481:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.185481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.186264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.186272:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.186275:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.186277:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.186278:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.186280:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.186281:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.186282:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.186284:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.186285:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.186286:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.186289:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.186290:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.186292:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.186293:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.186298:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.186299:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.186300:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.186301:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.186301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.186313:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.186315:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.186315:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.186317:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.186318:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.186320:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.186372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.186379:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677474.186381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:10.0:1619677474.186382:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.186383:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.186385:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.186387:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.186387:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677474.186389:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677474.186390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.186390:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:10.0:1619677474.186391:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.186393:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.186393:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.186394:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.186394:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.186395:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677474.186396:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.186397:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.186399:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.186400:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.186401:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677474.186406:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.186406:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.186409:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.186410:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.186412:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.186413:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.186413:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677474.186414:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.186414:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677474.186415:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.186416:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.186416:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.186416:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:10.0:1619677474.186417:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.186417:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677474.186418:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.186418:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:10.0:1619677474.186419:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.186420:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.186425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677474.186429:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.186431:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.186432:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.186433:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.186434:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677474.186434:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.186435:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677474.186436:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677474.186436:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677474.186436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677474.186437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677474.186437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.186438:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.186438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677474.186440:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.186443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677474.186451:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.186453:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.186454:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.186455:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.186455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.186457:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.187203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677474.187211:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.187213:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677474.187216:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.187217:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677474.187218:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.187220:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677474.187221:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.187222:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.187223:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.187224:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677474.187228:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.187230:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677474.187232:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.187233:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:31.0:1619677474.187233:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677474.187235:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:31.0:1619677474.187235:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.187236:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677474.187237:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:13.0:1619677474.187248:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677474.187251:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677474.187251:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.187253:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.187255:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.187256:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677474.187266:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677474.187269:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677474.187270:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677474.187270:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677474.187271:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677474.187273:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677474.187451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.187459:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.187461:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.187463:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.187465:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.187466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.187467:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.187468:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.187469:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.187470:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.187471:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.187474:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.187476:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.187479:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.187479:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.187480:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.187482:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.187482:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.187484:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.187485:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.187486:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.187491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.187497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.187500:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677474.187500:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.187501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.187501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.187502:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.187503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.187503:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.187504:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.187506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.187547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.187555:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.187558:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.187560:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.187562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.187563:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.187564:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.187565:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.187567:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.187568:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.187569:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.187572:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.187573:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.187576:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.187577:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.187578:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.187580:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.187580:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.187581:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.187582:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.187587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.187595:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.187597:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.187598:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.187600:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.187601:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.187603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.187848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.187853:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.187856:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.187857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.187859:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.187861:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.187862:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.187863:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.187865:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.187866:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.187867:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.187870:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.187872:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.187875:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.187875:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.187876:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.187878:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.187878:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.187879:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.2:1619677474.187879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677474.187880:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.187881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:2.0:1619677474.187885:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.2:1619677474.187886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677474.187888:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.187889:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677474.187891:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.187891:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:2.0:1619677474.187891:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.187892:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.187893:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.187893:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677474.187894:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.187894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:2.0:1619677474.187894:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:6.0:1619677474.187895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:2.0:1619677474.187895:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.187896:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.187897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:2.0:1619677474.187897:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.187897:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.187898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:2.0:1619677474.187899:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.187902:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.187904:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.187907:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.187908:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.187908:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.187910:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.187911:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.187912:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.187913:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.187914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677474.187919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.187923:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677474.187925:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677474.187926:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.187927:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.187927:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.187928:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677474.187929:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.187929:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677474.187930:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.187932:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677474.187945:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.187948:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.187949:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.187949:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.187950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.187951:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.188340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.188348:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.188351:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.188352:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.188354:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.188356:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.188358:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.188359:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.188360:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.188361:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.188362:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.188365:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.188367:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.188370:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.188371:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.188371:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.188373:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.188374:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.188374:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.188375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.188388:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.188391:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.188392:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.188392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.188394:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.188397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.188515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.188521:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.188524:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.188525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.188528:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.188530:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.188531:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.188532:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.188534:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.188535:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.188536:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.188539:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.188541:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.188544:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.188544:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.188545:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.188547:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.188547:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.188548:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.188550:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.188550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.188554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.188555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.188561:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677474.188562:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.188562:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.188563:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677474.188564:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677474.188564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:21.0:1619677474.188565:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.188567:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.188568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.188570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.188574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.188581:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.188584:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.188585:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.188586:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.188586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.188587:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.188824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.188832:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.188835:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.188837:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.188840:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.188840:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.188842:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.188843:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.188844:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.188845:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.188846:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.188850:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.188851:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.188854:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.188855:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.188856:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.188857:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.188858:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.188860:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.188861:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.188862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.188863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.188867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.188870:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.188872:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.188874:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.188875:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.188876:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.188877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.188878:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:20.0:1619677474.188878:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:31.0:1619677474.188879:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.188880:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677474.188914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.188919:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.188922:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.188923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.188925:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.188927:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.188929:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.188930:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.188931:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.188932:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.188933:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.188936:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.188938:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.188940:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.188941:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.188942:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.188943:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.188944:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.188944:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.188945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.188957:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.188959:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.188960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.188960:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.188962:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.188964:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.189177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.189182:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.189185:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.189185:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.189187:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.189189:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.189190:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.189191:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.189193:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.189194:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.189194:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.189197:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.189199:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.189201:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.189202:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.189203:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.189204:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.189205:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.189205:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.189206:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.2:1619677474.189211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.189218:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677474.189219:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.189221:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677474.189222:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:21.0:1619677474.189222:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.189224:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.189224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677474.189225:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.189226:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677474.189226:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677474.189228:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.189230:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.189231:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.189233:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.189234:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.189235:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.189238:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.189240:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.189243:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.189244:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.189244:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.189246:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.189247:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.189248:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.189248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.189260:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.189262:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.189263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.189264:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.189266:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.189268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.189575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.189583:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.189585:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.189587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.189588:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.189592:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.189593:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.189594:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.189596:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.189596:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.189598:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.189601:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.189603:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.189605:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.189606:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.189607:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.189608:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.189609:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.189609:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.189610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.189623:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.189626:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.189627:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.189627:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.189629:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.189635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.189636:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.189638:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.189639:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.189640:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.189640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.189641:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.189641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.189650:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.189653:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.189655:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.189657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.189658:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.189659:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.189661:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.189662:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.189663:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.189664:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.189667:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.189668:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.189670:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.189671:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.189671:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.189673:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.189673:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.189674:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.189675:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.189685:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.189688:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.189688:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.189690:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.189692:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.189693:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.189923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.189927:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.189929:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.189932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.189932:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.189934:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.189935:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.189937:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.189938:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.189939:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.189940:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.189943:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.189945:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.189948:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.189949:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.189949:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.189951:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.189952:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.189953:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.189953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.189957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.189964:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.189967:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.189968:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.189968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.189970:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.189973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.190316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.190324:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.190327:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.190328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.190330:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.190332:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.190333:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.190335:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.190336:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.190337:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.190338:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.190341:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.190343:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.190346:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.190347:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.190347:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.190349:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.190350:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.190350:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.190351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.190364:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.190367:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.190368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.190368:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.190371:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.190373:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.190588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.190594:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.190596:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.190599:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.190599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.190602:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.190603:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.190605:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.190606:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.190607:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.190608:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.190611:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.190613:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.190615:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.190616:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.190617:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.190618:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.190619:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.190620:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.190620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.190625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.190634:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.190636:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.190638:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.190638:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.190639:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.190642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.190743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.190751:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.190754:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.190756:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.190758:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.190758:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.190760:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.190761:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.190762:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.190763:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.190764:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.190768:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.190769:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.190772:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.190773:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.190773:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.190775:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.190775:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.190776:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.190777:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.190780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.190787:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.190789:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.190790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.190790:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.190792:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.190794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.191057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.191065:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.191068:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.191068:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.191071:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.191073:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.191074:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.191076:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.191077:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.191078:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.191079:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.191082:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.191084:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.191087:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.191087:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.191088:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.191090:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.191090:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.191091:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.191091:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.191096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.191104:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.191106:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.191108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.191108:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.191110:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.191113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.191131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.191136:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.191139:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.191140:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.191142:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.191144:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.191145:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.191147:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.191148:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.191149:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.191150:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.191153:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.191155:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.191158:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.191158:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.191159:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.191161:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.191161:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.191162:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.191162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.191174:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.191176:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.191177:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.191179:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.191180:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.191181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.191288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.191295:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.191301:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.191302:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677474.191305:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.191308:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.191309:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.191310:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.191312:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.191313:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.191314:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.191317:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.191319:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.191321:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.191322:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.191323:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.191325:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.191325:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.191326:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.191328:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.191328:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.191334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.191339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.191342:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.191342:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.191343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.191343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.191344:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.191345:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677474.191345:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.191347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.191348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.191631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.191639:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.191641:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.191642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.191644:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.191647:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.191648:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.191649:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.191651:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.191652:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.191652:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.191655:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.191657:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.191659:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.191659:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.191660:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.191661:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.191662:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.191662:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.191662:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.191667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.191673:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.191675:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.191676:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.191677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.191678:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.191680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.191686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.191692:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.191695:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.191695:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.191696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.191696:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.191697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.191736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.191744:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.191747:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.191749:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.191751:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.191752:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.191752:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.191753:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.191755:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.191756:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.191756:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.191759:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.191761:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.191763:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.191763:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.191764:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.191765:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.191766:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.191767:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.191767:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.191780:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.191783:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.191784:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.191785:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.191787:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.191788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.191807:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.191809:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.191810:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.191811:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.191811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.191812:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.191974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.191982:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.191984:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.191986:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.191988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.191988:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.191989:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.191990:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.191991:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.191992:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.191993:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.191996:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.191998:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 2328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.192001:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.192001:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.192002:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.192003:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.192004:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.192005:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.192005:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.192010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.192016:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.192018:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.192019:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.192019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.192021:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.192023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.192179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.192186:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.192189:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.192189:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.192192:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.192194:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.192195:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.192197:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.192198:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.192199:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.192200:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.192202:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.192204:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.192206:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.192207:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.192207:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.192209:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.192210:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.192211:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.192212:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.192212:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.192215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.192218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.192224:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.192225:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.192225:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.192226:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677474.192226:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.192227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.192228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.192228:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.192229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.192229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.192279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38ba00] (20)++
00000800:00000200:31.0:1619677474.192287:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38ba00] (21)++
00000800:00000200:31.0:1619677474.192290:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.45@o2ib3
00000400:00000200:31.0:1619677474.192292:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.45@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.192293:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38ba00] (22)--
00000400:00000200:31.0:1619677474.192300:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.192302:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:31.0:1619677474.192304:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.192305:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.192306:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.192307:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677474.192310:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.192312:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.192315:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.192316:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:31.0:1619677474.192317:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677474.192319:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677474.192319:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38ba00] (21)++
00000800:00000200:31.0:1619677474.192321:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38ba00] (22)++
00000800:00000200:31.0:1619677474.192322:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38ba00] (23)--
00000800:00000200:31.0:1619677474.192323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38ba00] (22)--
00000800:00000200:4.2:1619677474.192329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38ba00] (21)++
00000800:00000200:21.0:1619677474.192334:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.192337:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677474.192338:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38ba00] (22)++
00000400:00000200:21.0:1619677474.192338:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.192340:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38ba00] (23)--
00000400:00000200:21.0:1619677474.192340:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.192340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677474.192341:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38ba00] (22)--
00000800:00000200:30.0:1619677474.192342:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38ba00] (21)--
00000800:00000200:21.0:1619677474.192342:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.192353:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.192355:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.192356:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.192356:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.192357:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.192358:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.192567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.192573:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.192576:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.192576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.192578:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.192580:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.192582:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.192583:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.192584:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.192585:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.192586:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.192589:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.192591:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.192593:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.192594:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.192595:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.192596:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.192597:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.192597:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.192598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.192610:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.192612:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.192613:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.192615:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.192616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.192616:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.2:1619677474.193025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.193032:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.193035:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.193035:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.193037:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.193039:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.193041:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.193042:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.193043:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.193044:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.193045:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.193049:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.193051:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.193054:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.193055:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.193056:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.193057:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.193058:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.193059:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.193059:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677474.193070:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.193073:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.193074:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.193076:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.193078:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.193079:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.193109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.193117:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.193120:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.193121:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.193122:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.193124:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.193126:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.193127:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.193129:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.193130:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.193131:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.193134:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.193136:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.193139:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.193140:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.193140:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.193142:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.193142:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.193143:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.193145:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.193145:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.193151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.193156:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.193158:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.193159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.193159:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677474.193160:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.193161:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.193162:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.193162:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.193163:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.193164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.193248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.193255:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.193258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.193258:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677474.193260:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.193263:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.193264:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.193266:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.193267:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.193268:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.193269:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.193272:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.193273:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.193276:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.193276:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.193277:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.193279:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.193279:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.193280:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.193280:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.193286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.193294:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.193300:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.193301:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.193302:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.193303:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.193305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.193658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677474.193662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677474.193666:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677474.193668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:17.0:1619677474.193668:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677474.193669:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.193671:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.193671:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.193672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677474.193674:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.193674:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.193674:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.193675:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677474.193675:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.193676:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.193676:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.193677:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.193678:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.193678:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677474.193679:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.193679:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.193680:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677474.193681:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.193682:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.193683:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.193684:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677474.193685:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.193687:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.193688:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.193689:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96505616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.193689:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.193690:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.193691:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.193692:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677474.193693:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.193694:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.193695:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.193696:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.193696:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677474.193697:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.193698:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.193698:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:17.0:1619677474.193698:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.193698:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677474.193699:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677474.193699:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:31.0:1619677474.193700:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.193701:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.193701:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677474.193701:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.193701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677474.193702:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.193703:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.193704:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.193704:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.193704:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.193705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.193706:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.193706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:5.2:1619677474.193706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677474.193707:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.193707:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.193708:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677474.193708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.193711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677474.193712:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.193714:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.193715:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.193716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.193717:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:30.0:1619677474.193719:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677474.193719:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.193720:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.193720:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.193722:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.193722:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677474.193722:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.193722:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:6.0:1619677474.193722:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.193723:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677474.193723:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.193724:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.193724:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.193725:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:21.0:1619677474.193725:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.193725:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:22.0:1619677474.193727:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677474.193727:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.193728:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677474.193728:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677474.193729:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.193730:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.193731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.193731:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.193732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.193732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.193937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.193943:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.193945:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.193946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.193948:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.193950:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.193951:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.193952:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.193953:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.193954:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.193955:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.193959:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.193961:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.193963:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.193964:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.193965:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.193966:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.193967:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.193968:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.193969:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.193969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.193974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677474.193979:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.193979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.193981:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.193981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.193982:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.193982:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.193982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.193984:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.193985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.193987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.194056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39fa00] (20)++
00000800:00000200:8.0:1619677474.194063:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)++
00000800:00000200:24.0:1619677474.194067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)--
00000800:00000200:8.0:1619677474.194067:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.46@o2ib3
00000400:00000200:8.0:1619677474.194070:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.46@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.194073:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.194075:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:8.0:1619677474.194076:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.194078:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.194079:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.194080:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:8.0:1619677474.194083:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.194085:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:4.2:1619677474.194085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39fa00] (21)++
00000800:00000200:8.0:1619677474.194088:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.194089:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:8.0:1619677474.194089:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:8.0:1619677474.194091:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:24.0:1619677474.194092:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)++
00000800:00000200:24.0:1619677474.194093:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.46@o2ib3
00000800:00000200:8.0:1619677474.194093:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39fa00] (22)++
00000800:00000200:8.0:1619677474.194094:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39fa00] (24)++
00000800:00000200:8.0:1619677474.194097:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39fa00] (25)--
00000800:00000200:24.2:1619677474.194098:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:8.0:1619677474.194098:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (24)--
00000800:00000200:8.0:1619677474.194099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (23)--
00000400:00000200:24.0:1619677474.194101:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.46@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.194103:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.194103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39fa00] (22)++
00000400:00000200:24.0:1619677474.194104:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677474.194105:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)++
00000400:00000200:24.0:1619677474.194105:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96505613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.194107:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2fa00] (23)--
00000400:00000200:24.0:1619677474.194107:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.194108:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.194108:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.194109:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.194109:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)--
00000400:00000200:26.0:1619677474.194110:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.51@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.194111:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39fa00] (23)++
00000800:00000200:8.0:1619677474.194111:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39fa00] (24)--
00000800:00000200:8.0:1619677474.194112:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (23)--
00000800:00000200:26.0:1619677474.194113:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (21)--
00000400:00000200:24.0:1619677474.194114:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.194114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)--
00000800:00000200:24.0:1619677474.194116:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.194119:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.194119:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.194120:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.194121:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.194122:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39fa00] (21)++
00000800:00000200:24.0:1619677474.194123:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39fa00] (22)--
00000800:00000200:24.0:1619677474.194123:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)--
00000800:00000200:22.0:1619677474.194146:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.194148:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.194149:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.194149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.194152:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.194154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.194163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39fa00] (20)++
00000800:00000200:6.0:1619677474.194168:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.194170:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)++
00000800:00000200:6.0:1619677474.194170:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.194171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.194171:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.194172:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.194173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)--
00000800:00000200:8.0:1619677474.194173:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.46@o2ib3
00000800:00000200:6.0:1619677474.194173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:8.0:1619677474.194175:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.46@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.194178:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.194179:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677474.194180:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.194182:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.194183:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.194184:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677474.194186:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.194188:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677474.194191:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.194191:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677474.194192:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677474.194193:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677474.194194:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39fa00] (21)++
00000800:00000200:8.0:1619677474.194195:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39fa00] (22)--
00000800:00000200:8.0:1619677474.194195:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)--
00000800:00000200:8.2:1619677474.194199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.194199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39fa00] (20)++
00000800:00000200:24.0:1619677474.194207:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)++
00000800:00000200:21.0:1619677474.194207:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:24.0:1619677474.194210:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.46@o2ib3
00000800:00000200:21.0:1619677474.194210:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.194211:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.194211:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)--
00000400:00000200:24.0:1619677474.194212:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.46@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.194212:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.194213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:24.0:1619677474.194214:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.194215:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:24.0:1619677474.194217:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677474.194218:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.194220:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.194220:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.194221:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677474.194224:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.194225:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677474.194227:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.194228:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:24.0:1619677474.194228:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677474.194230:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:24.0:1619677474.194230:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39fa00] (21)++
00000800:00000200:24.0:1619677474.194231:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39fa00] (22)--
00000800:00000200:24.0:1619677474.194232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)--
00000800:00000200:13.0:1619677474.194243:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.194246:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.194247:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.194248:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.194251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.194252:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.194282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39fa00] (20)++
00000800:00000200:8.0:1619677474.194289:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)++
00000800:00000200:8.0:1619677474.194292:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.46@o2ib3
00000800:00000200:24.0:1619677474.194293:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)--
00000400:00000200:8.0:1619677474.194300:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.46@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.194302:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.194304:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677474.194305:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.194306:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.194307:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.194308:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677474.194311:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.194313:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.194314:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.194315:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677474.194316:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677474.194317:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677474.194318:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39fa00] (21)++
00000800:00000200:8.0:1619677474.194319:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39fa00] (22)++
00000800:00000200:8.0:1619677474.194320:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39fa00] (23)--
00000800:00000200:8.0:1619677474.194321:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)--
00000800:00000200:4.2:1619677474.194327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39fa00] (21)++
00000800:00000200:30.0:1619677474.194331:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.194333:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.194334:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.194334:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)++
00000400:00000200:30.0:1619677474.194335:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.194336:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39fa00] (23)--
00000800:00000200:24.0:1619677474.194337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)--
00000800:00000200:30.0:1619677474.194338:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677474.194338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)--
00000800:00000200:31.0:1619677474.194339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.194501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.194504:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.194507:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.194509:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.194510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.194511:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.194513:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.194514:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.194516:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.194516:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.194518:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.194521:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.194523:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.194525:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.194526:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.194527:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.194529:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.194529:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.194530:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.194530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677474.194542:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.194545:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.194546:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.194546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.194548:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.194550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.194553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.194561:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.194564:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.194566:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.194568:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.194569:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.194570:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.194572:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.194573:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.194575:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.194576:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:4.2:1619677474.194577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:30.0:1619677474.194579:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.194581:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.194584:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.194585:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.194585:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677474.194586:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.194588:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.194588:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.194589:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677474.194589:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:30.0:1619677474.194590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677474.194590:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.194591:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.194594:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677474.194594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:10.0:1619677474.194596:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677474.194597:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.194598:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.194599:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.194600:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:21.0:1619677474.194602:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:10.0:1619677474.194603:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.194604:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677474.194605:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.194606:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.194606:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677474.194607:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.194607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.194607:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.194607:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.194608:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.194609:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.0:1619677474.194610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677474.194610:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.194610:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.194611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:9.2:1619677474.194614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677474.194619:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.194620:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:20.0:1619677474.194621:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.194622:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:20.0:1619677474.194622:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000400:00000200:26.0:1619677474.194623:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.194623:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.194624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.194624:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677474.194625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:26.0:1619677474.194626:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.194627:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677474.194637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.194643:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.194645:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.194646:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.194646:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.194647:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.194648:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.194671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39fa00] (20)++
00000800:00000200:8.0:1619677474.194678:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)++
00000800:00000200:8.0:1619677474.194681:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.46@o2ib3
00000800:00000200:24.0:1619677474.194682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)--
00000400:00000200:8.0:1619677474.194683:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.46@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.194685:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.194687:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677474.194688:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.194689:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.194690:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.194691:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677474.194694:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.194695:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.194697:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.194697:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.194698:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.194699:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.194700:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39fa00] (21)++
00000800:00000200:8.0:1619677474.194700:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39fa00] (22)--
00000800:00000200:8.0:1619677474.194701:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)--
00000800:00000200:9.2:1619677474.194705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.194711:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.194713:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.194713:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.194713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39fa00] (20)++
00000400:00000200:24.0:1619677474.194715:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.194716:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677474.194716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.194717:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)++
00000800:00000200:8.0:1619677474.194719:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.46@o2ib3
00000800:00000200:24.0:1619677474.194720:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)--
00000400:00000200:8.0:1619677474.194721:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.46@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.194723:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.194724:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677474.194725:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.194726:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.194727:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.194728:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677474.194730:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.194731:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.194732:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.194733:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.194733:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.194735:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.194735:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39fa00] (21)++
00000800:00000200:8.0:1619677474.194736:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39fa00] (22)--
00000800:00000200:8.0:1619677474.194736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)--
00000800:00000200:9.2:1619677474.194740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.194746:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.194748:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.194749:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.194750:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.194750:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.194751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.194753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39fa00] (20)++
00000800:00000200:8.0:1619677474.194761:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)++
00000800:00000200:8.0:1619677474.194763:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.46@o2ib3
00000800:00000200:24.0:1619677474.194764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (22)--
00000400:00000200:8.0:1619677474.194765:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.46@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.194766:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.194767:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677474.194768:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.194769:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.194770:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.194771:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677474.194774:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.194775:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.194777:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.194778:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677474.194778:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677474.194779:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677474.194780:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39fa00] (21)++
00000800:00000200:8.0:1619677474.194780:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39fa00] (22)--
00000800:00000200:8.0:1619677474.194781:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39fa00] (21)--
00000800:00000200:31.0:1619677474.194792:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.194795:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.194795:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.194797:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.194799:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677474.194799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:4.2:1619677474.195042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.195045:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.195048:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.195050:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.195051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.195052:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.195054:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.195055:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.195057:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.195058:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.195059:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.195062:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.195063:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.195067:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.195068:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.195068:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.195070:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.195071:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.195071:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.195072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677474.195083:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.195086:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:4.2:1619677474.195086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:30.0:1619677474.195087:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.195089:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.195090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.195092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:12.0:1619677474.195094:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.195097:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.195099:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.195101:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.195101:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.195103:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677474.195104:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.195105:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.195106:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.195107:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677474.195110:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.195112:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677474.195114:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.195115:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677474.195116:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677474.195117:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677474.195118:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.195119:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.195119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.195123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.195129:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.195132:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.195133:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.195133:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677474.195135:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.195137:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677474.195286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.195293:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.195298:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.195301:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677474.195303:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.195304:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.195306:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677474.195306:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.195308:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.195308:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.195309:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677474.195312:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.195313:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.195315:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.195316:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.195316:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.195317:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.195318:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.195318:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.195319:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.195323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.195330:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.195332:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.195332:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.195333:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.195334:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.195335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.195547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.195554:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.195557:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.195558:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.195559:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.195561:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.195562:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.195563:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.195564:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.195565:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.195565:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.195568:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.195570:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.195572:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.195573:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.195573:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.195575:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.195575:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.195576:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.195577:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.195581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.195589:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.195591:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.195593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.195593:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.195595:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.195598:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.195599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677474.195606:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.195609:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.195610:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.195610:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.195611:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.195612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.195613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.195622:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.195624:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.195626:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.195629:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.195629:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.195631:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.195632:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.195633:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.195634:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.195635:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.195638:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.195640:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.195642:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.195643:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.195644:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.195645:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.195646:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.195647:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.195647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.195657:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.195660:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.195661:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.195662:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.195664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.195665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.195698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.195706:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.195708:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.195709:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.195711:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.195713:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.195714:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.195716:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.195717:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.195718:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.195719:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.195722:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.195723:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.195726:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.195726:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.195727:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.195728:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.195729:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.195729:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.195730:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.195734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.195741:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.195744:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.195745:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.195745:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.195747:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.195749:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.195880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.195887:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.195890:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.195890:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.195892:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.195895:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.195896:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.195897:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.195899:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.195900:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.195901:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.195904:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.195906:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.195908:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.195909:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.195910:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.195912:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.195912:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.195913:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.195913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.195925:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.195928:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.195928:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.195930:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.195931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.195932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.196073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.196080:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.196083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.196083:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.196085:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.196087:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.196089:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.196090:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.196092:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.196093:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.196094:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.196097:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.196098:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.196101:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.196102:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.196103:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.196104:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.196105:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.196105:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.196106:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.196118:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.196120:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.196121:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.196122:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.196124:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.196127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.196238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.196245:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.196248:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.196250:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.196251:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.196252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.196253:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.196254:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.196255:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.196256:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.196257:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.196260:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.196262:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.196264:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.196265:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.196266:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.196267:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.196268:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.196269:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.196270:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.196271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.196277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.196280:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.196283:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.196284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.196284:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.196285:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.196286:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.196288:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.196289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.196290:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.196290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.196557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.196560:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.196562:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.196565:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.196565:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.196567:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.196569:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.196570:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.196572:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.196573:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.196574:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.196577:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.196579:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.196581:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.196582:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.196582:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.196584:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.196585:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.196586:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.196587:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.196588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.196592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.196593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.196598:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.196600:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.196601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.196601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.196601:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.196604:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.196605:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.196605:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.2:1619677474.196607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:17.0:1619677474.196608:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.196611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.196612:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.196614:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.196615:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.196615:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.196616:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.196616:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.196725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.196733:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677474.196734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.196736:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.196739:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.196741:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.196741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677474.196742:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677474.196743:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:10.0:1619677474.196744:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:31.0:1619677474.196745:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.196745:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677474.196746:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.196748:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.196748:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.196749:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.196750:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.196751:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.196753:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.196753:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.196754:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.196755:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.196755:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.196756:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.196757:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.196758:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.196759:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677474.196759:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.196760:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.196761:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.196761:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.196762:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.196762:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677474.196763:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.196764:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.196764:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677474.196766:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:10.0:1619677474.196766:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.196767:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:9.2:1619677474.196767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677474.196768:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.196769:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.196773:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:4.2:1619677474.196774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.196775:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:24.0:1619677474.196776:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.196776:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)++
00000400:00000200:24.0:1619677474.196778:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.196778:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (24)--
00000400:00000200:8.0:1619677474.196778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.196780:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.196781:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677474.196781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.196782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677474.196782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.196783:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.196784:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.196785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.196785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.196791:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.196794:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.196795:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.196795:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.196796:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.196796:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.197416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.197422:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.197425:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.197426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.197428:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.197430:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.197431:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.197432:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.197434:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.197435:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.197436:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.197439:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.197441:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.197443:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.197444:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.197445:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.197446:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.197447:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.197448:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.197448:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.197462:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.197464:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.197465:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.197466:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.197467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.197469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.197479:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.197481:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.197482:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.197483:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.197484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.197485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.197652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.197656:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.197659:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.197661:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.197661:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.197663:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.197664:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.197665:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.197667:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.197668:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.197669:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.197672:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.197674:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.197676:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.197677:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.197677:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.197679:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.197680:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.197680:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.197681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.197692:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.197695:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.197696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.197696:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.197698:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.197700:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.197712:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.197715:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.197716:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.197716:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.197717:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.197717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.197905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.197913:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.197916:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.197917:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.197919:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.197921:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.197922:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.197924:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.197925:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.197926:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.197927:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.197930:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.197932:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.197935:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.197936:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.197936:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.197938:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.197939:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.197939:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.197940:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.197952:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.197954:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.197955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.197955:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.197958:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.197960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.197966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.197974:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.197977:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.197980:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.197982:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.197982:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.197984:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.197985:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.197987:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.197988:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.197988:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.197992:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.197993:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.197996:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.197997:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.197997:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.197999:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.198000:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.198000:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.198001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.198012:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.198015:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.198016:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.198016:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.198018:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.198021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.198086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.198092:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.198095:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.198096:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.198098:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.198101:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.198102:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.198104:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.198105:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.198106:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.198107:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.198110:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.198112:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.198114:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.198115:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.198116:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.198117:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.198118:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.198119:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.198120:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.198124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.198133:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.198135:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.198136:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.198136:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.198138:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.198141:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.198339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.198347:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.198350:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.198351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.198353:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.198356:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.198357:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677474.198359:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.198360:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.198361:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.198362:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677474.198364:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.198366:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.198368:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.198369:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.198369:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.198371:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.198371:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.198372:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.198373:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.198374:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677474.198377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:4.2:1619677474.198380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.198385:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.198386:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.198386:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.198388:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.198389:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.198390:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.198390:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.198390:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.198391:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677474.198391:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.198576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.198585:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.198587:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.198589:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.198592:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.198592:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.198593:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.198594:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.198595:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.198596:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.198597:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.198600:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.198602:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.198604:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.198604:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.198605:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.198606:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.198607:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.198608:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.198609:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.198610:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.198615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.198620:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.198622:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.198623:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.198624:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.198624:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.198625:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.198625:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.198626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.198627:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.198628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.198676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.198682:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.198684:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.198685:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.198687:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.198689:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.198691:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677474.198692:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.198694:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.198695:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.198696:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677474.198699:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.198701:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677474.198704:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.198705:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677474.198706:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677474.198708:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677474.198708:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.198709:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.198709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677474.198721:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.198724:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.198725:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.198727:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.198728:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.198729:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677474.198739:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.198742:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.198743:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.198744:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.198744:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.198746:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.198836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.198842:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.198845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.198845:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.198847:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.198849:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.198851:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.198852:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.198853:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.198854:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.198856:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.198859:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.198860:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.198863:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.198864:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.198864:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.198866:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.198867:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.198867:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.198868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.198880:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.198882:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.198883:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.198884:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.198885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.198887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677474.198900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.198906:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.198909:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.198910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.198912:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.198915:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.198916:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.198917:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.198919:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.198920:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.198921:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.198924:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.198926:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.198928:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.198929:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.198930:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.198931:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.198932:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.198932:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.198933:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.198945:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.198947:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.198948:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.198948:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.198950:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.198952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.199838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677474.199845:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.199848:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677474.199849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677474.199850:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.199853:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.199855:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:24.0:1619677474.199856:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.199857:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.199858:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.199859:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677474.199863:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.199865:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:24.0:1619677474.199867:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.199868:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:24.0:1619677474.199868:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:24.0:1619677474.199870:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:24.0:1619677474.199870:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.199871:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677474.199872:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677474.199873:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:9.2:1619677474.199876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:4.2:1619677474.199879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000800:00000200:10.0:1619677474.199884:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.199886:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:8.0:1619677474.199886:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:26.0:1619677474.199887:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.199887:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.199888:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000400:00000200:10.0:1619677474.199889:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.199889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.199890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:10.0:1619677474.199891:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.200033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.200041:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.200044:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.200046:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.200048:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.200049:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.200050:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.200051:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.200052:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.200053:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.200054:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.200057:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.200059:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677474.200059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:31.0:1619677474.200062:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.200063:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.200063:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.200063:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:31.0:1619677474.200065:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.200065:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:31.0:1619677474.200066:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.200066:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.200067:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:20.0:1619677474.200067:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.200068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.200070:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.200071:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.200072:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.200074:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.200074:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.200075:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.200078:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.200079:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.200080:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677474.200082:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.200083:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677474.200083:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.200083:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:6.0:1619677474.200083:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.200084:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677474.200085:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.200086:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.200086:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.200088:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677474.200088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677474.200089:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.200089:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.200092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.200094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677474.200098:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677474.200100:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677474.200101:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.200101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.200102:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.200102:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.200103:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:24.0:1619677474.200104:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.200104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677474.200107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.200119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.200125:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.200128:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.200128:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.200129:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.200130:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.200130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.200416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.200424:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.200426:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.200427:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.200429:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.200432:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.200433:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.200434:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.200435:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.200436:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.200437:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.200440:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.200442:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.200445:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.200446:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.200446:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.200448:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.200449:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.200449:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.200451:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.200451:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677474.200455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.200457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677474.200463:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.200465:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677474.200465:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:21.0:1619677474.200466:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.200467:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:21.0:1619677474.200468:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.200468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.200468:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.200469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677474.200472:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.200770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.200776:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.200779:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.200780:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.200782:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.200784:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.200785:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.200786:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677474.200787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000400:00000200:22.0:1619677474.200788:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.200789:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.200789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000400:00000200:22.0:1619677474.200790:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.200793:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.200794:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677474.200794:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:24.0:1619677474.200796:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:17.0:1619677474.200796:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:22.0:1619677474.200797:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.200797:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:22.0:1619677474.200798:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677474.200799:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:22.0:1619677474.200799:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.200800:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:17.0:1619677474.200800:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677474.200800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:22.0:1619677474.200801:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677474.200802:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.200802:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.200803:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.200803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:17.0:1619677474.200803:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.200804:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.200804:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.200806:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.200807:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:21.2:1619677474.200808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:17.0:1619677474.200808:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.200809:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96505576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.200809:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.200810:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.200811:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.200811:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.200813:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677474.200813:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.200815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:17.0:1619677474.200815:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677474.200815:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.200816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677474.200817:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.200817:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:17.0:1619677474.200817:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.200817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.200818:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.200820:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.200820:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.200820:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.200821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677474.200821:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.200822:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.200822:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.200823:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.200823:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677474.200824:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677474.200824:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.200824:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.200825:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:24.0:1619677474.200826:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677474.200826:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:17.0:1619677474.200826:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:24.0:1619677474.200827:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677474.200828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:22.0:1619677474.200837:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.2:1619677474.200837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677474.200840:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.200841:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.200841:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.200843:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:22.0:1619677474.200843:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.200845:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:22.0:1619677474.200845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.200846:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677474.200846:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.200848:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.200848:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.200850:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.200851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.200852:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:31.0:1619677474.200853:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.200853:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677474.200854:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.200854:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.200854:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.200855:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.200855:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:31.0:1619677474.200856:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.200856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.200857:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.200857:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:31.0:1619677474.200858:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.200862:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.200863:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.200865:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.200866:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.200867:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.200868:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.200868:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.200869:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.200870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.200880:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.200882:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.200883:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.200884:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.200886:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.200887:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.200919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.200927:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.200930:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.200930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.200933:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.200935:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.200936:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.200937:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.200939:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.200940:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.200941:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.200944:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.200946:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.200948:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.200949:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.200949:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.200951:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.200952:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.200952:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.200953:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.200957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.200964:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.200966:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.200966:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.200967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.200968:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.200970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.200976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.200983:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.200986:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.200987:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.200987:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.200987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.200988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.200988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.200996:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.200999:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.200999:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.201002:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.201004:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.201005:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.201007:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.201008:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.201009:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.201010:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.201014:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.201015:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.201018:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.201019:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.201020:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.201021:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.201022:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.201022:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.201023:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.201037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.201044:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.201046:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.201047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.201047:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.201049:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.201051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.201776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.201783:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.201785:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.201786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.201788:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.201790:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.201792:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.201793:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.201794:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.201795:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.201796:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.201799:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.201800:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.201803:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.201804:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.201805:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.201806:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.201807:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.201807:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.201808:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677474.201820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677474.201822:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.201824:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.201825:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.201826:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.201828:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677474.201828:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.201831:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677474.201831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677474.201833:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.201835:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.201836:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.201836:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.201837:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.201838:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.201839:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677474.201840:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.201840:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.201842:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.201843:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.201843:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677474.201844:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.201844:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.201845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.201846:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.201848:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.201849:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.201849:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.201851:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.201851:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.201852:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.201853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.201864:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.201866:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.201867:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.201868:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.201870:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.201872:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.201900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.201908:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.201910:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.201911:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.201913:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.201915:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.201917:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.201918:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.201919:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.201920:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.201921:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.201924:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.201925:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.201927:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.201928:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.201929:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.201930:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.201931:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.201931:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.201932:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.201947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.201953:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.201955:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.201956:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.201957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.201958:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.201961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.202164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.202172:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.202174:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.202176:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.202178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.202178:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.202180:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.202181:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.202182:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.202183:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.202184:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.202187:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.202188:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677474.202188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677474.202191:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.202192:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.202192:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.202193:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677474.202194:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.202194:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.202195:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.202195:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:30.0:1619677474.202196:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.202196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.202198:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.202200:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.202201:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.202203:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.202205:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.202206:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.202207:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.202209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.202211:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.202211:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.202212:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.202213:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.202214:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.202216:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.202217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.202217:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.202218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.202218:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.202220:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.202221:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.202222:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.202222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.202234:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.202237:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.202237:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.202239:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.202240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.202241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.202667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677474.202673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:26.0:1619677474.202674:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.202677:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677474.202677:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:10.0:1619677474.202678:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677474.202680:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:26.0:1619677474.202682:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.202683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.202683:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.202684:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.202685:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.202686:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677474.202687:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.202688:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.202688:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.202690:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.202691:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.202692:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.202692:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.202694:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677474.202694:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.202698:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.202700:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.202702:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.202702:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.202703:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.202704:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677474.202704:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.202705:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.202706:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.202706:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:26.0:1619677474.202707:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.202707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:26.0:1619677474.202708:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.202708:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.202710:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.202711:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.202711:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.202712:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.202722:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.202724:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.202724:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:22.0:1619677474.202725:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.202725:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.202726:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.202728:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.202728:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.202731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.202732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.202732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.202733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.202963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.202968:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.202970:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.202973:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.202973:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.202975:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.202977:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.202978:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.202980:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.202981:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.202982:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.202986:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.202988:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.202990:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.202991:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.202991:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.202993:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.202994:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.202994:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.202995:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.203000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.203009:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.203011:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.203013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.203013:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.203015:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.203017:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.203512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.203518:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.203520:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.203521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.203523:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.203525:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.203527:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.203528:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.203529:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.203530:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.203531:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.203535:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.203536:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.203538:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.203539:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.203540:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.203542:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.203542:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.203543:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.203543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.203556:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.203559:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.203560:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.203561:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.203563:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.203563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.203575:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.203578:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.203579:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.203580:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.203580:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.203581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677474.203612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.203619:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.203621:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.203622:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.203624:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.203626:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.203628:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.203629:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.203630:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.203632:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.203633:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.203636:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.203637:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.203640:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.203641:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.203641:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.203643:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.203643:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.203644:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.203644:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.203656:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.203659:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.203660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.203660:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.203662:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.203664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.203674:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.203677:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.203677:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.203678:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.203679:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.203679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.204236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.204245:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.204247:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.204250:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677474.204252:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.204253:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.204254:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.204256:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.204257:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.204258:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.204259:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.204262:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677474.204264:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.204267:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.204268:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.204269:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.204270:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.204271:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.204272:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.204274:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.204274:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.204280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.204284:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.204286:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.204287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.204287:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.204288:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.204289:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.204290:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.204291:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.204292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.204293:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.205053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677474.205059:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.205062:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.205063:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.205065:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.205067:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.205069:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.205070:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.205072:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.205073:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.205074:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.205077:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.205079:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.205082:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.205082:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.205083:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.205085:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.205085:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.205086:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.205086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677474.205103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.205110:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.205113:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.205114:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.205114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.205116:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.205119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.205361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.205367:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.205370:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.205372:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.205372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.205374:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.205375:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.205377:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.205378:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.205379:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.205380:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.205383:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.205384:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.205387:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.205387:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.205388:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.205390:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.205390:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.205391:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.205392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677474.205404:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.205407:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.205408:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.205408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.205410:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.205412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.205430:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.205432:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.205433:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.205433:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.205434:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.205435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.205440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:21.2:1619677474.205442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.205447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:10.0:1619677474.205448:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:22.0:1619677474.205449:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.205450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677474.205451:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.205451:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:22.0:1619677474.205452:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.205455:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.205455:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.205456:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.205457:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.205458:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.205459:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.205459:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677474.205460:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.205461:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677474.205462:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.205464:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.205465:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.205466:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.205467:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.205469:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:10.0:1619677474.205471:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.205473:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.205475:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.205475:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.205476:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677474.205477:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.205477:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.205478:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677474.205478:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.205479:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.205479:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.205480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:10.0:1619677474.205480:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.205481:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.205482:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.205482:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.0:1619677474.205491:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.205493:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.205494:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.205495:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677474.205496:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.205497:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.205498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677474.205498:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.205499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.205499:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.205501:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.205505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.205532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.205539:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.205542:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.205543:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.205545:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.205547:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.205548:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.205550:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.205551:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.205552:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.205553:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.205557:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.205558:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.205562:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.205563:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.205563:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.205565:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.205566:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.205567:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.205567:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677474.205578:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.205581:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.205582:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.205584:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.205586:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.205587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.206083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.206090:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.206093:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.206093:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.206096:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.206098:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.206099:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.206101:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.206102:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.206103:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.206104:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.206106:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.206108:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.206111:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.206112:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.206112:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.206114:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.206114:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.206115:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.206116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.206120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.206127:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.206129:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.206130:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.206130:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.206132:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.206134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.206147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.206153:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.206155:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.206156:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.206156:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.206156:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.206157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.206233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.206240:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.206242:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.206243:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.206245:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.206247:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.206248:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.206249:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.206251:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.206252:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.206253:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.206256:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.206258:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.206260:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.206261:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.206262:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.206263:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.206264:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.206264:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.206265:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.206277:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.206280:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.206281:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.206281:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.206283:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.206286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.206451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.206460:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.206462:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.206465:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677474.206467:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.206468:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.206469:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677474.206471:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.206472:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.206473:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.206474:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677474.206477:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.206479:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677474.206482:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.206483:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677474.206483:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677474.206485:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677474.206486:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.206487:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.206487:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677474.206492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.206501:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.206504:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.206505:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.206505:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.206507:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.206511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.206750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.206758:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.206761:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.206763:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.206766:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.206766:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.206767:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.206768:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.206770:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.206771:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.206771:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.206774:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.206776:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.206779:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.206780:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.206781:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.206782:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.206783:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.206784:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.206784:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.206798:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.206801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.206801:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.206802:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.206804:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.206807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.206821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.206829:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.206832:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.206833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.2:1619677474.206835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:10.0:1619677474.206835:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.206837:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.206838:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.206839:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.206840:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677474.206841:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.206841:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.206843:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:10.0:1619677474.206843:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:6.0:1619677474.206844:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.206847:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.206849:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.206849:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.206851:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677474.206851:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:22.0:1619677474.206853:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.206854:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.206854:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.206854:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:22.0:1619677474.206855:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.206855:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677474.206856:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.206856:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.206857:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.206858:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677474.206859:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.206860:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.206860:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677474.206861:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:9.2:1619677474.206862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.206863:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.206864:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.206865:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677474.206865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.206866:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.206867:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677474.206868:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677474.206868:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.206869:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.206870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677474.206871:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.206872:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.206872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.206873:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677474.206874:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.206875:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:21.2:1619677474.206875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677474.206876:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.206876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.206877:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.206880:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.206882:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.206882:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.206883:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.206884:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.206884:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677474.206885:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.206886:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:20.0:1619677474.206887:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677474.206890:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.206892:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:21.0:1619677474.206892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677474.206894:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677474.206895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677474.206896:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.206899:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.206900:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677474.206902:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.206903:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.206904:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.206905:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677474.206908:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.206910:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677474.206912:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.206912:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:22.0:1619677474.206913:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677474.206915:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677474.206915:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.206916:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677474.206917:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677474.206917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677474.206921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.206923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.206928:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:18.0:1619677474.206929:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:6.0:1619677474.206930:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:18.0:1619677474.206931:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:6.0:1619677474.206931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:18.0:1619677474.206932:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.206932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:2.0:1619677474.206932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.206934:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.206937:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.207051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677474.207052:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.207055:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.207056:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.207056:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.207057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.207058:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677474.207059:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:10.0:1619677474.207061:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.207062:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.207064:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.207066:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.207068:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.207069:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.207070:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.207071:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.207072:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.207076:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.207077:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.207080:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.207081:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.207082:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.207084:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.207084:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.207085:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.207085:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.207097:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.207100:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.207100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.207101:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.207103:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.207106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.207269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.207276:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.207279:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.207282:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.207283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.207285:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.207286:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.207287:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.207289:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.207290:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.207291:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.207297:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.207299:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.207301:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.207302:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.207302:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.207303:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.207304:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.207305:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.207305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.207317:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.207319:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.207320:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.207321:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.207323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.207324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.207578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.207582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.207584:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.207587:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.207588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.207589:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.207591:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.207592:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.207593:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.207594:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.207595:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.207599:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.207600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677474.207601:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.207605:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.207606:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.207606:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677474.207608:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677474.207608:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.207609:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.207610:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:26.0:1619677474.207611:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:20.0:1619677474.207611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:10.0:1619677474.207612:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.207614:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:9.2:1619677474.207614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:26.0:1619677474.207617:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.207618:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.207619:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.207621:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.207621:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:26.0:1619677474.207622:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.207623:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.0:1619677474.207624:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.207625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.207625:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.207626:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.207627:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677474.207627:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.207630:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.207630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677474.207631:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.207632:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.207633:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.207634:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.207634:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.207635:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677474.207646:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.207649:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.207649:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.207651:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.207653:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.207654:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.207698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.207704:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.207707:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.207708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.207710:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.207712:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.207713:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.207715:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.207716:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.207717:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.207718:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.207722:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.207724:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.207727:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.207728:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.207729:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.207731:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.207731:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.207732:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.207732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.207744:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.207746:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.207747:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.207749:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.207749:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.207752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677474.208406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.208411:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.208414:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.208415:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.208417:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.208419:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.208421:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.208422:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.208423:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.208424:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.208425:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.208428:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.208430:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.208432:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.208433:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.208434:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.208435:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.208436:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.208436:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.208437:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.208450:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.208453:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.208454:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.208454:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.208456:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.208458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.208682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.208690:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.208693:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.208696:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.208698:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.208698:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.208699:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.208701:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.208702:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.208703:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.208704:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.208707:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.208709:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.208713:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.208713:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.208714:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.208716:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.208716:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.208717:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.208718:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.208729:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.208731:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.208732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.208733:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.208735:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.208738:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.208747:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.208750:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.208751:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.208751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.208752:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.208753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.208761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.208766:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.208769:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.208769:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.208771:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.208773:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.208775:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.208775:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.208777:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.208778:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.208779:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.208782:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.208784:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.208787:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.208787:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.208788:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.208790:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.208790:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.208791:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.208791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.208804:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.208806:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.208807:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.208809:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.208810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.208811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.208827:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.208829:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.208830:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.208831:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.208832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.208833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.208925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.2:1619677474.208927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.208930:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.208932:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.208932:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.208933:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.208935:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677474.208936:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.208938:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.208939:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.208941:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677474.208942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000400:00000200:20.0:1619677474.208943:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.208945:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.208946:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.208948:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.208950:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000400:00000200:21.0:1619677474.208950:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.208950:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:21.0:1619677474.208951:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.208952:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.208953:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:21.0:1619677474.208953:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.208954:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.208954:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:21.0:1619677474.208955:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677474.208958:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.208958:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.208958:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.208959:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.208959:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:24.0:1619677474.208961:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.208961:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.208961:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.208962:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)++
00000400:00000200:24.0:1619677474.208963:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677474.208963:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (23)++
00000400:00000200:24.0:1619677474.208964:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96505538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.208964:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.208964:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.208964:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (24)--
00000800:00000200:21.0:1619677474.208965:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.208965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (23)--
00000400:00000200:24.0:1619677474.208966:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.208966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:24.0:1619677474.208967:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.208967:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.208967:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.208967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:9.2:1619677474.208967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:24.0:1619677474.208968:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.208968:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.208969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677474.208970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000400:00000200:24.0:1619677474.208972:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677474.208973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.208974:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677474.208975:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677474.208976:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.208976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.208977:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677474.208978:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.208979:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.208980:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.208980:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.208981:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.208982:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677474.208982:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:8.0:1619677474.208982:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.208983:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.208984:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:17.0:1619677474.208984:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:8.0:1619677474.208984:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.208985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:17.0:1619677474.208985:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.208986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.208986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.208988:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.208988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.208988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677474.208991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:2.0:1619677474.208996:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.208999:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.209000:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.209000:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.209003:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.209007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.209484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.209492:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.209495:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.209498:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.209500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.209501:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.209502:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.209504:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.209505:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.209506:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.209507:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.209510:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.209512:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.209515:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.209515:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.209516:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.209517:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.209518:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.209519:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.209521:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.209521:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.209524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.209528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.209531:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.209533:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.209534:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.209534:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.209536:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.209536:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.209539:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.209539:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.209539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.209540:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.209634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.209641:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.209643:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.209644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.209647:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.209649:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.209650:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.209651:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.209652:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.209654:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.209655:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.209658:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.209660:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.209662:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.209663:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.209663:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.209665:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.209665:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.209666:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.209667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.209678:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.209681:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.209681:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.209683:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.209684:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.209685:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.210402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.210409:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.210412:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.210412:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.210414:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.210417:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.210418:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.210420:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.210421:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.210422:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.210423:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.210426:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.210428:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.210430:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.210430:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.210431:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.210433:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.210433:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.210434:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.210435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.210440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.210447:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.210449:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.210450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.210450:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.210451:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.210452:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.210464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.210470:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.210473:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.210474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.210476:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.210478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.210479:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.210480:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.210482:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.210483:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.210484:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.210486:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.210488:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.210490:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.210491:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.210492:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.210493:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.210494:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.210495:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.210496:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.210496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.210501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.210506:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.210508:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.210509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.210509:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.210510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.210511:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.210512:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.210514:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.210515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.210516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.210755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.210761:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.210764:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.210765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.210767:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.210769:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.210771:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.210772:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.210773:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.210774:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.210775:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.210778:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.210780:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.210782:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.210783:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.210784:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.210785:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.210786:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.210787:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.210788:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.210789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.210792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.210794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.210799:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.210801:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677474.210801:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677474.210802:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.210803:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677474.210803:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.210804:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.210805:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.210806:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.210809:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.210811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.210818:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.210821:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.210821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.210822:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.210822:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.210823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.210843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677474.210845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.0:1619677474.210851:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677474.210853:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.210854:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.210855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677474.210856:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:8.0:1619677474.210857:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.210859:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.210860:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.210861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:8.0:1619677474.210861:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.210862:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.210863:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.210863:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.210864:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.210865:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.210865:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.210866:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.210867:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.210868:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.210869:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.210871:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.210873:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.210875:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.210876:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.210876:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.210877:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.210877:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.210879:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.210880:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.210880:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.210880:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677474.210881:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.210881:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.210882:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:31.0:1619677474.210883:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.210883:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.210883:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677474.210884:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.210885:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.210889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.210893:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:24.0:1619677474.210896:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.210896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:6.0:1619677474.210896:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677474.210897:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677474.210897:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.210898:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.210899:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.210899:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.210900:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.210900:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.210900:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.210903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.210904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.210905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.210909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.210913:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.210915:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.210917:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.210918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.210920:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.210922:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.210923:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.210925:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.210926:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.210927:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.210930:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.210932:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.210935:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.210936:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.210936:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.210938:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.210939:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.210940:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.210940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677474.210951:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.210954:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.210955:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.210957:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:30.0:1619677474.210957:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.210959:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.211200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.211208:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.211211:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.211212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.211214:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.211216:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.211218:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.211219:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.211220:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.211221:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.211222:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.211225:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.211227:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.211229:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.211230:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.211231:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.211232:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.211232:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.211233:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.211233:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.211238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.211244:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.211246:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.211247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.211247:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.211249:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.211251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.211488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.211494:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.211497:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.211497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.211499:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.211501:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.211503:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.211504:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.211505:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.211506:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.211507:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.211510:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.211512:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.211514:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.211515:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.211516:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.211517:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.211518:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.211519:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.211519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.211532:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.211534:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.211535:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.211537:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.211538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.211539:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.211893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.211901:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:4.2:1619677474.211902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.211904:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:26.0:1619677474.211907:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.211907:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.211908:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677474.211909:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.211909:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677474.211911:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.211912:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.211912:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.211914:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.211915:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.211916:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.211918:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.211920:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.211920:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.211921:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.211922:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.211923:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.211924:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.211924:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.211928:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.211929:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:10.0:1619677474.211929:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.211930:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.211931:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.211932:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.211933:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.211934:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.211934:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677474.211934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.211935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677474.211935:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.211936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.211936:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.211937:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.211938:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.211939:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.211939:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.2:1619677474.211939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.211940:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:26.0:1619677474.211941:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.211942:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.211942:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.211943:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.211943:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.211944:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.211945:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96505525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.211946:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.211947:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.211948:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677474.211948:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:4.2:1619677474.211948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:26.0:1619677474.211952:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.211952:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:6.0:1619677474.211952:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:20.0:1619677474.211953:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.211954:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677474.211954:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.211955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677474.211955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.211955:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.211955:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.211956:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.211956:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.211957:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.211957:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.211958:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677474.211958:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677474.211958:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.211958:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.211959:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.211960:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677474.211961:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.211961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677474.211962:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.211963:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.211965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.211971:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.211974:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.211975:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.211975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.211977:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.211979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.211989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.211996:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.211998:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.211999:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.211999:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.212000:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.212000:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.212344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.212350:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.212352:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.212353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.212355:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.212357:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.212358:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.212360:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.212361:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.212362:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.212363:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.212366:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.212368:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.212370:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.212371:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.212372:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.212373:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677474.212373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:22.0:1619677474.212374:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.212374:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.212375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.212378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.212381:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.212383:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.212385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.212386:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.212386:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:21.0:1619677474.212387:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677474.212387:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.212388:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.212388:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.212389:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.212390:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.212391:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.212392:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.212395:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.212397:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.212397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.212397:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.212399:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.212399:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.212400:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.212400:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.212400:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.212401:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.212401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.212402:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.212403:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.212403:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.212404:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.212405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.212416:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.212418:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.212419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.212419:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.212421:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.212424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.212578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.212586:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.212589:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.212591:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.212593:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.212594:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.212596:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.212597:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.212598:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.212599:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.212600:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.212604:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.212605:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.212609:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.212610:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.212610:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.212612:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.212612:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.212614:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.212615:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.212616:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.212622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.212625:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.212627:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.212628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.212629:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.212629:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.212631:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.212632:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.212633:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.212634:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.212634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.212643:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.212646:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.212647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.212647:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.212648:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.212649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.212929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.212937:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.212940:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.212940:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.212942:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.212945:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.212946:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.212947:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.212948:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.212949:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.212950:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.212953:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.212955:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.212957:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.212958:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.212959:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.212961:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.212961:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.212962:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.212962:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.212967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.212976:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.212978:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.212979:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.212980:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.212982:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.212985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677474.213163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.213169:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.213172:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.213173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.213175:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.213177:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.213178:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.213180:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.213181:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.213182:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.213183:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.213186:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.213188:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.213190:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.213191:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.213192:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.213193:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.213194:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.213195:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.213196:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.213196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.2:1619677474.213197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:5.2:1619677474.213201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.213204:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.213207:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:17.0:1619677474.213207:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.213209:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677474.213209:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:22.0:1619677474.213210:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.213210:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.213210:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.213211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.213211:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.213212:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.213213:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.213214:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.213215:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.213216:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.213218:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.213219:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.213221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.213222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.213223:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.213223:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.213224:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.213225:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.213225:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.213226:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.213227:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.213227:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.213228:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.213228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.213229:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.213231:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.213231:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.213232:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.213232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.213244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.213246:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.213247:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.213249:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.213250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.213251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.213389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.213397:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.213400:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.213402:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.213404:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.213405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.213406:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.213407:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.213408:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.213409:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.213410:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.213413:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.213414:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.213417:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.213418:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.213418:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.213420:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.213420:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.213421:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677474.213421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677474.213422:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.213426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.213431:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.213433:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.213434:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:24.0:1619677474.213435:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.213435:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.213436:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.213438:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.213438:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.213440:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.213441:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.213443:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.213444:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.213445:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.213445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.213447:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.213448:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.213448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677474.213450:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.213453:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.213454:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.213454:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.213456:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.213457:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.213457:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.213458:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.213462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.213471:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.213473:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.213474:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.213475:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.213477:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.213480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.214198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.214204:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.214207:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.214208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.214210:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.214212:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.214213:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.214214:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.214216:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.214217:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.214218:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.214221:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.214222:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.214225:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.214225:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.214226:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.214228:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.214228:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.214230:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.214231:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.214231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.214236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.214241:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.214243:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.214244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.214244:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.214245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.214246:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.214247:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.214248:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.214249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.214250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.214553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.214561:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.214564:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.214566:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.214568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.214569:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.214570:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.214572:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.214573:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.214574:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.214575:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.214579:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.214580:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.214583:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.214584:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.214585:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.214586:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.214587:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.214588:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.214589:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.214590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.214593:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.214596:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.214598:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.214599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.214600:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.214601:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677474.214602:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.214603:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.214604:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.214604:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.214605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:20.0:1619677474.214605:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.214605:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.214607:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677474.214608:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677474.214611:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677474.214611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677474.214613:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.214615:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.214616:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.214617:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.214619:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.214619:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.214620:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.214621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.214624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.214632:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.214635:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.214636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.214636:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.214638:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.214641:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.214645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.214651:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.214653:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.214654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.214655:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.214655:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.214656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.214902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.214907:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.214910:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.214912:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.214912:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.214915:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.214916:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.214918:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.214919:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.214920:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.214921:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.214924:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.214926:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.214929:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.214929:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.214930:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.214931:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.214932:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.214933:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.214933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.214938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.214946:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.214949:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.214950:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.214950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.214952:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.214954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.214959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.214966:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.214969:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.214971:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.214973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.214974:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.214976:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.214977:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.214978:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.214979:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.214980:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.214984:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.214985:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.214988:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.214989:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.214990:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.214992:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.214992:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.214993:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.214994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.214998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.215005:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.215008:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.215009:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.215011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.215011:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.215014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.215062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.215068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.215071:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.215072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.215074:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.215075:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.215077:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.215078:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.215079:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.215081:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.215081:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.215085:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.215086:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.215089:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.215090:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.215090:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.215092:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.215092:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.215093:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.215094:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.215105:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.215107:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.215108:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.215110:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.215111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.215112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.215165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e7a00] (20)++
00000800:00000200:31.0:1619677474.215174:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:31.0:1619677474.215177:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.19@o2ib3
00000400:00000200:31.0:1619677474.215179:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.1.19@o2ib3 : GET - for me
00000800:00000200:30.0:1619677474.215180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000400:00000200:31.0:1619677474.215182:0:19945:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.1.19@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:31.0:1619677474.215184:0:19945:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.1.19@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:31.0:1619677474.215186:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:31.0:1619677474.215188:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e7a00] (22)++
00000800:00000200:31.0:1619677474.215190:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (23)--
00000400:00000200:31.0:1619677474.215192:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.1.19@o2ib3 local destination
00000400:00000200:31.0:1619677474.215196:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.1.19@o2ib3(10.51.1.19@o2ib3:10.51.1.19@o2ib3) : REPLY try# 0
00000800:00000200:4.2:1619677474.215196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e7a00] (22)++
00000800:00000200:31.0:1619677474.215198:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.1.19@o2ib3
00000800:00000200:31.0:1619677474.215199:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a87ff47280] -> 10.51.1.19@o2ib3 (2) version: 12
00000800:00000200:31.0:1619677474.215200:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a87b3e7a00] (23)++
00000800:00000200:31.0:1619677474.215200:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e7a00] (24)++
00000800:00000200:31.0:1619677474.215201:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a87b3e7a00] (25)--
00000800:00000200:31.0:1619677474.215202:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (24)--
00000800:00000200:31.0:1619677474.215203:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (23)++
00000800:00000200:31.0:1619677474.215206:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e7a00] (24)--
00000800:00000200:31.0:1619677474.215206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (23)--
00000800:00000200:31.0:1619677474.215207:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)++
00000800:00000200:31.0:1619677474.215207:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e7a00] (23)--
00000400:00000200:31.0:1619677474.215208:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.215210:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.1.19@o2ib3: REPLY: OK
00000800:00000200:30.0:1619677474.215211:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000800:00000200:31.0:1619677474.215212:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)--
00000800:00000200:20.2:1619677474.215257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.215263:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.215266:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.215267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.215269:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.215271:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.215273:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.215274:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.215276:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.215277:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.215278:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.215282:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.215283:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.215286:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.215287:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.215288:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.215290:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.215290:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.215291:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.215291:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677474.215305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.215308:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.215310:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.215312:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.215313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.215315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.215542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677474.215549:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.215552:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.215552:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677474.215554:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.215557:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.215558:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677474.215560:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.215561:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.215562:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.215563:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677474.215567:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.215568:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677474.215572:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.215573:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677474.215573:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677474.215575:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677474.215575:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.215576:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.215576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677474.215582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.215589:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.215592:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.215593:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.215593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.215595:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.215599:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677474.215604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.215609:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.215612:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.215613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.215613:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.215614:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.215614:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.215923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.215928:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.215931:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.215931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.215932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:22.0:1619677474.215933:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.215936:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.215937:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.215938:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.215939:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677474.215940:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.215941:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.215941:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.215942:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677474.215946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.215947:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.215947:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.215949:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:30.0:1619677474.215951:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.215951:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677474.215952:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677474.215952:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.215953:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677474.215954:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.215954:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:30.0:1619677474.215955:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.215955:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677474.215956:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.215956:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.215956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:30.0:1619677474.215957:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.215960:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.215962:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.215964:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.215965:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.215966:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.215967:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.215968:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.215968:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.215969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.215969:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:4.2:1619677474.215969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677474.215972:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.215974:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.215976:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.215976:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.215977:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:4.2:1619677474.215978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677474.215979:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.215980:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:8.0:1619677474.215980:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.215981:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677474.215983:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.215983:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.215983:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.215985:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.215985:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.215986:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677474.215987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.215987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:8.0:1619677474.215987:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.215988:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.215989:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.215990:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677474.215990:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.215991:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677474.215991:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.215992:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.215992:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.0:1619677474.215994:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677474.215995:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.215995:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000400:00000200:26.0:1619677474.215996:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:21.0:1619677474.215996:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.215997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677474.215997:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.215998:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.215998:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677474.215999:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.215999:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677474.216000:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.216001:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.216002:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.216002:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.216003:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:26.0:1619677474.216004:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.216004:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.216005:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677474.216006:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.216006:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.216007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677474.216008:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.216009:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.216009:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.216011:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.216011:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677474.216011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677474.216012:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.216013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.216016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677474.216019:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.216022:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677474.216023:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677474.216023:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.216023:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.216025:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:1.0:1619677474.216025:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.216026:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.216027:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.216028:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.216028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677474.216031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.216308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.216316:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.216319:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.216321:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.216323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.216323:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.216324:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.216326:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.216327:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.216328:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.216329:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.216332:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.216334:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.216336:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.216337:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.216338:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.216339:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.216340:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.216341:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.216341:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.216365:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.216368:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.216369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.216369:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.216372:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.216375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.216795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.216801:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.216804:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.216805:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.216807:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.216810:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.216811:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.216812:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.216814:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.216814:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.216815:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.216818:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.216818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677474.216820:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.216822:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.216823:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.216824:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.216825:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.216826:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.216826:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:10.0:1619677474.216826:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677474.216827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:10.0:1619677474.216828:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.216829:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.216831:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677474.216832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:10.0:1619677474.216833:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.216835:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.216836:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.216838:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.216839:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.216840:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:10.0:1619677474.216840:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.216842:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.216843:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.216843:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677474.216845:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.216845:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.216846:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.216847:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677474.216848:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.216849:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.216850:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.216851:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.216852:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.216853:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.216854:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.216854:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.216860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.216867:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.216869:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.216869:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.216871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.216876:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.216878:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.216879:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.216880:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.216882:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.216885:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.216926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677474.216930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:10.0:1619677474.216932:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.216935:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677474.216935:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677474.216936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677474.216938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677474.216938:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.216938:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:10.0:1619677474.216941:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.216941:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.216942:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.216943:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677474.216944:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677474.216945:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.216946:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.216947:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.216948:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.216949:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.216950:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.216951:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677474.216951:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.216956:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.216957:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.216958:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.216960:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.216961:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.216961:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.216962:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677474.216963:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.216964:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.216964:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.216964:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.216965:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.216965:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677474.216966:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.216966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:10.0:1619677474.216967:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.216968:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.216968:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.0:1619677474.216977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.216979:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.216980:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.216981:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:21.0:1619677474.216981:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.216982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.216984:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:21.0:1619677474.216984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:22.0:1619677474.216985:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.216985:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.216987:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.216990:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.217508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.217514:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.217517:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.217518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.217520:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.217523:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.217524:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.217526:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.217527:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.217528:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.217529:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.217533:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.217534:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.217536:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.217537:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.217538:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.217539:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.217539:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.217540:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.217541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.217546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.217554:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.217556:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.217556:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.217557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.217559:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.217560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.217566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.217573:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.217575:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.217576:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.217576:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.217576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.217577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.217771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.217777:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.217780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.217780:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.217782:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.217785:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.217786:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.217787:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.217788:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.217790:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.217791:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.217794:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.217796:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.217798:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.217799:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.217800:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.217801:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.217802:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.217803:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.217804:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.217804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.217809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.217814:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.217815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.217816:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.217817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.217817:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677474.217818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.217818:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.217820:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.217821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.217822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677474.217961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.217967:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.217970:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.217971:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.217973:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.217975:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.217977:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.217978:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.217979:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.217980:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.217981:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.217984:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.217986:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.217989:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.217989:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.217990:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.217992:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.217992:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.217993:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.217993:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.218005:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.218008:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.218009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.218009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.218011:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.218013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.218173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.218180:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.218183:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.218184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.218186:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.218189:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.218190:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.218192:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.218193:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.218194:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.218195:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.218198:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.218199:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.218202:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.218203:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.218204:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.218206:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.218206:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.218207:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.218207:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.218212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.218221:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.218223:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.218224:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.218226:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.218226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.218229:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.218232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.218240:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.218242:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.218243:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.218244:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.218245:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677474.218245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.2:1619677474.218615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.218621:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.218624:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.218624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.218626:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.218628:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.218629:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.218631:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.218632:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.218633:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.218634:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.218637:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.218638:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.218641:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.218642:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.218642:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.218644:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.218644:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.218645:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.218645:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.2:1619677474.218655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677474.218658:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.218660:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.218661:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.218663:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.218664:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677474.218664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.218665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.218667:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.218669:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.218672:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.218672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.218673:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.218675:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.218676:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.218677:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.218678:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.218682:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.218684:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.218687:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.218687:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.218688:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.218690:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.218690:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.218691:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.218692:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.218703:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.218706:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.218707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.218707:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.218709:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.218712:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.218722:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.218725:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.218725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.218726:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.218727:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.218728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.219069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.219076:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.219079:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.219081:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.219083:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.219084:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.219085:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.219086:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.219088:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.219089:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.219089:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.219092:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.219094:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.219096:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.219096:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.219097:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.219098:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.219099:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.219101:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.219102:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.219103:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.219104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.219108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.219111:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.219114:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.219115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.219115:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.219116:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.219116:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.219119:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.219119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.219120:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.219121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.219559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.219566:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.219569:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.219571:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.219573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.219575:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.219576:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677474.219577:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.219579:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.219580:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.219580:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677474.219584:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.219586:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677474.219588:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.219589:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677474.219590:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677474.219591:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677474.219592:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.219593:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677474.219594:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677474.219595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677474.219597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677474.219601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:10.0:1619677474.219605:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.219607:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.219608:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.219608:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.219610:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.219611:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677474.219613:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:10.0:1619677474.219613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:12.0:1619677474.219614:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.219616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.219619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677474.219625:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.219627:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677474.219628:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.219628:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.219629:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.219630:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.219654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.219660:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.219663:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.219664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.219665:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.219667:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.219669:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.219670:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.219672:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.219673:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.219674:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.219677:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.219679:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.219682:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.219682:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.219683:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.219685:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.219685:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.219686:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.219686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.219699:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.219701:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.219702:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.219703:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.219704:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.219705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.219706:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.219707:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.219707:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.219708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (24)--
00000800:00000200:20.0:1619677474.219708:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.219709:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.219710:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.219766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.219769:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.219771:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.2:1619677474.219774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677474.219774:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.219776:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.219778:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677474.219779:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.219779:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.219780:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.219782:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.219783:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677474.219783:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.219786:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677474.219786:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.219789:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.219789:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.219790:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677474.219791:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.219792:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.219792:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.219793:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677474.219794:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677474.219794:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677474.219795:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.219795:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.219795:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000400:00000200:31.0:1619677474.219796:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.219796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:31.0:1619677474.219797:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.219798:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.219801:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.219803:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.219806:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.219807:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.219808:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677474.219808:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.219810:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.219810:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.219810:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.219811:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677474.219811:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.219811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.219812:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677474.219813:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.219816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677474.219827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.219836:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.219838:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.219839:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.219841:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.219841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.219844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.220269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.220277:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.220280:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.220282:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.220283:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.220285:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.220286:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.220287:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.220288:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.220290:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.220290:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.220298:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.220299:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.220301:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.220302:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.220302:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.220303:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.220304:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.220305:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.220305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.220322:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.220324:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.220325:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.220326:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.220328:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.220329:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.220349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.220357:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.220359:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.220361:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.220363:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.220363:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.220364:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.220365:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.220366:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.220367:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.220368:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.220371:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.220373:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.220375:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.220376:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.220377:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.220378:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.220379:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.220380:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.220382:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.220383:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.220388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.220392:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.220395:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.220396:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.220396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.220396:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.220399:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.220399:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.220399:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.220401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.220401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.220478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.220483:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.220485:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.220486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.220488:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.220490:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.220491:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.220492:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.220493:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.220494:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.220495:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.220498:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.220500:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.220502:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.220503:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.220504:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.220505:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.220506:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.220507:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.220507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.220519:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.220521:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.220522:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.220524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.220524:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.220526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.220546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.220552:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.220555:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.220556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.220558:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.220560:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.220562:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.220563:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.220564:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.220565:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.220566:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.220569:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.220571:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.220573:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.220574:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.220575:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.220576:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.220577:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.220578:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.220578:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.220583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.220590:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.220593:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.220593:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.220595:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.220596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.220598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.220601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.220609:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.220611:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.220612:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.220613:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.220614:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677474.220614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:4.2:1619677474.220663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.220670:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.220673:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.220674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.220676:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.220679:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.220680:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.220681:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.220683:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.220684:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.220685:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.220688:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.220689:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.220692:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.220693:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.220694:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.220696:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.220696:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.220697:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.220697:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677474.220698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.220705:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.220708:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677474.220709:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677474.220710:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.220711:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.220712:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.220712:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.220712:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.220713:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.220714:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.220716:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.220716:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.220718:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.220718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677474.220719:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.220720:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.220724:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.220725:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.220727:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.220728:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.220728:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.220730:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.220730:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.220731:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.220732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.220742:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.220745:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.220746:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.220747:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.220749:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.220750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.220762:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.220765:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.220766:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.220766:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.220767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.220768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.220962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.220970:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.220973:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.220974:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.220976:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.220978:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.220979:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.220981:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.220982:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.220983:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.220983:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.220986:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.220988:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.220990:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.220991:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.220991:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.220993:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.220993:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.220994:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.220995:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.220999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.221005:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.221007:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.221008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.221008:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.221010:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.221013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.221019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.221025:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.221027:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.221028:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.221028:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.221028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.221029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.221053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.221056:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.221059:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.221061:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.221062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.221064:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.221065:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.221066:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.221067:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.221068:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.221069:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.221073:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.221074:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.221077:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.221078:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.221079:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.221080:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.221081:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.221082:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.221082:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677474.221093:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.221095:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.221097:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.221098:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.221099:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.221100:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.221328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.221333:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.221336:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.221337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.221339:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.221341:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.221342:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.221343:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.221345:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.221346:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.221347:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.221350:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.221352:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.221354:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.221355:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.221356:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.221357:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.221358:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.221359:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.221360:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.221360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.221367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677474.221373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677474.221375:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.221378:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677474.221378:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:31.0:1619677474.221380:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.221380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.221380:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.221381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677474.221382:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.221382:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.221382:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.221382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677474.221383:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.221384:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.221385:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.221385:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.221386:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677474.221387:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.221388:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.221388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677474.221389:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.221392:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.221393:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.221396:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.221397:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.221397:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.221399:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.221400:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.221401:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.221403:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.221403:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.221404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.221409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.221411:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.221414:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.221415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.221415:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.221417:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.221417:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.221419:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.221419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.221420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.221421:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.221499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.221504:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.221506:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.221508:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.221509:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.221511:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.221512:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.221513:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.221515:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.221516:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.221517:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.221520:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.221522:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.221524:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.221525:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.221525:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.221527:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.221527:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.221529:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.221530:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.221530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677474.221535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677474.221541:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.221541:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.221543:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:22.0:1619677474.221544:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677474.221544:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:22.0:1619677474.221545:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.221545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.221546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.221547:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.221549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.221555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:6.0:1619677474.221559:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.221561:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.221561:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:22.0:1619677474.221562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.221562:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.221563:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.221564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677474.221565:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.221566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.221568:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.221570:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.221572:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677474.221573:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.221574:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.221576:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.221576:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677474.221578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:2.0:1619677474.221580:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.221582:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.221584:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.221585:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.221586:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:2.0:1619677474.221586:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.221588:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677474.221588:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:26.0:1619677474.221589:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:2.0:1619677474.221589:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.221589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:10.0:1619677474.221590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.221591:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.221595:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.221596:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677474.221597:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.221598:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.221599:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.221600:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:22.0:1619677474.221600:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.221602:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:26.0:1619677474.221603:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.221603:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.221604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.221606:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:22.0:1619677474.221606:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.221608:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.221608:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.221609:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677474.221609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677474.221610:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.221611:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.221612:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.221612:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.221616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.221622:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.221624:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.221625:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.221625:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.221627:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.221628:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.221662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.221669:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.221671:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.221673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.221674:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.221676:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.221677:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.221678:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.221679:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.221680:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.221681:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.221684:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.221685:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.221687:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.221688:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.221688:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.221690:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.221690:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.221691:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.221692:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.221695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.221701:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.221704:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.221705:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.221705:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.221706:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.221708:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.222344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.222350:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.222353:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.222354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.222356:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.222358:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.222359:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.222361:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.222362:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.222363:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.222364:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.222367:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.222369:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.222371:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.222372:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.222372:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.222374:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.222375:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.222375:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.222376:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.222389:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.222391:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.222392:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.222394:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.222395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.222396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.222472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.222480:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.222483:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.222485:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.222488:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.222488:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.222489:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.222491:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.222492:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.222493:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.222494:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.222497:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.222499:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.222501:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.222502:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.222503:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.222504:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.222505:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.222506:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.222506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.222518:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.222521:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.222522:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.222522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.222524:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.222527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.222736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.222743:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.222746:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.222747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.222750:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.222753:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.222754:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.222756:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.222757:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.222758:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.222759:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.222763:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.222765:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.222768:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.222769:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.222770:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.222772:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.222773:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.222774:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.222775:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677474.222785:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.222788:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.222789:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.222791:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.222791:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.222794:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677474.222807:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.222810:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.222810:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.222811:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.222811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.222812:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.223127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677474.223130:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.223133:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677474.223135:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.223136:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677474.223137:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.223139:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677474.223140:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.223142:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.223143:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.223144:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677474.223147:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.223148:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677474.223152:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.223153:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677474.223153:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677474.223155:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677474.223156:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.223157:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677474.223157:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:12.0:1619677474.223169:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.223172:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.223172:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.223174:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.223176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.223177:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677474.223211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.223217:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.223219:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.223220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.223222:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.223224:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.223225:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.223226:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.223228:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.223229:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.223230:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.223234:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.223235:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.223238:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.223238:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.223239:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.223241:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.223241:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.223242:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.223242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.223254:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.223256:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.223257:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.223259:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.223260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.223261:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.223271:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.223274:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.223275:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.223275:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.223276:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.223277:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.223812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.223820:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.223823:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.223823:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.223826:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.223828:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.223829:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.223831:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.223832:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.223833:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.223834:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.223837:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.223839:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.223842:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.223842:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.223843:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.223844:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.223845:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.223846:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.223848:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.223848:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677474.223854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677474.223858:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.223860:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.223861:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.223862:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:13.0:1619677474.223863:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677474.223864:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.223865:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677474.223865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.223866:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677474.223866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.224049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.224055:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.224058:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.224058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.224060:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.224063:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.224064:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.224065:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.224067:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.224068:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.224069:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.224072:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.224074:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.224076:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.224076:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.224077:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.224079:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.224079:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.224080:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.224080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.224092:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.224094:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.224095:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.224097:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.224098:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.224099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.224118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.224126:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.224129:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.224131:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.224134:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.224134:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.224135:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.224137:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.224138:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.224139:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.224140:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.224143:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.224145:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.224147:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.224148:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.224148:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.224150:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.224150:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.224151:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.224152:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.224162:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.224165:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.224165:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.224167:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.224169:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.224169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677474.224448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.224456:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.224458:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.224459:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.224461:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.224464:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.224465:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.224467:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.224468:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.224469:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.224470:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.224473:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.224475:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677474.224477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.0:1619677474.224478:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.224479:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.224479:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.224481:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.224482:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.224483:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.224484:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:10.0:1619677474.224485:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677474.224485:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677474.224488:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.2:1619677474.224488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677474.224489:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.224490:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.224490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677474.224493:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.224494:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.224495:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.224496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:10.0:1619677474.224497:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.224498:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:10.0:1619677474.224498:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.224498:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.224499:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677474.224500:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.224501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677474.224501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:10.0:1619677474.224501:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677474.224501:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.224503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677474.224503:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.224503:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.224506:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.224507:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.224507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677474.224508:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.224509:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.224510:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.224510:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.224511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677474.224512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.224520:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.224522:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.224524:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.224526:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.224526:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:31.0:1619677474.224527:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.224527:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.224529:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.224529:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.224529:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:30.0:1619677474.224530:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.224530:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.224531:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.224532:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677474.224533:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.224536:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.224538:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.224540:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.224541:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.224542:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.224542:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.224544:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.224544:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.224545:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.224546:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.224557:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.224559:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.224560:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.224560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.224562:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.224565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.224879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.224885:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.224887:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.224889:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.224890:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.224892:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.224894:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.224895:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.224896:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.224897:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.224898:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.224901:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.224903:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.224905:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.224906:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.224907:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.224909:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.224909:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.224911:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.224912:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.224913:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.224915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.224918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.224920:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.224923:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.224923:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.2:1619677474.224923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677474.224924:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.224927:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.224929:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.224930:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.224930:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.224931:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677474.224932:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.224934:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.224935:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.224936:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.224937:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677474.224937:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.224938:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.224939:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.224939:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.224939:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:17.0:1619677474.224940:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.224940:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.224941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:1.0:1619677474.224942:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.224943:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.224944:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.224947:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.224948:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.224951:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.224952:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.224952:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.224954:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.224954:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.224955:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.224956:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.224968:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.224970:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.224971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.224971:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.224973:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.224976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.224985:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.224987:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.224988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.224989:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.224990:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.224990:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.225035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.225042:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.225045:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677474.225045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:31.0:1619677474.225047:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.225049:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.225050:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.225051:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677474.225052:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:6.0:1619677474.225053:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:31.0:1619677474.225054:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.225054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677474.225056:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.225057:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.225057:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.225058:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677474.225059:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.225060:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.225062:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.225063:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.225064:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.225065:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677474.225065:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.225065:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.225069:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677474.225069:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.225070:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.225070:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677474.225071:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.225072:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.225073:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.225074:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.225074:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677474.225074:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.225075:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.225076:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.225077:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.2:1619677474.225078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677474.225078:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.225079:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.225079:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.225080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.225085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677474.225088:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:22.0:1619677474.225090:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.225090:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.225091:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.225092:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677474.225092:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.225093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.225093:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.225094:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.225094:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.225094:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.225096:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.225096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.225097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677474.225098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.225099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.225263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.225269:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.225272:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.225273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.225275:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.225277:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.225278:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.225280:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.225281:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.225282:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.225283:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.225286:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.225288:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.225290:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.225291:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.225291:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.225298:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.225298:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.225299:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.225299:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677474.225308:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.225311:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.225312:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.225312:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.225314:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.225316:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.225326:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.225328:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.225329:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.225329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.225330:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.225331:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.225342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.225348:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.225351:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.225352:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.225354:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.225357:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.225358:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.225360:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.225361:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.225362:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.225363:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.225366:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.225368:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.225370:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.225370:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.225371:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.225373:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.225374:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.225375:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.225376:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.225377:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.225379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.225382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.225388:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.225389:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.225390:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.225390:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.225390:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.225390:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.225391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.225392:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.225392:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677474.225393:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.225686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.225694:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.225697:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.225700:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.225702:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.225702:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.225704:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.225705:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.225706:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.225707:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.225708:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.225712:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.225713:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.225716:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.225716:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.225717:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.225718:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.225719:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.225720:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.225720:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.225731:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.225733:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.225734:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.225736:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.225737:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.225738:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.225901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.225907:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.225909:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.225910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.225912:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.225914:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.225916:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.225917:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.225918:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.225919:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.225920:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.225923:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.225925:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.225928:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.225928:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.225929:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.225931:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.225931:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.225932:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.225932:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.225944:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.225946:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.225947:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.225949:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.225950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.225951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.226003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.226011:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.226014:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.226015:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.226017:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.226020:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.226021:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.226023:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.226024:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.226025:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.226026:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.226029:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.226030:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.226033:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.226034:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.226035:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.226036:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.226037:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.226037:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.226038:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.226050:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.226052:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.226053:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.226054:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.226056:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.226059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.226173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.226181:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.226184:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.226185:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.226187:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.226190:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.226191:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.226193:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.226194:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.226195:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.226196:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.226199:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.226200:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.226203:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.226204:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.226204:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.226206:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.226206:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.226207:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.226207:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.226219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.226222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.226222:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.226223:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.226225:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.226228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.226264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.226271:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.226274:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.226276:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.226277:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.226278:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.226279:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.226281:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.226282:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.226283:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.226284:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.226286:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.226288:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.226291:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.226292:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.226292:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.226299:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.226299:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.226300:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.226301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.226315:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.226317:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.226318:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.226318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.226321:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.226323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.226334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.226341:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.226344:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.226345:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.226347:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.226350:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.226351:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677474.226352:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.226353:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.226354:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.226355:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677474.226358:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.226360:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677474.226363:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.226364:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677474.226364:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677474.226366:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677474.226366:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.226367:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.226368:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.226369:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677474.226372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677474.226375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:1.0:1619677474.226381:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677474.226382:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.226384:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:1.0:1619677474.226384:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:26.0:1619677474.226385:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:17.0:1619677474.226385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677474.226386:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:1.0:1619677474.226386:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.226388:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.226391:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.226392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.226400:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.226403:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.226404:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.226404:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.226405:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.226405:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.226469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.226478:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.226480:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.226483:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.226485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.226486:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.226488:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.226489:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.226490:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.226491:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.226492:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.226495:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.226497:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.226499:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.226500:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.226500:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.226502:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.226502:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.226504:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.226505:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.226506:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.226507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.226512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.226513:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.226516:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.226517:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.226517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.226519:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.226519:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.226521:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.226522:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.226523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.226524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.226775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.226780:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.226783:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.226784:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677474.226784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:6.0:1619677474.226786:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.226787:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.226789:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.226790:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.226791:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.226792:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677474.226793:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.226794:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:6.0:1619677474.226794:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.226795:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.226798:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.226799:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.226801:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:26.0:1619677474.226802:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.226803:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.226804:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677474.226804:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.226805:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:26.0:1619677474.226806:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.226807:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.226807:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.226807:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677474.226808:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.226808:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.226808:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:26.0:1619677474.226809:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.226812:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.226814:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.226817:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.226818:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.226819:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.226820:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.226820:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677474.226821:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.226822:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:21.0:1619677474.226822:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:26.0:1619677474.226823:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:21.0:1619677474.226823:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.226824:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:21.0:1619677474.226825:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.226826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.226827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.226829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.226834:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.226836:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677474.226837:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677474.226838:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.226838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.226839:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.226840:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677474.226840:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.226840:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677474.226840:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677474.226842:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.226843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677474.226843:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.226844:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.226844:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.226846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677474.226853:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.226856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.226856:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.226857:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.226858:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.226859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.227571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677474.227577:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.227579:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677474.227580:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677474.227583:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.227585:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.227586:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677474.227587:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.227589:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.227590:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.227591:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677474.227594:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.227596:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677474.227598:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.227599:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.227599:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.227601:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.227601:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.227602:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.227602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677474.227608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.227615:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.227617:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.227618:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.227619:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.227620:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.227621:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.227767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.227775:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.227778:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.227780:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.227783:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.227783:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.227784:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.227786:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.227787:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.227788:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.227789:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:21.2:1619677474.227789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:30.0:1619677474.227792:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.227794:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.227795:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677474.227796:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.227797:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.227798:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.227798:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:30.0:1619677474.227799:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677474.227799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677474.227800:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677474.227800:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.227801:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.227801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677474.227802:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.227804:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.227805:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.227806:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677474.227806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677474.227807:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.227808:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.227811:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.227812:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677474.227813:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.227815:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.227815:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.227816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:22.0:1619677474.227816:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.227816:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677474.227816:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.227818:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.227819:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.227819:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:8.0:1619677474.227819:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.227820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677474.227821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.227824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.227830:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.227833:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.227834:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677474.227834:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.0:1619677474.227834:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.227835:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.227836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677474.227836:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.227837:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.227838:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.227840:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.227841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.227963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.227969:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.227973:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.227974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.227975:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.227978:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.227979:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.227980:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.227982:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.227983:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.227984:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.227987:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.227989:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.227991:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.227992:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.227992:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.227994:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.227995:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.227995:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.227996:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.228000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.228008:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.228011:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.228012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.228012:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.228014:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.228016:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.228291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.228300:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.228303:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.228305:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.228307:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.228307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.228309:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.228310:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.228312:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.228313:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.228314:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.228317:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.228318:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.228322:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.228323:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.228323:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.228325:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.228325:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.228327:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.228329:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.228329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.228335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.228339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.228342:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.228343:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.228343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.228344:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.228346:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.228346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.228346:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.228348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.228349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.228455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.228463:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.228466:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.228466:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677474.228468:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.228472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.228473:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.228474:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.228476:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.228477:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.228478:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.228481:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.228482:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.228485:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.228486:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.228486:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.228488:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.228488:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.228489:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.228489:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.228494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.228503:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.228505:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.228506:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.228507:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.228508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.228510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.228623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.228629:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.228632:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.228632:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.228634:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.228636:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.228637:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.228639:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.228640:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.228641:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.228642:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.228645:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.228647:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.228650:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.228651:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.228651:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.228653:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.228653:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.228654:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.228655:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.228656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.228661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677474.228666:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.228666:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.228667:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677474.228668:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.228668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.228669:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.228669:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.228671:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.228672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.228673:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.229004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.229011:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.229014:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.229016:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.229018:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.229018:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.229020:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.229021:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.229023:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.229023:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.229025:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.229028:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.229030:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.229032:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.229033:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.229033:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.229035:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.229035:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.229036:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.229036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.229047:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.229050:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.229050:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.229052:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.229054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.229055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.229318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.229325:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.229328:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.229329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.229330:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.229333:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.229335:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.229336:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.229337:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.229338:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.229339:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.229342:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.229344:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.229347:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.229347:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.229348:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.229350:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.229350:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.229351:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.229351:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.229365:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.229367:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.229368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.229369:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.229371:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.229374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.229526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.229531:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.229534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.229534:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.229536:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.229538:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.229539:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.229540:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.229541:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.229542:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.229543:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.229547:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.229548:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.229551:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.229551:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.229552:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.229554:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.229554:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.229555:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.229555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.229567:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.229569:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.229570:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.229572:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.229573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.229574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.229869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.229877:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.229880:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.229882:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.229884:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.229884:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.229885:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.229887:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.229888:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.229889:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.229890:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.229893:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.229895:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.229897:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.229898:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.229899:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.229900:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.229901:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.229902:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.229902:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.229913:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.229915:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.229916:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.229918:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.229919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.229921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.229933:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.229936:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.229937:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.229937:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.229938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.229939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.230303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.230311:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.230313:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.230315:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.230318:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.230318:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.230319:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.230320:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.230321:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.230322:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.230323:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.230326:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.230327:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.230330:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.230331:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.230331:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.230333:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.230333:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.230334:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.230335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.230348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.230351:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.230351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.230352:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.230354:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.230357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.230598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.230604:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.230607:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.230608:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.230610:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.230612:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.230613:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.230614:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.230616:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.230617:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.230617:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677474.230620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:17.0:1619677474.230620:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.230622:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.230625:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.230625:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.230626:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.230626:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677474.230627:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.230628:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677474.230628:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.230629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:17.0:1619677474.230629:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.230629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.230631:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.230633:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.230634:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.230636:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.230638:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.230639:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.230640:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.230642:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.230643:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.230644:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.230644:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.230646:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.230646:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.230648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.230649:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.230650:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.230650:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.230651:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.230653:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.230654:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.230654:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.230655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.230666:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.230669:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.230669:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.230671:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.230672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.230673:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.230689:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.230692:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.230693:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.230694:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.230695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.230696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.230782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.2:1619677474.230786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.0:1619677474.230791:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.230793:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:30.0:1619677474.230794:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:21.0:1619677474.230796:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.230796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677474.230797:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:21.0:1619677474.230798:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.230799:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.230800:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677474.230800:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677474.230801:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.230801:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.230802:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.230803:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.230804:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677474.230804:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677474.230805:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.230806:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.230808:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.230809:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677474.230814:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677474.230815:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.230817:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.230818:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.230821:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.230821:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.230822:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.230823:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.230823:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.230823:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.230825:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.230825:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.230826:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.230826:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.230827:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677474.230828:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.230829:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:30.0:1619677474.230830:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.230830:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.230830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:4.2:1619677474.230831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677474.230832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.230835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.2:1619677474.230836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.230839:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677474.230841:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677474.230841:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677474.230841:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.230842:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677474.230843:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:21.0:1619677474.230843:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.230843:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677474.230844:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.230844:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:20.0:1619677474.230844:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.230844:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.230845:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.230845:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.230845:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677474.230846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.230846:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.230846:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.230847:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.230847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:10.0:1619677474.230847:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.230848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.230849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.230849:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:10.0:1619677474.230849:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.230850:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.230851:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677474.230851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.230852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:10.0:1619677474.230852:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.230853:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.230857:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.230858:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.0:1619677474.230859:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.230861:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.230861:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.230861:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:21.0:1619677474.230862:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677474.230862:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677474.230863:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677474.230863:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677474.230864:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.230864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677474.230864:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.230865:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.230866:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.230869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.230876:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.230878:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.230879:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.230880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.230881:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.230884:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.230894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.230899:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.230901:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.230903:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.230904:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.230906:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.230907:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.230908:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.230910:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.230911:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.230912:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.230915:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.230917:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.230919:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.230920:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.230920:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.230922:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.230923:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.230924:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.230924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.230928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.230936:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.230939:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.230940:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.230940:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.230942:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.230944:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.231545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.231551:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.231554:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.231555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.231557:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.231559:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.231560:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.231561:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.231563:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.231564:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.231565:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.231567:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.231569:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.231572:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.231572:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.231573:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.231575:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.231575:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.231576:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.231576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:4.2:1619677474.231579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.231586:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.231589:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677474.231589:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677474.231590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677474.231591:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:26.0:1619677474.231592:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.231592:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.231593:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.231595:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.231595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677474.231596:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677474.231596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677474.231597:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.231598:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.231600:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.231601:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.231604:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.231606:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.231608:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.231609:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.231610:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.231611:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.231612:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.231612:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.231613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.231618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.231625:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.231628:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.231629:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.231630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.231632:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.231634:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.231676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677474.231685:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.231688:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677474.231690:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.231692:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677474.231692:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.231694:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.231695:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.231697:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.231698:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.231698:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.231702:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.231703:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.231706:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.231707:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.231708:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.231709:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.231710:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.231711:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:30.0:1619677474.231713:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:30.0:1619677474.231713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:9.2:1619677474.231716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.231719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:18.0:1619677474.231726:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:31.0:1619677474.231727:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000800:00000200:31.0:1619677474.231729:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677474.231730:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:18.0:1619677474.231730:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:31.0:1619677474.231731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000400:00000200:18.0:1619677474.231732:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.231732:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.231735:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.231740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.231744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.231751:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.231755:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.231755:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.231757:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.231760:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.231761:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.231763:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.231764:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.231765:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.231766:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.231769:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.231771:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.2:1619677474.231773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:10.0:1619677474.231773:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.231774:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.231775:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.231776:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.231776:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.231777:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.231778:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677474.231781:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.231784:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677474.231786:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.231788:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677474.231789:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.231789:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:31.0:1619677474.231790:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677474.231792:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677474.231792:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:31.0:1619677474.231793:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.231793:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.231794:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.231794:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.231795:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677474.231797:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:31.0:1619677474.231798:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.231800:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677474.231800:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677474.231802:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.231803:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:31.0:1619677474.231803:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:31.0:1619677474.231805:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677474.231805:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.231806:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677474.231807:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:13.0:1619677474.231807:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.231810:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:9.2:1619677474.231810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:13.0:1619677474.231811:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.231811:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.231812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.231813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677474.231817:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.231819:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.231820:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.231820:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.231822:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.231824:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.232355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.232363:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.232365:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.232367:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.232370:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.232370:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.232372:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.232373:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.232374:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.232375:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.232376:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.232379:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.232380:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.232383:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.232384:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.232384:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.232386:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.232387:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.232387:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.232388:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.232401:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.232404:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.232405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.232405:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.232407:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.232410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.232436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.232441:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.232443:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.232444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.232446:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.232448:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.232449:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.232450:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.232451:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.232452:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.232453:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.232456:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.232458:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.232460:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.232461:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.232462:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.232463:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.232464:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.232465:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.232466:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.232466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.232471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.232477:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.232479:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.232479:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.232479:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.232480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.232481:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.232482:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.232483:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.232484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.232486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.232730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.232738:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.232741:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.232743:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.232745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.232746:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.232748:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.232749:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.232750:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.232751:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.232752:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.232755:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.232757:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.232760:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.232761:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.232762:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.232764:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.232764:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.232765:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.232766:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.232770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.232778:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.232781:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.232782:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.232783:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.232784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.232786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.233188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.233195:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.233198:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.233199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.233200:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.233203:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.233205:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.233206:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.233207:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.233208:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.233209:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.233213:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.233214:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.233217:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.233218:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.233218:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.233220:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.233220:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.233221:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.233221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.233233:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.233236:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.233237:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.233237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.233239:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.233242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.233249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:6.0:1619677474.233250:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.233252:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.233253:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.233254:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.233255:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.233255:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677474.233255:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.233258:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.233259:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.233261:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.233263:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.233265:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.233266:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.233267:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.233268:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.233269:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.233273:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.233275:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.233278:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.233279:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.233279:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.233281:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.233282:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.233283:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.233284:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.233284:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677474.233290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.233297:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.233300:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.233301:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677474.233301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677474.233302:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677474.233304:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.233305:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.233307:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.233309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.233310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.233526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.233532:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.233535:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.233536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.233538:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.233540:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.233541:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.233543:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.233544:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.233545:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.233546:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.233549:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.233550:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.233553:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.233553:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.233554:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.233555:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.233556:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.233557:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.233557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.233570:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.233572:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.233573:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.233575:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.233576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.233577:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.233608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.233616:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.233619:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.233622:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.233624:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.233625:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.233626:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.233627:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.233629:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.233630:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.233631:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.2:1619677474.233631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:30.0:1619677474.233633:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.233635:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.233638:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.233638:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.233639:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.233639:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.233640:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000800:00000200:30.0:1619677474.233641:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.233641:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677474.233642:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.233642:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.233643:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677474.233644:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.233644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677474.233646:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677474.233647:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677474.233647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:31.0:1619677474.233648:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.233649:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.233650:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677474.233653:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.233654:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:24.0:1619677474.233654:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.233656:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:31.0:1619677474.233657:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677474.233657:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.233657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.233658:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677474.233658:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:24.0:1619677474.233659:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.233660:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677474.233662:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:24.0:1619677474.233662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.233663:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:31.0:1619677474.233664:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:31.0:1619677474.233665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:9.2:1619677474.233666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.2:1619677474.233671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:2.0:1619677474.233674:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.233677:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:30.0:1619677474.233678:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:18.0:1619677474.233678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.233678:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.233680:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.233681:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:30.0:1619677474.233681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.233682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:2.0:1619677474.233684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677474.233787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.233793:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.233796:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.233796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.233798:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.233801:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.233802:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.233804:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.233805:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.233806:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.233807:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.233811:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.233813:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.233815:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.233815:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.233816:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.233818:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.233819:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.233819:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.233820:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.233824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.233831:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.233833:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.233833:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.233834:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.233835:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.233836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.233959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:21.2:1619677474.233964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:6.0:1619677474.233965:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.233968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.233968:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677474.233970:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.233971:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:6.0:1619677474.233973:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.233974:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:6.0:1619677474.233974:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677474.233976:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.233977:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.233978:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.233978:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.233979:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.233980:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.233980:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677474.233982:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.233983:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.233984:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.233985:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.233986:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.233987:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.233987:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677474.233989:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.233990:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:30.0:1619677474.233991:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.233991:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677474.233993:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677474.233994:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.233994:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.233994:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.233995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:30.0:1619677474.233997:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.233997:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.233998:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:9.2:1619677474.233999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677474.234000:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.234000:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.234001:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.234002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:18.0:1619677474.234007:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.234009:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.234010:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.234010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.234012:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.234013:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677474.234015:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:22.0:1619677474.234016:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.234017:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.234017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.234020:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.234022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.234107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.234115:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.234118:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.234119:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.234121:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.234123:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.234125:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.234126:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.234127:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.234129:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.234129:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.234133:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.234135:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.234138:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.234139:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.234139:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.234141:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.234141:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.234143:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.234144:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.234144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.234150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.234157:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.234158:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.234159:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.234160:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.234164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.234166:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.234167:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.234167:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.234170:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.234173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.234419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.234427:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677474.234427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677474.234430:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.234432:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.234433:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677474.234434:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.234435:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.234436:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677474.234436:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.234437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677474.234438:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.234440:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.234440:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.234441:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.234442:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.234442:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.234444:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.234446:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.234447:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.234447:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.234448:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.234450:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.234450:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.234452:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.234453:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.234453:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:22.0:1619677474.234453:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.234455:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.234455:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.234455:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.234456:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.234457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.234458:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.234459:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.234460:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.234461:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.234462:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.234462:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.234463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677474.234468:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.234471:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.234471:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.234473:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.234475:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677474.234475:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677474.234476:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.0:1619677474.234477:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.234478:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.234480:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.234481:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.234483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.234503:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.234506:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.234507:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.234508:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.234508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.234509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.234975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.234983:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.234986:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.234988:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.234990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.234991:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.234992:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.234993:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.234995:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.234996:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.234997:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.235000:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.235002:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.235004:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.235005:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.235006:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.235007:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.235008:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.235009:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.235011:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.235012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.235017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.235022:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.235023:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.235024:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.235025:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.235026:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.235027:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.235027:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.235028:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.235029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.235030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677474.235335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.235342:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.235345:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.235346:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.235348:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.235350:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.235351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:17.0:1619677474.235352:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.235353:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.235354:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.235355:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.235356:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.235356:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.235359:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.235360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:17.0:1619677474.235360:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.235362:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:6.0:1619677474.235364:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.235365:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.235366:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:6.0:1619677474.235366:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.235367:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677474.235367:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:17.0:1619677474.235368:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.235368:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.235369:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.235370:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:6.0:1619677474.235370:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.235371:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:6.0:1619677474.235371:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677474.235372:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677474.235372:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.235375:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677474.235377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677474.235378:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.235380:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.235381:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.235382:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677474.235382:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.235384:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.235384:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:1.0:1619677474.235384:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677474.235385:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.235385:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:1.0:1619677474.235385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.235385:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.235386:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.235386:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.235387:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:22.0:1619677474.235388:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.235388:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.235390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677474.235398:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.235400:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.235401:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.235402:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.235404:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.235404:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677474.235405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677474.235407:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.235407:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.235408:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.235408:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.235409:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.235598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.235605:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.235608:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.235609:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.235611:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.235614:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.235615:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.235617:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.235618:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.235619:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.235620:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.235622:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.235624:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.235626:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.235627:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.235628:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.235629:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.235630:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.235630:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.235631:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.235635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.235642:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.235645:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.235645:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.235646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.235647:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.235649:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.235658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.235664:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.235666:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.235667:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.235667:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.235668:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.235668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.236039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.236047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.236049:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.236051:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.236053:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.236054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.236054:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.236055:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.236057:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.236058:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.236058:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.236062:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.236063:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.236066:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.236067:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.236067:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.236069:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.236070:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.236071:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.236072:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.236073:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.236078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.236082:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.236085:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.236086:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.236086:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.236086:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.236088:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.236089:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.236090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.236091:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.236092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.236304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.236311:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.236314:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.236315:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.236317:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.236319:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.236320:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.236321:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.236323:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.236324:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.236324:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.236327:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.236329:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.236331:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.236331:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.236332:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.236333:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.236334:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.236335:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.236337:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.236337:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.236340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.236343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.236348:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.236350:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.236351:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.236351:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677474.236351:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.236352:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.236353:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.236354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677474.236354:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.236354:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.236454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.236459:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.236462:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.236463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.236465:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.236467:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.236468:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.236469:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.236470:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.236471:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.236472:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.236475:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.236477:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.236479:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.236480:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.236481:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.236482:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.236483:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.236484:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.236484:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.236485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.236490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.236495:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.236495:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.236497:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.236498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.236498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.236498:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.236499:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.236500:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.236501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.236503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.236939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677474.236947:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.236950:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677474.236952:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.236955:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.236955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677474.236957:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677474.236958:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.236960:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.236961:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.236962:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677474.236965:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.236966:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.236970:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.236970:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677474.236971:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677474.236972:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677474.236973:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.236974:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.236975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.236979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.236987:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.236990:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.236991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.236991:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.236993:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.236995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.236999:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.237000:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.237002:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.237003:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.237003:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.237004:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.237005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.237304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.237309:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.237312:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.237314:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.237315:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.237317:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.237318:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.237320:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.237321:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.237322:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.237323:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.237326:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.237328:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.237330:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.237331:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.237332:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.237333:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.237334:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.237334:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.237335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.237340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.237349:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.237352:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.237353:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.237353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.237355:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.237358:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.237389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.237397:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.237400:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.237402:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.237403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677474.237404:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.237405:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.237406:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.237408:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.237409:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.237409:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677474.237410:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.237411:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:6.0:1619677474.237412:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.237413:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.237416:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.237417:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.237418:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.237419:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:6.0:1619677474.237420:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.237421:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.237422:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.237422:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.237423:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:6.0:1619677474.237423:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.237424:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.237425:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.237425:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677474.237425:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.237426:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.237427:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677474.237428:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.237430:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:9.2:1619677474.237431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.237432:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.237433:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.237433:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.237435:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.237435:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.237436:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.237437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677474.237438:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.237440:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.237441:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.237441:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.237443:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.237445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677474.237449:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.237451:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.237452:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.237454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.237454:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.237456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.237718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.237726:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.237729:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.237731:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.237732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.237734:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.237735:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.237737:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.237738:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.237739:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.237740:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.237744:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.237745:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.237749:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.237749:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.237750:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.237752:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.237752:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.237753:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.237754:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.237766:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.237768:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.237769:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.237770:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.237772:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.237775:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.237860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.237867:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.237869:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.237871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.237872:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.237875:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.237876:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.237878:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.237879:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.237880:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.237881:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.237884:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.237886:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.237888:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.237889:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.237890:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.237891:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.237892:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.237892:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.237893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.237904:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.237907:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.237908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.237908:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.237911:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.237913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.237924:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.237926:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.237927:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.237927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.237928:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.237928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.238284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.238289:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.238291:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.238292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.238299:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.238300:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.238302:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.238303:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.238304:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.238305:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.238306:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.238309:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.238310:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.238312:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.238313:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.238314:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.238316:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.238316:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.238317:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.238317:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.238331:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.238333:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.238334:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.238335:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.238337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.238338:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.238353:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.238356:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.238357:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.238358:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.238359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.238360:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.238619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.238627:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.238629:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.238632:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.238634:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.238634:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.238635:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.238636:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.238637:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.238639:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677474.238639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677474.238640:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.238641:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.238643:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.238644:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.238645:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.238647:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.238647:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.238647:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677474.238648:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.238650:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.238650:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.238650:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677474.238651:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.238652:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.238653:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.238655:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.238655:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.238657:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.238657:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677474.238658:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.238658:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96505395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.238659:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.238659:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.238660:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.238660:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.238660:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677474.238660:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.238661:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677474.238661:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.238662:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.238662:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)++
00000400:00000200:10.0:1619677474.238662:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.238663:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)++
00000800:00000200:31.0:1619677474.238664:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (25)--
00000800:00000200:30.0:1619677474.238664:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (25)--
00000800:00000200:30.0:1619677474.238665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:10.0:1619677474.238665:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.238666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677474.238666:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677474.238667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.238668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.238669:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.238670:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.238670:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.238672:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.238673:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.238673:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.238674:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677474.238675:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:24.0:1619677474.238676:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:31.0:1619677474.238678:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:24.0:1619677474.238678:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:31.0:1619677474.238679:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.238679:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.238679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:31.0:1619677474.238680:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.238682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:24.0:1619677474.238682:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.238683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:24.0:1619677474.238684:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.238686:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.238689:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.238689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.238690:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.238692:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.238695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.239054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.239061:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.239064:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.239065:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.239067:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.239069:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.239071:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.239072:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.239073:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.239074:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.239075:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.239078:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.239080:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.239082:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.239083:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.239083:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.239085:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.239085:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.239086:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.239087:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.239091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.239099:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.239102:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.239103:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.239103:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.239106:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.239108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.239113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677474.239120:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.239123:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.239124:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.239124:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.239125:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.239126:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.239225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.239233:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.239236:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.239238:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.239240:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.239240:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.239241:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.239243:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.239244:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.239245:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.239246:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.239249:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.239250:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.239253:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.239254:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.239255:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.239256:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.239257:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.239258:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.239260:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.239260:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.239263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.239267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.239275:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.239275:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.239277:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677474.239278:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.239278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677474.239278:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.239279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677474.239280:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.239280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.239283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.239304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677474.239310:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.239313:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677474.239314:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677474.239316:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.239318:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.239319:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677474.239321:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677474.239322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000400:00000200:1.0:1619677474.239322:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.239323:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.239324:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677474.239327:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.239329:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:1.0:1619677474.239329:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677474.239330:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.239331:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677474.239332:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:1.0:1619677474.239332:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.239333:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.239333:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:31.0:1619677474.239334:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.239335:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677474.239336:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677474.239336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:31.0:1619677474.239337:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.239337:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677474.239338:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.239340:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677474.239340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:31.0:1619677474.239341:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.239342:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.239343:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677474.239343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:31.0:1619677474.239346:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.239348:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.239349:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.239350:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.239350:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:31.0:1619677474.239351:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.239351:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:17.0:1619677474.239351:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.239351:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:31.0:1619677474.239352:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.239352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677474.239353:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.239353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677474.239354:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.239354:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:17.0:1619677474.239354:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677474.239354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:31.0:1619677474.239356:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677474.239357:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:8.2:1619677474.239357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677474.239358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677474.239361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.239362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:4.2:1619677474.239363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:1.0:1619677474.239364:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677474.239367:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:1.0:1619677474.239367:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677474.239368:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:1.0:1619677474.239368:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.239368:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:8.0:1619677474.239369:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677474.239369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.239370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677474.239371:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:24.0:1619677474.239371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.239371:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.239371:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.239372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677474.239373:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677474.239374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:8.0:1619677474.239374:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.239375:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000400:00000200:6.0:1619677474.239375:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.239377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677474.239377:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.239378:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.239379:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677474.239380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:6.0:1619677474.239381:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.239382:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.239383:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.239384:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.239386:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:6.0:1619677474.239386:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.239389:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.239390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.239390:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677474.239391:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677474.239392:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.239392:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.239393:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677474.239394:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677474.239394:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.239395:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.239395:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:20.0:1619677474.239396:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.239396:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.239396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.239397:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.239398:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.239402:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.2:1619677474.239403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:20.0:1619677474.239403:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.239406:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.239406:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.239407:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.239409:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.239409:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:21.0:1619677474.239410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.239410:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.239410:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.239411:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.239411:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.239411:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677474.239412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:1.0:1619677474.239412:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.239413:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:21.0:1619677474.239414:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.239415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:1.0:1619677474.239415:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.239417:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677474.239417:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.239418:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.0:1619677474.239419:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.239420:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:1.0:1619677474.239420:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.239421:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.239421:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.239423:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.239423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:1.0:1619677474.239423:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.239424:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.239426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:1.0:1619677474.239429:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.239431:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.239433:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.239434:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.239435:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.239436:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.239437:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.239438:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.239439:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.239439:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.239442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.239444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677474.239450:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.239450:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677474.239452:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.239452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677474.239453:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.239453:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.239453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677474.239455:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.239456:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.239458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.239685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.239693:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.239696:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.239698:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.239700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.239701:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.239702:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.239704:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.239705:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.239706:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.239707:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.239710:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.239712:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.239714:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.239715:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.239716:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.239717:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.239718:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.239719:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.239719:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.239729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.239733:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.239735:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.239736:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.239737:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.239737:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.239738:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677474.239739:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.239740:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677474.239740:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.239741:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.239742:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000400:00000200:30.0:1619677474.239742:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.239743:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.239743:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.239744:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.239744:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677474.239745:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:30.0:1619677474.239745:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.239746:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.239747:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.239748:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.239751:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.239752:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.239755:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.239755:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.239756:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.239758:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.239758:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.239759:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.239759:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.239771:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.239774:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.239775:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.239775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.239777:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.239780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.240106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.240112:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.240115:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.240116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.240118:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.240120:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.240122:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.240123:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.240124:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.240125:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.240126:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.240129:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.240131:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.240133:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.240134:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.240134:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.240136:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.240137:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.240137:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.240138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.240150:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.240152:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.240153:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.240153:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.240155:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.240158:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.240264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.240269:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.240271:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.240272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.240274:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.240276:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.240277:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.240278:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.240280:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.240280:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.240282:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.240285:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.240287:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.240289:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.240290:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.240291:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.240292:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.240299:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.240300:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.240301:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.240302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.240307:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677474.240311:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.240313:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.240313:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:21.0:1619677474.240314:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.240315:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.240315:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.240315:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.240316:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.240317:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.240318:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.240327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.240335:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.240338:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.240339:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.240341:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.240343:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.240345:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.240346:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.240348:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.240348:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.240350:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.240353:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.240354:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.240357:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.240357:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.240358:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.240359:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.240360:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.240361:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.240361:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.240365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.240372:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.240374:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.240375:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677474.240375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677474.240376:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.240377:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.240379:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677474.240382:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:9.2:1619677474.240384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677474.240385:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.240386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.240388:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.240390:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.240390:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677474.240392:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.240393:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.240393:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.240394:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.240394:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677474.240395:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.240395:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:18.0:1619677474.240396:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.240396:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:18.0:1619677474.240397:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.240400:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.240402:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.240405:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.240406:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.240406:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.240408:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.240409:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.240410:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.240411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677474.240422:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.240424:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.240425:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.240427:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.240428:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.240430:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.240786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.240792:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.240795:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.240796:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.240798:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.240801:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.240803:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.240804:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.240806:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.240807:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.240808:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.240811:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.240812:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.240814:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.240815:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.240815:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.240817:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.240817:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.240818:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.240818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.240823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.240830:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.240832:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.240833:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.240834:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.240835:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.240836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.240944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.240952:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.240955:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.240957:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.240958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.240960:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.240961:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.240962:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.240963:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.240964:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.240965:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.240968:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.240970:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.240972:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.240973:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.240973:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.240975:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.240975:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.240976:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.240977:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.240980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.240986:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.240989:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.240990:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.240990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.240992:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.240994:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.241132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.241140:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.241143:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.241144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.241145:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.241147:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.241148:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.241150:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.241151:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.241152:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.241152:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677474.241153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:26.0:1619677474.241156:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.241157:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.241159:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677474.241160:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.241161:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.241161:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:26.0:1619677474.241162:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.241162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677474.241163:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.241164:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.241164:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677474.241164:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.241165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677474.241167:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.241168:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.241169:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.241170:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.241172:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.241173:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.241177:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.241178:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.241179:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.241179:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.241180:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.241181:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.241182:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:6.0:1619677474.241182:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.241183:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.241185:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.241185:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.241185:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.241186:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.241187:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677474.241187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677474.241198:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.241200:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.241201:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.241203:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.241204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.241205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.241231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.241237:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.241240:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.241241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.241242:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.241245:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.241246:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677474.241248:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.241249:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.241250:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.241251:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677474.241254:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.241256:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.241259:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.241259:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677474.241260:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.241262:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677474.241262:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.241263:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.241264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:6.0:1619677474.241275:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.241277:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.241278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.241279:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.241281:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.241283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.241412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.241421:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.241423:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.241426:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.241428:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.241428:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.241430:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.241431:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.241432:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.241433:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.241434:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.241437:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.241438:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.241442:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.241442:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.241443:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.241445:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.241446:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.241446:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.241447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.241460:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.241462:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.241463:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.241463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.241466:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.241468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.241477:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.241479:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.241480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.241480:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.241481:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.241481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.242224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.242229:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.242231:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.242232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.242234:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.242235:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.242237:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.242238:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.242239:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.242240:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.242241:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.242244:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.242245:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.242248:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.242248:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.242249:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.242251:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.242251:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.242252:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.242252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.242264:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.242267:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.242267:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.242269:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.242270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.242271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.242284:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.242286:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.242287:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.242288:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.242288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.242290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.242751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.242759:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.242762:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.242763:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.242765:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.242768:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.242769:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.242771:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.242772:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.242773:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.242774:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.242777:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.242779:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.242782:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.242783:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.242784:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.242786:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.242786:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.242787:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.242788:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.242789:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.242795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.242801:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.242803:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.242804:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.242805:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677474.242805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.242805:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.242806:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.242807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677474.242808:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.242811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.243163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.243169:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.243171:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.243173:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.243174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.243176:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.243178:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:21.0:1619677474.243181:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.243182:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.243183:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.243184:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677474.243184:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.243185:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677474.243186:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.243187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.243190:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.243190:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.243192:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677474.243193:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.243194:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.0:1619677474.243195:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677474.243196:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.243196:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.243196:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:22.0:1619677474.243197:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.243198:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.243198:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:22.0:1619677474.243199:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.243199:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.243200:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.243200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677474.243202:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.243204:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.2:1619677474.243205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.243206:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.243207:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.243208:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.243209:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.243210:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.243210:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.243211:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:17.0:1619677474.243215:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.243219:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.243221:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.243222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.243223:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.243225:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:17.0:1619677474.243225:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:20.0:1619677474.243226:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.243227:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.243228:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677474.243228:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677474.243230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.243321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.243329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.243332:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.243335:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.243336:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.243337:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.243339:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.243340:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.243341:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.243343:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.243343:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.243347:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.243349:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.243352:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.243352:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.243353:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.243355:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.243355:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.243356:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.243357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.243361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.243369:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.243371:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.243372:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.243374:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.243375:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.243377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.243397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.243405:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.243408:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.243410:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.243411:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.243413:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.243415:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677474.243416:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.243417:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.243418:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.243419:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677474.243423:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.243424:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677474.243427:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.243428:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677474.243429:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.243431:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677474.243431:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.243433:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.243434:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.243435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.243437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.2:1619677474.243441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.243448:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:18.0:1619677474.243448:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:30.0:1619677474.243451:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.243452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:18.0:1619677474.243452:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:30.0:1619677474.243453:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:18.0:1619677474.243454:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.243454:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.243457:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.243462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.243546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.243554:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.243558:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.243560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.243563:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.243564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677474.243566:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.243568:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.243570:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.243571:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677474.243573:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.243574:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.243575:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.243576:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677474.243576:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677474.243580:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.243583:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.243583:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.243584:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677474.243586:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677474.243587:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.243588:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.243589:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.243591:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.243591:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.243592:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.243593:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:10.0:1619677474.243595:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.243596:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.243596:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:10.0:1619677474.243597:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.243598:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.243599:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:10.0:1619677474.243600:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.243601:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.243601:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.2:1619677474.243601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677474.243603:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.243603:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.243604:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.243605:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677474.243609:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.243612:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.243613:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.243614:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.243615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.243617:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677474.243617:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.243619:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.243620:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.243621:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.243623:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.243626:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.243879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.243887:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.243890:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.243890:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.243893:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.243895:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.243896:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.243897:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.243899:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.243900:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.243900:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.243904:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.243905:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.243907:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.243908:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.243908:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.243910:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.243910:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.243911:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.243911:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677474.243923:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.243925:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.243926:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.243928:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.243930:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.243931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.243965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.243973:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.243976:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.243978:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.243980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.243981:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.243982:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.243983:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.243984:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.243985:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.243986:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.243988:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.243990:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.243992:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.243993:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.243993:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.243995:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.243995:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.243996:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.243998:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.243999:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.244004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.244009:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.244010:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.244011:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.244012:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.244012:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.244013:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.244014:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.244014:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.244015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.244016:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.244075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.244080:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.244083:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.244084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.244086:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.244088:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.244089:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.244090:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.244092:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.244093:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.244094:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.244096:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.244098:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.244100:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.244101:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.244102:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.244103:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.244104:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.244105:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.244106:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.244106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.244111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.244117:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.244118:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677474.244119:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.244119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.244120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.244121:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.244122:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.244124:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.244124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.244126:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.244289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.244298:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.244301:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.244303:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.244305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.244305:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.244307:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.244308:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.244310:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.244311:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.244312:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.244315:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.244317:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.244319:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.244320:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.244321:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.244322:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.244323:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.244324:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.244325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.244338:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.244341:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.244341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.244342:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.244344:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.244347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.244459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.244467:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.244469:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.244471:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.244473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.244474:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.244476:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.244477:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.244478:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.244479:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.244480:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.244483:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.244484:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.244487:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.244487:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.244488:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.244489:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.244490:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.244491:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.244491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.244495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.244503:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.244505:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.244506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.244506:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.244508:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.244511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.244955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.244962:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.244965:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.244965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.244968:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.244970:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.244971:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.244973:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.244974:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.244975:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.244976:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.244979:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.244981:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.244984:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.244984:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.244985:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.244987:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.244987:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.244988:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.244989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.245001:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.245003:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.245004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.245004:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.245007:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.245009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.245018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.245024:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.245027:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:22.0:1619677474.245028:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:18.0:1619677474.245028:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.245029:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677474.245030:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.245032:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.245032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:2.0:1619677474.245032:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.245033:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.245033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:2.0:1619677474.245033:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.245035:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.245036:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.245037:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.245038:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.245041:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.245043:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.245045:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.245046:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.245047:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.245048:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.245049:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.245050:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.245050:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.245054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.245062:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.245065:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.245066:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.245066:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.245068:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.245071:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.245082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.245090:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.245093:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.245095:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.245097:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.245098:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.245099:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.245100:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.245102:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.245103:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.245103:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.245106:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.245108:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.245110:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.245111:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.245112:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.245113:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.245114:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.245115:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.245115:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.245126:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.245128:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.245129:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.245130:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.245132:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.245133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.245148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.245153:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.245156:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.245157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.245159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:6.0:1619677474.245159:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.245161:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.245162:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.245164:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.245165:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.245166:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:6.0:1619677474.245166:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.245167:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.245170:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:6.0:1619677474.245171:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.245173:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.245173:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.245174:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:30.0:1619677474.245175:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.245176:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677474.245177:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.245177:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:30.0:1619677474.245178:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.245178:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677474.245179:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.245180:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.245180:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.245180:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677474.245181:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.245181:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.245182:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:30.0:1619677474.245185:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.245187:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.245190:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.245190:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.245191:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.245193:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.245193:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677474.245193:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677474.245194:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.245195:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.0:1619677474.245196:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.245196:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.245198:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.245199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.245200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677474.245206:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.245209:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.245209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.245210:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.245212:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.245215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.245442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.245449:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.245452:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.245453:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.245455:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.245458:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.245459:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.245461:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.245462:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.245463:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.245464:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.245467:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.245469:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.245472:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.245472:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.245473:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.245475:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.245475:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.245476:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.245476:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.245505:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.245508:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.245509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.245509:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.245511:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.245514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.245527:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.245529:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.245530:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.245530:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.245531:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.245532:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.246017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677474.246022:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.246024:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677474.246025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677474.246027:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.246029:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.246031:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677474.246032:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.246033:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.246034:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.246035:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677474.246039:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.246040:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677474.246043:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.246044:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677474.246045:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677474.246047:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677474.246047:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.246048:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677474.246049:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677474.246050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:21.2:1619677474.246055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:31.0:1619677474.246060:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:22.0:1619677474.246060:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677474.246062:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677474.246062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:31.0:1619677474.246063:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:22.0:1619677474.246063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:31.0:1619677474.246064:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.246065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:31.0:1619677474.246066:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.246067:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.246069:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:6.0:1619677474.246070:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.246072:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.246073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.246075:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.246078:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.246079:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.246080:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.246081:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.246082:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.246083:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.246086:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.246087:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.246089:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.246090:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.246091:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.246092:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.246092:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.246093:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.246093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.246106:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.246108:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.246109:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.246111:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.246112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.246113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.246139:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.246142:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.246143:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.246143:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.246144:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.246146:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.246241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.246249:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.246251:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.246252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.246254:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.246257:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.246258:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.246260:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.246261:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.246262:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.246263:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.246266:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.246268:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.246270:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.246271:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.246272:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.246273:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.246274:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.246275:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.246276:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.246277:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.246282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.246289:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.246291:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.246292:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.246293:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.246297:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.246300:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.246301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.246302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.246303:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.246306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.246979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.246987:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.246990:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.246992:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.246994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.246995:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.246996:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:21.2:1619677474.246996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:30.0:1619677474.246997:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.246998:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.246999:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.247000:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:6.0:1619677474.247001:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677474.247004:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.247004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.247004:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:30.0:1619677474.247007:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:6.0:1619677474.247008:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.247009:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.247010:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.247010:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:6.0:1619677474.247010:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.247011:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:30.0:1619677474.247012:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.247012:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677474.247012:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.247013:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.247014:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677474.247014:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.247015:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.247016:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:9.2:1619677474.247018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:6.0:1619677474.247019:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.247021:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.247023:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.247024:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677474.247025:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677474.247025:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.247026:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677474.247027:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677474.247027:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.247027:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.247028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:24.0:1619677474.247029:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.247029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.247030:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.247033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677474.247040:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.247043:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.247043:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.247045:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.247046:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.247047:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.247269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.247276:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.247279:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.247282:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.247283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.247284:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.247286:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.247287:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.247288:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.247289:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.247290:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.247298:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.247300:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.247303:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.247304:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.247305:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.247307:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.247307:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.247309:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.247309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.247335:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.247337:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.247338:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.247339:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.247341:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.247344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.247707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.247715:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.247718:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.247719:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.247720:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.247723:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.247725:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.247726:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.247727:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.247728:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.247729:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.247732:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.247733:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.247736:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.247737:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.247738:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.247739:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.247740:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.247740:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.247741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.247753:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.247756:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.247757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.247757:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.247759:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.247761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.248011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.248017:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.248019:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.248021:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.248022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.248024:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.248025:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.248027:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.248028:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.248029:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.248030:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.248033:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.248035:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.248037:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.248038:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.248038:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.248040:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.248041:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.248042:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.248042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.248046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.248054:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.248056:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.248057:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.248059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.248059:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.248061:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.248066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.2:1619677474.248073:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:21.0:1619677474.248077:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677474.248079:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677474.248080:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.248080:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:22.0:1619677474.248081:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.0:1619677474.248081:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.248082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:6.0:1619677474.248082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.248083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:22.0:1619677474.248084:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.248086:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.248088:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.248089:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.248090:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.248092:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.248092:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.248096:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.248097:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.248100:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.248101:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.248101:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.248103:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.248104:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.248105:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.248106:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.248106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.248111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.248117:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.248118:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.248119:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.248119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.248120:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.248120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677474.248121:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.248123:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.248124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.248125:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.248218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.248226:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.248231:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.248232:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.248236:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.248238:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.248241:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.248243:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.248245:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.248247:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.248249:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.248254:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.248257:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.248261:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.248262:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.248263:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.248265:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.248266:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.248267:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.248268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.248271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.248279:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.248282:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.248283:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.248284:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.248286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.248287:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.248304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677474.248312:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.248316:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677474.248318:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677474.248321:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.248324:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.248327:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677474.248329:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.248331:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.248333:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.248334:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677474.248340:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.248343:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677474.248347:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.248348:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677474.248349:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677474.248351:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677474.248352:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.248353:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.248354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:12.0:1619677474.248367:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.248371:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.248372:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.248375:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677474.248375:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.248378:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677474.248692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.248700:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.248705:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:20.2:1619677474.248706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.248706:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.248708:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:18.0:1619677474.248710:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.248711:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:18.0:1619677474.248714:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.248715:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.248716:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.248719:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.248722:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.248724:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.248725:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.248731:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.248734:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.248734:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.248737:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:18.0:1619677474.248738:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.248740:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:2.0:1619677474.248740:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677474.248741:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.248743:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.248743:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.248744:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.248746:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677474.248747:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:21.2:1619677474.248748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677474.248749:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.248749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.248751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677474.248753:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.248753:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.248755:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:30.0:1619677474.248756:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677474.248757:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.248757:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:18.0:1619677474.248759:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.248760:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677474.248763:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.248763:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.248764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.248764:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.248765:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677474.248766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.248766:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:30.0:1619677474.248768:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.248768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:30.0:1619677474.248769:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677474.248769:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:30.0:1619677474.248770:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.248771:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.248771:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:30.0:1619677474.248772:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.248772:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:30.0:1619677474.248773:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677474.248774:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:30.0:1619677474.248776:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677474.248776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.248778:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.248780:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.248781:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.248781:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.248783:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.248783:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.248784:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.248785:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.248786:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.248787:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.248787:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.248788:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.248789:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.248790:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.2:1619677474.248793:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.248797:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.0:1619677474.248798:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:31.0:1619677474.248799:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.248800:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.248801:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.248801:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.248802:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.2:1619677474.248802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.248803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.248804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.248804:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.248805:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.248809:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.248812:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.248813:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.248814:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.248815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.248816:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.248968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.248975:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.248978:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.248979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.248981:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.248981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:22.0:1619677474.248983:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.248984:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.248985:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.248987:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.248988:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:22.0:1619677474.248988:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.248989:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.248991:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:22.0:1619677474.248993:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.248994:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.248995:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.248995:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:30.0:1619677474.248996:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.248998:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677474.248998:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.248998:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:30.0:1619677474.248999:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.248999:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677474.249001:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.249001:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.249001:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677474.249002:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.249002:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.249002:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:30.0:1619677474.249003:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.249006:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.249008:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.249011:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.249012:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.249012:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.249014:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.249014:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677474.249015:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.249015:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:20.0:1619677474.249016:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677474.249017:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000400:00000200:20.0:1619677474.249017:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.249018:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:20.0:1619677474.249018:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.249020:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.249021:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.249023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.249028:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.249030:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.249031:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.249031:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.249032:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.249034:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.249034:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.249034:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.249035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.249036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.249046:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.249049:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.249050:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.249050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.249051:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.249052:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.249301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.249309:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.249312:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.249314:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.249316:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.249317:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.249318:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.249319:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.249321:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.249322:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.249322:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.249326:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.249327:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.249329:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.249330:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.249331:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.249332:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.249333:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.249334:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.249334:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.249345:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.249348:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.249349:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.249350:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.249352:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.249353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.249362:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.249364:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.249365:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.249366:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.249366:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.249367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.249558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.249564:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.249566:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.249568:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.249569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.249571:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.249573:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.249574:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.249575:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.249576:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.249577:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.249580:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.249582:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.249584:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.249585:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.249586:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.249587:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.249588:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.249589:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.249590:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.249591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.249594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.249596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.249599:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.249601:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.249602:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.249602:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.249603:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.249605:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.249606:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.249606:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.249608:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.249610:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.249687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.249694:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.249697:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.249698:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.249700:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.249703:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.249705:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.249706:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.249707:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.249708:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.249709:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.249713:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.249715:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.249718:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.249718:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.249719:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.249721:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.249721:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.249722:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.249722:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.249736:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.249738:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.249739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.249740:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.249742:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.249744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.249773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.249780:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.249783:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.249785:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.249787:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.249788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.249788:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.249789:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.249791:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.249792:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.249793:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.249796:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.249797:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.249799:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.249799:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.249800:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.249801:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.249802:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.249803:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.249803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.249814:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.249816:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.249817:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.249818:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.249820:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.249821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.249857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.249862:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.249865:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.249865:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.249867:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.249869:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.249871:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.249872:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.249873:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.249874:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.249875:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.249878:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.249880:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.249882:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.249883:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.249884:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.249885:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.249886:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.249886:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.249887:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.249899:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.249901:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.249902:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.249903:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.249905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.249906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.249921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.249924:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.249925:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.249926:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.249926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.249927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677474.250275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.250280:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.250283:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.250284:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.250286:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.250288:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.250290:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.250291:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.250292:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.250299:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.250300:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.250303:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.250305:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.250308:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.250308:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.250309:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.250311:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.250311:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.250312:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.250312:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677474.250321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677474.250325:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.250328:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.250329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.250329:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677474.250329:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.250331:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.250332:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.250333:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677474.250334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:10.0:1619677474.250335:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.250338:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.250339:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.250340:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.250342:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.250343:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.250344:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.250347:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.250349:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.250351:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.250352:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.250353:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.250354:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.250355:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.250356:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.250356:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.250360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.250367:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.250369:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.250370:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.250371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.250372:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.250375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.250381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.250387:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.250389:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.250390:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.250390:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.250391:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.250391:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.250680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.250688:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.250691:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.250693:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.250695:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.250696:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.250697:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.250698:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.250699:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.250700:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.250701:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.250704:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.250706:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.250709:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.250710:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.250710:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.250712:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.250712:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.250713:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.250714:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.250718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.250726:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.250729:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.250730:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.250731:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.250732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.250735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.250799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.250805:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.250808:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.250809:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.250811:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.250813:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.250815:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677474.250816:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.250817:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.250818:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.250819:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677474.250822:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.250824:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.250825:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.250826:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.250827:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.250829:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.250829:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.250831:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677474.250832:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677474.250832:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.2:1619677474.250835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.2:1619677474.250839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.250843:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.250844:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.250845:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:1.0:1619677474.250845:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.250846:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677474.250847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.250847:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.250848:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.250848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677474.250849:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:21.2:1619677474.250954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.250960:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.250963:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.250963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.250965:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.250967:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.250968:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.250970:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.250971:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.250972:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.250973:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.250976:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.250978:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.250981:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.250981:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.250982:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.250984:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.250984:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.250985:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.250985:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.250997:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.251000:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.251001:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.251002:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.251003:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.251004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.251076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.251084:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.251087:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.251089:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.251091:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.251092:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.251094:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.251095:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.251097:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.251098:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.251099:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677474.251100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677474.251102:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.251104:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.251107:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.251107:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677474.251108:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.251109:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.251109:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677474.251110:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.251110:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677474.251111:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.251112:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.251112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:26.0:1619677474.251112:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.251114:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.251115:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.251116:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.251118:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.251118:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.251119:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.251122:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.251124:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677474.251124:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.251126:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.251126:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.251127:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.251127:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.251128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.251128:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.251129:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.251129:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.251129:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.251130:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677474.251130:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.251133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677474.251134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.251140:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.251143:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.251144:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.251144:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.251146:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.251148:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.251284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.251291:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.251296:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.251299:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:18.0:1619677474.251301:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.251303:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.251305:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.251306:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.251307:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.251308:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.251309:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.251313:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.251315:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.251318:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.251319:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.251320:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.251321:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.251322:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.251323:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.251323:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677474.251338:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.251341:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.251342:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.251345:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.251346:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.251349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.251807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.251815:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.251818:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.251818:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.251821:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.251823:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.251825:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.251826:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.251827:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.251828:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.251829:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.251832:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.251834:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.251837:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.251838:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.251838:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.251840:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.251840:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.251841:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.251843:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.251843:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677474.251846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677474.251849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677474.251854:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677474.251856:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677474.251857:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:26.0:1619677474.251858:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:18.0:1619677474.251858:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.251859:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:2.0:1619677474.251859:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.251860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677474.251861:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.251864:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.251870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.251876:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.251879:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.251880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.251882:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.251884:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.251885:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.251886:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.251888:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.251889:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.251890:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.251893:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.251895:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.251897:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.251898:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.251899:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.251900:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.251901:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.251902:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.251903:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.251904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.251909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.251914:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.251916:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.251916:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.251917:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.251918:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.251918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677474.251919:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.251921:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.251922:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.251923:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.252609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.252617:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.252620:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.252622:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.252625:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.252625:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.252627:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.252628:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.252629:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.252630:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.252631:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.252634:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.252636:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.252638:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.252639:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.252640:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.252641:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.252642:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.252643:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.252645:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.252645:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.252648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.252649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:21.2:1619677474.252653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.252655:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677474.252655:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:8.0:1619677474.252658:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677474.252658:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:24.0:1619677474.252659:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677474.252659:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:8.0:1619677474.252659:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.252661:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:1.0:1619677474.252661:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.252662:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.252663:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:1.0:1619677474.252663:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.252664:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677474.252664:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677474.252665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677474.252665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:1.0:1619677474.252665:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.252666:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.252667:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.252669:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.252672:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.252673:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.252676:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.252677:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.252677:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.252679:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.252679:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.252680:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.252681:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.252685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.252693:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.252695:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.252696:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.252697:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.252699:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.252700:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.252777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.252783:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.252786:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.252787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.252789:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.252791:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.252792:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.252793:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.252795:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.252796:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.252797:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.252799:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.252801:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.252803:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.252804:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.252805:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.252806:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.252807:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.252807:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.252808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.252820:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.252822:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.252823:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.252824:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.252825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.252826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.253088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.253096:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.253099:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.253101:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.253102:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.253104:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.253105:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.253107:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.253108:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.253109:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.253110:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.253114:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.253115:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677474.253116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677474.253119:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.253119:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.253120:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.253122:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.253122:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.253123:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677474.253123:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677474.253124:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677474.253125:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.253127:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.253128:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.253131:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.253132:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.253133:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96505321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.253135:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.253135:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.253135:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:24.0:1619677474.253136:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.253138:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.253139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.253139:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.253140:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.253141:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.253142:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.253144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677474.253145:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.253145:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.253146:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.253148:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.253148:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.253149:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.253150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.253161:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.253164:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.253165:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.253165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.253167:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.253169:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.253738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.253747:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.253750:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.253752:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.253755:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.253755:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.253756:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.253757:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.253759:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.253760:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.253761:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.253764:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.253765:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.253767:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.253768:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.253768:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.253770:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.253770:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.253771:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.253772:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.253783:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.253786:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.253786:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.253788:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.253789:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.253790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.253824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677474.253832:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.253834:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677474.253835:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677474.253837:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.253840:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.253841:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.253842:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.253843:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.253844:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.253845:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.253848:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.253849:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.253852:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.253853:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.253853:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.253855:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.253856:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.253857:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677474.253858:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677474.253859:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:4.2:1619677474.253865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:6.0:1619677474.253869:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.253871:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.253872:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:22.0:1619677474.253872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.253873:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.253874:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677474.253875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:6.0:1619677474.253875:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.253876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677474.253878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.253886:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.253889:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.253890:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.253890:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.253891:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.253892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.254315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.254323:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.254327:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.254329:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.254331:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.254331:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.254332:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.254334:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.254335:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.254336:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.254337:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.254340:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.254342:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.254344:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.254344:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.254345:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.254346:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.254347:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.254348:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.254348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.254360:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.254362:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.254363:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.254364:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.254366:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.254367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.254496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.254504:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.254506:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.254507:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.254509:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.254512:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.254513:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.254515:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.254516:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.254517:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.254517:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.254520:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.254522:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.254525:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.254525:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.254526:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.254528:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.254528:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.254529:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.254530:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.254530:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.254536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.254541:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.254543:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.254544:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677474.254545:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677474.254545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.254545:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.254546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.254547:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.254547:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.254550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.255005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.255013:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.255016:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.255017:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.255019:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.255020:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.255021:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.255022:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.255023:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.255024:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.255025:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.255028:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.255030:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.255033:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.255034:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.255034:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.255036:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.255036:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.255037:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.255038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.255049:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.255052:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.255052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.255053:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.255055:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.255058:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.255165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.255173:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.255176:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.255178:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.255180:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.255181:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.255182:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.255184:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.255185:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.255186:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.255187:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.255190:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.255191:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.255193:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.255194:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.255195:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.255196:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.255197:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.255198:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.255198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.255202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.255209:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.255211:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.255212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.255212:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.255214:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.255217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.255292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.255301:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.255303:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.255304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.255306:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.255309:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.255310:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.255311:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.255312:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.255313:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.255314:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.255317:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.255318:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.255320:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.255321:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.255321:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.255323:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.255323:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.255324:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.255324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.255329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.255336:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.255338:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.255339:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.255339:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.255340:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.255342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.255347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.255353:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.255355:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.255356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.255356:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.255357:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.255357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.255502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.255508:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.255511:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.255512:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.255514:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.255516:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.255518:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.255519:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.255520:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677474.255521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:17.0:1619677474.255521:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.255522:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677474.255523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677474.255525:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.255526:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677474.255526:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.255527:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677474.255529:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677474.255530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:17.0:1619677474.255530:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.255530:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:17.0:1619677474.255531:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.255531:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677474.255532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.255532:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.255533:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.255533:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.255534:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:6.0:1619677474.255534:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.255535:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.255535:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.255536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677474.255536:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.255538:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677474.255538:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677474.255540:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.255540:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677474.255541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677474.255542:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.255543:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.255543:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.255544:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.255545:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677474.255546:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677474.255546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.255547:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.255548:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.255548:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.255548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677474.255549:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.255550:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.255550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.255551:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.255552:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.255552:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677474.255554:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.255555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677474.255555:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.255555:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677474.255556:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677474.255556:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677474.255557:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.255558:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677474.255558:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677474.255558:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.255559:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:20.0:1619677474.255559:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677474.255559:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.255560:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:20.0:1619677474.255560:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.255560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677474.255561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.255561:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.255561:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.255562:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.255563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.255563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.2:1619677474.255564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.255573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677474.255574:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:17.0:1619677474.255575:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677474.255576:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:17.0:1619677474.255576:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.255576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:21.0:1619677474.255577:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.255579:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:17.0:1619677474.255579:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.255580:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.255583:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677474.255583:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677474.255597:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.255600:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.255601:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.255602:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.255602:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.255603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.256075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.256083:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.256086:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.256086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.256088:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.256091:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.256092:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.256093:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.256095:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.256096:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.256097:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.256101:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.256102:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.256105:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.256106:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.256106:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.256108:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.256109:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.256110:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.256111:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.256111:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.256117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.256122:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.256123:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.256124:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677474.256125:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677474.256125:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.256126:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.256126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.256127:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677474.256128:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.256131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.256203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.256207:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.256210:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.256212:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.256213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.256214:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.256216:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.256217:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.256218:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.256219:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.256220:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.256224:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.256226:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.256229:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.256229:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.256230:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.256232:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.256233:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.256234:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.256234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677474.256246:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.256249:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.256250:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.256251:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.256253:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.256254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677474.256269:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.256272:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.256273:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.256273:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.256274:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.256275:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.256324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.256331:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.256334:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.256335:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.256337:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.256340:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.256341:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.256342:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.256343:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.256344:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.256345:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.256349:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.256351:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.256353:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.256354:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.256355:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.256356:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.256357:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.256358:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.256358:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.256363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.256372:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.256375:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.256376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.256377:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.256379:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.256382:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.256419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.256425:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.256428:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.256429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.256431:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.256433:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.256434:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.256436:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.256437:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.256438:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.256439:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.256442:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.256444:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.256446:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.256447:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.256448:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.256449:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.256450:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.256451:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.256452:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.256452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.256456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.256458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.256463:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677474.256465:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.256465:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.256465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.256466:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677474.256468:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.256468:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.256470:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.256472:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.256473:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.256630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.256638:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.256641:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.256643:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.256645:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.256645:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.256647:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.256648:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.256649:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.256651:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.256651:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.256655:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.256656:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.256659:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.256660:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.256660:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.256662:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.256663:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.256664:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.256666:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.256667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.256668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.256672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.256674:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.256676:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.256677:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.256678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.256679:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.256680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677474.256681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.256682:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.256683:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.256684:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.256746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.256753:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.256755:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.256757:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.256760:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.256760:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.256761:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.256763:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.256764:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.256765:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.256765:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.256769:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.256770:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.256773:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.256774:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.256774:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.256776:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.256776:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.256777:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.256779:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.256779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.256785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.256790:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.256793:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677474.256793:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.256794:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.256794:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.256796:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.256796:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.256796:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.256798:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.256799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.257262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.257267:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.257270:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.257271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.257273:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.257274:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.257276:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.257277:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.257278:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.257279:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.257280:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.257283:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.257285:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.257287:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.257287:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.257288:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.257290:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.257290:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.257291:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.257291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.257306:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.257309:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.257309:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.257311:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.257312:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.257313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.257791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.257797:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.257800:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.257801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.257803:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.257805:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.257807:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.257808:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.257809:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.257810:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.257811:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.257815:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.257816:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.257819:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.257819:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.257820:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.257822:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.257822:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.257824:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.257825:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.257825:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.257829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.257830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.257836:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677474.257837:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677474.257838:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.257839:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.257839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677474.257840:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.257841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.257841:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.257843:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:8.2:1619677474.257845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.257848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.257849:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.257851:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.257852:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.257853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677474.257853:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.257853:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.2:1619677474.257936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677474.257941:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.257944:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677474.257945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677474.257947:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.257948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.257950:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677474.257951:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.257952:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.257953:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.257954:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677474.257957:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.257959:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677474.257960:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.257961:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.257962:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.257963:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.257964:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.257964:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.257965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677474.257970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.257977:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.257979:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.257979:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.257981:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.257981:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.257982:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.257992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.257998:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.258001:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677474.258001:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.258002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.258002:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.258003:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.258016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.258024:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.258027:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.258028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.258029:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.258033:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.258034:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.258035:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.258037:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.258038:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.258039:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.258042:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.258044:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.258047:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.258048:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.258048:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.258050:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.258051:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.258051:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.258052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.258064:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.258067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.258067:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.258068:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.258070:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.258072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.258084:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.258086:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.258087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.258088:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.258089:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.258089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.258157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.258165:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.258167:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.258168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.258170:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.258173:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.258174:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677474.258175:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.258177:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.258178:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.258178:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677474.258181:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.258182:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.258183:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.258184:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.258184:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.258186:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.258186:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.258187:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.258187:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677474.258191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.258198:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.258199:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.258200:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.258201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.258202:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.258203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.258300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.258309:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.258311:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.258313:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.258316:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.258316:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.258317:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.258318:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.258320:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.258321:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.258322:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.258325:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.258326:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.258328:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.258329:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.258329:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.258331:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.258331:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.258332:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.258333:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.258344:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.258347:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.258348:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.258349:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.258351:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.258352:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.258364:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.258366:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.258367:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.258368:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.258368:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.258369:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.258721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.258727:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.258729:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.258730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.258732:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.258734:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.258735:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.258737:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.258738:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.258739:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.258740:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.258743:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.258745:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.258748:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.258748:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.258749:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.258751:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.258751:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.258753:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.258754:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.258754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.258760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.258764:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.258766:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.258767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.258767:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.258768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.258769:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.258770:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.258772:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.258773:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.258774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.258817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.258825:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.258827:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.258830:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.258831:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.258832:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.258833:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.2:1619677474.258833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:30.0:1619677474.258834:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.258836:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.258837:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.258838:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.258840:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.258841:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.258841:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.258843:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.258843:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.258845:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.258846:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.258847:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.258847:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.258847:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677474.258848:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.258849:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.258849:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677474.258850:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.258851:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.258854:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.258855:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.258856:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.258856:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.258856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677474.258858:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.258858:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.258858:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.258859:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.258860:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.258861:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.258862:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.258862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677474.258862:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.258865:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.258866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.258866:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.258869:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.258871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.258872:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.258875:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.258876:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.258878:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.258879:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.258880:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.259308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.259316:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.259319:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.259320:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.259322:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.259325:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.259326:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.259327:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.259329:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.259330:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.259330:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.259334:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.259335:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.259338:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.259339:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.259340:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.259342:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.259342:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.259343:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.259343:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.259356:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.259359:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.259360:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.259360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.259362:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.259365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.259394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.259403:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.259406:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.259408:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.259410:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.259410:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.259411:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.259412:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.259414:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.259415:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.259416:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.259419:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.259420:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.259422:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.259423:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.259423:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.259425:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.259425:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.259426:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.259427:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.259437:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.259439:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.259440:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.259441:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.259443:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.259444:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.259464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677474.259467:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677474.259470:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:21.0:1619677474.259472:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677474.259472:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.259474:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.259476:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.259477:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.259479:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.259480:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.259481:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.259484:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.259485:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.259488:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.259489:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.259489:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.259491:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.259491:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677474.259492:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:20.0:1619677474.259494:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677474.259494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677474.259498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.259501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677474.259504:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.259506:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000800:00000200:20.0:1619677474.259506:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.259506:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.259507:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000400:00000200:20.0:1619677474.259507:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.259508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677474.259509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.259510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677474.259511:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.259521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.259529:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.259531:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.259532:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.259532:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.259533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.259534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.259715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.259722:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.259725:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.259728:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677474.259730:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.259732:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.259732:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677474.259733:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.259735:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.259736:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.259737:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677474.259740:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.259742:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677474.259745:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.259746:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677474.259746:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677474.259748:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677474.259749:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.259750:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.259751:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.259754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.259763:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.259766:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.259767:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.259767:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.259769:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.259773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.260037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.260044:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.260046:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.260047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.260049:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.260052:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.260054:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.260055:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.260056:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.260057:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.260058:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.260061:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.260062:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.260064:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.260065:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.260066:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.260067:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.260068:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.260068:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.260069:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.260073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.260079:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.260082:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.260083:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.260083:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.260084:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.260087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.260209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.260216:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.260218:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.260219:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.260221:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.260223:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.260224:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.260225:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.260227:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.260228:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.260229:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.260232:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.260234:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.260236:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.260237:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.260238:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.260239:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.260240:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.260241:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.260241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.260253:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.260255:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.260256:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.260257:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.260259:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.260261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.260538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677474.260543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677474.260547:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.260549:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677474.260549:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677474.260552:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.260552:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.260552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677474.260554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.260554:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.260555:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.260556:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.260557:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.260557:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.260559:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.260559:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.260560:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.260561:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.260562:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.260563:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.260563:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.260564:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.260569:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.260570:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.260572:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.260573:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.260575:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.260575:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677474.260576:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.260576:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677474.260577:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.260578:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.260578:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.260579:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.260579:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677474.260580:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.260580:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.260580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677474.260581:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.260582:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.260591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:20.0:1619677474.260593:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.260595:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.260596:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.260597:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:20.0:1619677474.260598:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.260599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.260600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677474.260600:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.260601:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.260602:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.260603:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.260605:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:1.0:1619677474.260605:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.260606:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:1.0:1619677474.260606:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677474.260607:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.260607:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.260609:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.260609:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.260610:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.260611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:1.0:1619677474.260611:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.260615:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.260617:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.260619:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.260620:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.260621:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.260622:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.260623:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.260623:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.260624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.260629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.260637:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.260639:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.260640:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.260642:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.260643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.260644:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.260785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.260789:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.260792:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.260794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.260794:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.260795:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.260797:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.260798:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.260800:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.260801:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.260802:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.260805:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.260807:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.260809:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.260810:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.260810:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.260812:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.260813:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.260814:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.260814:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677474.260825:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.260828:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.260829:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.260829:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.260831:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.260833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.260916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.260924:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.260927:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.260928:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.260930:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.260932:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.260934:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.260935:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.260936:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.260937:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.260938:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.260941:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.260943:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.260946:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.260947:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.260948:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.260950:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.260950:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.260951:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.260951:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.260962:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.260965:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.260966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.260966:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.260968:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.260971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.260984:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.260987:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.260988:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.260988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.260989:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.260990:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.261459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.261467:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.261470:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.261473:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.261475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.261476:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.261477:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.261478:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.261479:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.261480:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.261481:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.261484:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.261486:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.261488:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.261489:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.261490:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.261491:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.261492:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.261493:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.261495:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.261495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.261498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.261501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.261505:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.261507:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.261508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.261508:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.261509:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.261510:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.261511:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.261512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677474.261512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.261513:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.261612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.261619:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.261622:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.261623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.261624:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.261627:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.261628:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677474.261629:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.261630:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.261631:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.261632:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677474.261634:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.261636:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.261637:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.261638:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.261639:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.261640:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.261640:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.261641:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.261641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677474.261645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.261651:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.261653:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.261653:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.261654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.261655:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.261656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.261978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.261984:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.261987:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.261989:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.261989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.261991:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.261993:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.261994:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.261995:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.261997:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.261997:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.262000:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.262002:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.262004:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.262005:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.262006:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.262008:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.262008:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.262009:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.262010:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.262014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.262022:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.262024:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.262025:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.262026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.262028:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.262030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.262193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.262198:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.262201:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.262202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.262204:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.262206:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.262208:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.262209:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.262210:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.262211:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.262212:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.262216:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.262218:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.262220:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.262221:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.262221:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.262223:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.262224:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.262224:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.262225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.262236:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.262239:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.262239:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.262241:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.262242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.2:1619677474.262243:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.0:1619677474.262247:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677474.262249:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.262252:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.262253:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.262255:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.262256:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.262258:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677474.262258:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.262259:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000400:00000200:2.0:1619677474.262259:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:21.0:1619677474.262260:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000400:00000200:2.0:1619677474.262260:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.262261:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.262262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:2.0:1619677474.262262:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.262263:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.262264:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.262267:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.262269:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.262271:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.262272:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.262272:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.262274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.262274:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.262274:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.262276:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.262277:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.262278:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.262280:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:9.2:1619677474.262281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.262282:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:20.2:1619677474.262283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.262284:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.262284:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.262285:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.262286:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.262287:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.262287:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.262288:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.262288:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.262290:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677474.262290:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.262291:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.262292:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.262299:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.262300:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.262301:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.262302:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.262302:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.262303:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.262304:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.262304:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.262305:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.262305:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.262306:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.262307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.262307:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.262308:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677474.262308:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.262308:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.262311:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.262311:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.262312:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.262312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677474.262325:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.262328:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.262329:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.262331:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.262334:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.262335:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.262810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.262819:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.262822:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.262824:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.262826:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.262826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.262827:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.262829:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.262830:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.262831:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.262832:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.262835:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.262836:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.262839:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.262840:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.262840:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.262842:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.262843:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.262844:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.262844:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.262855:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.262857:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.262858:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.262859:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.262861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.262862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.262916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.262923:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.262926:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.262928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.262929:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.262932:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.262933:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.262935:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.262937:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.262938:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.262938:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.262941:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.262943:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.262946:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.262947:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.262947:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.262949:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.262949:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.262950:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.262952:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.262952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.2:1619677474.262957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:4.2:1619677474.262958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.262962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.262965:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.262965:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.262966:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.262967:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:22.0:1619677474.262967:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.262967:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677474.262967:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.262968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.262969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.262970:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.262971:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.262972:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.262973:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.262973:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677474.262975:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.262976:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.262977:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.262978:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.262979:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.262982:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.262984:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.262987:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.262987:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.262988:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.262990:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.262990:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.262991:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.262992:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.263003:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.263005:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.263006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.263007:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.263009:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.263011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.263377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.263386:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.263389:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.263391:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.263393:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.263394:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.263395:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.263397:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.263398:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.263399:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.263400:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.263403:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.263404:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.263406:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.263407:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.263407:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.263409:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.263409:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.263410:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.263411:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.263422:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.263424:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.263425:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.263427:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.263428:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.263429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.263438:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.263441:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.263442:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.263442:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.263443:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.263443:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.263598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.263604:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.263607:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.263608:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.263610:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.263612:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.263614:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.263615:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.263616:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.263617:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.263618:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.263621:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.263623:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.263625:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.263626:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.263626:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.263628:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.263629:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.263629:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.263630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.263642:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.263644:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.263645:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.263646:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.263647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.263649:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.263905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.263912:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.263915:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.263917:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.263919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.263919:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.263921:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.263922:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.263924:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.263924:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.263926:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.263929:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.263930:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.263932:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.263932:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.263933:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.263934:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.263935:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.263936:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.263936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.263947:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.263950:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.263951:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.263952:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.263954:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.263955:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.264136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.264144:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.264147:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.264149:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.264150:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.264153:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.264155:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.264156:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.264158:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.264159:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.264159:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.264162:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.264163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677474.264164:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.264167:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.264169:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.264169:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.264171:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.264171:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.264172:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.264172:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677474.264173:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.264174:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677474.264175:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:10.0:1619677474.264175:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.264177:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.264179:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.264180:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.264181:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.264182:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.264182:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.264185:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.264185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.264186:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677474.264187:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.264188:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.264188:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.264190:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.264191:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.264191:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:6.0:1619677474.264191:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.264193:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677474.264193:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677474.264194:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677474.264195:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:10.0:1619677474.264196:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677474.264197:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:8.2:1619677474.264199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.264203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:20.0:1619677474.264208:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677474.264210:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:20.0:1619677474.264211:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677474.264212:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000400:00000200:20.0:1619677474.264212:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.264213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:20.0:1619677474.264213:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.264214:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:21.0:1619677474.264214:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.264217:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.264463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.264470:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.264473:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.264474:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.264475:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.264478:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.264479:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.264480:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.264482:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.264483:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.264484:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.264487:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.264489:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.264491:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.264492:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.264493:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.264494:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.264495:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.264495:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.264496:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.264501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.264509:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.264512:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.264512:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.264514:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.264515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.264516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.264530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.264538:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.264540:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.264542:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.264545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.264545:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.264546:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.264548:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.264549:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.264550:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.264551:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.264554:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.264556:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.264559:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.264559:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.264560:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.264561:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.264562:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.264564:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.264565:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.264565:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.264568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.264570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:21.2:1619677474.264571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.264576:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.264578:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:4.2:1619677474.264578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:31.0:1619677474.264579:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677474.264579:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000400:00000200:8.0:1619677474.264579:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.264581:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677474.264581:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.3.3@o2ib3
00000400:00000200:8.0:1619677474.264581:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.264582:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677474.264582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:31.0:1619677474.264583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:24.0:1619677474.264583:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : REPLY - routed
00000800:00000200:8.0:1619677474.264584:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677474.264585:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:8.0:1619677474.264585:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.264586:0:19939:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000400:00000200:20.0:1619677474.264587:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677474.264587:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:21.0:1619677474.264588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677474.264588:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:20.0:1619677474.264589:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.264589:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.264589:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.264590:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:20.0:1619677474.264591:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677474.264591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677474.264592:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39c600] (21)++
00000400:00000200:20.0:1619677474.264592:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.264593:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.264594:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (22)++
00000400:00000200:20.0:1619677474.264594:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.264595:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (23)--
00000800:00000200:24.0:1619677474.264595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:20.0:1619677474.264595:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.264599:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.264600:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.264603:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677474.264603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000800:00000200:20.0:1619677474.264604:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.264606:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.264608:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.264609:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.264610:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.264610:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677474.264611:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:8.0:1619677474.264613:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:8.0:1619677474.264614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:22.0:1619677474.264622:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.264624:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.264625:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.264625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.264627:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.264629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.264658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.264666:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.264669:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.264671:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.264672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.264673:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.264675:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.264676:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.264678:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.264679:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.264680:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.264683:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.264685:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.264688:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.264689:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.264689:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.264691:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.264692:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.264693:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.264695:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.264695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.264702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.264704:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.264707:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.264708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.264708:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.264709:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.264710:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.264712:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.264713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.264713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.264714:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.264721:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.264723:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.264724:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.264724:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.264725:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.264726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.264767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.264773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.264776:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.264777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.264779:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.264781:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.264782:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.264783:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.264785:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.264786:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.264787:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.264789:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.264791:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.264794:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.264794:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.264795:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.264797:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.264797:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.264798:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.264798:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:4.2:1619677474.264809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:6.0:1619677474.264810:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.264812:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.264813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.264813:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.264815:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.264817:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:6.0:1619677474.264818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677474.264820:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.264821:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.264823:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.264827:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.264828:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.264829:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.264831:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.264832:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.264833:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.264836:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.264837:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.264840:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.264841:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.264842:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.264843:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.264844:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.264844:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.264845:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.264849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.264858:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.264862:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.264863:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.264863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.264865:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.264868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.264871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677474.264878:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.264881:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.264882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.264882:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.264883:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.264883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677474.265003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.265009:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.265012:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.265013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.265015:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.265017:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.265018:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.265019:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.265021:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.265022:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.265022:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.265025:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.265027:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.265030:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.265030:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.265031:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.265032:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.265033:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.265034:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.265034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.265046:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.265048:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.265049:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.265049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:4.2:1619677474.265049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000400:00000200:22.0:1619677474.265052:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.265054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677474.265057:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:24.0:1619677474.265059:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.3.3@o2ib3
00000800:00000200:8.0:1619677474.265060:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (23)++
00000800:00000200:8.0:1619677474.265061:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.3.3@o2ib3
00000800:00000200:24.0:1619677474.265062:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39c600] (24)--
00000400:00000200:24.0:1619677474.265063:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.265064:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677474.265064:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : REPLY - routed
00000800:00000200:6.0:1619677474.265064:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:24.0:1619677474.265066:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.265067:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.265067:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.265068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.265068:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000400:00000200:24.0:1619677474.265069:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.265069:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000400:00000200:24.0:1619677474.265070:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677474.265070:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677474.265072:0:19938:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000400:00000200:24.0:1619677474.265074:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:24.0:1619677474.265076:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.265077:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39c600] (23)++
00000800:00000200:24.0:1619677474.265078:0:19939:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:8.0:1619677474.265079:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (24)++
00000800:00000200:8.0:1619677474.265080:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (25)--
00000800:00000200:8.0:1619677474.265080:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (24)--
00000800:00000200:8.0:1619677474.265082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:4.2:1619677474.265085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (22)++
00000800:00000200:24.0:1619677474.265092:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.265093:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677474.265093:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677474.265094:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (23)++
00000800:00000200:8.0:1619677474.265094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (24)--
00000800:00000200:24.0:1619677474.265095:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677474.265096:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (22)++
00000800:00000200:8.0:1619677474.265096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:24.0:1619677474.265097:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (23)--
00000800:00000200:24.0:1619677474.265098:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:30.0:1619677474.265116:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.265118:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.265119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.265157:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.265159:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.265163:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.265166:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.265179:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.265179:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:4.2:1619677474.265179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.265187:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.265189:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.265190:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.265192:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.265195:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.265196:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.265197:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.265199:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.265200:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.265200:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.265204:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.265205:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.265208:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.265209:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.265209:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.265211:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.265211:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.265212:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.265213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677474.265308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:22.0:1619677474.265310:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.265312:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.265313:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.265313:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677474.265313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.265315:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.265316:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:22.0:1619677474.265318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:21.0:1619677474.265318:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.265318:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.265321:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.265322:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677474.265324:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.265325:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.265326:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.265327:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677474.265330:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.265332:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.265335:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.265336:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677474.265336:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677474.265338:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677474.265339:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.265340:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.265341:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.265342:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:4.2:1619677474.265347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.265351:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.265353:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.265354:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.265355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677474.265417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.265426:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.265429:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.265430:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.265430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.265432:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.265434:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.265462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.265467:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.265470:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.265471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.265473:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.265475:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.265476:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:4.2:1619677474.265477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000400:00000200:6.0:1619677474.265478:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.265479:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.265480:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.265481:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.265484:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.265485:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:6.0:1619677474.265486:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677474.265488:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:8.0:1619677474.265488:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.3.3@o2ib3
00000800:00000200:6.0:1619677474.265489:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.265489:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.0:1619677474.265490:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39c600] (22)--
00000800:00000200:6.0:1619677474.265490:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677474.265491:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.265492:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.265492:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677474.265493:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677474.265494:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96505259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.265494:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:8.0:1619677474.265495:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.265495:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:8.0:1619677474.265496:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.265496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:8.0:1619677474.265497:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:21.2:1619677474.265500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677474.265501:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:8.0:1619677474.265502:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.265504:0:19938:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:22.0:1619677474.265505:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.265507:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.265508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.265509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677474.265513:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.265514:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677474.265514:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:8.0:1619677474.265516:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:8.0:1619677474.265516:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677474.265517:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000800:00000200:8.0:1619677474.265518:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:20.0:1619677474.265609:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.265611:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.265612:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.265614:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.265615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.265616:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.265747:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:31.0:1619677474.265751:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (24)--
00000400:00000200:31.0:1619677474.265752:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.265754:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: REPLY: OK
00000800:00000200:30.0:1619677474.265754:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:30.0:1619677474.265755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (24)--
00000800:00000200:30.0:1619677474.265756:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.265757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.265781:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.265784:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.265788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.265788:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:30.0:1619677474.265802:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.265802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.266361:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.266364:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.266365:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.266366:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: REPLY: OK
00000800:00000200:30.0:1619677474.266367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.266368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.266433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.266441:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.266444:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.266446:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.266447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.266449:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.266450:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.266451:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.266453:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.266454:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.266455:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.266458:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.266460:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.266463:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.266464:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.266465:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.266466:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.266467:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.266468:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.266468:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677474.266474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.266482:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.266484:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.266485:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.266487:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.266488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.266490:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.266501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.266509:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.266512:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.266513:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.266514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:10.0:1619677474.266515:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.266518:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:10.0:1619677474.266518:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.266519:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677474.266520:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:10.0:1619677474.266521:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.266523:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.266523:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.266523:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.266524:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.266525:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.266526:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.266527:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.266529:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.266529:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677474.266530:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.266531:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.266531:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.266532:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677474.266534:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.266535:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:20.0:1619677474.266536:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.266536:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.266537:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.266538:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677474.266539:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677474.266539:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.266540:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:20.0:1619677474.266541:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.266541:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677474.266542:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.266543:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.266544:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.266545:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:4.2:1619677474.266546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677474.266547:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.266547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.266550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677474.266551:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.266554:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.266554:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.266556:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677474.266556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.266556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.266557:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.266557:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:26.0:1619677474.266558:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677474.266558:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.266560:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.266561:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.266561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677474.266561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:24.0:1619677474.266563:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.266566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.266567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.266573:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.266576:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.266577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.266577:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.266578:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.266578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.267151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.267159:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.267162:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.267164:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.267166:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.267167:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.267168:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.267170:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.267171:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.267172:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.267173:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.267176:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.267178:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.267181:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.267182:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.267182:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.267184:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.267184:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.267185:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.267186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.267189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.267196:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.267198:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.267199:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.267199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.267201:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.267203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.267289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.267296:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.267299:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.267300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.267302:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.267304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.267305:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.267306:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.267307:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.267309:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.267309:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.267313:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.267315:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.267317:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.267318:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.267319:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.267320:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.267321:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.267322:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.267322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.267336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.267338:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.267339:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.267340:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.267341:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.267343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.267503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.267511:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.267514:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.267516:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.267518:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.267519:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.267521:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.267522:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.267523:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.267524:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.267525:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.267529:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.267531:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.267534:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.267535:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.267535:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.267537:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.267537:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.267538:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.267539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.267551:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.267553:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.267554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.267555:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.267557:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.267563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.267565:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.267567:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.267568:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.267568:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.267569:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.267570:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.267766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.267774:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.267777:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.267778:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.267780:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.267783:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.267784:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677474.267785:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.267787:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.267788:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.267789:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677474.267792:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.267794:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.267795:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.267796:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.267796:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.267798:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.267798:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.267799:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.267799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.267814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.267820:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.267822:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.267822:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.267823:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.267824:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.267826:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677474.267845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.267848:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.267851:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.267853:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.267854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.267855:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.267857:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.267858:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.267860:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.267861:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.267862:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.267865:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677474.267867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677474.267867:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.267869:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.267870:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.267870:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.267872:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.267872:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.267873:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.267874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677474.267875:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.267878:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677474.267879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:31.0:1619677474.267880:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.267881:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.267883:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.267884:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.267886:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.267887:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.267887:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:31.0:1619677474.267888:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.267889:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677474.267890:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.267891:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.267891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.267893:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.267894:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.267896:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677474.267896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677474.267899:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.267900:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.267900:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.267902:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.267903:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.267904:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.267904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.267915:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.267917:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.267918:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.267919:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.267921:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.267922:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.268220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.268227:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.268230:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.268231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.268233:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.268236:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.268237:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.268238:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.268240:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.268241:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.268242:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.268245:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.268246:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.268249:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.268250:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.268250:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.268252:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.268252:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.268253:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.268253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.268266:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.268268:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.268269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.268270:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.268272:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.268275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.268368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.268376:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.268379:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.268381:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.268384:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.268384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.268385:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.268386:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.268388:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.268389:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.268390:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.268393:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.268394:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.268396:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.268397:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.268398:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.268399:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.268399:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.268400:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.268401:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.268405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.268412:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.268415:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.268416:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.268416:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.268418:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.268420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.268524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.268531:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.268534:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.268536:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.268538:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.268538:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.268540:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.268541:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.268542:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.268543:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.268544:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.268547:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.268548:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.268551:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.268552:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.268553:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.268554:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.268555:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.268555:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.268556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.268568:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.268570:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.268571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.268572:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.268573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:6.0:1619677474.268574:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.268576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677474.268579:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.268582:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.268583:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.268585:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.268587:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.268588:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.268590:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.268591:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.268592:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.268593:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.268596:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.268597:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.268600:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.268600:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.268601:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.268603:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.268603:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.268604:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.268604:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.268609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.268617:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.268620:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.268621:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.268622:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.268624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.268625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.268679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.268684:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.268687:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.268687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.268689:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.268691:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.268693:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.268694:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.268696:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.268696:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.268698:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.268701:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.268703:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.268705:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.268706:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.268707:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.268708:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.268709:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.268709:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.268710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.268721:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.268724:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.268725:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.268726:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.268727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.268728:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.268741:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.268743:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.268744:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.268745:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.268746:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.268746:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.269054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.269062:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.269064:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.269067:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.269068:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.269070:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.269071:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.269072:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.269074:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.269075:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.269076:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.269079:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.269081:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.269083:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.269084:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.269085:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.269086:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.269087:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.269088:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.269090:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.269090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.269096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.269100:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.269102:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.269103:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.269104:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.269105:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.269105:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.269106:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.269107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.269108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.269109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.269276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.269283:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.269286:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.269287:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.269289:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.269291:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.269292:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.269300:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.269301:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.269302:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.269303:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677474.269306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:10.0:1619677474.269306:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.269308:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.269311:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.269312:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.269312:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677474.269312:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677474.269314:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.269314:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.269315:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.269315:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677474.269315:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.269317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.269319:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.269321:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.269323:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.269324:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.269326:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.269327:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.269328:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677474.269329:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.269331:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:2.0:1619677474.269331:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677474.269332:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.269333:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.269333:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677474.269335:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.269336:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.269337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677474.269337:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.269338:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.269340:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.269340:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.269341:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.269342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677474.269356:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.269359:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.269360:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.269363:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.269364:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.269367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677474.269376:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.269379:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.269380:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.269380:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.269381:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.269382:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677474.270130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.270136:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.270139:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.270140:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.270141:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.270144:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.270146:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.270147:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.270148:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.270149:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.270150:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.270153:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.270154:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.270157:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.270158:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.270158:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.270160:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.270161:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.270161:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.270162:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.270173:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.270176:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.270177:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.270177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.270179:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.270181:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.270490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.270495:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.270498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.270498:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.270500:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.270502:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.270503:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.270504:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.270506:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.270507:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.270508:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.270510:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.270512:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.270514:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.270515:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.270516:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.270517:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.270518:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.270518:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.270519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.270532:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.270535:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.270535:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.270537:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.270538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.270539:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.270590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677474.270594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:26.0:1619677474.270598:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.270601:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677474.270602:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:10.0:1619677474.270602:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.270604:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.270605:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.270607:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.270607:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.270608:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.270609:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.270609:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.270610:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.270610:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:26.0:1619677474.270611:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.270612:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.270612:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.270614:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.270614:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.270615:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.270616:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.270618:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.270620:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.270622:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.270623:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.270624:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.270624:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.270624:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.270626:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.270627:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.270627:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677474.270628:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.270628:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.270628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677474.270629:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.270630:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.270631:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.270632:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.270634:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.270634:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.270640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:21.2:1619677474.270640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.270642:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677474.270642:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000400:00000200:22.0:1619677474.270644:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.270644:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677474.270644:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.270646:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.270646:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.270648:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.270650:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:22.0:1619677474.270650:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.270650:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.270651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.270651:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.270653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.270662:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.270664:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.270666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.270666:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.270667:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.270667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.271033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.271042:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.271046:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.271049:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.271051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.271053:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.271055:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.271057:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.271059:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.271061:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.271062:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.271068:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.271070:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.271074:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.271075:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.271076:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.271078:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.271079:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.271080:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.271081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.271084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.271091:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.271094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.271094:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.271095:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.271097:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.271099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.271423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.271431:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.271433:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.271434:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.271436:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.271439:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.271440:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.271441:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.271442:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.271443:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.271444:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.271447:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.271448:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.271451:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.271451:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.271452:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.271453:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.271454:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.271454:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.271455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.271459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.271466:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.271468:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.271469:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.271470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.271471:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.271473:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.271477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.271483:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.271486:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.271486:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.271487:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.271487:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.271488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.271679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.271686:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.271689:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.271690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.271692:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.271695:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.271696:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.271697:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.271699:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.271700:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.271701:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.271703:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.271705:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.271707:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.271707:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.271708:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.271710:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.271711:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.271712:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.271712:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.271716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.271723:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.271726:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.271726:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.271727:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.271728:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.271729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.271886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677474.271892:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.271895:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677474.271896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677474.271898:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.271900:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.271902:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677474.271903:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.271905:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.271906:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.271907:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677474.271910:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.271912:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677474.271914:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.271915:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677474.271915:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677474.271917:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677474.271917:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.271918:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.271919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677474.271923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.271931:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.271933:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.271934:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.271934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.271936:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.271939:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.272187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.272193:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.272196:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.272197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.272198:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.272201:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.272202:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.272203:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.272205:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.272206:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.272207:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.272210:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.272212:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.272215:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.272215:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.272216:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.272218:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.272218:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.272220:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.272221:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.272221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.272226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677474.272231:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.272231:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.272233:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.272233:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677474.272234:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.272234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.272235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.272236:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.272237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.272238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.272346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.272354:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.272358:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.272360:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.272362:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.272362:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.272363:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.272365:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.272366:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.272367:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.272368:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.272372:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.272374:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.272377:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.272377:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.272378:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.272380:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.272380:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.272381:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.272382:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.272394:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.272397:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.272398:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.272398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677474.272398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.272400:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.272403:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.272405:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.272408:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.272410:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.272412:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.272413:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.272414:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.272415:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.272417:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.272418:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.272419:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.272422:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.272424:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.272426:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.272427:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.272427:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.272429:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.272429:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.272430:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.272431:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.272434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.272441:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.272444:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.272445:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.272445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.272447:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.272449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.272888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.272894:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.272896:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.272898:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.272899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.272900:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.272902:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.272903:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.272904:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.272905:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.272906:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.272909:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.272911:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.272913:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.272914:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.272914:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.272916:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.272917:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.272917:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.272919:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.2:1619677474.272920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:8.2:1619677474.272922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.272926:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.272929:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677474.272929:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:20.0:1619677474.272930:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:17.0:1619677474.272931:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.272932:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.272933:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.272934:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.272935:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.272936:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:17.0:1619677474.272936:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677474.272937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:17.0:1619677474.272937:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.272938:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.272939:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.272940:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677474.272943:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:8.2:1619677474.272943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.272944:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677474.272947:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.272948:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677474.272949:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677474.272950:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.272950:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677474.272951:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.272952:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:21.0:1619677474.272953:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.272953:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:21.0:1619677474.272954:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.272954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:21.0:1619677474.272955:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.272955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.272956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677474.272956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.272962:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.272965:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.272966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.272966:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.272969:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.272971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.272981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677474.272987:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677474.272989:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677474.272990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677474.272990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:21.2:1619677474.273077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.273082:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.273085:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.273085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.273087:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.273090:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.273091:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.273092:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.273094:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.273095:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.273095:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.273099:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.273099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677474.273100:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.273102:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.273103:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.273103:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.273105:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677474.273106:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677474.273106:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.273107:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:24.0:1619677474.273108:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677474.273108:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.273108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:24.0:1619677474.273111:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677474.273111:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677474.273114:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.273114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:8.0:1619677474.273115:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.273116:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.273118:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96505227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.273119:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.273119:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000400:00000200:24.0:1619677474.273120:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.273121:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.273121:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677474.273122:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677474.273122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.273123:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:24.0:1619677474.273124:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.273124:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.273125:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.273126:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677474.273127:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677474.273127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677474.273129:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.273130:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.273130:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677474.273131:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.273132:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.273133:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.273133:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677474.273133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677474.273134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677474.273135:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.273138:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.273140:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.273141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.273142:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.273143:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.273145:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.273146:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.273147:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.273148:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.273152:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.273153:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.273155:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.273156:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.273156:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677474.273156:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.273158:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.273158:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677474.273158:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.273159:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677474.273159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.273159:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.273160:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677474.273162:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.273165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.273170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.273173:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.273174:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.273176:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.273177:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.273178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.273187:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.273190:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677474.273190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:31.0:1619677474.273191:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.273191:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.273192:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.273193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:12.0:1619677474.273198:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.273200:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.273203:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.273205:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.273206:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.273207:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677474.273208:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.273209:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.273210:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.273211:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677474.273214:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.273216:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677474.273218:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.273218:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677474.273219:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677474.273220:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677474.273221:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.273222:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.273222:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.273226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.273233:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.273235:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.273236:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.273236:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677474.273238:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.273241:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.273532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.273538:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.273541:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.273542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.273544:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.273546:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.273548:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677474.273549:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.273550:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.273551:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.273552:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677474.273555:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.273557:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677474.273560:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.273561:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677474.273561:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677474.273563:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677474.273564:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.273565:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.273565:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.273570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.273577:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.273580:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.273580:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.273581:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.273583:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.273586:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677474.273586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.2:1619677474.273587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.0:1619677474.273591:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677474.273593:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677474.273593:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:22.0:1619677474.273595:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.0:1619677474.273596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.273596:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677474.273597:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.273598:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.273598:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.273600:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.273600:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.273601:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.273602:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.273603:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.273603:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.273604:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.273605:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.273606:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.273607:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.273607:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.273613:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677474.273614:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677474.273617:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.273617:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.273619:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.273620:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.273620:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.273621:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.273621:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.273622:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.273624:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.273624:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.273624:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677474.273625:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.273626:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.273626:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677474.273627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.273627:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.273628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677474.273633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.273637:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.273638:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.273639:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.273639:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:21.0:1619677474.273640:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.273640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.273641:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.273641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.273641:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.273642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:21.0:1619677474.273643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677474.273643:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.273644:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.273644:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.273646:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.273649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.273830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.273832:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677474.273835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677474.273835:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.273837:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.273840:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.273841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.273841:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.273842:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.273843:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677474.273843:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.273844:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.273846:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677474.273846:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677474.273850:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.273850:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677474.273851:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.273853:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.273853:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677474.273854:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677474.273856:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.273857:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.273857:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.273857:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:31.0:1619677474.273858:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.273859:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.273859:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:31.0:1619677474.273860:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677474.273860:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.273860:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.273861:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.273862:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677474.273864:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.273865:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.2:1619677474.273866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677474.273867:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677474.273867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677474.273868:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.273869:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.273870:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.273871:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.273872:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.273872:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677474.273874:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.273874:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677474.273874:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:31.0:1619677474.273875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.273875:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.273876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677474.273877:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.273878:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.273879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.2:1619677474.273880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677474.273880:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.273882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677474.273885:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.273886:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.273887:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.273888:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.273889:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.273890:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.273890:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.273891:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.273892:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.273893:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.274047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.274054:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.274057:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.274058:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.274059:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.274062:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.274063:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.274064:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.274066:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.274067:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.274068:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.274071:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.274073:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.274076:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.274077:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.274077:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.274079:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.274080:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.274081:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.274082:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.274083:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.274088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.274093:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.274095:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.274096:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.274096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.274097:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.274097:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.274098:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677474.274099:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.274099:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.274102:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.274249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.274257:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.274260:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.274260:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.274262:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.274265:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.274267:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.274268:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.274269:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.274270:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.274271:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.274275:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.274276:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.274279:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.274279:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.274280:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.274282:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.274282:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.274283:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.274283:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.274288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.274296:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.274299:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.274300:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.274301:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.274302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.274305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.274306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.274314:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.274317:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.274317:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.274318:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.274319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.274320:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.274696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.274704:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.274707:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.274709:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.274711:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.274711:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.274712:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.274713:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.274715:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.274716:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.274716:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.274720:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.274721:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.274724:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.274725:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.274725:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.274727:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.274728:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.274728:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.274729:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.274742:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.274745:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.274746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.274746:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.274748:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.274754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.274756:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.274758:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.274759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.274759:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.274760:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.274761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.274918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.274926:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.274929:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.274931:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.274932:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.274934:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.274936:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.274937:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.274938:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.274939:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.274940:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.274943:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.274945:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.274948:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.274948:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.274949:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.274950:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.274951:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.274952:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.274952:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.274956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.274963:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.274966:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.274967:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.274967:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.274969:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.274970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:8.0:1619677474.274971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677474.274977:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.274979:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.274980:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.274982:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.274985:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.274987:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677474.274988:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.274989:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.274990:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.274991:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677474.274995:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677474.274996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.274997:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677474.275001:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.275002:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677474.275002:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677474.275003:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.275004:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.275005:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677474.275006:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:1.0:1619677474.275006:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.275007:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.275007:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677474.275008:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.275008:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.275010:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:9.2:1619677474.275011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:1.0:1619677474.275011:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.275012:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.275013:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.275014:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.275016:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.275018:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:1.0:1619677474.275018:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.275020:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.275021:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:1.0:1619677474.275021:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.275021:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.275022:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.275022:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.275023:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.275023:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677474.275024:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.275024:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.275025:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.275026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677474.275027:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:5.2:1619677474.275031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677474.275032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677474.275036:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.275037:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.275038:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677474.275038:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677474.275039:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.275039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.275040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677474.275041:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677474.275041:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.275041:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677474.275041:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.275042:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.275043:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:22.0:1619677474.275043:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.275044:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677474.275045:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.275055:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.275057:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.275058:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.275059:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.275059:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.275060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.275120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.275126:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.275129:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.275130:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.275132:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.275134:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.275135:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.275137:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.275138:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.275139:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.275140:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.275144:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.275146:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.275149:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.275150:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.275150:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.275152:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.275153:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.275154:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.275154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677474.275165:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.275168:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.275169:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.275170:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.275172:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.275173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.275440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.275446:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.275449:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.275450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.275451:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.275454:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.275455:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.275456:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.275457:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.275458:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.275460:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.275463:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.275465:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.275467:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.275468:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.275469:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.275470:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.275471:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.275471:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.275472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.275485:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.275487:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.275488:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.275490:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.275491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.275492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.275510:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.275512:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.275513:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.275514:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.275514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.275515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.275543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.275551:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.275554:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.275554:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.275556:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.275559:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.275560:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.275562:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.275563:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.275564:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.275565:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.275569:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.275570:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.275573:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.275573:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.275574:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.275576:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.275576:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.275577:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.275577:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.275581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.275588:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.275591:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.275592:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.275592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.275594:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.275596:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.275600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.275606:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.275609:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.275609:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.275610:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.275611:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.275611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.276310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.276318:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.276321:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.276322:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.276323:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.276326:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.276327:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.276328:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.276329:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.276330:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.276331:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.276334:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.276335:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.276337:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.276338:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.276338:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.276340:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.276340:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.276341:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.276343:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.276343:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.276346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.276349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.276352:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.276354:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.276355:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.276355:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.276356:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677474.276357:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.276359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.276359:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.276359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.276360:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.276415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677474.276421:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.276424:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677474.276425:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677474.276426:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.276429:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.276430:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677474.276432:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.276433:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.276434:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.276435:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677474.276439:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.276440:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677474.276443:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.276444:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677474.276445:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.276447:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677474.276447:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.276448:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.276448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677474.276452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.276459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.0:1619677474.276459:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.276461:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.276462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.276463:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.276465:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.276467:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.276467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.276470:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.276473:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.276475:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.276475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.276476:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.276477:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.276479:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.276480:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.276481:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.276484:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.276486:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.276490:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.276490:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.276491:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.276493:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.276493:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.276494:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.276495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.276506:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.276509:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.276510:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.276510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.276512:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.276515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.276525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.276532:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.276534:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.276535:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.276537:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.276539:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.276541:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677474.276542:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.276543:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.276544:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.276545:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.276549:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.276551:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.276553:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.276553:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677474.276554:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.276555:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.276556:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.276557:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.276557:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:30.0:1619677474.276568:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.276570:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.276571:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.276572:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.276574:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.276576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.276878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.276885:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.276888:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.276890:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.276892:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.276892:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.276894:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.276895:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.276896:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.276897:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.276898:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.276901:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.276903:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.276904:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.276905:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.276906:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.276907:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.276908:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.276908:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.276909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.276919:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.276922:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.276922:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.276924:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.276926:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.276926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.2:1619677474.276959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677474.276964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.276967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.276967:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677474.276969:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.276972:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.276973:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677474.276975:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.276976:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.276977:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.276978:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677474.276981:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.276983:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677474.276985:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.276986:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677474.276987:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677474.276989:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677474.276989:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.276990:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.276990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677474.276995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.277003:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.277006:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.277007:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.277007:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.277009:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.277011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.277015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.277022:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.277025:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.277026:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.277026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.277027:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.277028:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.277088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.277094:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.277097:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.277097:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.277099:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.277102:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.277103:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.277104:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.277105:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.277106:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.277106:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.277109:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.277111:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.277113:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.277113:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.277114:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.277115:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.277116:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.277117:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.277117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.277129:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.277131:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.277132:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.277134:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.277135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.277136:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.277406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.2:1619677474.277411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:21.0:1619677474.277415:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:1.0:1619677474.277417:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:21.0:1619677474.277418:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.277420:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677474.277420:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:20.0:1619677474.277421:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:17.0:1619677474.277421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:21.0:1619677474.277423:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.277423:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.277425:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.277425:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.277426:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.277427:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.277428:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.277428:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.277429:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.277430:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.277431:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.277431:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.277433:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.277435:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677474.277437:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.277438:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.277440:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.277440:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.277441:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.277441:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.277442:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.277443:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.277443:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.277444:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:1.0:1619677474.277444:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.277445:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.277446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:1.0:1619677474.277446:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.277446:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.277447:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.277448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.277452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677474.277457:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.277460:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:6.0:1619677474.277460:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.277461:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.277462:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.277462:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.277463:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.277464:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.277466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.277466:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.277467:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677474.277469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.277647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.277655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.277657:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.277660:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.277662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.277662:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.277663:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.277664:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.277666:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.277667:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.277668:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.277671:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.277673:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.277675:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.277676:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.277676:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.277678:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.277678:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.277680:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.277680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.277683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.277690:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.277693:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.277694:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.277694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.277696:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.277698:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.278106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.278113:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.278115:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.278116:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.278119:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.278121:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.278122:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.278124:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.278125:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.278126:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.278127:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.278130:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.278132:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.278134:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.278135:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.278136:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.278137:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.278138:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.278139:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.278139:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.278143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.278150:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.278152:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.278153:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.278154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.278154:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.278155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.278403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.278411:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.278414:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.278416:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.278419:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.278419:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.278420:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.278421:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.278422:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.278423:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.278424:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677474.278425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677474.278427:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.278429:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.278431:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.278432:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.278433:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677474.278433:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677474.278434:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.278435:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.278435:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677474.278436:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677474.278436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.278437:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.278438:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.278438:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677474.278440:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.278442:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.278443:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96505198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.278444:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677474.278445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677474.278445:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.278446:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.278449:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.278450:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:8.0:1619677474.278450:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.278451:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.278452:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.278453:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677474.278453:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.278454:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.278454:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.278454:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.278454:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.278455:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.278456:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.278457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.0:1619677474.278457:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.278457:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677474.278458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677474.278458:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.278459:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677474.278466:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.278468:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.278469:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.278470:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.278470:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.278471:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.278471:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.278473:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.278474:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.278475:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.278477:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.278480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.278485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.278490:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.278493:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.278494:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.278496:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.278497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.278499:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.278500:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.278501:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677474.278502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000400:00000200:6.0:1619677474.278502:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.278503:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.278506:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.278507:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677474.278508:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.278510:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:6.0:1619677474.278510:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.278511:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.278511:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677474.278511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.278513:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.278513:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.278513:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.278514:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:18.0:1619677474.278515:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.278515:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:18.0:1619677474.278516:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:6.0:1619677474.278516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:18.0:1619677474.278517:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.278518:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.278519:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.278520:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.2:1619677474.278521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:18.0:1619677474.278523:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.278525:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677474.278526:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:18.0:1619677474.278527:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.278528:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.278528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.278528:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677474.278528:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.278528:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677474.278529:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:18.0:1619677474.278529:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677474.278530:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:18.0:1619677474.278530:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000400:00000200:21.0:1619677474.278531:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.278531:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677474.278532:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000400:00000200:21.0:1619677474.278533:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.278533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677474.278533:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:21.0:1619677474.278535:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677474.278536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.278538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.278544:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.2:1619677474.278545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:18.0:1619677474.278545:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677474.278546:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.278546:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.278546:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.278547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.278548:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.278548:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.278549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677474.278550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.278553:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.278556:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:9.2:1619677474.278557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000400:00000200:31.0:1619677474.278558:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.278560:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.278561:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.278562:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.278563:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677474.278564:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:31.0:1619677474.278565:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.278566:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.278567:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677474.278568:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.278568:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.278569:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.278569:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.278570:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:31.0:1619677474.278571:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.278573:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.278577:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.278578:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.278578:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.278580:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.278580:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.278582:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.278583:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.278584:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.278590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.278593:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.278596:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.278596:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.278597:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.278597:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.278599:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.278600:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.278601:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.278601:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.278602:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.278610:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.278613:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.278613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.278614:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.278615:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.278616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.278916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.278924:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.278927:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.278928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.278930:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.278932:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.278934:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.278935:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.278937:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.278937:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.278939:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.278941:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.278943:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.278946:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.278947:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.278947:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.278949:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.278950:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.278950:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.278951:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.278963:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.278966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.278966:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.278967:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.278969:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.278972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.279455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.279462:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.2:1619677474.279463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677474.279465:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.279466:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.279467:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.279470:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.279471:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677474.279471:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.279472:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.279473:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677474.279474:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.279475:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.279476:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.279476:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677474.279478:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.279479:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.279479:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677474.279481:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677474.279481:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:30.0:1619677474.279482:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.279484:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.279484:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.279484:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:30.0:1619677474.279485:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.279485:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:30.0:1619677474.279486:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.279486:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.279487:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.279488:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:30.0:1619677474.279489:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.279489:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.279489:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677474.279490:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.279493:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.279494:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.279495:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.279496:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:4.2:1619677474.279496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:30.0:1619677474.279497:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.279498:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.279498:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:13.0:1619677474.279500:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.279502:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:26.0:1619677474.279503:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:13.0:1619677474.279503:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677474.279503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:13.0:1619677474.279504:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677474.279505:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.279505:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.279506:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677474.279507:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.279508:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.0:1619677474.279512:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.279515:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.279515:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.279517:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.279518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.279520:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677474.279726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.279733:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.279735:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.279736:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.279738:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.279740:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.279742:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.279743:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.279745:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.279746:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.279747:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.279750:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.279752:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.279754:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.279755:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.279756:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.279757:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.279758:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.279758:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.279759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.279771:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.279773:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.279774:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.279774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.279776:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.279779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.279891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.279897:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.279900:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.279901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.279902:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.279905:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.279907:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.279908:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.279909:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.279910:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.279911:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.279915:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.279917:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.279920:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.279921:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.279922:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.279923:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.279924:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.279925:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.279926:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.279927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677474.279932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677474.279936:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677474.279938:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677474.279939:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.279940:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.279940:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.279940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.279941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677474.279942:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.279943:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.279945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.279990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.279998:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.280001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.280001:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677474.280003:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.280006:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.280008:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.280009:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.280010:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.280011:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.280012:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.280016:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.280018:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.280020:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.280021:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.280022:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.280024:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.280024:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.280025:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.280026:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.280027:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:4.2:1619677474.280033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677474.280037:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.280039:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:22.0:1619677474.280040:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.280040:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.280041:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:6.0:1619677474.280041:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.280042:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.280043:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:6.0:1619677474.280043:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.280046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.280125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677474.280132:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.280136:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677474.280136:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677474.280138:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.280141:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.280142:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677474.280143:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.280145:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.280146:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.280147:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677474.280150:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.280152:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677474.280154:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.280155:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677474.280155:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677474.280157:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677474.280157:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.280158:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677474.280159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677474.280163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.280169:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.280171:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.280172:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.280172:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677474.280174:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:26.0:1619677474.280176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.280264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.280272:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.280274:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.280276:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.280279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.280279:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.280281:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.280282:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.280283:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.280284:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.280285:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.280288:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.280289:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.280292:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.280293:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.280298:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.280299:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.280300:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.280301:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.280301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.280314:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.280317:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.280317:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.280318:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.280320:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.280323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.280389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.280394:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.280397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.280397:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.280399:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.280401:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.280403:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.280403:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.280405:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.280406:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.280407:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.280410:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.280412:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.280414:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.280415:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.280415:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.280417:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.280418:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.280418:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.280419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.280431:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.280433:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.280434:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.280435:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.280436:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.280438:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.280603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.280610:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.280613:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.280614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.280615:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.280618:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.280620:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.280621:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.280622:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.280623:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.280624:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.280628:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.280630:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.280632:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.280633:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.280634:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.280636:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.280636:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.280637:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.280637:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.280649:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.280651:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.280652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.280653:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.280655:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.280658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.280813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.280821:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.280824:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.280826:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.280828:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.280829:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.280830:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.280831:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.280833:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.280834:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.280835:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.280838:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.280839:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.280842:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.280842:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.280843:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.280844:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.280845:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.280846:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.280846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.280850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.280857:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.280859:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.280860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.280860:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.280862:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.280865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.280874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.280880:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.280882:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.280883:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.280883:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.280884:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.280885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.280919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.280926:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.280928:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.280929:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.280931:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.280934:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.280935:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.280937:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.280938:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.280939:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.280940:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.280943:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.280944:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.280947:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.280948:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.280948:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.280950:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.280950:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.280951:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.280952:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677474.280964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.280966:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.280967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.280968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.280969:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.280972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.280982:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.280985:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.280986:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.280986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.280987:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.280987:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.281373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.281381:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.281384:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.281385:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.281386:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.281389:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.281391:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.281392:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.281393:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.281394:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.281395:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.281397:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.281399:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.281401:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.281401:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.281402:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.281403:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.281404:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.281405:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.281405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.281409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.281417:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.281419:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.281420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.281420:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.281421:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.281424:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.281579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.281587:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.281590:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.281592:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.281594:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.281595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.281595:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.281596:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.281598:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.281598:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.281599:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.281602:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.281604:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.281607:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.281607:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.281608:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.281610:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.281610:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.281611:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.281611:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.281616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.281625:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.281627:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.281628:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.281630:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.281631:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.281633:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.282086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.282092:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.282094:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.282095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.282097:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.282099:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.282100:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.282101:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.282103:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.282104:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.282105:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.282108:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.282109:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.282112:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.282112:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.282113:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.282115:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.282115:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.282116:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.282116:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.282128:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.282131:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.282131:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.282133:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.282134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.282135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.282149:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.282151:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.282153:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.282153:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.282154:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.282155:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.282348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.282356:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.282359:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.282361:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.282363:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.282364:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.282365:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.282367:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.282368:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.282369:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.282370:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.282374:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.282376:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.282379:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.282380:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.282380:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.282382:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.282383:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.282384:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.282384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.282388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.282396:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:22.0:1619677474.282397:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.282399:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677474.282399:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.282400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:22.0:1619677474.282401:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.282401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:1.0:1619677474.282401:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.282403:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.282404:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.282405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:1.0:1619677474.282405:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.282407:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.282408:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.282409:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.282410:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.282413:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.282414:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.282417:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.282417:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.282418:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.282419:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.282420:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.282421:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.282421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.282426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.282434:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.282437:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.282437:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.282439:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.282440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.282442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.282642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.282650:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.282653:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.282655:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.282656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.282658:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.282659:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.282661:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.282662:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.282663:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.282664:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.282667:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.282669:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.282671:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.282672:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.282673:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.282674:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.282675:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.282676:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.282678:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.282678:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.282684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.282688:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.282690:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.282691:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.282692:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.282692:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.282693:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.282694:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.282695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.282696:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.282696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.283290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.283298:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.283301:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.283302:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.283304:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.283306:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.283307:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.283308:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.283310:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.283311:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.283311:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.283314:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.283316:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.283319:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.283319:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.283320:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.283322:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.283322:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.283323:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.283325:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.283325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.283331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.283336:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.283339:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.283339:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.283339:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.283340:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.283341:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.283342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677474.283342:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.283343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.283345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.283484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.283490:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.283492:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.283493:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.283495:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.283497:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.283498:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.283499:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.283500:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.283502:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.283502:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.283506:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.283508:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.283510:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.283511:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.283511:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.283513:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.283514:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.283514:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.283515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.283526:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.283529:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.283529:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.283531:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.283532:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.283533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.283713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.283721:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.283724:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.283727:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.283729:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.283729:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.283730:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.283731:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.283733:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.283734:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.283735:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.283738:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.283740:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.283742:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.283743:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.283743:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.283745:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.283745:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.283746:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.283747:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.283751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.283757:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.283760:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.283761:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.283761:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.283763:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.283766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.283804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.283811:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.283814:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.283815:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.283817:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.283819:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.283820:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.283821:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.283823:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.283824:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.283825:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.283828:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.283830:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.283833:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.283833:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.283834:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.283836:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.283836:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.283837:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.283837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.283849:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.283852:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.283852:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.283853:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.283855:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.283858:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.284106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.284114:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.284117:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.284119:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.284120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.284122:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.284123:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.284125:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.284126:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.284127:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.284128:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.284131:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.284133:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.284136:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.284137:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.284138:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.284139:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.284140:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.284141:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.284141:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.284153:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.284155:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.284156:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.284156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.284159:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.284161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.284452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.284458:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.284461:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.284462:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.284464:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.284466:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.284468:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.284469:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.284470:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.284471:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.284472:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.284475:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.284477:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.284479:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.284480:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.284481:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.284482:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.284483:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.284484:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.284484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.284497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.284500:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.284501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.284501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.284503:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.284505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.284516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.284519:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.284520:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.284520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.284521:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.284521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.285296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.285301:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.285304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.285304:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.285306:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.285308:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.285309:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.285310:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.285312:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.285313:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.285314:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.285317:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.285319:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.285321:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.285322:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.285322:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.285324:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.285324:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.285325:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.285326:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.285327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.285332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.285337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.285338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.285339:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.285339:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.285340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.285340:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.285341:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.285343:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.285344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.285345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.285422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.285430:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.285433:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.285434:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.285435:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.285439:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.285440:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.285441:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.285443:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.285444:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.285445:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.285448:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.285450:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.285452:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.285453:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.285454:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.285455:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.285455:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.285456:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.285457:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.285461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.285468:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.285470:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.285471:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.285471:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.285473:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.285474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.285555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.285563:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.285565:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.285567:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.285570:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.285570:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.285571:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.285572:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.285574:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.285574:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.285575:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.285578:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.285580:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.285583:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.285583:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.285584:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.285586:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.285586:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.285587:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.285588:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.285592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.285600:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.285602:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.285603:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.285604:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.285605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.285607:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.285629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.285638:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.285642:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.285644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.285647:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.285651:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.285653:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.285655:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.285657:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.285659:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.285661:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.285665:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.285668:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.285671:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.285672:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.285673:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.285675:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.285676:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.285677:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.285678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.285681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.285690:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.285693:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.285694:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.285696:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.285697:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.285699:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.285874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.285878:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.285881:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.285883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.285883:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.285885:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.285887:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.285888:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.285889:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.285890:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.285892:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.285894:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.285896:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.285898:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.285899:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.285900:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.285901:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.285902:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.285903:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.285903:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.285915:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.285917:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.285918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.285918:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.285920:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.285922:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.285932:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.285935:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.285936:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.285936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.285937:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.285938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.286155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.286162:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.286165:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.286166:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.286168:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.286171:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.286172:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.286173:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.286174:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.286175:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.286176:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.286179:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.286181:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.286183:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.286184:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.286185:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.286186:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.286186:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.286187:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.286189:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.286189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.286192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.286195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.286198:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.286200:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.286201:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.286202:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.286202:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.286203:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.286204:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.286204:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.286205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.286206:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.286212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:21.2:1619677474.286217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.0:1619677474.286218:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.286221:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.286222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.286222:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.286223:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.286223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.286224:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.286227:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.286229:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.286231:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.286232:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.286234:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.286235:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.286236:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.286237:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.286238:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.286241:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.286242:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.286246:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.286246:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.286247:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.286249:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.286249:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.286251:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.286252:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.286253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.286259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.286263:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.286266:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677474.286266:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.286267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.286267:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.286269:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.286270:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.286270:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.286271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.286272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.286358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.286366:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.286369:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.286371:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.286373:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.286374:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.286375:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.286376:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.286377:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.286378:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.286379:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.286383:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.286384:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.286387:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.286388:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.286388:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.286390:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.286391:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.286391:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.286392:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.286397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.286406:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.286408:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.286409:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.286411:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.286412:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.286414:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.286414:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.286415:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.286415:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.286416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.286416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.286548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677474.286553:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.286555:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.286558:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.286558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.286560:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.286562:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.286563:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.286564:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.286565:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.286566:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.286570:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.286571:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.286574:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.286575:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.286575:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.286577:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.286578:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.286578:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677474.286579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:22.0:1619677474.286592:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.286594:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.286595:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.286595:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.286597:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.286600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.286956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.286964:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.286967:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.286970:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.286971:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.286971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:31.0:1619677474.286972:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.286973:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.286975:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.286976:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.286976:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677474.286977:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.286978:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677474.286979:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.286980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677474.286983:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.286983:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.286985:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677474.286986:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.286987:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677474.286988:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.286989:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:6.0:1619677474.286989:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.286990:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:6.0:1619677474.286990:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.286991:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:6.0:1619677474.286991:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.286992:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677474.286992:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.286993:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.286995:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.286996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677474.286996:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.286997:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.287000:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677474.287001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677474.287001:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.287002:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.287003:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.287004:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.287004:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.287005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677474.287006:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.287007:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.287008:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.287010:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.287010:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.287011:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.287011:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.287012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.287013:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.287014:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.287016:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.287019:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.287020:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.287021:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.287022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.287024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.287187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.287195:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.287198:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.287199:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.287201:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.287203:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.287205:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.287206:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.287207:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.287208:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.287209:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.287213:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.287214:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.287217:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.287218:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.287218:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.287220:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.287221:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.287221:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.287221:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.287226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.287235:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.287238:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.287238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.287239:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.287241:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.287244:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677474.287420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.287426:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.287428:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.287431:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.287431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.287433:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.287435:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.287436:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.287437:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.287439:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.287440:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.287443:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.287445:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.287447:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.287448:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.287448:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.287450:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.287451:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.287452:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.287452:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.287457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.287466:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.287468:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.287470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.287470:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.287472:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.287474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.287502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677474.287508:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.287510:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677474.287511:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677474.287513:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.287515:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.287517:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677474.287518:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.287519:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.287520:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.287521:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677474.287524:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.287526:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677474.287529:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.287529:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677474.287530:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677474.287532:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677474.287533:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.287534:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677474.287535:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677474.287535:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:9.2:1619677474.287539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.2:1619677474.287540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.287546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:2.0:1619677474.287547:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677474.287547:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677474.287548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.287549:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677474.287550:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.287551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.287551:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.287553:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.287555:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.287605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.287613:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.287616:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.287619:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.287621:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.287621:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.287622:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.287623:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.287625:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.287626:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.287627:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.287630:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.287632:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.287634:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.287635:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.287636:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.287637:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.287638:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.287639:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.287639:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.287643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.287649:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.287652:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.287653:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.287653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.287655:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.287657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.287808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.287814:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.287817:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.287818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.287820:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.287822:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.287823:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.287825:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.287826:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.287827:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.287828:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.287831:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.287832:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.287836:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.287836:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.287837:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.287839:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.287840:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.287841:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.287841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677474.287852:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.287855:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.287856:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.287858:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.287859:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.287860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.288287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.288294:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.288300:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677474.288302:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.288302:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.288304:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.288306:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.288307:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.288308:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.288309:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.288310:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.288313:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.288315:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.288318:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.288319:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.288319:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.288321:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.288321:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.288322:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.288323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.288336:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.288338:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.288339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.288340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.288342:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.288345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.288372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.288377:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.288379:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.288380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.288382:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.288384:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.288385:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.288386:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.288388:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.288389:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.288390:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.288393:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.288395:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.288397:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.288398:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.288399:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.288400:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.288401:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.288401:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.288402:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.288414:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.288416:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.288417:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.288418:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.288420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.288421:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677474.288433:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.288435:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.288436:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.288437:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.288437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.288438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.288592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.288599:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.288602:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.288603:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.288605:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.288608:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.288610:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.288611:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.288612:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.288613:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.288614:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.288618:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.288619:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.288622:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.288623:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.288624:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.288625:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.288626:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.288627:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.288627:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.288638:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.288641:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.288642:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.288642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.288644:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.288644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677474.288647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677474.288651:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.288654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.288654:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.288656:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.288659:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.288660:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.288661:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.288663:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.288664:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.288665:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.288668:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.288670:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.288672:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.288673:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.288674:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.288675:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.288676:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.288677:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.288677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.288695:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.288697:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.288698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.288698:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.288700:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.288703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.288722:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.288724:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.288725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.288726:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.288727:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.288727:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.289586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.289594:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.289597:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.289599:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.289602:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.289602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.289603:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.289605:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.289606:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.289607:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.289608:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:5.2:1619677474.289608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677474.289611:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.289613:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677474.289614:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677474.289615:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.289616:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.289617:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677474.289617:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677474.289618:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677474.289618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677474.289619:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677474.289619:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.289620:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.289620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677474.289622:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.289623:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.289624:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677474.289624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677474.289626:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.289626:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.289627:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.289630:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.289631:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677474.289632:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677474.289634:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:17.0:1619677474.289634:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677474.289635:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.289635:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:8.0:1619677474.289635:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677474.289636:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:24.0:1619677474.289637:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.289637:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.289638:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677474.289639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677474.289639:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.289639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.289651:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.289654:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.289655:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.289655:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.289657:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.289659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.289938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.289946:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.289949:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.289951:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.289953:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.289954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.289955:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.289956:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.289957:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.289959:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.289959:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.289963:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.289964:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.289967:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.289968:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.289969:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.289970:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.289971:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.289972:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.289972:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.289983:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.289986:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.289987:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.289987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.289990:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.289992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.290212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.290217:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.290219:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.290220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.290222:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.290223:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.290225:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.290226:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.290227:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.290228:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.290229:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:4.2:1619677474.290229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:6.0:1619677474.290232:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.290234:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.290236:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.290237:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.290237:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.290237:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.290239:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.290239:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677474.290240:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:6.0:1619677474.290240:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.290240:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:10.0:1619677474.290241:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.290242:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.290245:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.290247:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.290248:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.290249:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.290250:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.290251:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.290252:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:26.0:1619677474.290254:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.290255:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.290255:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.290256:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.290257:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.290258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677474.290259:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.290259:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.290259:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677474.290260:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.290262:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.290262:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.290263:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.290263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.290275:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.290277:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.290278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.290279:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.290281:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.290284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.290361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.290367:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.290370:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.290371:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.290373:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.290376:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.290377:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.290379:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.290380:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.290381:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.290382:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.290386:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.290387:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.290391:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.290392:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.290393:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.290395:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.290395:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.290397:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677474.290398:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677474.290398:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677474.290404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:12.0:1619677474.290409:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677474.290410:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677474.290412:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:12.0:1619677474.290412:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677474.290413:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:12.0:1619677474.290413:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.290413:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:12.0:1619677474.290415:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.290418:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.290419:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.290819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.290826:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.290829:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.290830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.290832:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.290834:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.290835:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.290837:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.290838:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.290839:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.290840:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.290843:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.290845:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.290847:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.290848:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.290849:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.290850:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.290851:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.290852:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.290852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.290857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.290865:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.290867:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.290868:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.290870:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.290871:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.290873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.290879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.290887:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.290890:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.290891:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.290891:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.290892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.290893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.290972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.290980:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.290983:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.290986:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.290988:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.290989:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.290991:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.290992:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.290993:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.290994:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.290995:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.290999:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.291000:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.291003:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.291003:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.291004:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.291005:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.291006:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.291007:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.291008:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.291011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.291017:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.291020:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.291021:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.291021:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.291023:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.291025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.291213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.291220:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.291223:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.291224:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.291226:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.291228:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.291229:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.291230:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.291232:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.291233:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.291233:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.291236:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.291238:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677474.291240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:10.0:1619677474.291241:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.291241:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.291242:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.291244:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.291244:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677474.291245:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677474.291245:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.291245:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.0:1619677474.291247:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.291249:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.291250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.291252:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.291254:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.291255:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.291256:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.291257:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.291258:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677474.291258:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.291260:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.291261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.291261:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.291262:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.291264:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:6.0:1619677474.291264:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.291267:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.291267:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.291268:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.291268:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.291271:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.291271:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.291272:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.291273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677474.291283:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.291286:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.291287:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.291289:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.291290:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.291292:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677474.291308:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.291311:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.291312:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.291312:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.291313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.291314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.291336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.291340:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.291342:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.291344:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.291345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.291348:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.291349:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.291351:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.291352:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.291353:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.291354:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.291357:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.291359:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.291361:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.291362:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.291363:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.291365:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.291365:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.291366:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.291367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677474.291379:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.291381:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.291382:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.291382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.291384:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.291386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.291394:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.291397:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.291398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.291398:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.291399:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.291400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.291411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.291419:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.291422:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.291424:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.291426:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.291426:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.291428:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.291429:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.291431:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.291431:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.291433:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.291436:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.291438:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.291441:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.291442:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.291442:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.291444:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.291445:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.291446:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.291446:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.291458:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.291460:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.291461:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.291462:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.291464:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.291466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.291518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.291526:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.291530:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.291530:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.291532:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.291535:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.291536:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.291538:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.291539:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.291540:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.291541:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.291544:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.291546:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.291548:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.291549:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.291549:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.291551:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.291551:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.291552:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.291552:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677474.291564:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.291567:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.291567:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.291569:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.291571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.291572:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677474.291583:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.291585:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.291586:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.291587:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.291587:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.291589:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.291734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.291741:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.291744:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.291746:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.291748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.291749:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.291750:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.291752:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.291753:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.291754:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.291755:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.291758:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.291760:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.291761:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.291762:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.291763:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.291764:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.291764:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.291766:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.291767:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.291768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.291774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.291778:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.291780:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.291780:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.291781:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677474.291782:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.291782:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.291783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.291784:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.291785:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.291786:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.291921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.291926:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.291929:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.291929:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.291931:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.291933:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.291935:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.291936:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.291937:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.291938:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.291939:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.291942:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.291943:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.291946:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.291947:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.291947:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.291949:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.291949:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.291950:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.291951:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.291952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.291957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.291962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.291964:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.291964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.291964:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.291965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.291966:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.291967:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.291969:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.291969:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.291971:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.292298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.292307:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.292309:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.292312:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.292314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.292314:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.292315:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.292316:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.292318:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.292319:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.292320:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.292323:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.292325:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.292328:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.292329:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.292330:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.292331:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.292332:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.292333:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.292334:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.292338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.292347:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.292349:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.292350:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.292352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.292352:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.292355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.292462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.292470:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.292473:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.292475:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.292477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.292478:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.292480:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677474.292481:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.292482:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.292484:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.292484:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677474.292487:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.292489:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677474.292492:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.292493:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677474.292493:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677474.292495:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677474.292496:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.292496:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.292497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.292502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.292511:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.292514:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.292514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.292515:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.292517:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.292521:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.292591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf3200] (20)++
00000800:00000200:10.0:1619677474.292599:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf3200] (21)++
00000800:00000200:10.0:1619677474.292602:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.53@o2ib3
00000800:00000200:26.0:1619677474.292603:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf3200] (22)--
00000400:00000200:10.0:1619677474.292605:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.53@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.292608:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.292609:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.292611:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.292612:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.292613:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.292614:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.292617:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.53@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.292618:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.292621:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.292622:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.292623:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.292624:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.292625:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf3200] (21)++
00000800:00000200:10.0:1619677474.292626:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf3200] (22)++
00000800:00000200:10.0:1619677474.292628:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf3200] (23)--
00000800:00000200:10.0:1619677474.292628:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf3200] (22)--
00000800:00000200:8.2:1619677474.292630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.292634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf3200] (21)++
00000800:00000200:21.0:1619677474.292638:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677474.292641:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf3200] (22)++
00000800:00000200:21.0:1619677474.292641:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.292642:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.292643:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf3200] (23)--
00000400:00000200:21.0:1619677474.292643:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.292644:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf3200] (22)--
00000800:00000200:20.0:1619677474.292644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.292646:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf3200] (21)--
00000800:00000200:21.0:1619677474.292647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.292723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:20.2:1619677474.292726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:2.0:1619677474.292729:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677474.292731:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:2.0:1619677474.292731:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.292732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677474.292733:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:2.0:1619677474.292734:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.292736:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677474.292736:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.292737:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.292738:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.292739:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.292740:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677474.292741:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.292742:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.292744:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.292744:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.292745:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.292745:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.292747:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677474.292747:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.292753:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.292754:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.292756:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677474.292757:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.292759:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.292760:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677474.292760:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.292761:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677474.292761:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.292762:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.292763:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.292764:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677474.292764:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:2.0:1619677474.292764:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.292764:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677474.292765:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:2.0:1619677474.292765:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.292766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.292769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.292777:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:21.0:1619677474.292777:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.292779:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677474.292780:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:21.0:1619677474.292780:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.292781:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.292782:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.292782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:31.0:1619677474.292784:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.292784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.292784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:31.0:1619677474.292788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.293005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.293013:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.293015:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.293018:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.293019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.293020:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.293022:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.293023:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.293025:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.293026:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.293027:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.293030:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.293032:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.293035:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.293036:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.293037:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.293038:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.293039:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.293040:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.293040:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.293044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.293052:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.293054:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.293055:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.293057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.293057:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.293060:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.293062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.293069:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.293071:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.293073:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.293073:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.293074:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.293075:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.293180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.293187:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.293190:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.293192:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.293194:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.293195:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.293197:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.293198:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.293199:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.293200:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.293201:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.293205:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.293206:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.293209:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.293210:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.293211:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.293212:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.293213:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.293215:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.293216:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.293217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.293222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.293230:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.293232:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.293233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.293234:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.293237:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.293240:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.293241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.293242:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.293244:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.293247:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.293315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.293321:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.293323:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.293324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.293326:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.293328:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.293329:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.293330:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.293332:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.293332:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.293333:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.293336:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.293338:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.293340:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.293341:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.293342:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.293343:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.293344:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.293344:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.293345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.293358:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.293360:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.293361:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.293363:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.293363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.293365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.293512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.293520:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.293524:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.293525:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.293526:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.293529:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.293531:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.293532:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.293534:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.293535:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.293536:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.293539:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.293541:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.293543:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.293544:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.293545:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.293547:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.293547:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.293548:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.293549:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.293550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.293555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.293560:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.293562:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.293563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.293563:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677474.293564:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677474.293564:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.293565:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.293566:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.293566:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.293568:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.293644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.293647:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.293650:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.293652:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.293653:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.293654:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.293656:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.293657:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.293659:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.293659:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.293661:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.293664:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.293666:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.293668:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.293669:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.293669:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.293671:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.293671:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.293672:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.293673:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.293678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.293687:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.293690:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.293691:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.293691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.293693:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.293695:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.293716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.293723:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.293726:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.293728:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.293729:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.293733:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.293734:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.293735:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.293737:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.293738:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.293739:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.293743:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.293744:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.293747:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.293748:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.293748:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.293750:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.293751:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.293751:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.293752:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.293774:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.293776:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.293777:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.293778:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.293780:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.293783:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.293787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.293794:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.293796:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.293797:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.293799:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.293802:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.293804:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.293805:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.293806:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.293807:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.293808:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.293812:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.293814:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.293815:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.293816:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.293817:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.293819:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.293820:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.293821:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.293822:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.293823:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.293825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.293828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.293833:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.293835:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.293835:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.293836:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.293836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.293836:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.293838:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.293838:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.293839:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.293839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.294282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.294288:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.294291:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.294292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.294299:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.294302:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.294303:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.294304:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.294306:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.294307:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.294308:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.294311:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.294313:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.294315:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.294316:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.294316:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.294318:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.294319:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.294320:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.294321:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.294322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.294327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677474.294333:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.294334:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.294335:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.294336:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677474.294337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.294337:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677474.294337:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.294338:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.294339:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.294341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.294352:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.294355:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.294355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.294356:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.294357:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.294357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.294633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677474.294641:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.294645:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677474.294647:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.294649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677474.294650:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.294651:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.294653:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.294654:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.294656:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.294656:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.294660:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.294662:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.294665:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.294666:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.294667:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.294668:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.294669:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.294670:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.294670:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677474.294674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.294684:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.294687:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.294688:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.294688:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.294690:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.294694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.294696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.294704:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.294706:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.294707:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.294708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.294709:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.294709:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.295014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.295022:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.295025:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.295027:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.295029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.295030:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.295031:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.295033:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.295034:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.295035:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.295036:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.295039:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.295040:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.295044:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.295044:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.295045:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.295047:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.295047:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.295048:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.295049:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.295060:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.295063:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.295064:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.295064:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.295066:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.295069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.295079:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.295082:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.295083:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.295083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.295084:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.295084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.295198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.295206:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.295208:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.295210:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.295212:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.295213:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.295214:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.295216:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.295217:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.295218:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.295219:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.295221:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.295223:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.295225:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.295225:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.295226:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.295227:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.295228:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.295229:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.295229:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.295240:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.295243:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.295243:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.295245:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.295247:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.295248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.295257:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.295259:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.295260:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.295261:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.295261:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.295262:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.295516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.295524:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.295527:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.295528:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.295530:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.295533:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.295534:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.295535:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.295536:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.295537:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.295538:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.295541:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.295543:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.295545:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.295546:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.295546:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.295547:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.295548:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.295549:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.295549:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.295554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.295561:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:4.2:1619677474.295562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:8.0:1619677474.295563:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.295564:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.295564:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.295566:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.295567:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677474.295568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677474.295570:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:9.2:1619677474.295571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:21.0:1619677474.295572:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.295573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.295575:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.295576:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677474.295577:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:21.0:1619677474.295577:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.295579:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.295580:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:21.0:1619677474.295580:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.295581:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:21.0:1619677474.295581:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.295581:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.295582:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.295583:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:21.0:1619677474.295585:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.295586:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.295589:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.295590:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.295590:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.295592:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.295592:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.295594:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.295595:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.295596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677474.295604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677474.295606:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.295607:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677474.295608:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.295609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.295609:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:6.0:1619677474.295609:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.295610:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.295611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677474.295611:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.295613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.296162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.296171:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.296173:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.296176:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.296178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.296178:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.296179:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.296181:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.296182:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.296183:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.296184:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.296187:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.296189:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.296191:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.296192:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.296192:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.296194:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.296194:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.296196:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.296197:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.296198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.296199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.296203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.296206:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.296208:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.296209:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.296209:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.296211:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.296212:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.296214:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.296214:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.296215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.296216:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.296285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.296293:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.296297:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.296301:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677474.296303:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.296305:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.296306:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.296307:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.296308:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.296309:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.296310:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.296313:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.296314:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.296316:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.296317:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.296318:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.296319:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.296320:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.296321:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.296322:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.296323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.296325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.296329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.296332:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.296334:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.296335:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.296335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.296337:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.296337:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.296338:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677474.296339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.296339:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.296340:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677474.296425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.296432:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.296434:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.296435:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.296437:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.296439:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.296440:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.296442:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.296443:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.296444:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.296444:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.296447:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.296449:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.296451:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.296452:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.296453:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.296454:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.296455:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.296455:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.296456:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.296461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.296469:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.296471:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.296472:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.296474:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.296475:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.296477:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.297024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.297032:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.297035:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.297038:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.297040:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.297040:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.297041:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.297043:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.297044:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.297045:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.297046:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.297049:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.297051:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.297053:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.297053:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.297054:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.297055:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.297056:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.297057:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.297057:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.297068:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.297071:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.297071:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.297073:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.297075:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.297076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.297164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.297169:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.297172:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.297174:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.297174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.297177:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.297178:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.297179:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.297181:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.297182:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.297183:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.297186:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.297188:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.297190:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.297191:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.297192:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.297193:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.297193:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.297194:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.297195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677474.297207:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.297210:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.297211:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.297211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.297213:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.297215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.297267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.297275:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.297277:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.297280:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.297282:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.297282:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.297283:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.297285:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.297286:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.297287:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.297288:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.297292:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.297298:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.297301:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.297302:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.297302:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.297304:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.297305:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.297305:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.297306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.297319:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.297322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.297322:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.297323:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.297325:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.297328:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.297577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.297585:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.297588:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.297590:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.297592:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.297593:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.297594:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.297596:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.297597:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.297598:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.297599:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.297602:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.297603:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.297606:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.297606:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.297607:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.297608:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.297609:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.297610:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.297610:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.297620:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.297623:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.297624:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.297625:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.297627:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.297628:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.298275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.298283:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.298286:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.298287:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.298289:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.298291:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.298293:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.298299:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.298300:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.298301:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.298302:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.298305:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.298306:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.298309:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.298310:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.298310:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.298312:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.298313:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.298313:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.298314:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.298327:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.298330:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.298330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.298331:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.298333:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.298336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.298471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.298480:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.298484:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.298486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.298489:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.298492:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.298494:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.298496:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.298499:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.298500:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.298502:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.298507:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.298510:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.298514:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.298515:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.298516:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.298518:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.298519:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.298521:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.298522:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.298525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.298536:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.298540:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.298542:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.298543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.298545:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.298548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.298560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.298569:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.298571:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.298574:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.298575:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.298576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.298577:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.298578:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.298579:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.298581:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.298582:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.298585:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.298587:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.298589:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.298590:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.298590:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.298592:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.298592:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.298593:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.298594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.298598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.298604:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.298607:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.298607:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.298608:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.298609:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.298612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.298986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677474.298995:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.298998:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677474.298999:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.299001:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.299001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677474.299003:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.299004:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.299006:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.299006:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.299007:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.299010:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.299011:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.299014:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.299014:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.299015:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.299017:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.299017:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.299018:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.299019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.299022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.299023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:24.0:1619677474.299029:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:30.0:1619677474.299030:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.299031:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.299032:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.299032:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.299033:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:24.0:1619677474.299033:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.299035:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.299036:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.299037:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.299038:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.299039:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.299040:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.299041:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.299042:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.299043:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.299046:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.299048:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.299050:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677474.299050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677474.299051:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.299052:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.299053:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.299053:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677474.299054:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.299054:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.299055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.0:1619677474.299056:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.299058:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.299059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.299061:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.299062:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.299064:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.299065:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.299066:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.299067:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677474.299067:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677474.299070:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.299070:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.299071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.299071:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.299072:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:6.0:1619677474.299073:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.299075:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.299076:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.299076:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677474.299076:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677474.299078:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.299078:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.299079:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.299080:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.299085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.299089:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.299092:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.299092:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.299093:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677474.299093:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677474.299094:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.299094:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677474.299096:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.299097:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.299097:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.299099:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.299101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.299140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.299147:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.299150:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.299151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.299153:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.299155:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.299157:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.299158:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.299160:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677474.299160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:8.0:1619677474.299161:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.299162:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.299165:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.299167:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.299170:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.299170:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:12.0:1619677474.299171:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:8.0:1619677474.299171:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.299173:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.299173:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:12.0:1619677474.299174:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:8.0:1619677474.299174:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.299174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:12.0:1619677474.299178:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.299179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.299182:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.299184:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677474.299186:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.299186:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:12.0:1619677474.299188:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.299189:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.299190:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:12.0:1619677474.299190:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.299190:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.299192:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677474.299192:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.299195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:12.0:1619677474.299198:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.299201:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677474.299205:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.299206:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677474.299207:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677474.299209:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677474.299210:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.299212:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677474.299214:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677474.299215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677474.299215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.299220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:2.0:1619677474.299223:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.299226:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.299227:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677474.299228:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:2.0:1619677474.299228:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.299230:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:2.0:1619677474.299230:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.299231:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.299232:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677474.299233:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.299459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.299465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.299468:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.299469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.299471:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.299473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.299475:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.299476:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.299477:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.299478:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.299479:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.299483:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.299485:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.299487:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.299488:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.299489:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.299490:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.299491:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.299492:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.299492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.299505:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.299507:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.299508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.299509:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.299511:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.299513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.299959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.299965:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.299968:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.299969:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.299970:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.299973:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.299975:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.299976:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.299977:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.299978:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.299978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000400:00000200:17.0:1619677474.299979:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.299981:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.299983:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677474.299985:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:17.0:1619677474.299985:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.299986:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.299986:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.299988:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:26.0:1619677474.299989:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:17.0:1619677474.299989:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:10.0:1619677474.299989:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:17.0:1619677474.299990:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677474.299991:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000400:00000200:26.0:1619677474.299992:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.299992:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:26.0:1619677474.299994:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677474.299995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:26.0:1619677474.299996:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:21.2:1619677474.299996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:26.0:1619677474.299997:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.299999:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.300000:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.300001:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677474.300002:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000400:00000200:26.0:1619677474.300004:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.300004:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.300004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677474.300005:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677474.300005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:26.0:1619677474.300006:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677474.300007:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.300007:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:26.0:1619677474.300008:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:21.0:1619677474.300008:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.300009:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:21.0:1619677474.300009:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.300010:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.300010:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:20.0:1619677474.300010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:26.0:1619677474.300011:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677474.300012:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:21.0:1619677474.300012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677474.300016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.300022:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.300024:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.300025:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.300025:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.300027:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.300029:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677474.300044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677474.300050:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.300052:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677474.300053:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.300053:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.300054:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.300054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.300167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.300175:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.300178:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.300180:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.300182:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.300183:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.300184:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.300185:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.300187:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.300188:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.300189:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.300192:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.300194:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.300196:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.300196:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.300197:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.300198:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.300199:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.300200:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.300202:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.300202:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.300208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.300213:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.300214:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.300215:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.300216:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.300217:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.300217:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.300218:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.300219:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.300220:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.300221:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.300291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.300298:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.300301:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.300302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.300303:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.300305:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.300306:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.300307:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.300308:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.300309:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.300310:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.300313:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.300315:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.300317:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.300317:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.300318:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.300320:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.300320:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.300321:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.300321:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.300326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.300333:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.300335:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.300336:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.300336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.300337:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.300340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.300345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.300351:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.300354:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.300354:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.300354:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.300355:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.300356:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.300653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.300660:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.2:1619677474.300663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:18.0:1619677474.300663:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.300664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677474.300665:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000400:00000200:18.0:1619677474.300666:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.300668:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:18.0:1619677474.300668:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.300669:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677474.300670:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.300671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:18.0:1619677474.300671:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.300672:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.300673:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.300673:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.300674:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677474.300675:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677474.300676:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.300678:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.300679:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.300679:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677474.300680:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677474.300681:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:20.0:1619677474.300684:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.300684:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.300685:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.300685:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677474.300687:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 984 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677474.300687:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.300688:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677474.300689:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.300689:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.300689:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.0:1619677474.300690:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.300690:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.300692:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.300693:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.300693:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:9.2:1619677474.300693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677474.300694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:2.0:1619677474.300700:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.300702:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.300703:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.300704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.300705:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.300706:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677474.300707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677474.300708:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.300709:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.300710:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.300711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.300712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.300727:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.300730:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.300730:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.300731:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.300732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.300733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.300920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.300928:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.300931:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.300934:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.300936:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.300936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.300937:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.300939:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.300940:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.300941:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.300942:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.300945:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.300947:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.300950:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.300951:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.300952:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.300954:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.300954:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.300956:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.300957:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.300958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.300963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.300967:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.300970:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.300971:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.300971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.300971:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.300973:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677474.300973:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.300974:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.300975:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.300976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.301048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.301056:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.301058:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.301059:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.301061:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.301063:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.301064:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.301066:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.301067:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.301068:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.301069:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.301071:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.301073:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.301075:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.301076:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.301076:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.301078:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.301078:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.301079:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.301079:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.301083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.301089:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.301092:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.301093:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.301093:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.301094:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.301096:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.301269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.301276:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.301279:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.301281:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.301283:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.301283:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.301285:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.301286:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.301287:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.301288:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.301289:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.301292:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.301297:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.301299:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.301300:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.301300:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.301302:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.301302:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.301303:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.301303:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.301316:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.301318:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.301319:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.301321:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.301322:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.301323:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.301326:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.301327:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.301327:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.301328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.301329:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.301389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.301395:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.301398:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.301399:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.301401:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.301403:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.301405:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.301406:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.301407:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.301408:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.301409:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.301412:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.301414:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.301417:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.301417:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.301418:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.301420:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.301420:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.301421:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677474.301422:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677474.301423:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:21.2:1619677474.301429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:22.0:1619677474.301433:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677474.301435:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:22.0:1619677474.301436:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.301437:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.301437:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:6.0:1619677474.301437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.301438:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:22.0:1619677474.301439:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.301439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677474.301441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.301718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.301723:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.301726:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.301728:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.301728:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.301729:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.301731:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.301732:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.301734:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.301735:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.301736:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.301739:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.301741:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.301744:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.301745:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.301745:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.301747:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.301748:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.301749:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.301750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.301753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.301760:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.301763:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.301764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.301764:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.301766:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.301769:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.301821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.301825:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.301827:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.301829:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.301830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.301832:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.301833:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.301835:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.301836:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.301837:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.301838:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.301841:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.301842:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.301844:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.301845:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.301846:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.301847:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.301848:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.301849:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.301849:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.301853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.301861:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.301863:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.301864:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.301866:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.301866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.301868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.302166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.302173:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.302176:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.302178:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.302180:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.302180:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.302182:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.302183:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.302185:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.302185:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.302187:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.302190:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.302192:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.302194:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.302194:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.302195:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.302196:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.302197:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.302198:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.302198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.302209:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.302212:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.302212:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.302214:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.302215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.302216:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.302682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.302690:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.302692:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.302694:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.302696:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.302697:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.302697:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.302698:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.302699:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.302700:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.302701:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.302704:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.302705:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.302708:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.302709:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.302710:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.302712:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.302712:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.302713:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.302714:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.302727:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.302729:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.302730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.302731:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.302733:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.302735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.302744:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.302747:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.302748:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.302748:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.302749:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.302750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.303073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.303081:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.303083:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.303084:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.303086:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.303088:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.303090:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.303091:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.303093:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.303094:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.303094:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.303098:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.303099:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.303102:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.303103:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.303103:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.303105:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.303105:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.303106:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.303106:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.303118:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.303121:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.303122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.303122:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.303124:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.303127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.303146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.303154:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.303157:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.303159:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.303160:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.303162:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.303163:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.303164:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.303166:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.303167:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.303168:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.303170:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.303172:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.303175:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.303175:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.303176:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.303177:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.303178:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.303179:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.303179:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.303183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.303190:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.303192:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.303193:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.303193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.303195:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.303197:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.303626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677474.303634:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.303637:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677474.303639:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.303641:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.303641:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677474.303643:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677474.303644:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.303645:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.303646:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.303647:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677474.303650:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.303651:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677474.303655:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.303655:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677474.303656:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677474.303658:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677474.303658:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.303659:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677474.303660:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677474.303665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.303674:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.303676:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.303677:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.303678:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.303680:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.303683:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.303688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.303696:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.303698:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.303699:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.303699:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.303700:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.303700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.303722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.303733:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.303737:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.303740:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.303743:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.303744:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.303747:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677474.303749:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.303751:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.303753:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.303754:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677474.303759:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.303762:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677474.303765:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.303766:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677474.303772:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677474.303774:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677474.303775:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.303776:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.303777:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:30.0:1619677474.303787:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.303790:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.303791:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.303792:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.303794:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.303795:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.303973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.303980:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.303983:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.303984:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.303985:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.303988:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.303989:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.303990:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.303992:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.303992:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.303993:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.303996:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.303998:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.304000:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.304001:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.304001:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.304003:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.304003:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.304005:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.304006:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.304006:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677474.304012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677474.304015:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.304018:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.304018:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.304019:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:12.0:1619677474.304020:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:10.0:1619677474.304021:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:13.0:1619677474.304022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.304022:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.304023:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677474.304026:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677474.304173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.304179:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.304183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.304183:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677474.304185:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.304187:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.304189:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.304190:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.304191:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.304192:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.304193:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.304197:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.304198:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.304201:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.304201:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.304202:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.304204:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.304204:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.304205:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.304205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.304217:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.304220:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.304221:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.304221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.304223:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.304225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.304239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.304242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.304242:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.304243:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.304244:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.304244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.304310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.304318:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.304321:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.304323:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.304325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.304326:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.304328:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.304330:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.304331:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.304332:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.304333:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.304337:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.304339:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.304342:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.304343:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.304344:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.304346:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.304346:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.304348:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.304350:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.304350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.304352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.304356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.304361:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.304364:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677474.304364:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.304365:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.304366:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.304367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.304367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.304367:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.304368:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.304370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.304464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.304471:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.304474:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.304476:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.304478:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.304479:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.304480:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.304482:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.304483:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.304484:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.304485:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.304488:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.304489:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.304492:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.304493:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.304493:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.304495:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.304495:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.304496:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.304497:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.304509:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.304512:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.304513:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.304513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.304515:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.304518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.304533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.304539:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.304541:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:4.2:1619677474.304541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:21.0:1619677474.304543:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.304545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.304546:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.304547:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677474.304548:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677474.304549:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.304550:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.304552:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:21.0:1619677474.304552:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.304552:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:21.0:1619677474.304553:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677474.304555:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.304557:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.304558:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.304559:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:8.0:1619677474.304560:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.304561:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96505078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.304562:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.304563:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.304563:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:8.0:1619677474.304563:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.304564:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.304565:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.304565:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677474.304565:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.304567:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.304568:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:8.0:1619677474.304568:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.304569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.304571:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.304574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677474.304575:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.304576:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.304578:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:8.0:1619677474.304578:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.304578:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.304579:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677474.304579:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:8.0:1619677474.304579:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.304580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677474.304581:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.304581:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677474.304582:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.304583:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.304583:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677474.304583:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677474.304584:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677474.304585:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.304585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677474.304588:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.304590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.304594:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.304597:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.304597:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.304597:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.304598:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.304600:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:6.0:1619677474.304600:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.304601:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.304602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.304603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.305048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.305052:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.305054:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.305057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.305057:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.305059:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.305060:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.305062:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.305063:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.305064:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.305065:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.305068:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.305070:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.305073:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.305073:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.305074:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.305076:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.305077:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.305077:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.305078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.305082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.305089:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.305091:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.305092:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.305092:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.305095:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.2:1619677474.305097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677474.305097:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.305105:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.305108:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.305111:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.305112:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.305114:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.305115:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.305117:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.305118:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.305119:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.305120:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.305123:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.305125:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.305127:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.305128:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.305129:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.305130:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.305131:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.305132:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.305132:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.305142:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.305145:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.305145:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.305147:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.305149:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.305150:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.305182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.305190:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.305193:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.305194:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.305196:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.305198:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.305199:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.305201:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.305202:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.305203:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.305204:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.305207:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.305209:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.305211:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.305212:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.305213:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.305214:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.305215:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.305216:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.305216:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.305228:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.305231:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.305231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.305232:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.305234:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.305237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.305638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.305644:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.305647:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.305648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.305650:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.305652:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.305654:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.305655:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.305656:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.305657:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.305658:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.305662:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.305663:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.305666:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.305667:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.305668:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.305670:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.305670:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.305671:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.305671:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.305676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.305683:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.305686:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.305687:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.305687:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.305689:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.305692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.305694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.305700:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.305702:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.305703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.305703:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.305704:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.305705:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.305737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.305744:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.305747:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.305748:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.305750:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.305751:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.305753:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.305754:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.305756:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.305756:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.305758:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.305760:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.305762:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.305765:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.305766:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.305767:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.305769:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.305770:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.305771:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.305772:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.305772:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677474.305777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677474.305783:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677474.305783:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.305785:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:13.0:1619677474.305786:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:2.0:1619677474.305786:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:13.0:1619677474.305787:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.305787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677474.305789:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.305791:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677474.305791:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:4.2:1619677474.305798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.305806:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.305808:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.305809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.305811:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.305814:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.305816:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677474.305817:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.305818:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.305819:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.305820:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.305823:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.305825:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.305827:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.305828:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677474.305829:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.305830:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.305831:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.305831:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.305832:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677474.305844:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.305847:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.305847:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.305849:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.305852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.305852:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.305869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8665a00] (20)++
00000800:00000200:24.0:1619677474.305876:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8665a00] (21)++
00000800:00000200:24.0:1619677474.305879:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.51@o2ib3
00000800:00000200:8.0:1619677474.305880:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8665a00] (22)--
00000400:00000200:24.0:1619677474.305882:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.51@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.305884:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.305886:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677474.305887:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.305889:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.305889:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.305890:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677474.305894:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.51@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.305895:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.305898:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.305899:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677474.305899:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677474.305901:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677474.305901:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8665a00] (21)++
00000800:00000200:24.0:1619677474.305902:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8665a00] (22)--
00000800:00000200:24.0:1619677474.305903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8665a00] (21)--
00000800:00000200:8.2:1619677474.305907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.305915:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.305918:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.305919:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.305919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.305921:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.305924:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.306069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.306076:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.306079:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.306080:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.306081:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.306084:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.306086:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.306087:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677474.306087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:26.0:1619677474.306089:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.306089:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.306091:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.306093:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.306093:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:26.0:1619677474.306095:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677474.306095:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.306096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:26.0:1619677474.306098:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.306099:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:18.0:1619677474.306099:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.306100:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.306101:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.306102:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677474.306102:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.306103:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.306104:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.306104:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677474.306104:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677474.306105:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.306106:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.306107:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677474.306107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:18.0:1619677474.306108:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:4.2:1619677474.306110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677474.306112:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.306113:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.306113:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.306115:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.306116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677474.306117:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.306117:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.306117:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677474.306117:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.306118:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677474.306119:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677474.306119:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.306120:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:10.0:1619677474.306120:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677474.306121:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677474.306121:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:18.0:1619677474.306122:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.306122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.0:1619677474.306122:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:9.2:1619677474.306126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.306132:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.306135:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.306136:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.306136:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.306137:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.306140:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.306145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.306151:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.306153:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.306154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.306154:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.306155:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.306155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.306379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.306386:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.306389:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677474.306389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:31.0:1619677474.306391:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.306393:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.306394:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.306395:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.306395:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:31.0:1619677474.306396:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.306397:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.306397:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.306398:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.306399:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.306402:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.306404:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.306405:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.306406:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.306407:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:30.0:1619677474.306407:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.306408:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.306409:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677474.306409:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.306410:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.306411:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.306412:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.306413:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.306414:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.306414:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.306415:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.306416:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.306416:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.306417:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.306418:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.306418:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.306419:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.306420:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.306421:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.306421:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.306422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.306422:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.306423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.306423:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.306424:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.306425:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.306425:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.306437:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.306440:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677474.306440:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.306441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.306441:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.306444:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677474.306444:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.306445:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.306446:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.306446:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677474.306447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.306448:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.306456:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.306459:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.306460:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.306460:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.306461:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.306461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.306928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677474.306934:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.306937:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.306938:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.306939:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.306942:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.306943:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.306945:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.306946:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.306947:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.306948:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.306951:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.306953:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.306956:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.306957:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.306957:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.306959:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.306960:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.306961:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677474.306962:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677474.306962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:21.2:1619677474.306968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.306973:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:17.0:1619677474.306975:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:17.0:1619677474.306976:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:9.2:1619677474.306976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.306976:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:24.0:1619677474.306984:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.306986:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.306987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.306988:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.306990:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.306992:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.307324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677474.307332:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.307335:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677474.307337:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.307339:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.307339:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677474.307341:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677474.307342:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.307344:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.307345:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.307346:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677474.307349:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.307351:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677474.307353:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.307354:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:31.0:1619677474.307354:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677474.307356:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:31.0:1619677474.307357:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.307358:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677474.307358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:12.0:1619677474.307370:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.307372:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.307373:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.307375:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.307377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677474.307378:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:4.2:1619677474.307755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.307762:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.307764:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.307766:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.307768:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.307768:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.307769:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677474.307770:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.307772:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.307773:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.307773:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677474.307776:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.307778:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.307780:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.307780:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.307781:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.307782:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.307783:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.307784:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.307784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:21.2:1619677474.307793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677474.307799:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.307802:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677474.307803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:31.0:1619677474.307804:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:17.0:1619677474.307805:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677474.307806:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.307807:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.307807:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.307809:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.307810:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.307810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:17.0:1619677474.307811:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.307812:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:17.0:1619677474.307812:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.307814:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.307814:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.307818:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677474.307820:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.307822:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.307823:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677474.307824:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.307826:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677474.307826:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.307827:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677474.307828:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677474.307832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.307839:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.307842:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.307842:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.307843:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.307845:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.307847:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.307968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.307975:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.307978:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.307979:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.307981:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.307983:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.307985:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.307986:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.307987:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.307988:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.307989:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.307992:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.307994:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.307996:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.307997:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.307998:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.308000:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.308000:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.308001:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.308001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.308014:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.308017:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.308017:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.308018:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.308019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677474.308020:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.308023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.308026:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.308029:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.308031:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.308032:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.308033:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.308035:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.308036:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.308037:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.308039:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.308039:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.308042:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.308044:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.308047:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.308047:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.308048:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.308049:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.308050:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.308051:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.308051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.308055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.308061:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.308064:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.308065:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.308065:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.308066:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.308069:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.308178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.308184:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.308186:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.308188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.308190:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.308192:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.308193:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.308195:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.308196:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.308197:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.308198:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.308201:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.308203:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.308206:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.308206:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.308207:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.308209:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.308209:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.308210:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.308210:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677474.308215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.308223:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.308225:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.308226:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.308226:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.308229:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.308231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.308791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.308799:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.308802:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.308802:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.308805:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.308807:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.308808:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.308810:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.308811:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.308812:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.308813:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.308817:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.308818:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.308821:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.308822:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.308823:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.308824:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.308825:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.308826:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.308826:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:4.2:1619677474.308826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.308831:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:9.2:1619677474.308831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677474.308834:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.308836:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.308837:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.308839:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.308839:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:21.0:1619677474.308840:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.308842:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677474.308842:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:21.0:1619677474.308843:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.308843:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:21.0:1619677474.308844:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.308844:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.308845:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677474.308846:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:21.0:1619677474.308849:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.308849:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677474.308850:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677474.308850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:21.0:1619677474.308853:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.308853:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.308854:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.308856:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.308857:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:2.0:1619677474.308857:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677474.308858:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.308859:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.308860:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:2.0:1619677474.308860:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.308861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.308861:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.308862:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.308863:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.308865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677474.308869:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.308869:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.308871:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:22.0:1619677474.308872:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677474.308872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:22.0:1619677474.308873:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.308873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.308873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.308875:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.308878:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.308978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.308984:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.308987:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.308988:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.308989:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.308992:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.308994:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.308995:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677474.308996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:17.0:1619677474.308996:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.308997:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.308998:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.309001:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.309003:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.309004:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677474.309005:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.309006:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677474.309007:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:17.0:1619677474.309007:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.309008:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.309008:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677474.309009:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.309009:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.309010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677474.309011:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.309011:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.309013:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.309014:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.309016:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.309016:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.309018:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.309021:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.309021:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:31.0:1619677474.309023:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.309024:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.309025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:31.0:1619677474.309026:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677474.309026:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.309027:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.309028:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677474.309028:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.309030:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.309030:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.309030:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.309032:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.309033:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.309034:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.309040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.309043:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.309045:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.309046:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.309046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.309047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.309048:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.309050:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:22.0:1619677474.309050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.309051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.309052:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.309178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.309185:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.309188:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.309190:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.309192:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.309193:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.309194:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.309196:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.309197:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.309198:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.309199:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.309202:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.309204:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.309206:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.309207:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.309207:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.309209:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.309210:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.309210:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.309212:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.309212:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.309215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.309218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.309222:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.309224:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.309225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.309225:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.309226:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.309227:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.309229:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.309229:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677474.309229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.309230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.309281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.309288:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.309290:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.309291:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.309297:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.309300:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.309301:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.309302:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.309304:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.309305:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.309305:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.309308:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.309310:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.309312:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.309313:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.309313:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.309315:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.309316:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.309317:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.309317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.309322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.309329:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.309332:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.309332:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.309333:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.309334:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.309335:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.309457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.309461:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.309463:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.309466:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.309467:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.309468:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.309470:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.309471:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.309472:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.309473:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.309474:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.309477:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.309479:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.309481:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.309482:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.309482:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.309484:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.309485:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.309485:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.309487:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.309487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677474.309491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.309493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.309498:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.309499:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.309499:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.309500:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:20.0:1619677474.309500:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.309501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.309502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.309502:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.309503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.309504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:4.2:1619677474.309513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.309521:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.309523:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.309524:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.309526:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.309529:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.309530:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.309532:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.309533:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.309534:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.309535:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.309538:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.309540:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.309542:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.309543:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.309544:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.309546:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.309546:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.309547:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.309548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.309560:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.309562:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.309563:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.309564:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.309566:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.309568:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.309783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.309791:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.309794:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.309796:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.309798:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.309798:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.309799:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.309800:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.309802:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.309803:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.309804:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.309807:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.309808:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.309810:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.309811:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.309811:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.309812:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.309813:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.309814:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.309815:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.309825:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.309828:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.309828:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.309830:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.309832:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.309832:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.2:1619677474.310376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.310383:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.310386:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.310386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.310388:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.310390:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.310392:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.310393:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.310395:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.310395:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.310397:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.310400:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.310402:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.310405:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.310406:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.310407:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.310409:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.310409:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.310410:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.310411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677474.310424:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.310426:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.310427:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.310429:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.310431:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.310431:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677474.310441:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.310444:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.310445:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.310446:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:4.2:1619677474.310446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.310447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.310448:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:26.0:1619677474.310454:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.310457:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.310458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.310460:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.310463:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.310464:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.310465:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.310467:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.310468:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.310469:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.310472:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.310473:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.310476:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.310476:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.310477:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.310479:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.310479:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.310480:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.310480:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.310484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.310491:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.310494:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.310495:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.310495:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.310496:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.310498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.310643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.310651:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.310653:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.310655:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.310657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.310657:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.310658:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.310660:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.310661:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.310662:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.310662:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.310666:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.310667:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.310670:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.310671:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.310672:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.310673:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.310674:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.310675:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.310675:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.310688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.310690:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.310691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.310692:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.310694:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.310697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.310707:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.310710:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.310711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.310711:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.310712:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.310712:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.310864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.310872:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.310874:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.310876:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.310878:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.310879:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.310881:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.310882:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.310884:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.310884:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.310885:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.310889:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.310890:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.310892:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.310893:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.310893:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.310895:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.310895:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.310896:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.310897:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.310907:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.310910:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.310910:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.310912:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.310913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.310914:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.311207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677474.311215:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.311217:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677474.311219:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.311221:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.311222:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677474.311223:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.311224:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.311225:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.311226:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.311227:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.311230:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.311231:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.311233:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.311234:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.311235:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.311236:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.311237:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.311238:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.311238:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677474.311242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.311248:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.311250:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.311251:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.311251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.311253:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.311255:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.311263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.311270:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.311272:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.311273:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.311273:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.311274:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.311274:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.311428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.311435:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.311438:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.311439:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.311440:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.311443:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.311444:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.311445:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.311446:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.311447:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.311448:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.311451:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.311453:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.311456:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.311456:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.2:1619677474.311457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677474.311457:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.311459:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.311459:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.311460:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.311460:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677474.311466:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.311469:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.311470:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.311473:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.311473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.311474:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677474.311474:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677474.311475:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.311476:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.311477:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.311477:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:31.0:1619677474.311478:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.311478:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.311478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.311481:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.311483:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.311485:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.311485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.311487:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.311488:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.311488:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.311490:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.311490:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.311491:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.311492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.311502:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.311504:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.311505:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.311507:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.311508:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.311509:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.311888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.311896:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.311898:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.311899:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.311901:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.311904:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.311905:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.311907:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.311908:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.311909:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.311910:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.311912:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.311914:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.311917:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.311917:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.311918:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.311920:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.311920:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.311921:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.311921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677474.311934:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.311937:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.311938:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.311941:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.311944:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.311945:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.312698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.312706:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.312709:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.312711:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.312713:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.312713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.312714:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.312715:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.312716:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.312717:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.312718:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.312722:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.312723:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.312726:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.312727:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.312728:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.312729:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.312730:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.312731:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.312731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.312755:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.312758:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.312759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.312759:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.312761:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.312764:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.312919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.312923:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.312926:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.312928:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.312929:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.312930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.312931:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.312932:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.312933:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.312934:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.312935:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.312939:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.312940:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.312943:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.312943:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.312944:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.312945:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.312946:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.312947:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.312947:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677474.312959:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.312961:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.312962:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.312962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.312964:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.312966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.313382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.313389:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.313392:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.313393:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.313395:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.313397:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.313398:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.313400:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.313401:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.313402:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.313403:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.313406:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.313408:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.313410:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.313411:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.313412:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.313413:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.313414:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.313415:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.313415:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.313420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.313430:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.313433:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.313434:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.313437:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.313437:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677474.313440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.313443:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.313445:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.313448:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.313448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.313449:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.313450:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.313451:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.313810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.313818:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.313821:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.313823:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.313826:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.313826:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.313828:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.313829:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.313830:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.313831:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.313832:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.313836:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.313837:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.313840:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.313840:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.313841:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.313843:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.313843:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.313845:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.313846:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.313847:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.313849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.313853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.313855:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.313858:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.313859:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.313859:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.313861:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.313862:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677474.313863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.313864:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.313865:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.313866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.313997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.314005:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.314008:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.314009:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.314010:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.314013:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.314014:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.314016:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.314017:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.314018:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.314018:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.314021:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.314023:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.314026:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.314026:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.314027:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.314029:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.314029:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.314030:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.314030:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.314035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.314043:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.314045:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.314046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.314048:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.314048:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.314051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.314744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.314751:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677474.314751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:26.0:1619677474.314754:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.314755:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.314757:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.314759:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:26.0:1619677474.314759:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.314761:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677474.314762:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:26.0:1619677474.314763:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.314764:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.314764:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.314765:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.314766:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677474.314767:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.314767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.314769:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.314770:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.314770:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.314771:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.314772:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.314772:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677474.314773:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.314775:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.314776:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.314776:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.314777:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.314778:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.314779:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.314779:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.314780:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677474.314781:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.314781:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:31.0:1619677474.314782:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.314782:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677474.314783:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.314784:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.314785:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.314786:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.314786:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.314788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.314796:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.314796:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.314798:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677474.314798:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677474.314799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.314800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.314800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:6.0:1619677474.314800:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.314801:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.314802:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.314802:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.314804:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.314805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.314806:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.314807:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.314808:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.314810:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.314811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.314811:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.314812:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.314812:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.314845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.314851:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.314854:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.314854:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.314857:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.314859:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.314860:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.314862:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.314863:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.314865:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.314866:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.314869:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.314871:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.314873:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.314874:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.314875:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.314876:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.314877:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.314878:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.314879:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.314880:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.314885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.314890:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.314890:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.314892:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.314892:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.314893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677474.314894:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.314894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.314894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.314896:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.314898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.314906:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.314909:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.314910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.314910:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.314911:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.314911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.314925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.314933:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.314936:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.314936:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.314938:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.314941:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.314943:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.314944:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.314945:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.314946:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.314947:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.314951:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.314952:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.314955:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.314956:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.314956:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.314958:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.314959:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.314959:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.314960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.314971:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.314974:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.314975:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.314975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.314977:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.314980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.315011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.315019:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.315022:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.315024:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.315026:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.315026:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.315028:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.315029:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.315030:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.315031:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.315032:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.315035:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.315037:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.315039:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.315039:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.315040:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.315041:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.315042:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.315043:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.315043:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.315047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.315054:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.315056:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.315057:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.315057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.315059:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.315061:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.315503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.315509:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.315512:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.315513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.315515:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.315518:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.315519:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.315521:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.315522:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.315523:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.315524:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.315527:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.315529:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.2:1619677474.315531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.315532:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.315533:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.315534:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.315537:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.315538:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.315538:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.315539:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.315539:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.315540:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:18.0:1619677474.315541:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677474.315541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.315544:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.315545:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.315546:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.315547:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.315548:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.315549:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677474.315552:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:18.0:1619677474.315552:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.315553:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677474.315554:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:18.0:1619677474.315555:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.315556:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.315556:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.315556:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:31.0:1619677474.315557:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.315558:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.315558:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.315559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.315559:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.315559:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677474.315561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:8.2:1619677474.315564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.315573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.315576:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.315577:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.315577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.315579:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.315582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.315591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.315599:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.315602:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.315604:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.315605:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.315606:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.315608:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.315609:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.315611:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.315611:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.315613:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.315616:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.315618:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.315622:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.315622:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.315623:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.315625:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.315625:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.315626:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.315627:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.315631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.315639:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.315641:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.315642:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.315642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:21.0:1619677474.315644:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.315646:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.315647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677474.315650:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.2:1619677474.315650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.315653:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.315654:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.315655:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.315657:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000400:00000200:10.0:1619677474.315659:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.315660:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.315661:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000400:00000200:10.0:1619677474.315661:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677474.315662:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000400:00000200:10.0:1619677474.315662:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.315663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.315664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:10.0:1619677474.315664:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.315665:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.315665:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.315669:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.315671:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.315673:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.315674:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.315674:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.315676:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.315676:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.315677:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.315677:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.315681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.315687:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.315690:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.315691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.315691:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.315693:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.315695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.316033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.316041:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.316044:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.316046:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.316048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.316049:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.316050:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.316051:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.316053:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.316053:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.316054:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.316057:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.316059:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.316061:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.316062:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.316062:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.316064:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.316064:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.316065:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.316066:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.316069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.316075:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.316078:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.316079:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.316079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.316081:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.316083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.316090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.316096:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.316098:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.316099:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.316099:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.316100:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.316101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.316532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.316536:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.316539:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.316541:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.316541:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.316543:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.316545:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.316546:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.316547:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.316548:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.316549:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.316552:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.316554:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.316557:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.316557:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.316558:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.316560:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.316560:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.316561:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.316562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.316566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.316573:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.316576:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.316577:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.316577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.316579:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.316582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.316589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.316597:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.316600:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.316601:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.316602:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.316605:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.316606:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.316607:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.316609:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.316610:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.316610:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.316613:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.316615:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.316618:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.316619:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.316619:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.316621:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.316621:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.316622:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.316624:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.316624:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.316629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.316635:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.316637:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.316637:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.316638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.316639:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.316639:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.316639:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.316640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:10.0:1619677474.316641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677474.316641:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.316644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.316645:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.316647:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.316650:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.316650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.316652:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.316654:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.316655:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.316657:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.316658:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.316659:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.316663:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.316664:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.316667:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.316668:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.316669:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.316671:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.2:1619677474.316673:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.0:1619677474.316676:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.316677:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.316678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677474.316682:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.316683:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677474.316685:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:30.0:1619677474.316685:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.316686:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.316687:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.316687:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.316689:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.316690:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000400:00000200:30.0:1619677474.316690:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.316692:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.316693:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.316694:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.316697:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.316699:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.316700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.316701:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.316702:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677474.316702:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.316702:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.316703:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.316705:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.316706:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.316707:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.316709:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.316709:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.316711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.316715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.316719:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.316722:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677474.316723:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677474.316723:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.316724:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.316725:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.316725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.316726:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.316727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.316728:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.316761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.316769:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.316772:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.316774:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.316776:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.316777:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.316779:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.316781:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.316782:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677474.316783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000400:00000200:31.0:1619677474.316784:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.316785:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.316788:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.316789:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.316789:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.316792:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:31.0:1619677474.316793:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.316793:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:31.0:1619677474.316795:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.316795:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:2.0:1619677474.316795:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.316797:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.316798:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:2.0:1619677474.316798:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.316799:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.316800:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000400:00000200:2.0:1619677474.316800:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:31.0:1619677474.316801:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:2.0:1619677474.316801:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96505022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.316803:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.316804:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.316805:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:21.2:1619677474.316807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:2.0:1619677474.316808:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.316810:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677474.316810:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:22.0:1619677474.316813:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.316814:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.316814:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677474.316814:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.316815:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:2.0:1619677474.316815:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677474.316815:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:30.0:1619677474.316817:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677474.316817:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.316817:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:30.0:1619677474.316818:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:2.0:1619677474.316818:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:30.0:1619677474.316819:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.316819:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677474.316819:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.316820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:4.2:1619677474.316821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:9.2:1619677474.316823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.316830:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:12.0:1619677474.316830:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677474.316833:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:12.0:1619677474.316833:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:26.0:1619677474.316834:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.316834:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:12.0:1619677474.316835:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.316836:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.316837:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.316837:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.316838:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677474.316839:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:12.0:1619677474.316840:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.316841:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.316842:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.316843:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677474.316846:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.316848:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677474.316850:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.316851:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677474.316852:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677474.316854:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677474.316854:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.316855:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.316856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.316860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.316867:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.316870:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.316871:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.316871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.316873:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.316877:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.316885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.316892:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.316895:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.316896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.316896:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.316897:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.316898:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.317250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.317258:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.317260:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.317262:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.317264:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.317265:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.317267:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677474.317268:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.317269:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.317270:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.317271:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677474.317274:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.317276:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.317279:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.317279:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677474.317280:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677474.317282:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677474.317282:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.317283:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.317284:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.317288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.317299:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.317301:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.317302:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.317303:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.317305:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.317308:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677474.317603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.317609:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.317611:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.317612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.317614:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.317616:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.317618:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.317619:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.317620:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.317621:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.317622:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.317625:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.317627:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.317629:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.317629:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.317630:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.317632:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.317632:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.317633:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.317633:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.317638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.317645:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.317647:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.317648:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.317649:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.317649:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.317650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.318469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.318477:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.318480:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.318482:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.318484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.318485:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.318487:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.318488:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.318489:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.318490:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.318491:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.318494:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.318496:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.318499:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.318499:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.318500:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.318502:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.318502:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.318503:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.318504:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.318517:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.318520:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.318521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.318521:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.318523:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.318528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.318529:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.318531:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.318532:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.318533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.318533:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.318533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.319384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.319390:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.319393:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.319395:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.319395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.319397:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.319399:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.319400:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.319401:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.319402:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.319404:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.319406:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.319408:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.319411:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.319411:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.319412:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.319414:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.319415:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.319416:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.319416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.319420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.319428:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.319430:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.319431:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.319433:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.319434:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.319435:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677474.319521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.319529:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.319533:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.319535:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.319538:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.319541:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.319542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677474.319543:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.319545:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.319548:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.319549:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.319550:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677474.319551:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677474.319553:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.319556:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.319557:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677474.319557:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.319559:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.319561:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677474.319561:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:30.0:1619677474.319563:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.319564:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.319565:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677474.319565:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677474.319566:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677474.319566:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.319567:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:30.0:1619677474.319570:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.319570:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.319570:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.319571:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677474.319572:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677474.319573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677474.319575:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.319575:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.319576:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.319578:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.319578:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.319579:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.319580:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.319584:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.319587:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.319588:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.319588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:31.0:1619677474.319590:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:22.0:1619677474.319590:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.319592:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.319593:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.319594:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.319595:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.319596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.319597:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.319610:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.319613:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.319614:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.319614:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.319615:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.319616:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.319648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.319656:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.319658:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.319659:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.319661:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.319664:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.319666:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.319667:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.319668:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.319669:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.319670:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.319674:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.319675:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.319678:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.319679:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.319680:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.319681:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.319682:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.319683:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.319683:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.319695:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.319697:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.319698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.319699:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.319701:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.319704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.319743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.319751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.319753:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.319755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.319756:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.319758:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.319760:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.319761:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.319762:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.319764:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.319764:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.319768:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.319769:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.319772:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.319773:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.319773:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.319775:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.319776:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.319777:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.319777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.319789:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.319791:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.319792:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.319792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:4.2:1619677474.319792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000400:00000200:22.0:1619677474.319795:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.319797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677474.319799:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.319802:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677474.319803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677474.319804:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.319807:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.319809:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677474.319810:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96505012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.319811:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.319813:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.319813:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677474.319816:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.319817:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.319819:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.319820:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677474.319820:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677474.319822:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677474.319822:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.319823:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677474.319824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:4.2:1619677474.319829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:31.0:1619677474.319834:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.319837:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.319838:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.319839:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.319839:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:30.0:1619677474.319841:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.319842:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:13.0:1619677474.319842:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.319847:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.319847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.319851:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.319853:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677474.319855:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.319858:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.319859:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.319860:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677474.319866:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.319869:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677474.319873:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.319874:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677474.319875:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677474.319877:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677474.319878:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.319880:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.319880:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677474.319884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.319894:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.319899:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.319901:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.319901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.319905:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.319905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.319912:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.319914:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.319916:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.319917:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.319918:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.319920:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.319921:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.320200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.320207:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.320210:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.320210:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.320212:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.320215:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.320216:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.320217:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.320219:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.320220:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.320221:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.320224:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.320226:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.320228:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.320229:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.320230:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.320231:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.320232:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.320232:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.320233:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677474.320244:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.320246:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.320247:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.320249:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.320251:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.320252:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677474.320266:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.320269:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.320270:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.320270:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.320271:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.320272:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.320408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.320417:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.320419:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.320421:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.320423:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.320424:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.320424:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.320426:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.320427:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.320428:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.320429:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.320432:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.320433:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.320435:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.320436:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.320436:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.320438:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.320438:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.320439:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.320440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.320451:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.320454:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.320454:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.320456:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.320458:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.320458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.320483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.320491:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.320494:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:4.2:1619677474.320494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677474.320496:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.320498:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.320498:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.320499:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.320500:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.320501:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.320502:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.320502:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677474.320503:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.320505:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.320505:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:31.0:1619677474.320507:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.320508:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.320509:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.320510:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.320512:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.320512:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:31.0:1619677474.320513:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.320513:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.320513:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96505007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.320515:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677474.320515:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.320515:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.320516:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.320516:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000400:00000200:10.0:1619677474.320516:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.320517:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:10.0:1619677474.320519:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.320521:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.320523:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.320524:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.320524:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.320526:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.320526:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.320527:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.320527:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.320529:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:9.2:1619677474.320531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.320532:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.320533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.320533:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.320535:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.320538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.320539:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.320541:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.320542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.320542:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.320544:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.320547:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.320831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.320839:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.320842:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.320844:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.320846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.320846:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.320848:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.320849:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.320850:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.320851:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.320852:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.320855:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.320857:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.320859:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.320859:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.320860:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.320861:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.320862:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.320862:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.320863:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.320873:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.320876:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.320877:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.320878:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.320880:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.320881:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.321308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.321316:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.321319:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.321321:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.321323:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.321324:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677474.321324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.321325:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.321327:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.321327:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.321328:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.321331:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.321333:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.321336:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.321337:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.321338:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.321340:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.321340:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.321342:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.321343:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.321344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.321347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.321350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.321358:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.321359:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.321361:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.321361:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677474.321362:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.321362:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677474.321362:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.321364:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.321364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677474.321367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.321368:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677474.321372:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.321375:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.321375:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.321377:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.321379:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.321380:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.321382:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.321383:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.321384:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.321385:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.321388:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.321389:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.321391:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.321392:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.321393:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.321394:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.321394:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.321395:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.321396:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.321400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.321406:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.321409:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.321409:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.321410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.321411:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.321413:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.321964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.321971:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.321974:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.321976:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.321978:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.321979:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.321980:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.321981:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.321982:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.321983:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.321984:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.321987:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.321989:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.321992:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.321993:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.321994:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.321995:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.321996:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.321997:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.321997:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.322001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.322010:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.322012:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.322013:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.322015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.322015:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.322018:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.322169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.322177:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.322180:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.322182:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.322183:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.322185:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.322186:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.322188:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.322189:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.322190:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.322191:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.322194:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.322195:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.322198:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.322199:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.322200:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.322201:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.322202:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.322203:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.322203:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.322216:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.322219:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.322220:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.322220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.322222:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.322225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.322944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.322954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.322958:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:13.0:1619677474.322962:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.322962:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677474.322965:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.322967:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677474.322969:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96505001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.322971:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.322973:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.322975:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677474.322980:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.322983:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677474.322987:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.322988:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:12.0:1619677474.322989:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:12.0:1619677474.322991:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:12.0:1619677474.322992:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.322994:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677474.322996:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677474.322997:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677474.322998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677474.323003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:1.0:1619677474.323009:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:13.0:1619677474.323010:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677474.323013:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677474.323013:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.323014:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:1.0:1619677474.323014:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.323016:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.323016:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.323020:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.323024:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.323204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.323212:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.323216:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.323218:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.323221:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.323224:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.323227:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.323229:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96505000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.323232:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.323233:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.323235:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.323241:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.323244:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.323248:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.323249:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.323251:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.323253:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.323254:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.323256:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.323257:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677474.323268:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.323271:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.323272:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.323273:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.323275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.323277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677474.323288:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.323291:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.323292:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.323295:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.323299:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.323299:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.323562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919075bebe00] (20)++
00000800:00000200:18.0:1619677474.323571:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919075bebe00] (21)++
00000800:00000200:18.0:1619677474.323576:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.60@o2ib3
00000800:00000200:2.0:1619677474.323577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919075bebe00] (22)--
00000400:00000200:18.0:1619677474.323581:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.60@o2ib3 : GET - for me
00000400:00000200:18.0:1619677474.323585:0:19933:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.60@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:18.0:1619677474.323589:0:19933:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.60@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:18.0:1619677474.323591:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919075bebe00] (21)++
00000800:00000200:18.0:1619677474.323594:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919075bebe00] (22)++
00000800:00000200:18.0:1619677474.323595:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919075bebe00] (23)--
00000400:00000200:18.0:1619677474.323598:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.60@o2ib3 local destination
00000800:00000200:21.2:1619677474.323601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919075bebe00] (22)++
00000400:00000200:18.0:1619677474.323606:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.60@o2ib3(10.51.5.60@o2ib3:10.51.5.60@o2ib3) : REPLY try# 0
00000800:00000200:18.0:1619677474.323609:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.60@o2ib3
00000800:00000200:2.0:1619677474.323610:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919075bebe00] (23)++
00000800:00000200:2.0:1619677474.323611:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919075bebe00] (24)--
00000800:00000200:18.0:1619677474.323612:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91906ffa8f00] -> 10.51.5.60@o2ib3 (2) version: 12
00000800:00000200:2.0:1619677474.323612:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919075bebe00] (23)--
00000800:00000200:18.0:1619677474.323614:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919075bebe00] (23)++
00000800:00000200:2.0:1619677474.323614:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919075bebe00] (22)--
00000800:00000200:18.0:1619677474.323615:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919075bebe00] (23)++
00000800:00000200:18.0:1619677474.323616:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919075bebe00] (23)--
00000800:00000200:18.0:1619677474.323618:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919075bebe00] (22)--
00000800:00000200:21.2:1619677474.323622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919075bebe00] (21)++
00000800:00000200:2.0:1619677474.323629:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919075bebe00] (22)++
00000800:00000200:2.0:1619677474.323632:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919075bebe00] (23)--
00000400:00000200:2.0:1619677474.323634:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.323635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919075bebe00] (22)--
00000400:00000200:2.0:1619677474.323637:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.60@o2ib3: REPLY: OK
00000800:00000200:2.0:1619677474.323639:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919075bebe00] (21)--
00000800:00000200:21.2:1619677474.324107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.324115:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.324118:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.324120:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.324122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.324123:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.324124:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.324125:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.324127:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.324128:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.324129:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.324132:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.324133:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.324136:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.324137:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.324138:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.324139:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.324140:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.324141:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.324142:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:21.2:1619677474.324143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.324146:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.324147:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.324148:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677474.324148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677474.324150:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.324151:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.324152:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.324152:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677474.324153:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:30.0:1619677474.324153:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.324154:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.324155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.324155:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.324155:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.324155:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.324156:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:30.0:1619677474.324156:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677474.324157:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.324158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677474.324159:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.324159:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.324161:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677474.324161:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.324163:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.324164:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.324164:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677474.324165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.324166:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.324166:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.324167:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.324168:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.324173:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.324176:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.324176:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.324177:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.324178:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:22.0:1619677474.324178:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.324178:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.324181:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.324182:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.324184:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.324185:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.324186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.324353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.324359:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.324362:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.324363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.324365:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.324367:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.324369:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.324371:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.324372:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.324373:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.324374:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.324377:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.324379:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.324381:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.324382:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.324383:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.324385:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.324385:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.324386:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.324387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.324392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.324400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.324402:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.324403:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.324404:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.324406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.324407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.324477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.324484:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.324487:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.324488:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.324490:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.324493:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.324494:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.324495:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.324497:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.324497:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.324499:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.324502:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.324504:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.324507:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.324507:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.324508:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.324510:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.324510:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.324511:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.324511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.324524:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.324527:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.324528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.324528:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.324530:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.324533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.324669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.324674:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.324677:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.324679:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.324680:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.324683:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.324684:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.324685:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.324687:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.324688:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.324689:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.324692:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.324694:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.324696:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.324697:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.324698:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.324699:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.324700:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.324700:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.324701:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.324712:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.324715:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.324716:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.324716:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.324718:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.324720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.324731:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.324734:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.324734:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.324734:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.324736:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.324736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.324790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677474.324798:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.324801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.324801:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:8.0:1619677474.324803:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677474.324806:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.324807:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677474.324808:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.324810:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.324811:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.324812:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677474.324815:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.324817:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.324818:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.324819:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.324820:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.324821:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.324822:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.324822:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.324823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677474.324826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.324833:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.324835:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.324835:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.324836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.324838:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.324839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.325036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.325043:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.325045:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.325046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.325047:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.325049:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.325051:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.325052:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.325053:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.325054:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.325055:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.325058:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.325060:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.325062:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.325063:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.325063:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.325065:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.325066:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.325067:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.325068:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.325068:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.325074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.325079:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.325081:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.325081:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.325082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.325083:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677474.325083:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.325084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.325085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.325085:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.325087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.325297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.325300:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.325303:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.325306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.325306:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.325307:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.325309:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.325310:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.325312:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.325313:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.325313:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.325317:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.325318:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.325321:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.325322:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.325322:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.325324:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.325324:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.325325:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.325326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.325331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.325340:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.325342:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.325343:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.325343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.325345:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.325348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.325350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.325357:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.325360:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.325361:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.325361:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.325362:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.325362:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.325567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.325574:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.325577:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.325578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.325579:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.325582:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.325583:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.325584:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.325586:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.325587:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.325588:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.325591:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.325592:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.325594:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.325595:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.325595:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.325597:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.325597:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.325598:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.325599:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.325599:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.325604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.325606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.325612:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.325613:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.325614:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.325614:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.325614:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.325615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.325616:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.325616:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.325617:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.325617:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.325640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.325648:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677474.325648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677474.325651:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.325652:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.325654:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.325656:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677474.325657:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.325658:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.325659:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677474.325660:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.325661:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.325662:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.325663:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.325663:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.325664:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.325665:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677474.325665:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.325666:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.325668:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.325668:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.325669:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.325670:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677474.325671:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.325673:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.325674:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.325675:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677474.325675:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.325675:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.325676:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.325677:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677474.325677:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.325677:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677474.325678:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677474.325678:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677474.325679:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677474.325679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677474.325680:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.325680:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.325681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677474.325691:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.325694:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.325695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.325695:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.325697:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.325700:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.325711:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.325714:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.325715:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.325717:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.325718:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.325719:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.325742:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.325745:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.325746:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.325746:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.325747:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.325748:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.326031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.326038:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.326040:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.326042:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.326043:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.326046:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.326047:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.326049:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.326050:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.326051:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.326052:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.326055:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.326057:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.326059:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.326060:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.326061:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.326062:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.326063:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.326064:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.326065:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.326066:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677474.326068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677474.326071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677474.326076:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677474.326078:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:18.0:1619677474.326079:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.326080:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.326080:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.326081:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.326082:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:18.0:1619677474.326082:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:10.0:1619677474.326082:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677474.326085:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.326128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.326136:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.326139:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.326141:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.326143:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.326144:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.326145:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.326146:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677474.326147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:30.0:1619677474.326148:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.326149:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.326149:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.326152:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.326153:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677474.326154:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.326155:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:30.0:1619677474.326157:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.326158:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:21.0:1619677474.326158:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.326158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677474.326159:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.326161:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.326161:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:21.0:1619677474.326161:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.326162:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.326163:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:21.0:1619677474.326163:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.326164:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.326165:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.326166:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.326167:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.326170:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.326172:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.326173:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.326174:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.326174:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.326175:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.326176:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:22.0:1619677474.326177:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677474.326177:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:22.0:1619677474.326178:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.326178:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:6.0:1619677474.326178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.326179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:22.0:1619677474.326180:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.326182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677474.326183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677474.326191:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.326193:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.326194:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.326196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.326196:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.326198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.326415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.326423:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.326426:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.326427:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.326429:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.326432:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.326433:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.326435:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.326436:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.326437:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.326438:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.326441:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.326443:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.326445:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.326446:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.326447:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.326449:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.326449:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.326450:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.326451:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.326452:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.326457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.326463:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.326464:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.326466:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677474.326466:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.326467:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.326467:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.326468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677474.326468:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.326469:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.326470:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (23)++
00000400:00000200:6.0:1619677474.326470:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.326471:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677474.326472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.326473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.326473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.2:1619677474.326483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.326491:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.326494:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.326496:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.326498:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.326499:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.326500:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.326502:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.326503:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.326504:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.326505:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.326508:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.326509:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.326512:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.326513:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.326514:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.326515:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.326516:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.326517:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.326519:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.326520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.326522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.326526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.326533:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.326536:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.326536:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.326536:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.326537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677474.326539:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.326540:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.326541:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.326542:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.326543:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.326544:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:21.0:1619677474.326545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.326545:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (24)--
00000800:00000200:20.0:1619677474.326546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.326547:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.327055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.327063:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.327066:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.327068:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.327070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.327071:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.327072:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.327073:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.327075:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.327076:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.327077:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.327080:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.327082:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.327084:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.327085:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.327086:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.327087:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.327088:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.327089:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.327089:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.327093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.327099:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.327102:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.327103:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.327103:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.327105:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.327108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.327114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.327119:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.327122:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.327123:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.327123:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.327124:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.327124:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.327855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.327864:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.327866:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.327868:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.327870:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.327871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.327872:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.327873:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.327874:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.327875:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.327876:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.327879:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.327880:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.327884:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.327884:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.327885:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.327887:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.327887:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.327888:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.327889:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.327894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.327902:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.327905:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.327906:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.327907:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.327908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.327910:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.328144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.328150:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.328153:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.328154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.328155:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.328158:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.328160:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.328161:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.328162:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.328163:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.328165:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.328168:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.328170:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.328173:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.328174:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.328174:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.328176:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.328177:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.328178:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.328179:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.328180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677474.328185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.328191:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677474.328191:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677474.328193:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.328193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.328194:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677474.328194:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.328195:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.328197:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.328199:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.328201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.328215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.328223:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.328226:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.328228:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.328229:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.328230:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.328232:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.328233:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.328235:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.328235:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.328237:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.328240:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.328242:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.328245:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.328246:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.328246:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.328248:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.328249:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.328249:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.328250:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.328263:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.328265:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.328266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.328267:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.328269:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.328271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.328870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.328878:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.328881:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.328883:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.328885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.328886:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.328887:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.328888:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.328889:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.328890:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.328891:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.328894:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.328895:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.328897:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.328898:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.328898:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.328900:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.328900:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.328901:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.328902:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.328913:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.328915:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.328916:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.328917:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.328919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.328920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677474.329374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.329380:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677474.329382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677474.329383:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.329384:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.329386:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.329388:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.329390:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677474.329390:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677474.329391:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.329392:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.329393:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.329394:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677474.329394:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.329394:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.329397:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.329398:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.329399:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.329400:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.329401:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.329402:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.329403:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.329404:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.329404:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677474.329404:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.329405:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677474.329406:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:10.0:1619677474.329406:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.329407:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.329407:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.329408:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677474.329409:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.329411:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.329414:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.329415:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.329415:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (23)++
00000800:00000200:10.0:1619677474.329417:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (24)--
00000800:00000200:10.0:1619677474.329417:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.329418:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.329419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.329421:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:22.0:1619677474.329423:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000800:00000200:6.0:1619677474.329424:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000400:00000200:22.0:1619677474.329425:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.329425:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000400:00000200:6.0:1619677474.329426:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.329427:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.329427:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.329430:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.329430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.329431:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.329825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.329833:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.329836:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.329837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.329838:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.329841:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.329843:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.329844:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.329846:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.329847:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.329848:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.329851:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.329852:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.329855:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.329856:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.329856:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.329858:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.329858:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.329859:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.329860:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.329871:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.329874:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.329875:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.329875:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.329877:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.329880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.329913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.329922:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.329924:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.329926:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.329928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.329929:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.329930:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.329931:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.329933:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.329934:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.329935:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.329938:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.329940:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.329943:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.329943:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.329944:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.329946:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.329946:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.329947:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.329948:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.329959:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.329962:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.329963:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.329963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.329965:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.329968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.330487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.330495:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.330498:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.330499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.330501:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.330504:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.330505:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.330506:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.330508:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.330509:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.330510:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.330512:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.330514:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.330516:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.330517:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.330517:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.330519:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.330519:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.330520:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.330520:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.330525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.330532:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.330534:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.330535:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.330536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.330537:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.330539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.330957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.330965:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.330968:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.330970:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.330973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.330973:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.330974:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.330975:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.330976:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.330977:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.330978:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.330981:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.330983:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.330985:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.330986:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.330986:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.330988:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.330988:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.330989:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.330991:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.330991:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.330997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.331001:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.331003:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.331004:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.331005:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.331005:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.331005:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.331006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.331007:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.331008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.331009:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.331219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677474.331225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.331228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.331228:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677474.331230:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.331232:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.331234:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677474.331235:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.331236:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.331237:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.331238:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677474.331241:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.331243:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677474.331246:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.331247:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677474.331247:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677474.331247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677474.331249:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677474.331250:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.331250:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.331251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:10.0:1619677474.331254:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677474.331255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.2:1619677474.331257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677474.331257:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.331258:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.331260:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.331262:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.331263:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677474.331263:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:2.0:1619677474.331263:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:10.0:1619677474.331264:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.331266:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677474.331266:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:10.0:1619677474.331267:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.331267:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.331268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:20.0:1619677474.331269:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677474.331269:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.331270:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.331270:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677474.331270:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.331273:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.331274:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:21.0:1619677474.331275:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677474.331275:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677474.331277:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.331278:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.331278:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:21.0:1619677474.331279:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.331280:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677474.331280:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.331281:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.331281:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.331283:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:21.0:1619677474.331284:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.331284:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.331284:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.331285:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.0:1619677474.331286:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.331288:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677474.331288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677474.331289:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.331289:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.331291:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.331292:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.331292:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.331293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677474.331295:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.331300:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.2:1619677474.331302:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677474.331307:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.331308:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.331309:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.331310:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:8.0:1619677474.331311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677474.331312:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.331313:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.331314:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.331316:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.331318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.331530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.331537:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.331540:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.331541:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.331543:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.331545:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.331547:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.331548:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.331550:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.331551:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.331552:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.331555:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.331557:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.331559:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.331560:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.331561:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.331562:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.331563:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.331564:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.331565:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.331565:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:4.2:1619677474.331572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:13.0:1619677474.331576:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.331578:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.331579:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.331579:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:13.0:1619677474.331580:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.331581:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.331582:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:13.0:1619677474.331583:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.331583:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677474.331584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.331916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.331924:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.331927:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.331929:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.331931:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.331931:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.331933:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.331934:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.331935:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.331936:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.331937:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.331941:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.331942:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.331946:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.331946:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.331947:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.331949:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.331949:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.331951:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.331952:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.331952:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.331958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.331962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.331965:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.331966:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.331966:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.331966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.331968:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677474.331968:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.331969:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.331970:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.331971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.331980:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.331983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.331983:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.331984:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.331985:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.331985:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.332023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.332031:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.332033:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.332036:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.332037:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.332038:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.332040:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.332041:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.332042:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.332043:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.332044:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.332047:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.332048:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.332051:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.332051:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.332052:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.332054:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.332054:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.332055:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.332056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.332059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.332066:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.332068:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.332069:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.332070:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.332071:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.332074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.332151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.332158:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.332160:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.332161:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.332163:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.332165:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.332167:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.332168:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.332169:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.332170:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.332171:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.332174:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.332176:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.332179:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.332179:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.332180:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.332182:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.332182:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.332183:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.332183:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.332188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677474.332189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.332196:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:20.0:1619677474.332197:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:18.0:1619677474.332199:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.332199:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677474.332200:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.332201:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.332202:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.332203:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.332204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:18.0:1619677474.332204:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.332206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:18.0:1619677474.332206:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.332207:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.332208:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.332210:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.332210:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.332214:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.332215:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.332217:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.332218:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.332218:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.332220:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.332221:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.332221:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.332222:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.332226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.332233:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.332236:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.332236:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.332237:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.332238:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.332239:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.332328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677474.332334:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.332337:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.332339:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.332340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.332342:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.332343:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677474.332345:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.332346:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.332347:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.332348:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677474.332351:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.332353:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.332356:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.332357:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677474.332357:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.332359:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.332360:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.332361:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677474.332362:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677474.332363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.2:1619677474.332369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.332376:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677474.332376:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677474.332376:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.332377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677474.332377:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.332378:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:20.0:1619677474.332378:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.332379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.332380:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.332382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.332720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677474.332725:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.332728:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:4.2:1619677474.332729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:21.0:1619677474.332730:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.332732:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.332732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677474.332733:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.0:1619677474.332733:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:21.0:1619677474.332734:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.332735:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.332736:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.332737:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.332738:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677474.332741:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.332742:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.332744:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677474.332744:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.332745:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677474.332745:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677474.332746:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.332747:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.332748:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677474.332749:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.332750:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.332751:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.332752:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.332753:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.332753:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677474.332754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000400:00000200:20.0:1619677474.332755:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.332756:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.332757:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.332757:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.332757:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:21.0:1619677474.332758:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.332760:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.332760:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.332761:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677474.332761:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677474.332762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.332763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.332763:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677474.332764:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.332765:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.332765:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677474.332769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.332777:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.2:1619677474.332779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:18.0:1619677474.332780:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.332781:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.332781:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.332783:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.332784:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:18.0:1619677474.332786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677474.332787:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677474.332789:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.332789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:9.2:1619677474.332790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000400:00000200:21.0:1619677474.332791:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.332793:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.332794:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.332795:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.332796:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.332797:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.332797:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.332800:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677474.332801:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.332801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.332801:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.332802:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.332802:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677474.332803:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.332805:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.332806:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.332806:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.332808:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.332809:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.332810:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677474.332810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:20.2:1619677474.332812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677474.332815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677474.332818:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677474.332820:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.332821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:22.0:1619677474.332822:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677474.332822:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.332823:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:22.0:1619677474.332824:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.332825:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.332825:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.332826:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.332826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.332828:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677474.332828:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.332829:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.2:1619677474.332830:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:22.0:1619677474.332834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.332836:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:20.0:1619677474.332836:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.332837:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:20.0:1619677474.332838:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:21.0:1619677474.332839:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.332841:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677474.332842:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.332842:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:20.0:1619677474.332842:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:21.0:1619677474.332844:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677474.332844:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:21.0:1619677474.332845:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.332846:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.332847:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.332848:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677474.332849:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (23)++
00000800:00000200:9.2:1619677474.332850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.0:1619677474.332851:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (24)++
00000400:00000200:21.0:1619677474.332852:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.332852:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (25)--
00000800:00000200:21.0:1619677474.332853:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677474.332853:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (24)--
00000800:00000200:20.0:1619677474.332853:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (23)++
00000800:00000200:20.0:1619677474.332855:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:21.0:1619677474.332857:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.332857:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:20.0:1619677474.332857:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.332857:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:21.0:1619677474.332858:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677474.332860:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:20.0:1619677474.332860:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.332860:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.332861:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:20.0:1619677474.332861:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.332861:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.332863:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96504958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.332864:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:20.0:1619677474.332865:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.332865:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.332867:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.0:1619677474.332868:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (24)++
00000800:00000200:21.0:1619677474.332869:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (25)--
00000800:00000200:21.0:1619677474.332870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (24)--
00000400:00000200:20.0:1619677474.332870:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.332870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.0:1619677474.332871:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (23)++
00000800:00000200:21.0:1619677474.332872:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (24)--
00000800:00000200:21.0:1619677474.332872:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677474.332872:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.0:1619677474.332873:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000800:00000200:30.0:1619677474.332874:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.0:1619677474.332874:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:20.0:1619677474.332875:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.332876:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:30.0:1619677474.332877:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:21.0:1619677474.332877:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.332877:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:30.0:1619677474.332878:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.332879:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.332879:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:20.0:1619677474.332879:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (23)++
00000400:00000200:30.0:1619677474.332880:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.332880:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (24)--
00000800:00000200:31.0:1619677474.332881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677474.332881:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677474.332881:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (23)--
00000400:00000200:21.0:1619677474.332882:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.332882:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000800:00000200:20.0:1619677474.332883:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677474.332884:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.332885:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.332886:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.332889:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.332890:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677474.332892:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:13.0:1619677474.332892:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:21.0:1619677474.332893:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.332894:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:20.0:1619677474.332894:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.332895:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:13.0:1619677474.332895:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:21.0:1619677474.332896:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.332896:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.332896:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.332897:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (23)++
00000400:00000200:13.0:1619677474.332897:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.332898:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (24)--
00000800:00000200:21.0:1619677474.332898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (23)--
00000400:00000200:20.0:1619677474.332898:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:21.0:1619677474.332899:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677474.332899:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677474.332899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:20.0:1619677474.332901:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677474.332902:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:8.2:1619677474.332905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:20.0:1619677474.332906:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.332907:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:21.0:1619677474.332909:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000800:00000200:13.0:1619677474.332909:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:20.0:1619677474.332910:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.332911:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:20.0:1619677474.332911:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:21.0:1619677474.332913:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.332914:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.332915:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:21.0:1619677474.332916:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.332916:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:21.0:1619677474.332917:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.332918:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96504955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.332918:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:21.0:1619677474.332920:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.332921:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.332922:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.332923:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (23)++
00000800:00000200:20.0:1619677474.332924:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (24)++
00000800:00000200:9.2:1619677474.332924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:21.0:1619677474.332925:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.332926:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.332926:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (25)--
00000800:00000200:20.0:1619677474.332926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (24)--
00000800:00000200:20.0:1619677474.332928:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.332929:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.332929:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:26.0:1619677474.332930:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:21.0:1619677474.332930:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:20.0:1619677474.332930:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.332931:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:20.0:1619677474.332931:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.332933:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:20.0:1619677474.332933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:26.0:1619677474.332934:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.332934:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677474.332934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (23)++
00000800:00000200:10.0:1619677474.332934:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:20.0:1619677474.332935:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (24)--
00000800:00000200:20.0:1619677474.332935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (23)--
00000400:00000200:26.0:1619677474.332936:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.332936:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.332937:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.332937:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.332938:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:26.0:1619677474.332939:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.0:1619677474.332939:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (22)++
00000800:00000200:20.0:1619677474.332939:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.332940:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677474.332940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.332941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677474.332942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:31.0:1619677474.332943:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.332943:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677474.332944:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677474.332946:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:31.0:1619677474.332947:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.332948:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.332949:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.332949:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.332950:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.332950:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.332951:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.332953:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.332954:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.332955:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.332958:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.332958:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.332959:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.332960:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.332960:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677474.332962:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.332963:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.332964:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.332965:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.332966:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.332967:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.332967:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.332971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.332977:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.332980:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.332980:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.332981:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.332982:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.332985:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.332993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.332999:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.333002:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.333003:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.333003:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.333004:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.333004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.333064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:4.2:1619677474.333067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:18.0:1619677474.333071:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.333074:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:8.0:1619677474.333074:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:2.0:1619677474.333075:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.333077:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677474.333077:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677474.333078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:18.0:1619677474.333079:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.333080:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.333081:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.333082:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.333083:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.333083:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.333084:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.333085:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.333085:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677474.333088:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.333090:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677474.333090:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.333091:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677474.333093:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.333093:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677474.333096:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.333097:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.333098:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:8.0:1619677474.333098:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.333099:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.333100:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677474.333100:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:18.0:1619677474.333101:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.333101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677474.333103:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.333104:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.2:1619677474.333106:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.333109:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677474.333111:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677474.333112:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.333113:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677474.333114:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:17.0:1619677474.333115:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677474.333115:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.2:1619677474.333117:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.333118:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.333119:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.333120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000400:00000200:17.0:1619677474.333121:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.333121:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.333123:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.333124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677474.333124:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.333125:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.333125:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.333127:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000400:00000200:1.0:1619677474.333127:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.333130:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:1.0:1619677474.333130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677474.333131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677474.333131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:21.2:1619677474.333684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677474.333688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:30.0:1619677474.333692:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677474.333694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.333695:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677474.333695:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:30.0:1619677474.333697:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.333698:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:30.0:1619677474.333699:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.333699:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:31.0:1619677474.333700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677474.333700:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.333701:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677474.333701:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.333702:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.333702:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.333703:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.333704:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.333704:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677474.333705:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.333706:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677474.333706:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.333707:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.333709:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.333710:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.333710:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677474.333712:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.333713:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.333714:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.333715:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.333715:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.333717:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96504949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.333717:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677474.333719:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.333720:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677474.333720:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.333720:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:31.0:1619677474.333721:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.333721:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677474.333723:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677474.333723:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:30.0:1619677474.333724:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.333724:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:31.0:1619677474.333725:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.333725:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:17.0:1619677474.333725:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:30.0:1619677474.333726:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.333727:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.333727:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.333728:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.333729:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.333730:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.333730:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.333730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.333731:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.333731:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.333731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.333733:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.333734:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.2:1619677474.333734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.333735:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.333735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:2.0:1619677474.333737:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.333740:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.333741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.333741:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.333743:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:2.0:1619677474.333743:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.333745:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.333746:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.333746:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:6.0:1619677474.333747:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677474.333748:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.333749:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677474.333750:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.333751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.333751:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.333752:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:6.0:1619677474.333753:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.333754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677474.333756:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677474.333762:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.333765:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677474.333765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.333766:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.333767:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.333768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.334167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.334174:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.334177:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.334178:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.334180:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.334182:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.334184:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.334185:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.334186:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.334187:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.334188:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.334191:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.334193:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.334196:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.334196:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.334197:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.334198:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.334199:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.334200:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.334202:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.334202:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.334208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.334212:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.334215:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.334215:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.334215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.334216:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677474.334216:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.334217:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.334218:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677474.334218:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.334221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.334352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.334360:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.334363:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.334365:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.334367:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.334368:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.334369:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.334370:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.334372:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.334373:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.334374:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.334377:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.334378:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.334381:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.334381:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.334382:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.334383:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.334384:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.334386:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.334387:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.334388:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.334390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.334394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.334396:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.334399:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.334400:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.334400:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.334402:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.334402:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.334404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.334405:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.334405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.334406:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677474.334559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.334565:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.334568:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.334569:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.334571:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.334573:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.334574:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.334576:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.334577:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.334578:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.334579:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.334582:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.334583:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.334586:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.334586:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.334587:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.334588:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.334589:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.334590:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.334591:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.334592:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.334597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.334602:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.334602:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.334604:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.334605:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.334605:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.334606:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.334606:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.334606:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.334608:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.334610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.334622:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.334625:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.334625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.334626:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.334627:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.334627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.334642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.334650:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.334652:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.334653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.334655:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.334657:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.334659:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.334660:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.334662:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.334662:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.334664:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.334667:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.334668:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.334671:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.334672:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.334672:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.334674:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.334675:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.334675:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.334675:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.334687:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.334689:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.334690:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.334691:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.334693:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.334695:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.335609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.335615:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.335618:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.335620:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.335620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.335623:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.335625:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.335626:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.335627:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.335628:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.335629:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.335633:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.335634:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.335637:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.335637:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.335638:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.335639:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.335640:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.335642:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.2:1619677474.335644:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:8.2:1619677474.335646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.335647:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.335648:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.335650:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677474.335650:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:21.0:1619677474.335652:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.335652:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677474.335652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677474.335653:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:21.0:1619677474.335654:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.335654:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677474.335655:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.335657:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.335657:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:1.0:1619677474.335657:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.335659:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:1.0:1619677474.335659:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.335660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.335661:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677474.335661:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.335662:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.335663:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.335664:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.335665:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.335669:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.335671:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.335673:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.335674:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.335674:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.335676:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.335676:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.335677:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.335678:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.335682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.335690:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.335692:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.335693:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.335695:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.335696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.335697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.335785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.335793:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.335796:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.335797:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.335799:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.335802:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.335804:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.335805:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.335806:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.335807:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.335808:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.335811:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.335813:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.335815:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.335816:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.335817:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.335818:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.335819:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.335819:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.335820:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.335823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:9.2:1619677474.335824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.335831:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:31.0:1619677474.335832:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.335834:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:31.0:1619677474.335835:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677474.335835:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.335836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:31.0:1619677474.335838:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.335839:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.335840:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.335841:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.335841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.335841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:31.0:1619677474.335843:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.335844:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.335845:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.335846:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.335847:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.335848:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.335849:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.335850:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.335852:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.335853:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.335854:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.335855:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.335855:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677474.335855:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.335857:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677474.335857:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.335858:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.335859:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.335860:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.335861:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.335862:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.335863:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.335863:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677474.335864:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.335865:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.335865:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.335866:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.335867:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.335868:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.335868:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.335869:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.335869:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.335870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.335870:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.335871:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.335871:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.335872:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.335873:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.335873:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.335885:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.335888:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.335888:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.335889:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.335889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.335892:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.335892:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000400:00000200:22.0:1619677474.335892:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.335893:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.335894:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.335895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.335895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.335904:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.335906:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.335907:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.335908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.335908:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.335909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.336470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.336478:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.336480:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.336481:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.336483:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.336485:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.336487:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.336488:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.336489:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.336490:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.336491:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.336494:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.336496:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.336499:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.336499:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.336500:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.336502:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.336502:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.336503:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.336504:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.336509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.336517:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.336519:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.336520:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.336522:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.336523:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.336525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.336528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.336536:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.336538:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.336539:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.336540:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.336541:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.336541:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.336560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.336568:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.336570:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.336571:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.336573:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.336575:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.336577:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.336578:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.336579:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.336580:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.336581:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.336584:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.336586:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.336588:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.336588:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.336589:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.336590:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.336591:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.336591:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.336593:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.336593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.336596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.336599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.336603:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.336605:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677474.336606:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677474.336606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.336606:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.336608:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.336608:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677474.336608:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.336610:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.336610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.337859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.337867:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.337870:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.337872:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.337875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.337875:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.337876:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.337877:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.337879:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.337879:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.337880:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.337883:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.337885:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.337887:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.337887:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.337888:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.337889:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.337890:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.337891:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.337891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.337903:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.337906:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.337906:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.337908:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.337909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.337910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.337952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.337960:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.337962:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.337964:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.337965:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.337967:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.337967:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.337968:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.337969:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.337970:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.337971:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.337974:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.337976:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.337979:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.337980:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.337980:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.337982:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.337983:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.337983:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.337984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.337999:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.338003:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.338005:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.338005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.338009:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.338014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.338531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.338539:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.338542:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.338544:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.338546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.338547:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.338549:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.338550:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.338552:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.338553:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.338553:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.338557:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.338559:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.338561:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.338561:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.338562:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.338563:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.338564:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.338565:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.338565:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.338565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:9.2:1619677474.338570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677474.338571:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.338573:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.338575:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.338576:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.0:1619677474.338576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.338577:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.338578:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:24.0:1619677474.338579:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:21.0:1619677474.338579:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.338580:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.338580:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:21.0:1619677474.338581:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.338582:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.338582:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.338583:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677474.338585:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:21.0:1619677474.338586:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.338588:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.338591:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677474.338591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:21.0:1619677474.338592:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.338593:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.338595:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.338595:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.338596:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.338597:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:8.0:1619677474.338597:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677474.338598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:24.0:1619677474.338600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.2:1619677474.338601:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.338605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677474.338606:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.338607:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.338607:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.338608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677474.338610:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.338611:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.338611:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.338612:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:20.0:1619677474.338612:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.338613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.338614:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.338614:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.338615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677474.338616:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.338977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.338986:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.338989:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.338992:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.338993:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.338994:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.338996:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677474.338997:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.338999:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.339000:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.339001:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677474.339005:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.339006:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677474.339009:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.339010:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677474.339011:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677474.339012:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677474.339013:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.339014:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.339014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.339019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.339028:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.339032:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.339033:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.339034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.339036:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.339039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.339078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.339086:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677474.339088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677474.339089:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.339091:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.339093:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.339093:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.339095:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:10.0:1619677474.339095:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677474.339096:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.339097:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.339098:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.339098:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677474.339099:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.339099:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.339101:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.339102:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.339103:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.339104:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.339105:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677474.339106:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.339106:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.339107:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:10.0:1619677474.339107:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.339108:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677474.339108:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.339109:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677474.339109:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.339110:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.339111:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677474.339112:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.339113:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.339113:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677474.339113:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677474.339115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.339116:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.339117:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.339118:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.2:1619677474.339119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.339119:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.339120:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.339120:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677474.339121:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677474.339121:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677474.339123:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.339124:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.339124:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.339126:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.339127:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677474.339128:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.339129:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.339130:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.339131:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677474.339133:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.339136:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.339137:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.339138:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.339140:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.339143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.339348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.339354:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.339357:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.339358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.339360:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.339362:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.339364:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.339365:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.339366:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.339367:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.339368:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.339372:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.339373:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.339376:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.339376:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.339377:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.339379:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.339379:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.339380:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.339380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.339393:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.339396:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.339397:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.339397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.339399:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.339401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.339451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.339459:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.339462:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.339463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.339464:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.339467:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.339469:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.339471:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.339472:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.339473:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.339474:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.339478:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.339480:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.339483:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.339483:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.339484:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.339486:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.339486:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.339487:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.339488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.339499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.339502:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.339503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.339503:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.339505:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.339507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.339591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.339600:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.339603:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.339605:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.339607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.339607:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.339609:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.339610:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.339612:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.339613:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.339613:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.339617:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.339618:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.339621:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.339622:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.339623:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.339624:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.339625:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.339626:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.339628:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.339628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.339634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.339638:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.339640:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.339641:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.339641:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.339642:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.339643:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.339645:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.339646:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.339646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.339647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.340017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.340025:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.340028:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.340030:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.340032:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.340033:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.340034:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.340036:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.340037:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.340038:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.340039:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.340042:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.340044:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.340045:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.340046:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.340046:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.340048:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.340049:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.340049:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.340050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.340060:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.340063:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.340063:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.340065:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.340067:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.340068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.340470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.340475:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.340478:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.340480:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.340481:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.340482:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.340484:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.340485:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.340487:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.340488:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.340489:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.340492:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.340494:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.340497:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.340497:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.340498:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677474.340498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:21.0:1619677474.340500:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.340500:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.340501:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.340502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677474.340505:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.2:1619677474.340507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.340508:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.340509:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.340511:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.340514:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.340515:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:26.0:1619677474.340516:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677474.340517:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.340517:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:26.0:1619677474.340518:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.340519:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677474.340519:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.340519:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.340520:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677474.340521:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.340524:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.340526:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677474.340527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677474.340529:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.340530:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677474.340531:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.340533:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.340533:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.340534:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.340535:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.340539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.340546:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.340549:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.340550:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.340550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.340552:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.340554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.340556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.340563:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.340566:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.340567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.340567:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.340568:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.340569:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.340819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.340826:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.340828:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.340829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.340831:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.340833:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.340835:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.340836:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.340838:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.340838:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.340839:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.340843:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.340845:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.340848:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.340848:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.340849:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.340851:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.340852:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.340853:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.340853:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677474.340865:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.340868:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.340869:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.340871:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.340873:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.340874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.340886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.340893:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.340895:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.340896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.2:1619677474.340897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:10.0:1619677474.340898:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.340901:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.340902:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:2.0:1619677474.340902:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677474.340903:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.340905:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677474.340906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677474.340906:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.340906:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:10.0:1619677474.340908:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677474.340909:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.340912:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.340912:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.340914:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:2.0:1619677474.340914:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.340915:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.340917:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.340918:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.340918:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:2.0:1619677474.340918:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.340919:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:2.0:1619677474.340919:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677474.340921:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.340921:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.340922:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.340922:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677474.340923:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.340924:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.2:1619677474.340926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677474.340926:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.340927:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.340928:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.340930:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.340930:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.340931:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677474.340932:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:2.0:1619677474.340932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677474.340935:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.340936:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.340936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.340936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.340938:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.340940:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677474.340943:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.340945:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.340946:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.340947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.340948:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.340950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.341627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.341635:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.341637:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.341638:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.341640:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.341643:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.341644:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.341646:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.341647:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.341648:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.341649:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.341652:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.341654:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.341657:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.341657:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.341658:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.341660:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.341660:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.341661:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.341661:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.341666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.341673:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.341675:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.341676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.341676:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.341678:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.341680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.0:1619677474.341680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.341688:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.341691:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.341693:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.341695:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.341695:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.341697:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.341698:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.341699:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.341700:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.341701:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.341704:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.341705:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.341709:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.341709:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.341710:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.341712:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.341712:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.341713:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.341714:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.341726:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.341729:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.341730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.341730:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.341733:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.341735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.341743:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.341746:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.341747:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.341747:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.341748:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.341748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.342068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.342071:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.342073:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.342075:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.342076:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.342078:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.342080:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.342081:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.342082:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.342083:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.342084:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.342088:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.342089:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.342092:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.342093:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.342093:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.342095:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.342096:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.342096:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.342097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.342101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.342109:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.342112:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.342113:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.342114:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.342115:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.342117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.342532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.342540:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.342543:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.342545:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.342548:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.342548:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.342549:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.342550:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.342551:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.342552:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.342553:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.342556:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.342558:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.342560:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.342561:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.342561:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.342562:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.342563:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.342564:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.342565:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.342576:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.342578:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.342579:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.342580:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.342582:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.342583:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.342691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.342697:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.342700:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.342701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.342702:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.342705:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.342706:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.342708:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.342709:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.342710:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.342711:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.342714:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.342716:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.342719:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.342719:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.342720:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.342722:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.342723:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.342724:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.342725:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.342726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.342729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.342731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.342736:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677474.342738:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.342739:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.342739:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677474.342740:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677474.342740:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:21.0:1619677474.342741:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.342743:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.342744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.342745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.342754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.342761:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.342764:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.342765:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.342765:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.342766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.342767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.343364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.343372:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.343375:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.343377:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.343379:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.343379:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.343381:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.343382:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.343383:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.343384:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.343385:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.343389:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.343391:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.343394:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.343395:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.343396:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.343397:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.343398:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.343399:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.343399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.343412:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.343415:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.343416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.343416:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.343418:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.343421:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.343766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677474.343774:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.343777:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677474.343780:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.343781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677474.343782:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.343784:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.343785:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.343787:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.343788:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.343789:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.343792:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.343794:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.343797:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.343798:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.343799:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.343800:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.343801:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.343802:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.343802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677474.343807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.343815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.343817:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.343818:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.343820:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.343821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.343823:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.343857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.343864:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.343867:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.343869:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.343870:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.343873:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.343875:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.343876:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.343877:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.343878:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.343879:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.343882:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.343884:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.343886:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.343887:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.343888:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.343890:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.343890:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.343891:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.343891:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.343896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.343904:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.343907:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.343909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.343909:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.343911:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.343914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.344019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.344026:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:5.2:1619677474.344027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677474.344028:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.344029:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.344031:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.344033:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677474.344034:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.344036:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:1.0:1619677474.344036:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677474.344037:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.344037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.344039:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.344040:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.344041:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.344041:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.344042:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.344043:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.344045:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.344046:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677474.344046:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.344047:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.344048:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.344048:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.344051:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.344052:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:1.0:1619677474.344052:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.344053:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.344054:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.344055:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.344055:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677474.344055:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.344056:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677474.344058:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.344058:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.344059:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.344061:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.344061:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.344062:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.344063:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.344068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:22.0:1619677474.344070:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000800:00000200:6.0:1619677474.344071:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (24)++
00000800:00000200:6.0:1619677474.344071:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000400:00000200:22.0:1619677474.344072:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.344072:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.344074:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.344074:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.344077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.344077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.344078:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.344089:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.344092:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.344093:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.344093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.344094:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.344094:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.344103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.344110:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.344113:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.344115:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.344117:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.344118:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.344120:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.344121:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.344123:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.344124:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.344125:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.344128:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.344129:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.344132:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.344133:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.344133:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.344135:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.344136:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.344137:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.344137:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.344141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.344147:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.344150:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.344151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.344151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.344153:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.344155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.344161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.344167:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.344170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.344170:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.344171:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.344172:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.344172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.344291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.344300:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.344302:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.344303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.344304:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.344307:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.344308:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.344309:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.344311:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.344312:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.344313:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.344316:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.344317:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.344321:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.344321:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.344322:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.344323:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.344324:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.344325:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.344326:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.344327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.344333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.344338:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.344341:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.344341:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.344342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.344342:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.344343:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.344344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677474.344344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677474.344344:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.344347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.344550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.344559:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.344562:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.344564:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.344566:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.344566:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.344567:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.344569:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.344570:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.344571:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.344572:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.344575:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.344577:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.344580:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.344581:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.344581:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.344583:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.344584:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.344585:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.344587:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.344587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.344589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.344595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.344602:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677474.344603:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.344604:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.344605:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.344606:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.344606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677474.344607:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.344607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.344608:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677474.344610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.345189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.345197:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.345200:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.345202:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.345203:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.345205:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.345206:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.345208:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.345209:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.345210:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.345211:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.345214:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.345216:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.345219:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.345220:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.345220:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.345222:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.345222:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.345223:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.345224:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.345235:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.345238:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.345239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.345240:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.345242:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.345245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.345503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677474.345503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.345510:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:30.0:1619677474.345511:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677474.345513:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.345514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:30.0:1619677474.345515:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677474.345516:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.345518:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.345519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677474.345519:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.345520:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:30.0:1619677474.345521:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.345522:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.345523:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.345524:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.345524:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.345525:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.345526:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.345526:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:30.0:1619677474.345527:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.345528:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677474.345531:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.345533:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.345534:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677474.345536:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.345539:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.345539:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.345540:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677474.345540:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:30.0:1619677474.345541:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677474.345541:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:30.0:1619677474.345543:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677474.345543:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:30.0:1619677474.345544:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677474.345544:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:30.0:1619677474.345545:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677474.345545:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:30.0:1619677474.345546:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677474.345546:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677474.345549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:9.2:1619677474.345553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677474.345556:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.345559:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.345560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.345560:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:8.0:1619677474.345560:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.345562:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.345563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.345563:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.345564:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.345565:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:10.0:1619677474.345566:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.345569:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:5.2:1619677474.345613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677474.345619:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.345622:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677474.345623:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677474.345625:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.345628:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.345629:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677474.345631:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.345632:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.345633:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.345634:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677474.345637:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.345639:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677474.345642:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.345643:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:17.0:1619677474.345644:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677474.345645:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677474.345646:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.345647:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677474.345647:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677474.345651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.345659:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.345661:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.345662:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.345662:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.345665:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.345667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.345849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.345855:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.345858:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.345858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.345860:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.345863:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.345864:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.345865:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.345866:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.345867:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.345868:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677474.345868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:18.0:1619677474.345871:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.345873:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677474.345876:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677474.345876:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.345877:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.345877:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677474.345878:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:18.0:1619677474.345879:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.345879:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677474.345879:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677474.345880:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677474.345881:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.345881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:26.0:1619677474.345884:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.345885:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.2:1619677474.345885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:26.0:1619677474.345886:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.345887:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.345888:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.345889:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.345892:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.345893:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677474.345894:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.345896:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:26.0:1619677474.345897:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.345897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.345897:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.345898:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.345898:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677474.345899:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:26.0:1619677474.345900:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.345900:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.345901:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677474.345901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677474.345902:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.345914:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.345917:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.345918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.345918:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.345921:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.345923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.345932:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.345934:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.345935:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.345935:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.345936:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.345936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.346257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.346261:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.346263:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.346266:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.346267:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.346268:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.346270:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.346271:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.346273:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.346274:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.346275:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.346278:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.346280:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.346283:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.346284:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.346284:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.346286:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.346287:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.346288:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.346288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677474.346300:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.346303:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.346304:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.346305:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.346307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.346309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.346711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.346719:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.346722:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.346723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.346725:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.346727:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.346729:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.346730:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.346731:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.346732:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.346733:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.346736:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.346737:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.346740:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.346740:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.346741:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.346742:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.346743:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.346744:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.346745:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.346746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.346748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.346751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.346755:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.346757:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.346758:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.346759:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.346759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.346760:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.346761:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.346761:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.346762:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677474.346763:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.346958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677474.346965:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.346967:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677474.346968:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677474.346969:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677474.346971:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.346973:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.346974:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.346975:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.346976:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.346977:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.346980:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.346982:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.346985:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.346985:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.346986:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.346988:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.346988:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.346989:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677474.346989:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677474.347002:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.347005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.347005:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.347006:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.347008:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.347011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.347110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.347118:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.347121:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.347123:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.347125:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.347125:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.347127:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.347128:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.347129:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.347130:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.347131:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.347135:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.347136:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.347140:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.347140:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.347141:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.347143:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.347143:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.347145:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.347146:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.347147:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.347153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.347155:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.347158:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.347159:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.347159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.347161:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.347161:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.347163:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.347164:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.347164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.347165:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.348373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.348380:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.348383:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.348385:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.348388:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.348388:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.348390:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.348391:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.348392:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.348393:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.348394:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.348397:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.348399:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.348400:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.348401:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.348402:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.348403:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.348404:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.348405:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.348405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.348416:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.348419:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.348420:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.348421:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.348423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.348424:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.348434:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.348436:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.348437:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.348437:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.348438:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.348439:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677474.348736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.348742:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.348745:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.348746:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.348748:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.348750:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.348751:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.348752:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.348754:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.348755:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.348756:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.348759:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.348761:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.348763:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.348764:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.348765:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.348767:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.348767:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.348768:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.348768:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.348780:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.348783:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.348784:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.348784:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.348786:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.348788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.348865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.348873:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.348876:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.348877:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.348879:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.348881:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.348883:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.348884:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.348885:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.348887:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.348887:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.348891:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.348892:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.348895:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.348896:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.348897:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.348898:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.348899:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.348899:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.348900:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.348912:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.348915:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.348915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.348916:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.348918:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.348921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.349041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.2:1619677474.349044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:26.0:1619677474.349048:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677474.349050:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:26.0:1619677474.349051:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.349052:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:1.0:1619677474.349053:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:26.0:1619677474.349054:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677474.349054:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.349056:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.349057:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.349059:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677474.349059:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.349060:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.349061:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677474.349063:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.349063:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.349064:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.349065:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.349066:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.349067:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677474.349068:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677474.349073:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677474.349073:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.349075:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677474.349077:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677474.349078:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.349078:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:26.0:1619677474.349079:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.349079:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677474.349080:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.349081:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677474.349081:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:26.0:1619677474.349082:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677474.349082:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:26.0:1619677474.349083:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677474.349083:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.349083:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:26.0:1619677474.349084:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.349085:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.349086:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677474.349088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.349091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677474.349097:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:13.0:1619677474.349098:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677474.349098:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:21.0:1619677474.349099:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.349100:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.349101:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.349101:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677474.349101:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:20.0:1619677474.349102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:13.0:1619677474.349102:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.349102:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.349103:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677474.349104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:13.0:1619677474.349105:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.349108:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.349109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677474.349118:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.349122:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.349124:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.349125:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.349125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.349127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.349167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.349174:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.349177:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.349179:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.349181:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.349182:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.349183:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.349185:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.349186:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.349187:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.349188:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.349191:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.349193:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.349196:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.349197:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.349198:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.349199:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.349200:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.349201:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.349201:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.349213:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.349215:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.349216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.349217:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.349219:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.349221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.349610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.349618:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.349621:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.349622:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.349623:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.349626:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.349627:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.349628:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.349630:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.349631:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.349632:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.349635:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.349637:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.349640:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.349640:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.349641:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.349643:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.349643:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.349644:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.349644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677474.349647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.349655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.349657:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.349658:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.349660:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.349660:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.349661:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.349661:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677474.349662:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.349663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677474.349663:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.349664:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.349665:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.349666:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.349666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677474.349667:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.349668:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.349671:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.349673:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677474.349674:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.349675:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.349676:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.349676:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.349677:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.349678:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.349678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.349678:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.349679:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677474.349679:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.349680:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677474.349680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.349681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.349683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.349690:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.349692:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.349693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.349693:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.349696:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.349698:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.350242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.350246:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.350249:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.350251:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.350252:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.350253:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.350255:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.350256:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.350257:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.350259:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.350260:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.350263:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.350264:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.350267:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.350268:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.350268:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.350270:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.350270:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.350271:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.350272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677474.350294:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.350298:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.350302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.350302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.350304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.350306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.350310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.350316:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.350319:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.350320:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.350322:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.350325:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.350326:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.350327:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.350329:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.350330:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.350331:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.350334:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.350336:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.350337:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.350338:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.350339:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.350340:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.350341:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.350342:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.350342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.350347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.350354:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.350357:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.350357:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.350358:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.350359:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.350360:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.350367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.350374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.350377:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.350377:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.350378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.350378:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.350378:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.350381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.350386:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.350389:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.350390:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.350391:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.350392:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.350394:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.350395:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.350396:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.350397:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.350397:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.350400:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.350402:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.350405:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.350406:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.350406:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.350408:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.350408:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.350410:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.350411:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.350411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677474.350417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.350422:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677474.350423:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677474.350424:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:13.0:1619677474.350425:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:2.0:1619677474.350425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.350425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677474.350426:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.350428:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.350430:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677474.350430:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:4.2:1619677474.350553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.350561:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.350564:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.350565:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.350566:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.350569:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.350571:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.350572:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.350573:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.350574:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.350575:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.350578:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.350580:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.350583:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.350584:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.350584:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.350586:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.350586:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.350588:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677474.350589:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677474.350589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:4.2:1619677474.350597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:6.0:1619677474.350600:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.350602:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.350603:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.350603:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.350604:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000400:00000200:6.0:1619677474.350605:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.350606:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677474.350607:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677474.350608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677474.350608:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.350738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.350746:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.350749:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.350749:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.350751:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.350752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677474.350754:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.350755:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.350757:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.350758:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.350759:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.350760:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677474.350760:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.350763:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677474.350764:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.350765:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.350766:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.350768:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.350768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.350769:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:26.0:1619677474.350769:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.350769:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:31.0:1619677474.350770:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.350770:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677474.350772:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.350772:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.350772:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677474.350773:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.350773:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677474.350774:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.350774:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677474.350777:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.350778:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.350781:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.350781:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.350782:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.350783:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.350784:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.350785:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677474.350785:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.350787:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.350787:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677474.350788:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.350789:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.350789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.350789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.350791:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.350793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.350794:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677474.350796:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.350798:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.350799:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.350799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.350801:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.350801:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.350803:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677474.350803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.350804:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.350805:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.350810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.350816:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.350818:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.350819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.350819:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.350820:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.350821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.350904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.350911:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.350914:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.350916:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.350918:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.350918:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.350920:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.350921:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.350922:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.350923:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.350924:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.350927:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.350929:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.350932:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.350933:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.350933:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.350935:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.350936:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.350936:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.350937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.350949:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.350952:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.350953:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.350953:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.350955:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.350958:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.350967:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.350969:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.350970:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.350970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.350971:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.350972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.351521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.351529:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.351531:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.351532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.351534:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.351537:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.351538:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.351539:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.351541:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.351542:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.351543:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.351546:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.351547:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.351550:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.351551:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.351551:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.351553:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.351553:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.351554:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.351554:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.351568:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.351571:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.351572:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.351572:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.351574:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.351576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.351689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.351697:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.351700:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.351702:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.351704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.351705:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.351707:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.351708:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.351709:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.351710:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.351711:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.351714:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.351716:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.351719:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.351720:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.351720:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.351722:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.351723:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.351723:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.351724:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.351729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.351737:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.351740:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.351741:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.351742:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.351744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.351745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.351986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.351991:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.351994:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677474.351996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.351996:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.351999:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.352000:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.352002:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.352003:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.352004:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.352006:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.352010:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.352012:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.352015:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.352016:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.352017:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.352019:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.352019:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.352020:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.352021:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677474.352031:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.352034:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.352035:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.352037:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.352038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.352040:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677474.352053:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.352056:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.352057:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.352057:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.352058:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.352059:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.352169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.352176:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.352179:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.352181:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.352183:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.352184:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.352185:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.352186:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.352188:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.352189:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.352190:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.352193:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.352194:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.352197:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.352198:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.352198:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.352200:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.352200:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.352201:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.352202:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.352205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.352212:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.352214:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.352215:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.352215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.352217:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.352219:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.352656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.352664:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.352667:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.352669:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.352672:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.352672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.352673:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.352674:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.352675:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.352676:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.352677:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.352680:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.352681:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.352683:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.352684:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.352685:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.352686:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.352687:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.352687:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.352688:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.352699:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.352702:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.352703:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.352704:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.352706:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.352707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.352816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677474.352824:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.352827:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677474.352829:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.352831:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.352831:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677474.352833:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677474.352834:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.352835:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.352836:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.352837:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677474.352840:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.352841:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677474.352844:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.352845:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677474.352845:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.352847:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677474.352848:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.352849:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677474.352849:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677474.352853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.352862:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.352864:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.352865:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.352866:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.352868:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.352871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.352925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.352932:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.352935:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.352938:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.352939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.352941:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.352942:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.352943:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.352945:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.352946:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.352946:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.352949:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.352951:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.352954:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.352955:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.352955:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.352957:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.352957:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.352958:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.352959:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.352971:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.352974:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.352975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.352976:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.352978:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.352981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.353160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.353167:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.353170:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.353171:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.353173:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.353176:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.353178:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.353179:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.353180:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.353181:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.353182:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.353185:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.353187:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.353190:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.353190:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.353191:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.353193:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.353193:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.353194:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.353195:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.353196:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.353201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.353206:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.353209:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.353209:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.353210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.353210:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.353211:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.353211:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.353212:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.353212:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.353215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.353388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677474.353395:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.353398:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677474.353399:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677474.353401:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677474.353404:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.353405:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677474.353406:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.353407:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.353408:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.353409:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677474.353412:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.353414:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.353416:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.353416:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.353417:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.353418:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.353419:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.353419:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677474.353420:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677474.353424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.353430:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.353432:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.353433:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.353434:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.353435:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.353436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.353443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.353449:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.353451:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.353452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.353452:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.353453:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.353453:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677474.353881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.353887:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.353890:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.353891:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.353893:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.353895:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.353897:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.353898:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.353899:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.353900:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.353901:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.353905:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.353906:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.353909:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.353909:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.353910:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.353912:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.353912:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.353913:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.353914:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.353915:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.353920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.353925:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.353925:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.353927:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677474.353928:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.353928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677474.353929:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.353929:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.353930:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.353931:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.353933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.353947:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.353950:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.353951:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.353951:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.353952:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.353952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.354100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9194733a7a00] (20)++
00000800:00000200:21.0:1619677474.354106:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (21)++
00000800:00000200:21.0:1619677474.354108:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.6.21@o2ib3
00000400:00000200:21.0:1619677474.354111:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.354111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000400:00000200:21.0:1619677474.354114:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.354115:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.354117:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.354118:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.354119:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.354120:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.354124:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.354125:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.354129:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.354129:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.354130:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.354132:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:4.2:1619677474.354132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9194733a7a00] (21)++
00000800:00000200:21.0:1619677474.354133:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (22)++
00000800:00000200:21.0:1619677474.354133:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (23)--
00000800:00000200:21.0:1619677474.354134:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000800:00000200:21.0:1619677474.354135:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (21)++
00000800:00000200:21.0:1619677474.354137:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000800:00000200:9.2:1619677474.354138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:21.0:1619677474.354139:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.354140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000400:00000200:21.0:1619677474.354141:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.354142:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:21.0:1619677474.354143:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.354144:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:21.0:1619677474.354144:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.354145:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.354145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:21.0:1619677474.354146:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677474.354147:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.354148:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.354148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677474.354149:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.354150:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.354151:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.354152:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:24.0:1619677474.354153:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677474.354153:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.354154:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:21.0:1619677474.354154:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.354155:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:21.0:1619677474.354156:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.354156:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:21.0:1619677474.354157:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (21)++
00000800:00000200:21.0:1619677474.354157:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (22)--
00000400:00000200:26.0:1619677474.354158:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.354158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (21)--
00000400:00000200:26.0:1619677474.354160:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.354162:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:9.2:1619677474.354162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:26.0:1619677474.354163:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.354164:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.354165:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.354165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9194733a7a00] (20)++
00000400:00000200:26.0:1619677474.354166:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.354169:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.354169:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (21)++
00000800:00000200:26.0:1619677474.354170:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677474.354170:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:20.0:1619677474.354171:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000800:00000200:2.0:1619677474.354172:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:20.0:1619677474.354173:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.354173:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.354174:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.354174:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000800:00000200:18.0:1619677474.354174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.354175:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.354175:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:2.0:1619677474.354176:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.354177:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.354177:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:20.0:1619677474.354177:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.354178:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677474.354178:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677474.354179:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677474.354180:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.354181:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.354182:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.354183:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677474.354183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677474.354185:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:20.0:1619677474.354186:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.354188:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.354191:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.354191:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.354192:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:4.2:1619677474.354192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9194733a7a00] (21)++
00000800:00000200:21.0:1619677474.354193:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.354193:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677474.354194:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.354195:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.354195:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.354196:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (22)++
00000800:00000200:20.0:1619677474.354196:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (23)--
00000800:00000200:20.0:1619677474.354197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000800:00000200:21.0:1619677474.354199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.354200:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (21)++
00000800:00000200:20.0:1619677474.354200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.354202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.354203:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000400:00000200:21.0:1619677474.354205:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.354207:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.354207:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000800:00000200:20.0:1619677474.354208:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677474.354209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677474.354209:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.354210:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.354210:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677474.354211:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677474.354211:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000400:00000200:22.0:1619677474.354212:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.354212:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.354212:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.354212:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677474.354212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:21.0:1619677474.354213:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.354213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:21.0:1619677474.354214:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677474.354215:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.354217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:21.0:1619677474.354218:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.354220:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.354222:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.354223:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.354224:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.354226:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.354226:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (21)++
00000800:00000200:21.0:1619677474.354228:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9194733a7a00] (22)++
00000800:00000200:21.0:1619677474.354229:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (23)--
00000800:00000200:21.0:1619677474.354229:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000800:00000200:8.2:1619677474.354232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677474.354234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9194733a7a00] (21)++
00000800:00000200:20.0:1619677474.354238:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)++
00000800:00000200:20.0:1619677474.354240:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9194733a7a00] (23)--
00000800:00000200:17.0:1619677474.354240:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677474.354241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000800:00000200:20.0:1619677474.354242:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (21)++
00000800:00000200:17.0:1619677474.354242:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:20.0:1619677474.354243:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000400:00000200:17.0:1619677474.354243:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.354244:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)++
00000800:00000200:1.0:1619677474.354244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.354245:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000400:00000200:17.0:1619677474.354246:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:20.0:1619677474.354247:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.354248:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.354248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:20.0:1619677474.354250:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.354252:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.354253:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.354254:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.354255:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.354256:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.354257:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.354260:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.354262:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.354263:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:21.0:1619677474.354264:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.354265:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.354266:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.354267:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677474.354269:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677474.354270:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.354270:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.354271:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.354273:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677474.354273:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.354273:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (23)++
00000800:00000200:20.0:1619677474.354274:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (24)--
00000800:00000200:21.0:1619677474.354275:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.354275:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)--
00000800:00000200:21.0:1619677474.354276:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:20.0:1619677474.354276:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)++
00000800:00000200:21.0:1619677474.354277:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677474.354277:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000800:00000200:8.2:1619677474.354278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.354279:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:20.0:1619677474.354279:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.354280:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (23)++
00000800:00000200:21.0:1619677474.354281:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (24)--
00000800:00000200:21.0:1619677474.354281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)--
00000800:00000200:21.0:1619677474.354282:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)++
00000400:00000200:20.0:1619677474.354282:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.354284:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000400:00000200:20.0:1619677474.354284:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677474.354285:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96504866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.354287:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.354288:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.354288:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:21.0:1619677474.354292:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.354293:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:20.0:1619677474.354297:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677474.354299:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.354300:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000800:00000200:20.0:1619677474.354300:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.354301:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:21.0:1619677474.354301:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:31.0:1619677474.354302:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.354303:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96504865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.354304:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.354304:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.354306:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.354306:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:31.0:1619677474.354309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.0:1619677474.354310:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677474.354311:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.354311:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:20.0:1619677474.354312:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677474.354313:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677474.354314:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677474.354314:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (23)++
00000800:00000200:20.0:1619677474.354315:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (24)--
00000800:00000200:21.0:1619677474.354316:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.354316:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:20.0:1619677474.354316:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)--
00000800:00000200:21.0:1619677474.354317:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.0:1619677474.354317:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.354318:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.354319:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:20.0:1619677474.354319:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.354320:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (22)++
00000400:00000200:20.0:1619677474.354320:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.354321:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9194733a7a00] (23)++
00000800:00000200:21.0:1619677474.354322:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (24)--
00000800:00000200:20.0:1619677474.354322:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.354323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)--
00000800:00000200:20.0:1619677474.354324:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)++
00000800:00000200:21.0:1619677474.354325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677474.354325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.0:1619677474.354326:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)++
00000800:00000200:20.0:1619677474.354326:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000800:00000200:30.0:1619677474.354327:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.0:1619677474.354327:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000400:00000200:20.0:1619677474.354328:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.354330:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:21.0:1619677474.354330:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.354330:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.354331:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.354332:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677474.354332:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:31.0:1619677474.354334:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677474.354334:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:20.0:1619677474.354334:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96504864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.354334:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.354335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:20.0:1619677474.354335:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.354336:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.354336:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.354338:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677474.354338:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:20.0:1619677474.354341:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.354343:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.354344:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.354346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:10.0:1619677474.354347:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:21.0:1619677474.354348:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:20.0:1619677474.354349:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.354349:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:21.0:1619677474.354350:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.354350:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:9.2:1619677474.354350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:21.0:1619677474.354351:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.354351:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677474.354352:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.354353:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677474.354354:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (24)++
00000800:00000200:20.0:1619677474.354354:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (25)--
00000800:00000200:20.0:1619677474.354355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (24)--
00000800:00000200:26.0:1619677474.354356:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.0:1619677474.354356:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)++
00000400:00000200:21.0:1619677474.354357:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677474.354357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.354358:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677474.354358:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000800:00000200:26.0:1619677474.354360:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677474.354360:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.354360:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.354361:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:21.0:1619677474.354361:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.354362:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:21.0:1619677474.354362:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.354363:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.0:1619677474.354363:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:20.0:1619677474.354363:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.354364:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.0:1619677474.354365:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:20.0:1619677474.354366:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96504862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.354367:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.354368:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.354369:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:9.2:1619677474.354370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:20.0:1619677474.354372:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.354373:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (24)++
00000800:00000200:20.0:1619677474.354373:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.354374:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (25)--
00000800:00000200:21.0:1619677474.354374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (24)--
00000800:00000200:21.0:1619677474.354375:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)++
00000800:00000200:20.0:1619677474.354376:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.354376:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.354377:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000800:00000200:20.0:1619677474.354377:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (23)++
00000800:00000200:10.0:1619677474.354377:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:21.0:1619677474.354379:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.354379:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (24)--
00000800:00000200:20.0:1619677474.354379:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (24)++
00000800:00000200:10.0:1619677474.354379:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.354380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:20.0:1619677474.354380:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (25)--
00000800:00000200:20.0:1619677474.354380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (24)--
00000400:00000200:10.0:1619677474.354380:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.354381:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.354382:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000400:00000200:21.0:1619677474.354383:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677474.354383:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:20.0:1619677474.354383:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.354383:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.354385:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96504861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.354385:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.354387:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.354388:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.354388:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677474.354391:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:21.0:1619677474.354392:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.354392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.354393:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)++
00000800:00000200:21.0:1619677474.354394:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677474.354394:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9194733a7a00] (24)--
00000800:00000200:20.0:1619677474.354395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)--
00000800:00000200:20.0:1619677474.354395:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.354396:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.354396:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.354397:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677474.354397:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:20.0:1619677474.354397:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.354398:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.354399:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677474.354400:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (22)++
00000800:00000200:20.0:1619677474.354400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.354401:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)++
00000800:00000200:21.0:1619677474.354402:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9194733a7a00] (24)++
00000800:00000200:20.0:1619677474.354402:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000800:00000200:21.0:1619677474.354404:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (25)--
00000800:00000200:21.0:1619677474.354404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (24)--
00000400:00000200:20.0:1619677474.354404:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.354405:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:20.0:1619677474.354405:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.354405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:20.0:1619677474.354406:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:21.0:1619677474.354407:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.354408:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000400:00000200:20.0:1619677474.354408:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.354409:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.354409:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.354410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)++
00000400:00000200:20.0:1619677474.354410:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.354410:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.354411:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000400:00000200:20.0:1619677474.354412:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677474.354413:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:21.0:1619677474.354415:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.354416:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:20.0:1619677474.354417:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677474.354417:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.354417:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:21.0:1619677474.354418:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.354419:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677474.354419:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.354420:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:21.0:1619677474.354421:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96504859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.354423:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.354424:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.354425:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:18.0:1619677474.354427:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677474.354428:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677474.354429:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.354429:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:20.0:1619677474.354429:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677474.354431:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677474.354431:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677474.354432:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (24)++
00000800:00000200:20.0:1619677474.354433:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (25)--
00000800:00000200:21.0:1619677474.354434:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.354434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (24)--
00000800:00000200:21.0:1619677474.354435:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.0:1619677474.354435:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.354436:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677474.354436:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)++
00000800:00000200:20.0:1619677474.354437:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000800:00000200:21.0:1619677474.354438:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677474.354438:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (24)++
00000800:00000200:21.0:1619677474.354439:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (25)--
00000400:00000200:20.0:1619677474.354439:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.354440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (24)--
00000800:00000200:21.0:1619677474.354441:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)++
00000400:00000200:20.0:1619677474.354441:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.354442:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000400:00000200:20.0:1619677474.354443:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:9.2:1619677474.354443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:20.0:1619677474.354444:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96504858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.354445:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:20.0:1619677474.354445:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.354446:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.354447:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.354448:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.354449:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.354450:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:21.0:1619677474.354451:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.354452:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.354452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677474.354452:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.354453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:20.0:1619677474.354453:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677474.354454:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.354454:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.354456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677474.354456:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677474.354456:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:8.0:1619677474.354456:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.354457:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96504857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.354459:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.354460:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.354461:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677474.354463:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.354464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000400:00000200:21.0:1619677474.354464:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.354466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.354466:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677474.354466:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677474.354466:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.354466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.354467:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677474.354467:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.354467:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.354468:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.354469:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.354469:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:20.0:1619677474.354469:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.354470:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (24)++
00000800:00000200:20.0:1619677474.354471:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (25)--
00000800:00000200:20.0:1619677474.354471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (24)--
00000800:00000200:20.0:1619677474.354472:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)++
00000800:00000200:21.0:1619677474.354473:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.354473:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9194733a7a00] (24)--
00000800:00000200:21.0:1619677474.354474:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:21.0:1619677474.354474:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677474.354474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (23)--
00000800:00000200:20.0:1619677474.354475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000800:00000200:6.0:1619677474.354475:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:25.2:1619677474.354476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.354476:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.0:1619677474.354477:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (21)++
00000800:00000200:22.0:1619677474.354478:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.354478:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:21.0:1619677474.354479:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9194733a7a00] (22)++
00000800:00000200:21.0:1619677474.354480:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (23)--
00000800:00000200:21.0:1619677474.354480:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000400:00000200:6.0:1619677474.354480:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677474.354482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677474.354483:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:6.0:1619677474.354483:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.354485:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:4.2:1619677474.354485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9194733a7a00] (21)++
00000800:00000200:31.0:1619677474.354486:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:6.0:1619677474.354486:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96504856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.354487:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.354487:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.354489:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)++
00000400:00000200:6.0:1619677474.354489:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.354490:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:6.0:1619677474.354490:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677474.354491:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.354491:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:20.0:1619677474.354492:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9194733a7a00] (23)--
00000800:00000200:4.2:1619677474.354492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677474.354493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000800:00000200:1.0:1619677474.354493:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:20.0:1619677474.354494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (21)--
00000400:00000200:6.0:1619677474.354494:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677474.354494:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.354495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:17.0:1619677474.354495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677474.354497:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677474.354497:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.354499:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.354500:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.354500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677474.354501:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:24.0:1619677474.354502:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677474.354502:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677474.354503:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677474.354504:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:24.0:1619677474.354505:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.354505:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.354506:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.354506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:24.0:1619677474.354507:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.354509:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.354510:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677474.354511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:24.0:1619677474.354511:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.354512:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.354513:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.354517:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.354518:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677474.354518:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.354521:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:24.0:1619677474.354522:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.354522:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:12.0:1619677474.354522:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.354523:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:12.0:1619677474.354524:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.354525:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.354525:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677474.354525:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:24.0:1619677474.354526:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.354526:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677474.354527:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.354531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.354538:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.354541:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.354541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.354542:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.354544:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.354547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.354550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.354557:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.354560:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.354561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.354561:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.354562:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.354562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.354668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.354675:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.354678:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.354681:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.354682:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.354684:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.354685:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.354687:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.354688:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.354689:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.354690:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:21.2:1619677474.354691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:30.0:1619677474.354693:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.354695:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677474.354697:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:30.0:1619677474.354698:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.354698:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.354699:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.354700:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677474.354700:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.354700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:30.0:1619677474.354701:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.354702:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:17.0:1619677474.354702:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.354704:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:17.0:1619677474.354704:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.354705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677474.354706:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:9.2:1619677474.354706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677474.354707:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.354708:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.354709:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677474.354710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677474.354710:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.0:1619677474.354712:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677474.354713:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.354714:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677474.354715:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.354716:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.354716:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.354717:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677474.354718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:17.0:1619677474.354718:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:8.0:1619677474.354718:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.354719:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.354720:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677474.354720:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.354721:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677474.354721:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:31.0:1619677474.354722:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:17.0:1619677474.354722:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:31.0:1619677474.354723:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677474.354723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:31.0:1619677474.354724:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.354726:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677474.354726:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.354727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:31.0:1619677474.354728:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.354728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.354730:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.354732:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.354733:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.354735:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.354735:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.354735:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:31.0:1619677474.354737:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.354737:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.354738:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.354740:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.354741:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.354741:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.354742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677474.354743:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.354747:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.354748:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.354748:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.354750:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.354751:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.354752:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.354753:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.354754:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.354756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.354759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.354764:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.354766:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.354767:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677474.354767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.354768:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.354770:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.354770:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677474.354770:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.354771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.354773:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.354864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9194733a7a00] (20)++
00000800:00000200:21.0:1619677474.354869:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (21)++
00000800:00000200:21.0:1619677474.354872:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000400:00000200:21.0:1619677474.354874:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.354874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000400:00000200:21.0:1619677474.354876:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.354878:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677474.354879:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.354881:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.354881:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.354883:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677474.354886:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.354887:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.354889:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.354890:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677474.354891:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677474.354893:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677474.354893:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (21)++
00000800:00000200:21.0:1619677474.354894:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (22)--
00000800:00000200:21.0:1619677474.354895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (21)--
00000800:00000200:9.2:1619677474.354899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.354907:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:4.2:1619677474.354909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9194733a7a00] (20)++
00000800:00000200:2.0:1619677474.354910:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.354911:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.354911:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.354913:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9194733a7a00] (21)++
00000400:00000200:2.0:1619677474.354913:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.354915:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.21@o2ib3
00000800:00000200:2.0:1619677474.354915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:20.0:1619677474.354917:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.21@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.354918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (22)--
00000400:00000200:20.0:1619677474.354920:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.354921:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677474.354922:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.354923:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.354924:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.354925:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677474.354929:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.354931:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677474.354933:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.354934:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:20.0:1619677474.354935:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677474.354937:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677474.354937:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9194733a7a00] (21)++
00000800:00000200:20.0:1619677474.354938:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9194733a7a00] (22)--
00000800:00000200:20.0:1619677474.354939:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9194733a7a00] (21)--
00000800:00000200:9.2:1619677474.354942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.354949:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.354952:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.354952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.354953:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.354955:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.354958:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.355093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.355099:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.355101:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.355103:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.355104:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.355106:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.355107:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.355109:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.355110:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.355111:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.355112:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.355115:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.355117:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.355120:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.355120:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.355121:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.355123:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.355123:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.355125:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.355126:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.355127:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.355129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.355131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677474.355135:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677474.355135:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677474.355138:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677474.355138:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.355139:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677474.355139:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.355140:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.355140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:24.0:1619677474.355142:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.355144:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.355554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.355557:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.355560:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677474.355562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677474.355562:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.355564:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.355566:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.0:1619677474.355567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.355567:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.355568:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.355569:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.355570:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677474.355570:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677474.355573:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677474.355573:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.355574:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677474.355575:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.355576:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.355577:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:31.0:1619677474.355578:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.355578:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.355578:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.355579:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:30.0:1619677474.355580:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677474.355580:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677474.355581:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.355581:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.355581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677474.355582:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.355583:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.355584:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.2:1619677474.355586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.355587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677474.355588:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.355589:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.355592:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.355592:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.355593:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677474.355594:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:30.0:1619677474.355595:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.355595:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.355595:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677474.355596:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.355596:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:30.0:1619677474.355597:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677474.355597:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.355599:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:21.0:1619677474.355599:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.355599:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677474.355600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.355601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677474.355601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677474.355602:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.355604:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.355606:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.355607:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.355607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:10.0:1619677474.355609:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.355610:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.355610:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.355611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677474.355611:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.355611:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.355612:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.355614:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.355614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.355616:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.355619:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.355619:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.355620:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.355622:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.355622:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677474.355622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.355623:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.355623:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.355628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.355628:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.355630:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.355631:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.355631:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.355632:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.355632:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677474.355636:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.355639:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.355640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.355640:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.355642:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.355645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.356520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.356528:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.356530:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.356531:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.356533:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.356536:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.356538:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.356539:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.356540:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.356541:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.356542:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.356544:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.356546:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.356548:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.356549:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.356549:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.356551:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.356552:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.356553:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.356554:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.356555:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.356558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.356561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.356566:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.356568:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.356569:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.356569:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.356570:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.356570:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.356571:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677474.356572:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.356572:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.356575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.356786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.356794:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.356797:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.356800:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.356802:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.356802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.356804:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.356805:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.356806:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.356807:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.356808:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.356812:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.356813:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.356816:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.356817:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.356818:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.356818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.356819:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.356820:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.356821:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.356821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677474.356823:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.356824:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.356826:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.356826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.356828:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.356830:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.356831:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.356832:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.356832:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.356833:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:25.2:1619677474.356833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677474.356836:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.356837:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.356839:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.356840:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.356840:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.356840:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.356842:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.356843:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.356843:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677474.356843:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.356844:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.356844:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.356844:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.356846:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.356848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.356849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.356855:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:25.2:1619677474.356857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.356858:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.356859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.356859:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.356861:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.356863:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.356864:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.356867:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.356868:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.356868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.356869:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.356870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.357514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.357522:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.357525:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.357527:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.357529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.357530:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.357531:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.357532:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.357534:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.357535:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.357535:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.357538:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.357540:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.357542:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.357542:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.357543:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.357544:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.357545:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.357546:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.357547:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.357551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.357557:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.357560:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.357561:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.357561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.357563:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.357565:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.357599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.357607:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.357612:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.357613:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.357617:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.357620:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.357622:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.357624:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.357627:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.357628:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.357630:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.357635:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.357638:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.357641:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.357642:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.357643:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.357646:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.357647:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.357648:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.357650:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.357650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.357652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.357655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.357662:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.357665:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.357665:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.357666:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677474.357666:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.357667:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.357669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.357669:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.357671:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.357672:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.358036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.358044:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.358047:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.358048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.358050:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.358053:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.358055:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.358056:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.358057:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.358058:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.358060:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677474.358061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677474.358062:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.358064:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.358067:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.358068:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.358068:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.358069:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677474.358070:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:31.0:1619677474.358071:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677474.358071:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.358071:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.358072:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:31.0:1619677474.358073:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.358075:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.358075:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.358076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:31.0:1619677474.358077:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.358078:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.358079:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.358080:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.358081:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.358084:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.358085:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677474.358085:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:31.0:1619677474.358087:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.358088:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.358088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.358088:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:31.0:1619677474.358089:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:6.0:1619677474.358089:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.358090:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.358091:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677474.358091:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.358093:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677474.358093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.358094:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.358095:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.358096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.358100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.358105:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.358107:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.358108:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.358109:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.358109:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.358110:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.358110:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.358111:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.358112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.358113:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.358434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.358441:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.358444:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.358444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.358446:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.358448:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.358450:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.358451:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.358452:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.358453:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.358455:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677474.358456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:18.0:1619677474.358457:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.358459:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.358461:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.358462:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.358463:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.358464:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677474.358464:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:18.0:1619677474.358465:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.358466:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.358466:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677474.358466:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.358467:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.358469:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677474.358471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:10.0:1619677474.358472:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.358474:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677474.358475:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.358476:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.358477:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.358478:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677474.358480:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:10.0:1619677474.358481:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.358482:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:10.0:1619677474.358483:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:1.0:1619677474.358483:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.358484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.358486:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.358487:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.358487:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677474.358488:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677474.358488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677474.358490:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677474.358490:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.358491:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.358491:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.358497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.2:1619677474.358497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.358505:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.358505:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:17.0:1619677474.358508:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.358508:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.358509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677474.358509:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000400:00000200:2.0:1619677474.358509:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.358509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.358510:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.358511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:2.0:1619677474.358511:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.358515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.358519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.358526:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.358529:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.358530:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.358530:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.358531:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.358532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.358568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.358576:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.358579:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.358581:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.358583:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.358583:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.358584:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.358586:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.358587:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.358588:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.358589:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.358593:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.358594:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.358598:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.358598:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.358599:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.358601:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.358601:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.358602:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.358603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.358607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:5.2:1619677474.358611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677474.358615:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677474.358617:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.358618:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.358619:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.358619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.358620:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.358621:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677474.358622:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.358623:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.358625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:1.0:1619677474.358625:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.358627:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.358628:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.358630:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.358631:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.358632:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.358635:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.358637:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.358639:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.358640:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.358641:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.358642:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.358643:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.358644:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.358644:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.358648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.358656:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.358658:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.358659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.358660:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.358662:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.358664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.358816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.358824:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.358827:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.358829:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.358831:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.358833:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.358834:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.358836:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.358837:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.358838:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.358839:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.358842:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.358844:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.358846:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.358847:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.358848:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.358849:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.358850:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.358851:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.358851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.358855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.358861:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.358864:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.358864:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.358866:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.358868:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.358869:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.359392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.359400:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.359402:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.359403:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.359405:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.359408:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.359409:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.359410:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.359411:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.359412:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.359413:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.359416:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.359417:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.359420:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.359421:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.359422:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.359423:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.359424:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.359425:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.359425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.359430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.359438:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.359441:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.359441:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.359442:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.359444:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.359446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.359550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.359556:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.359559:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.359560:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.359561:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.359564:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.359565:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677474.359567:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.359568:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.359569:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.359570:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677474.359574:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.359575:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.359578:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.359579:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677474.359579:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.359581:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677474.359582:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.359583:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677474.359584:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677474.359585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:25.2:1619677474.359587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.2:1619677474.359590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:6.0:1619677474.359595:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677474.359596:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:6.0:1619677474.359597:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:18.0:1619677474.359598:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:6.0:1619677474.359598:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.359599:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677474.359599:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.359600:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:6.0:1619677474.359600:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.359603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.359973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.359977:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.359979:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.359982:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.359983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.359984:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.359986:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.359987:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.359988:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.359989:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.359991:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.359994:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.359996:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.359999:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.360000:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.360000:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.360002:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.360003:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.360003:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.360004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.360008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.360014:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.360017:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.360018:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.360018:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.360020:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.360023:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.360250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.360255:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.360257:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.360259:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.360260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.360262:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.360263:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.360265:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.360266:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.360267:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.360268:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.360271:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.360272:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.360275:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.360275:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.360276:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.360278:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.360279:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.360280:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.360281:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.360282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.360284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.360287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.360290:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.360292:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677474.360292:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677474.360293:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.360297:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677474.360297:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.360300:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.360301:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.360303:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.360305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.360428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.360436:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.360439:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.360440:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.360442:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.360445:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.360446:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.360447:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.360449:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.360449:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.360450:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.360453:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.360455:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.360458:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.360458:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.360459:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.360461:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.360461:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.360462:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.360462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.360468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.360476:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.360478:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.360479:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.360480:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.360482:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.360484:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.360486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.360493:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.360496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.360496:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.360497:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.360498:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.360498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.360702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.360710:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.360713:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.360715:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.360717:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.360717:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.360718:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.360720:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.360721:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.360722:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.360723:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.360726:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.360727:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.360730:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.360731:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.360732:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.360733:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.360734:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.360735:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.360735:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.360739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.360747:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.360750:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.360751:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.360751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.360753:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.360756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.360875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.360883:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.360885:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.360886:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.360888:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.360891:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.360893:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.360894:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.360896:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.360897:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.360898:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.360901:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.360903:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.360905:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.360906:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.360907:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.360908:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.360909:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.360910:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.360910:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.360915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.360923:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.360925:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.360927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.360927:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.360929:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.360932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.360934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677474.360941:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.360943:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677474.360944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.360945:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.360946:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.360946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.361533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.361539:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.361542:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.361543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.2:1619677474.361544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677474.361545:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.361547:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.361548:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.361550:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.361551:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.361552:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677474.361552:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.361553:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677474.361556:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677474.361557:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.361558:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.361559:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677474.361560:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.361561:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.361562:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.361563:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:1.0:1619677474.361563:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.361563:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:31.0:1619677474.361564:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.361565:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.361565:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:31.0:1619677474.361566:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.361566:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677474.361566:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.361567:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:31.0:1619677474.361568:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.361571:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677474.361572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:31.0:1619677474.361573:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.361575:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.361576:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.361576:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677474.361577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.361578:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.361578:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.361579:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.361580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677474.361580:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.361583:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.361583:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.361584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677474.361585:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.361586:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677474.361586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.361587:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677474.361589:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.361590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677474.361591:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:26.0:1619677474.361591:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.361594:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.361595:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.361595:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.361596:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.361597:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.361597:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.361598:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.361599:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:26.0:1619677474.361599:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.361600:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.361601:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.361604:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.361606:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.361608:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.361609:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.361609:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.361611:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.361611:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.361612:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.361613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.361616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.361623:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.361625:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.361626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.361626:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.361628:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.361630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.361644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.361647:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.361650:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677474.361653:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.361653:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.361655:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.361657:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.361658:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.361659:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.361660:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.361662:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.361665:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.361666:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.361669:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.361670:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.361671:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.361673:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.361673:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.361674:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.361675:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.361679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.361686:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.361689:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.361689:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.361691:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.361692:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.361694:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.361710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.361715:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.361718:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.361720:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.361720:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.361722:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.361724:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.361725:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.361727:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.361728:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.361729:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.361731:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.361733:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.361735:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.361736:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.361737:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.361739:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.361739:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.361740:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.361741:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.361745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.361753:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.361755:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.361756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.361756:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.361758:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.361761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.362161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.362168:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.362171:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.362173:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.362175:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.362175:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.362176:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.362177:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.362178:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.362179:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.362180:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.362183:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.362185:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.362187:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.362188:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.362189:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.362190:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.362191:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.362192:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.362192:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.362196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.362202:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.362205:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.362206:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.362207:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.362209:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.362210:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.362218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.362224:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.362227:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.362228:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.362228:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.362229:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.362230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.362420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.362427:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.362430:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.362432:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.362434:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.362435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.362435:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.362437:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.362438:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.362439:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.362439:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.362442:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.362444:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.362447:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.362448:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.362448:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.362450:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.362451:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.362452:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.362454:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.362454:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.362457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.362462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.362465:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.362468:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.362469:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.362469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.362470:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.362472:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.362473:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.362474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.362474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.362475:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.362616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.362623:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.362626:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.362628:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.362630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.362631:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.362633:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.362634:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.362635:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.362636:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.362637:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.362640:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.362641:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.362643:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.362644:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.362645:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.362646:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.362647:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.362648:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.362649:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.362650:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.362652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.362655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.362659:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.362661:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.362662:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.362663:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.362663:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.362664:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.362665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.362665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.362666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.362667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.362774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.362781:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.362784:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.362785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.362786:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.362789:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.362790:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.362792:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.362793:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.362794:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.362794:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.362798:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.362799:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.362802:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.362803:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.362804:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.362805:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.362806:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.362806:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.362807:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.362811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.362819:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.362821:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.362822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.362823:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.362825:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.362828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.363036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.363043:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.363046:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.363047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.363049:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.363052:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.363053:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677474.363054:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.363056:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.363057:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.363057:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677474.363060:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.363061:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677474.363064:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.363064:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:26.0:1619677474.363065:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677474.363067:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677474.363067:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.363068:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.363068:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.363073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677474.363080:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677474.363082:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677474.363083:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.363085:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.363087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677474.363088:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:25.2:1619677474.363094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:12.0:1619677474.363100:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677474.363103:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677474.363103:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.363104:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677474.363105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.2:1619677474.363106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677474.363106:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:31.0:1619677474.363114:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.363117:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.363119:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.363121:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.363121:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.363123:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.363124:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.363125:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.363126:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.363127:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.363131:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.363132:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.363135:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.363135:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.363136:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.363137:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.363138:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.363139:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.363139:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.363143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.363150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.363152:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.363153:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.363154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.363155:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.363158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.363162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.363168:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.363171:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.363172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.363172:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.363173:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.363173:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.363247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.363254:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.363256:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.363257:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.363260:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.363262:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.363263:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.363265:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.363266:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.363267:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.363268:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.2:1619677474.363270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:17.0:1619677474.363271:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.363273:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.363275:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.363276:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.363276:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677474.363276:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677474.363278:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677474.363278:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677474.363279:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.363279:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677474.363280:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677474.363280:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677474.363281:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:25.2:1619677474.363284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:2.0:1619677474.363284:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.363286:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.363287:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.363289:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.363290:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.363291:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677474.363292:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.363297:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.363298:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.363299:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.363299:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.363300:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:25.2:1619677474.363301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677474.363301:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.363304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677474.363304:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.363305:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677474.363305:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.363305:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677474.363308:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677474.363308:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.363308:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677474.363309:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.363309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677474.363309:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677474.363310:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677474.363310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:22.0:1619677474.363311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.363314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.363321:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.363324:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.363325:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.363327:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.363328:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.363330:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.364352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.364360:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.364362:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.364363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.364365:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.364368:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.364370:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.364371:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.364372:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.364373:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.364375:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.364377:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.364379:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.364383:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.364383:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.364384:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.364386:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.364386:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.364387:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.364389:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.364389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.364395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677474.364403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.364403:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.364404:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.364405:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.364406:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.364411:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.364414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.364414:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.364415:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.364417:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.364419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.364426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.364433:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.364436:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.364436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.364437:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.364438:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.364438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.364633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.364641:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.364644:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.364647:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.364649:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.364649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.364650:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.364651:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.364653:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.364654:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.364655:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.364658:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.364659:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.364662:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.364663:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.364664:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.364666:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.364666:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.364667:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.364668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.364682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.364690:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.364692:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.364693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.364694:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.364696:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.364699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.364724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677474.364731:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.364734:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677474.364735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677474.364737:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.364739:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.364740:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677474.364742:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.364743:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.364744:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.364745:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677474.364748:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.364749:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.364751:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.364751:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.364752:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.364753:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.364753:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.364754:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677474.364755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677474.364758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.364764:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.364766:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.364767:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.364768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.364769:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.364770:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.365112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.365119:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.365122:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.365124:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.365127:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.365127:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.365129:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.365130:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.365131:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.365132:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.365133:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.365136:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.365138:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.365141:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.365142:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.365142:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.365144:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.365144:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.365145:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.365146:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.365150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.365158:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.365160:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.365162:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.365163:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.365164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.365167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.365361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.365369:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.365371:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.365372:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.365374:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.365376:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.365378:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.365379:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.365380:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.365381:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.365382:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.365385:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.365387:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.365390:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.365391:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.365391:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.365393:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.365393:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.365394:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.365394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.365399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.365408:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.365411:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.365411:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.365412:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.365414:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.365418:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.365761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.365768:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.365771:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.365772:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.365774:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.365776:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.365778:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.365779:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.365780:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.365782:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.365782:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.365786:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.365787:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.365790:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.365791:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.365791:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.365793:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.365793:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.365794:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.365794:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.365799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.365807:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.365810:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.365811:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.365811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.365814:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.365817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.366159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.366168:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.366171:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.366173:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.366175:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.366176:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.366177:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.366179:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.366180:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.366181:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.366182:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.366185:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.366186:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.366188:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.366189:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.366189:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.366191:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.366191:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.366192:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.366193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.366196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.366203:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.366205:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.366206:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.366208:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.366209:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.366210:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.366253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.366259:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.366261:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.366264:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.366264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.366266:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.366268:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.366269:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.366270:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.366271:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.366272:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.366276:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.366278:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.366280:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.366281:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.366282:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.366283:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.366284:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.366285:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.366286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.366290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.366300:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.366302:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.366304:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.366304:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.366306:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.366308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.366310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.366318:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.366320:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.366321:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.366321:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.366322:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.366323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.366373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.366381:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.366384:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.366386:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.366388:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.366389:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.366390:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.366391:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.366393:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.366394:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.366394:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.366398:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.366399:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.366403:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.366403:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.366404:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.366406:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.366406:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.366407:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.366408:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.366412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.366421:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.366424:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.366425:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.366425:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.366427:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.366430:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.366444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.366450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.366453:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.366454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.366455:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.366458:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.366460:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677474.366461:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.366463:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.366464:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.366465:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677474.366468:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.366470:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677474.366474:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.366475:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677474.366475:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677474.366477:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677474.366478:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.366478:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.366479:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677474.366483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.366490:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.366493:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.366494:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.366495:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.366496:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.366499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.366618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.366625:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.366628:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.366629:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.366631:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.366633:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.366635:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.366635:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.366637:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.366638:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.366639:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.366642:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.366643:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.366646:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.366646:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.366647:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.366649:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.366649:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.366650:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.366652:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.366652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.366654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.366658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.366661:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.366663:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.366664:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.366664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.366665:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677474.366666:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.366666:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.366667:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.366668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.366668:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.366739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.366746:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.366749:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.366751:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.366753:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.366753:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.366755:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.366756:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.366757:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.366758:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.366759:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.366761:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.366763:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.366765:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.366766:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.366766:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.366768:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.366768:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.366770:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.366771:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.366772:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.366773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.366778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.366779:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.366782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.366782:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.366783:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.366785:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.366785:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.366787:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.366788:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.366789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.366790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.367497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.367501:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.367504:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.367506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.367506:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.367508:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.367509:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.367511:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.367512:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.367513:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.367514:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.367517:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.367519:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.367521:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.367522:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.367523:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.367524:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.367525:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.367526:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.367526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.367531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.367538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.367541:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.367542:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.367542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.367544:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.367546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.367548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.367555:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.367557:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.367558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.367558:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.367559:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.367560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.367626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.367634:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.367636:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.367637:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.367639:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.367641:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.367643:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.367644:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.367645:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.367646:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.367647:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.367651:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.367652:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.367655:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.367656:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.367657:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.367658:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.367659:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.367659:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.367660:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.367665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.367672:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.367676:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.367676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.367677:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.367679:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.367682:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.367861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:13.0:1619677474.367869:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677474.367872:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677474.367874:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : GET - for me
00000800:00000200:12.0:1619677474.367876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:13.0:1619677474.367877:0:19942:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.103@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:13.0:1619677474.367881:0:19942:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.103@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:13.0:1619677474.367883:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677474.367884:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:13.0:1619677474.367887:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:13.0:1619677474.367893:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:13.0:1619677474.367896:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:13.0:1619677474.367899:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.367900:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677474.367901:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677474.367903:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677474.367904:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:25.2:1619677474.367908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.2:1619677474.367911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:12.0:1619677474.367915:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:18.0:1619677474.367918:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677474.367918:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.367919:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.367920:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:18.0:1619677474.367921:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.367921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677474.367922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:18.0:1619677474.367923:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:21.2:1619677474.367924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677474.367924:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:18.0:1619677474.367926:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.367928:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.367929:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.367931:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.367932:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677474.367932:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.367933:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677474.367935:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677474.367937:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677474.367938:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.367939:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677474.367940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.367941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.367942:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:18.0:1619677474.367943:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.367944:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677474.367944:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:31.0:1619677474.367945:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677474.367945:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:31.0:1619677474.367946:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677474.367946:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:31.0:1619677474.367947:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:18.0:1619677474.367947:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.367948:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.367949:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677474.367951:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.367953:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.367955:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.367956:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.367957:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.367958:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.367959:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.367960:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.367961:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.367963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:9.2:1619677474.367964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.367970:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:13.0:1619677474.367971:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:24.0:1619677474.367973:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.367974:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.367974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.367975:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.367975:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.367976:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.367978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:13.0:1619677474.367979:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.367979:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.367982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677474.367983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:9.2:1619677474.367985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677474.367989:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677474.367991:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677474.367992:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.367993:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677474.367995:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.367995:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.367996:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:1.0:1619677474.367996:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.367997:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.367997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:1.0:1619677474.367998:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.367999:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.368001:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.368002:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.368003:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.368004:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.368007:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.368009:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.368012:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.368013:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.368013:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.368015:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.368015:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.368016:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.368017:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.368021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.368028:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.368032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.368032:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.368033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.368035:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.368037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.368113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.368118:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.368121:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.368122:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.368124:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.368126:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.368128:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.368129:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.368130:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.368131:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.368132:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.368135:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.368136:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.368138:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.368139:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.368140:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.368141:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.368141:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.368143:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677474.368144:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.368144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.368147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.368150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.368155:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.368157:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677474.368158:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677474.368158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.368158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677474.368160:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.368161:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.368163:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.368163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.368165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.368167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.368175:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.368178:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.368178:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.368179:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.368180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.368181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.368394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.368402:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.368405:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.368407:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.368409:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.368409:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.368411:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.368412:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.368414:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.368415:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.368416:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.368419:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.368421:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.368424:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.368425:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.368426:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.368427:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.368428:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.368430:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677474.368431:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677474.368432:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677474.368434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.368438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:2.0:1619677474.368442:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.368445:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:31.0:1619677474.368446:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:18.0:1619677474.368446:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.368446:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.368448:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000400:00000200:2.0:1619677474.368448:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.368449:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.368450:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:2.0:1619677474.368452:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.368498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.368506:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.368509:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.368511:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.368513:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.368514:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.368516:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.368517:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.368518:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.368519:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.368520:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.368523:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.368524:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.368526:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.368527:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.368527:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.368529:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.368530:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.368530:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.368531:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.368535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.368542:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.368544:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.368545:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.368545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.368547:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.368549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.369120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.369127:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.369130:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.369132:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.369134:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.369134:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.369135:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.369136:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.369138:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.369138:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.369139:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.369142:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.369144:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.369146:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.369146:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.369147:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.369149:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.369149:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.369150:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.369150:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.369154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.369161:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.369163:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.369164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.369164:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.369165:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.369167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.369304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.369311:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.369313:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.369314:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.369316:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.369319:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.369320:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.369321:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.369322:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.369323:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.369324:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.369327:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.369329:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.369332:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.369332:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.369333:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.369335:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.369335:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.369336:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.369336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.369342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.369350:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.369352:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.369353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.369354:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.369356:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.369358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.369365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.369373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.369375:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.369376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.369376:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.369377:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.369378:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.369499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.369504:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.369506:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.369507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.369509:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.369511:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.369513:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677474.369514:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.369516:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.369517:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.369518:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.369521:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.369523:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.369526:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.369527:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.369528:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.369530:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.369530:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.369531:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.369531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677474.369536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.369543:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.369546:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.369547:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.369549:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.369550:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.369552:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.369973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.369980:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.369983:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.369983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.369986:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.369989:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.369990:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.369991:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.369993:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.369994:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.369995:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.369998:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.370000:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.370002:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.370002:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.370003:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.370005:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.370005:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.370006:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.370007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.370011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.370018:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.370021:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.370021:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.370022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.370023:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.370024:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.370113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.370121:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.370124:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.370126:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.370127:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.370128:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.370130:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.370131:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.370133:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.370134:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.370135:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.370138:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.370140:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.370142:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.370143:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.370143:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.370145:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.370146:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.370147:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.370149:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.370149:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.370151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.370155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.370157:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.370160:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.370160:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.370161:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.370163:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677474.370163:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.370165:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.370166:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.370166:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.370167:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.370606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.370614:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.370617:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.370618:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.370619:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.370622:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.370624:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.370625:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.370626:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.370627:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.370628:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.370631:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.370632:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.370635:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.370636:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.370637:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.370638:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.370639:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.370639:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.370640:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.370645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.370654:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.370657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.370657:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.370658:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.370660:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.370663:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.370689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.370695:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.370697:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.370699:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.370700:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.370702:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.370703:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.370704:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.370706:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.370707:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.370707:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.370710:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.370712:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.370714:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.370715:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.370716:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.370717:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.370718:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.370718:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.370719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.370724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.370733:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.370736:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.370737:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.370737:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.370739:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.370741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.371130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.371138:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.371141:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.371144:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.371146:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.371146:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.371147:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.371149:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.371150:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.371151:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.371152:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.371156:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.371157:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.371160:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.371160:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.371161:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.371163:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.371163:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.371164:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.371165:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.371168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.371175:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.371177:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.371178:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.371180:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.371182:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.371183:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.371189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.371196:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.371199:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.371200:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.371200:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.371201:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.371202:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.371282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.371288:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.371291:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.371291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.371297:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.371299:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.371301:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677474.371302:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.371304:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.371305:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.371305:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.371309:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.371310:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.371313:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.371314:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.371315:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.371317:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.371317:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.371319:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677474.371320:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677474.371320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:25.2:1619677474.371323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.371326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:31.0:1619677474.371331:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:6.0:1619677474.371332:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:31.0:1619677474.371334:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:6.0:1619677474.371334:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000400:00000200:31.0:1619677474.371335:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.371335:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.371335:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:31.0:1619677474.371337:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.371339:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.371340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.371597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.371605:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.371608:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.371608:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.371610:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.371613:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.371615:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677474.371616:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.371618:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.371619:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.371620:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677474.371622:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.371624:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677474.371626:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.371627:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677474.371627:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.371629:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.371629:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.371630:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.371631:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.371632:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677474.371634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677474.371637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677474.371642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.371644:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.371645:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.371646:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677474.371646:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.371647:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.371648:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.371649:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.371649:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:4.2:1619677474.371650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:26.0:1619677474.371651:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.371652:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:10.0:1619677474.371652:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.371653:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.371653:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.371654:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.371655:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.371655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:26.0:1619677474.371656:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.371656:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.371657:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.371657:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.371658:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.371660:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.371661:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.371663:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.371663:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.371665:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.371665:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677474.371666:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:10.0:1619677474.371666:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.371667:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677474.371668:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.371668:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.371669:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.371670:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677474.371670:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.371671:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.371671:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677474.371672:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:10.0:1619677474.371673:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.371674:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677474.371674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.371675:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.371678:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.371679:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.371679:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677474.371681:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677474.371681:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.371682:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.371682:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:24.0:1619677474.371683:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:10.0:1619677474.371683:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:24.0:1619677474.371684:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.371684:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.0:1619677474.371684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.371685:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.371687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677474.371688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.371689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.371697:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677474.371697:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677474.371700:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:20.0:1619677474.371700:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677474.371701:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677474.371701:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.371702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677474.371703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.371703:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.371706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.371714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.371721:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.371724:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.371726:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.371728:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.371728:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.371730:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.371731:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.371732:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.371733:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.371734:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.371737:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.371738:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.371741:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.371741:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.371742:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.371744:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.371744:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.371745:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.371747:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.371747:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.371750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.371753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.371757:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.371759:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.371759:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.371761:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.371761:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.371762:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.371762:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.371763:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.371764:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.371765:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.371935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.371943:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.371946:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.371947:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.371948:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.371951:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.371952:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.371953:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.371955:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.371956:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.371957:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.371959:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.371961:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.371964:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.371964:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.371965:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.371967:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.371967:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.371968:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.371970:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.371970:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.371975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.371976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.371982:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.371983:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.371985:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677474.371985:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.371986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677474.371986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.371987:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.371987:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677474.371989:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.371992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.372062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.372067:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.372070:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.372072:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.372072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.372075:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.372076:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.372078:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.372079:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.372080:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.372081:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.372085:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.372086:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.372089:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.372090:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.372090:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.372092:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.372093:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.372094:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.372095:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.372096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.372098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.372101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677474.372104:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677474.372105:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.2:1619677474.372106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677474.372107:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:24.0:1619677474.372108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677474.372108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677474.372108:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677474.372109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677474.372109:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.372111:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.372113:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.372114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.372116:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.372118:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.372120:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.372120:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.372121:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.372122:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.372124:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.372125:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.372126:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.372129:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.372130:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.372133:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.372134:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.372134:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.372136:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.372136:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.372137:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.372138:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.372141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.372148:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.372150:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.372151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.372151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.372153:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.372155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.372159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.372166:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.372168:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.372169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.372169:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.372170:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.372170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.372293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.372302:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.372304:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.372307:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.372309:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.372309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.372311:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.372312:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.372313:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.372314:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.372315:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.372318:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.372319:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.372321:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.372322:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.372322:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.372324:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.372325:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.372325:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.372326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.372330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.372337:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.372340:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.372341:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.372342:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.372344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.372345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677474.373250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.373256:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.373259:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.373260:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.373262:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.373264:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.373266:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.373267:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.373268:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.373269:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.373270:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.373273:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.373275:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.373277:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.373278:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.373279:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.373280:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.373281:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.373282:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.373283:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.373283:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.373287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.373288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.373294:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.373294:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.373300:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677474.373301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.373301:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.373301:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.373304:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (22)--
00000800:00000200:25.2:1619677474.373305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (22)++
00000400:00000200:6.0:1619677474.373306:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.373309:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.373311:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.373313:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.373314:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.373314:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.373315:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.373317:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.373317:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.373754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.373757:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.373760:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.373762:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.373763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.373765:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.373766:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.373768:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.373769:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.373770:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.373771:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.373774:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.373776:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.373777:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.373778:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.373779:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.373781:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.373781:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.373782:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.373782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677474.373784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.2:1619677474.373787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.373791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.373794:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.373796:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.373798:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.373799:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:31.0:1619677474.373800:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677474.373800:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.373802:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.373802:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:21.0:1619677474.373802:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:30.0:1619677474.373803:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.373803:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.373805:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.373805:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.373805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:30.0:1619677474.373806:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.373807:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.373807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:31.0:1619677474.373811:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.373812:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.373814:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.373814:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.373815:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.373816:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.373817:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.373818:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.373819:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.373822:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.373822:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.373822:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.373823:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.373824:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.373825:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.373825:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.373826:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.373826:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.373827:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.373827:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.373828:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.373828:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677474.373829:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.373829:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.373830:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:9.2:1619677474.373830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.373831:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.373831:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.373835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.373840:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.373841:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.373842:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:24.0:1619677474.373843:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.373843:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.373844:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.373844:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.373845:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.373847:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.373848:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677474.373848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.373849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.374652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.374660:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.374663:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.374664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.374665:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.374668:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.374669:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.374670:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.374672:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.374672:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.374673:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.374676:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.374678:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.374681:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.374681:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.374682:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.374684:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.374684:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.374685:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.374685:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.374690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.374699:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.374702:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.374703:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.374703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.374705:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.374708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.375268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.375276:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.375279:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.375281:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.375283:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.375284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.375285:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.375286:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.375287:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.375288:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.375289:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.375292:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.375299:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.375301:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.375302:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.375302:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.375304:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.375304:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.375305:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.375306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.375310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.375317:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.375319:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.375320:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.375320:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.375322:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.375324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.375327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.375333:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.375336:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.375337:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.375337:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.375337:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.375338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.375432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.375440:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.375443:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.375444:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.375445:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.375448:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.375449:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.375451:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.375452:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.375453:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.375453:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.375457:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.375458:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.375461:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.375462:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.375462:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.375464:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.375465:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.375465:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.375466:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.375470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.375478:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.375481:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.375482:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.375482:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.375484:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.375487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.375703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.375710:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.375713:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.375714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.375716:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.375718:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.375719:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.375721:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.375722:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.375723:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.375724:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.375727:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.375728:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.375731:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.375731:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.375732:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.375734:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.375734:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.375735:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.375736:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.375737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.375740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.375742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.375747:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677474.375748:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677474.375749:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.375750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677474.375750:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677474.375751:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677474.375751:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.375752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.375754:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.375756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.375764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.375772:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.375775:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.375777:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.375779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.375780:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.375781:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.375782:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.375783:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.375784:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.375785:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.375788:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.375790:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.375792:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.375793:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.375794:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.375795:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.375796:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.375797:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.375798:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.375801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.375808:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.375810:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.375811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.375811:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.375814:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.375815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.375950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.375958:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.375961:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.375963:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.375965:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.375965:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.375966:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.375967:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.375968:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.375969:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.375970:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.375973:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.375975:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.375976:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.375977:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.375977:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.375979:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.375979:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.375981:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.375982:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.375983:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.375984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.375988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.375993:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.375994:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.375995:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.375996:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.375996:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.375997:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.375997:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.375998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.375999:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.376000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.376003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.376010:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.376013:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.376014:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.376014:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.376015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.376016:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.376242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.376245:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.376248:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.376250:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.376251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.376252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.376253:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.376255:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.376256:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.376257:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.376258:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.376261:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.376263:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.376265:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.376266:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.376267:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.376268:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.376269:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.376270:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.376271:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.376272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.376275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.376277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.376282:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677474.376283:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677474.376284:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.376285:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677474.376285:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677474.376286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677474.376286:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.376286:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.376288:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.376291:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.376465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677474.376471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677474.376473:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.376476:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.376477:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.376479:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.376480:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677474.376481:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.376483:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677474.376483:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.376484:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.376485:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.376486:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.376487:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.376487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677474.376487:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.376488:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677474.376488:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.376490:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.376492:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.376493:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.376493:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.376494:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.376495:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.376497:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.376498:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.376498:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.376499:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.376499:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.376500:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.376501:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.376501:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677474.376501:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677474.376502:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.376502:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677474.376503:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.376503:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677474.376504:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.376504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677474.376505:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.376505:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.376507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:25.2:1619677474.376510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677474.376510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.376514:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.376517:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:30.0:1619677474.376518:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:26.0:1619677474.376518:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677474.376518:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.376518:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.376519:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.376520:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:30.0:1619677474.376521:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:26.0:1619677474.376521:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677474.376522:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.376522:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677474.376522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:30.0:1619677474.376523:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.376525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.376526:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.376706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.376713:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.376716:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.376717:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.376718:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.376721:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.376722:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.376723:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.376725:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.376725:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.376726:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.376729:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.376731:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.376734:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.376735:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.376735:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.376737:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.376738:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.376739:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.376739:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.376743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.376751:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.376755:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.376755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.376756:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.376758:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.376761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.376816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677474.376824:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.376827:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677474.376828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677474.376829:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.376832:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.376834:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.376835:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.376837:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.376837:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.376838:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.376841:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.376843:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.376846:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.376846:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.376847:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.376849:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.376849:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.376850:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677474.376850:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677474.376855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.376863:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.376866:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.376867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.376868:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.376869:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.376873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.377253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.377260:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.377263:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.377265:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.377267:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.377267:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.377268:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.377269:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.377270:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.377271:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.377272:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.377275:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.377277:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.377279:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.377279:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.377280:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.377281:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.377282:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.377283:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.377283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.377287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.377294:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.377301:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.377302:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.377302:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.377304:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.377305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.377427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.377435:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.377437:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.377439:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.377441:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.377442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.377443:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.377444:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.377445:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.377446:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.377447:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.377450:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.377451:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.377454:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.377454:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.377455:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.377456:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.377457:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.377458:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.377460:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.377460:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.377460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:9.2:1619677474.377462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.377465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:21.2:1619677474.377466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:1.0:1619677474.377468:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:24.0:1619677474.377469:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:17.0:1619677474.377469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.377471:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.377472:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.377473:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.377473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:1.0:1619677474.377473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.377474:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.377475:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.377475:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677474.377476:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.377476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:1.0:1619677474.377476:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.377477:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677474.377478:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:1.0:1619677474.377478:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.377479:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.377480:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.377482:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.377484:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.377487:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.377487:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.377488:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.377490:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.377490:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.377491:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.377492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.377496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.377505:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.377507:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.377508:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.377509:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.377510:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.377512:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.377581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.377588:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.377591:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.377592:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.377594:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.377597:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.377598:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.377599:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.377601:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.377602:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.377603:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.377606:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.377608:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.377611:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.377612:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.377612:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.377614:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.377615:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.377615:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.377616:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.377620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.377628:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.377631:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.377632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.377632:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.377634:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.377638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.377825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.377829:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.377832:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.377834:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.377835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.377836:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.377838:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.377839:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.377841:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.377842:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.377843:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.377846:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.377848:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.377850:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.377851:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.377852:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.377853:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.377854:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.377855:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.377856:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.377857:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677474.377859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.377862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.377867:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.377868:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.377868:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.377868:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.377869:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.377870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.377870:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.377871:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.377872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.377873:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:5.2:1619677474.378026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.378032:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.378034:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.378035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.378037:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.378039:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.378041:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.378042:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.378043:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.378044:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.378045:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.378048:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.378050:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.378052:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.378053:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.378053:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.378055:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.378056:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.378056:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.378057:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.378061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.378069:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.378071:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.378072:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.378073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.378074:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.378077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.378635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:20.2:1619677474.378636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:22.0:1619677474.378640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.378643:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:18.0:1619677474.378643:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677474.378644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677474.378645:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.378646:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.378647:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:22.0:1619677474.378648:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.378649:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.378649:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.378650:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.378651:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.378652:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.378652:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677474.378654:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.378654:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.378655:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.378656:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.378657:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.378658:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.378661:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.378663:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:22.0:1619677474.378666:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.378668:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.378668:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.378669:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.378670:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677474.378671:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.378672:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677474.378672:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (23)++
00000800:00000200:18.0:1619677474.378672:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.378673:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:22.0:1619677474.378674:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.378674:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.378674:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:22.0:1619677474.378675:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.378676:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677474.378677:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677474.378677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677474.378679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.2:1619677474.378682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:2.0:1619677474.378686:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000800:00000200:6.0:1619677474.378688:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677474.378689:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:2.0:1619677474.378689:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000800:00000200:18.0:1619677474.378690:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000800:00000200:6.0:1619677474.378690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:2.0:1619677474.378690:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.378691:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000800:00000200:6.0:1619677474.378691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:18.0:1619677474.378692:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.378692:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:18.0:1619677474.378693:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.378694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.378695:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.378696:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.378702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.378709:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.378711:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.378712:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.378713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.378714:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.378714:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.379123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.379127:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.379129:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.379131:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.379132:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.379133:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.379134:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.379136:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.379137:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.379138:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.379139:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.379143:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.379144:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.379147:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.379147:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.379148:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.379150:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.379150:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.379151:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.379152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.379156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.379164:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.379166:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.379167:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.379167:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.379169:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.379172:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.379175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.379182:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.379185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.379185:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.379186:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.379187:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.379187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.379344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677474.379349:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.379352:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677474.379354:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.379354:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677474.379356:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.379358:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.379359:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.379360:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.379361:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.379362:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.379365:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.379367:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.379369:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.379370:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.379371:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.379372:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.379373:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.379374:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677474.379374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.379379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.379386:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.379389:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.379389:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.379391:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.379392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.379393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.379398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.379406:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.379408:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.379409:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.379410:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.379410:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.379411:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.379423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.379431:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.379434:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.379436:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.379439:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.379439:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.379440:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.379442:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.379443:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.379444:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.379445:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.379448:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.379450:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.379452:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.379453:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.379454:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.379455:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.379456:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.379457:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.379458:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.379461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.379469:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.379471:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.379472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.379472:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.379474:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.379476:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.379497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.379503:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.379505:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.379506:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.379506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.379507:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.379508:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.379530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677474.379534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677474.379537:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677474.379540:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677474.379540:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.379541:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677474.379542:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:8.0:1619677474.379542:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.379544:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.379545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677474.379546:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.379547:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.379547:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.379549:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677474.379549:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.379550:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.379551:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.379551:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.379552:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.379553:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.379553:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.379555:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.379560:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.379560:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.379563:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677474.379563:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.379566:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.379566:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.379567:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677474.379567:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.379568:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677474.379568:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.379570:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.379571:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677474.379571:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677474.379572:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677474.379572:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677474.379573:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.379573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677474.379573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.379576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.2:1619677474.379577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677474.379584:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.379585:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:22.0:1619677474.379587:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677474.379587:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:22.0:1619677474.379588:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.379588:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.379588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677474.379590:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:22.0:1619677474.379591:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677474.379591:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.379593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677474.379595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.379667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.379674:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.379677:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.379680:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.379681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.379682:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.379684:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.379685:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.379686:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.379687:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.379689:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.379692:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.379694:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.379696:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.379697:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.379698:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.379699:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.379700:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.379701:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.379702:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.379705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.379711:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.379714:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.379715:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.379715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.379716:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.379719:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.380070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.380076:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.380079:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.380080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.380082:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.380084:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.380086:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.380087:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.380088:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.380089:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.380091:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.380094:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.380096:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.380098:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.380099:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.380100:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.380101:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.380102:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.380102:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.380103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677474.380108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.380115:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.380118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.380118:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.380119:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.380121:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.380123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.380246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.380254:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.380257:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.380258:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.380260:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.380263:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.380264:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.380266:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.380267:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.380268:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.380269:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.380272:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.380272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8667000] (20)++
00000800:00000200:10.0:1619677474.380273:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.380276:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.380277:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.380278:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.380279:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.380280:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.380280:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.380280:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667000] (21)++
00000800:00000200:10.0:1619677474.380281:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.380283:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.67@o2ib3
00000800:00000200:24.0:1619677474.380284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667000] (22)--
00000400:00000200:8.0:1619677474.380285:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.67@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.380288:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.380290:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:8.0:1619677474.380291:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.380292:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677474.380297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:8.0:1619677474.380300:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.380301:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677474.380304:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.380305:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.380305:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:4.2:1619677474.380306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8667000] (21)++
00000800:00000200:22.0:1619677474.380308:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.380308:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677474.380309:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.380309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.380310:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:22.0:1619677474.380311:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.380311:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677474.380313:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677474.380313:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667000] (22)++
00000800:00000200:22.0:1619677474.380314:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677474.380315:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8667000] (23)++
00000800:00000200:8.0:1619677474.380315:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667000] (23)--
00000800:00000200:24.0:1619677474.380316:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.67@o2ib3
00000800:00000200:8.0:1619677474.380316:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667000] (23)--
00000800:00000200:8.0:1619677474.380317:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667000] (22)--
00000400:00000200:24.0:1619677474.380318:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.67@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677474.380319:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:24.0:1619677474.380321:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.380323:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677474.380324:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.380326:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.380327:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.380328:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.380328:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:24.0:1619677474.380330:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.67@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.380330:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.380331:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.380332:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:21.0:1619677474.380333:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.380334:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677474.380335:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.380335:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677474.380336:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.380336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677474.380338:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677474.380338:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8667000] (21)++
00000800:00000200:24.0:1619677474.380339:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8667000] (22)--
00000800:00000200:24.0:1619677474.380340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8667000] (21)--
00000800:00000200:21.2:1619677474.380341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677474.380344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677474.380349:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.380351:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677474.380352:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677474.380353:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:31.0:1619677474.380354:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.380354:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.380356:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.380356:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.380357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.380357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677474.380358:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677474.380358:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677474.380359:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.380360:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.380361:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.380362:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.380365:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.380367:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.380370:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.380370:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.380371:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.380372:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.380373:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.380374:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.380376:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.380377:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.380378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.380382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.380385:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.380388:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.380389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.380389:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.380390:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.380391:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.380392:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.380393:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677474.380393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.380394:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.380838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.380845:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.380848:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.380849:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.380851:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.380853:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.380854:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.380856:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.380857:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.380858:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.380859:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.380863:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.380865:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.380867:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.380868:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.380869:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.380871:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.380872:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.380873:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.380873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.380877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.380884:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.380887:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.380887:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.380889:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.380891:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.380892:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.380898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.380905:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.380908:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.380909:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.380910:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.380911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.380912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.380934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.380942:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.380945:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.380947:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.380949:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.380950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.380950:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.380952:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.380953:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.380954:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.380955:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.380958:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.380960:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.380962:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.380963:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.380963:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.380965:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.380965:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.380966:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.380967:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.380970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.380977:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.380979:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.380980:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.380981:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.380983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.380984:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.381277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.381285:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.381288:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.381289:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.381291:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.381299:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.381301:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.381302:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.381303:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.381304:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.381305:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.381307:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.381309:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.381312:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.381312:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.381313:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.381314:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.381315:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.381316:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.381316:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.381321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.381329:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.381332:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.381333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.381333:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.381335:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.381338:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.381340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.381346:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.381349:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.381350:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.381350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.381351:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.381352:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.381389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.381397:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.381400:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.381402:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.381404:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.381404:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.381405:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.381406:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.381408:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.381409:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.381410:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.381413:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.381414:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.381417:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.381418:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.381418:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.381420:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.381420:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.381421:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.381422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.381425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.381432:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.381435:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.381436:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.381436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.381438:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.381440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.381451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.381458:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.381461:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.381462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.381464:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.381467:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.381468:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.381469:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.381471:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.381471:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.381472:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.381475:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.381477:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.381479:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.381480:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.381481:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.381482:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.381483:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.381483:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.381484:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.381488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.381496:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.381498:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.381499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.381500:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.381502:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.381505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.381662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.381669:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.381672:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.381674:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.381676:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.381676:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.381678:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.381679:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.381680:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.381681:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.381682:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.381685:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.381687:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.381689:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.381690:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.381690:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.381692:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.381692:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.381693:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.381694:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.381697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.381704:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.381706:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.381707:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.381707:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.381709:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.381711:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.381721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.381727:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.381730:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.381730:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.381731:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.381732:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.381732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.382061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.382068:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.382071:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.382071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.382073:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.382076:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.382077:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.382079:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.382080:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.382081:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.382082:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.382085:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.382086:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.382089:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.382090:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.382090:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.382092:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.382092:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.382093:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.382093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.382098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.382106:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.382109:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.382110:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.382110:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.382112:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.382115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.382843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.382851:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.382854:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.382856:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.382858:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.382858:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.382860:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.382861:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.382863:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.382864:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.382864:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.382868:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.382869:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.382873:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.382873:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.382874:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.382876:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.382876:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.382877:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.382878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.382883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.382892:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.382894:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.382895:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.382897:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.382897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677474.382900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.382903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.382904:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.382906:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.382907:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.382908:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.382908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.382909:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.383090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f1800] (20)++
00000800:00000200:30.0:1619677474.383097:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)++
00000800:00000200:30.0:1619677474.383100:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.4.68@o2ib3
00000400:00000200:30.0:1619677474.383103:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.68@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.383104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)--
00000400:00000200:30.0:1619677474.383106:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.383107:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.383109:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.383110:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.383111:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.383112:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.383115:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.68@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.383116:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.383119:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.383120:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.383121:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.383123:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.383123:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f1800] (21)++
00000800:00000200:30.0:1619677474.383124:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f1800] (22)--
00000800:00000200:30.0:1619677474.383125:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)--
00000800:00000200:8.2:1619677474.383129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.383136:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.383139:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.383140:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.383142:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.383145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.383146:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.383200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677474.383207:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.383210:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677474.383211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677474.383213:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.383216:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.383218:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677474.383220:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.383221:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.383222:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.383223:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677474.383226:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.383228:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677474.383230:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.383231:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677474.383232:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.383233:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677474.383234:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.383235:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677474.383235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677474.383240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.383250:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.383254:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.383255:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.383256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.383259:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.383262:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.383410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.383419:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.383422:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.383424:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.383426:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.383427:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.383429:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.383430:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.383432:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.383433:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.383434:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.383437:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.383439:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.383442:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.383443:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.383443:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.383445:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.383446:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.383447:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.383447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.383447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:9.2:1619677474.383451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677474.383454:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.383457:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677474.383458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:8.0:1619677474.383458:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:6.0:1619677474.383460:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.383461:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.383462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.383462:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.383463:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.383464:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.383464:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677474.383466:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.383467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677474.383467:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.383468:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.383469:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677474.383473:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.383474:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677474.383477:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.383478:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677474.383479:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677474.383481:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677474.383481:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.383482:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.383483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677474.383487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.383493:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.383496:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.383497:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.383499:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.383500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.383503:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.383767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.383774:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.383777:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:20.0:1619677474.383780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.383781:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.383784:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.383787:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.383789:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.383791:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.383793:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.383795:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.383800:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.383803:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.383807:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.383808:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.383809:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.383812:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.383813:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.383814:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.383815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.383817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.383824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.383826:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.383827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.383828:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.383830:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.383832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.383922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677474.383930:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.383932:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677474.383935:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.383937:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.383937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677474.383939:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677474.383940:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.383941:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.383942:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.383943:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677474.383947:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.383948:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.383952:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.383952:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677474.383953:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677474.383955:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677474.383956:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.383957:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:31.0:1619677474.383959:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677474.383959:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:9.2:1619677474.383961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.383965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:18.0:1619677474.383970:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.383972:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:30.0:1619677474.383973:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000800:00000200:2.0:1619677474.383973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.383974:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.383976:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:30.0:1619677474.383976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:18.0:1619677474.383976:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.383978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:18.0:1619677474.383979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.384589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.384596:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.384599:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.384600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.384602:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.384604:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.384606:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.384607:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.384608:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.384609:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.384610:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.384613:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.384615:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.384617:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.384618:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.384619:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.384620:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.384621:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.384623:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.384624:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.384624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.384627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.384630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.384635:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.384637:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.384638:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.384639:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.384639:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.384640:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.384640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.384641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.384641:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.384644:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.384773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.384781:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.384784:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.384786:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.384789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.384789:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.384790:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.384791:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.384793:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.384794:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.384795:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.384798:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.384799:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.384802:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.384802:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.384803:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.384804:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.384805:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.384806:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.384808:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.384808:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.384810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.384814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.384817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.384819:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.384820:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.384820:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.384822:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.384822:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.384824:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.384824:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.384825:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.384826:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.384857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.384860:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.384862:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.384865:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.384865:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.384867:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.384868:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.384870:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.384871:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.384872:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.384873:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.384876:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.384878:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.384880:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.384880:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.384881:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.384883:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.384884:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.384884:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.384885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.384889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.384897:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.384899:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.384900:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.384901:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.384902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.384903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.385015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677474.385021:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.385023:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677474.385024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677474.385026:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.385028:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.385030:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677474.385031:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.385033:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.385034:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.385035:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677474.385038:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.385040:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677474.385043:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.385044:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677474.385044:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677474.385046:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677474.385047:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.385047:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.385048:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677474.385052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677474.385056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.385061:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.385062:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.385064:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.385064:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:31.0:1619677474.385065:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.385066:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677474.385068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.385069:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:30.0:1619677474.385069:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.385071:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.385071:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.385073:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.385074:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.385075:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.385076:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.385077:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.385080:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.385082:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.385085:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.385086:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.385086:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.385088:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.385088:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.385090:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.385091:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.385092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.385093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.385097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.385100:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.385102:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.385103:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.385103:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.385105:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.385105:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.385107:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677474.385107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.385108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.385109:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.385113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.385120:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.385122:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.385123:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.385123:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.385125:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.385125:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.385588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.385597:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.385599:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.385602:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.385604:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.385604:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.385605:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.385607:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.385608:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.385609:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.385610:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.385613:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.385614:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.385616:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.385617:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.385617:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.385619:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.385619:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.385620:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.385621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.385627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.385634:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.385637:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.385637:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.385639:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.385641:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.385642:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.385655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.385662:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.385664:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.385665:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.385666:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.385666:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.385667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.385700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.385708:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.385710:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.385711:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.385713:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.385716:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.385717:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.385718:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.385720:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.385721:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.385721:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.385724:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.385726:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.385729:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.385729:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.385730:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.385731:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.385732:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.385732:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.385733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.385738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.385745:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.385748:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.385749:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.385750:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.385752:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.385754:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.385824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.385830:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.385833:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.385834:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.385835:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.385838:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.385839:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.385840:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.385842:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.385843:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.385844:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.385847:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.385849:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.385852:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.385853:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.385854:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.385856:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.385856:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.385857:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.385857:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.385862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.385871:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.385875:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.385876:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.385879:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:12.0:1619677474.385879:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.385881:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.385882:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)++
00000800:00000200:13.0:1619677474.385882:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (24)--
00000800:00000200:12.0:1619677474.385882:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (24)--
00000800:00000200:13.0:1619677474.385884:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.385885:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677474.385990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.385996:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.385999:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.386000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.386002:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.386004:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.386006:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.386007:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.386008:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.386010:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.386010:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.386014:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.386015:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.386016:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.386017:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.386017:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.386019:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.386020:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.386021:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.386021:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.386025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.386033:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.386035:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.386036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.386036:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.386038:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.386039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.386269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.386275:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.386278:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.386279:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.386281:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.386283:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.386284:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.386286:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.386287:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.386288:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.386289:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.386292:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.386298:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.386301:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.386302:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.386302:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.386304:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.386304:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.386305:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.386305:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.386311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.386319:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.386321:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.386322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.386322:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.386325:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.386327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.386331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:21.2:1619677474.386336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677474.386337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.386340:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.386341:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.386341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.386342:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.386342:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.386344:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.386347:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.386349:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.386350:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.386351:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.386352:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.386353:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.386354:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.386355:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.386356:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.386360:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.386361:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.386363:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.386364:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.386365:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.386366:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.386366:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.386368:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.386368:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.386382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.386389:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.386391:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.386392:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.386394:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.386395:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.386396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.386642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.386650:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.386653:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.386654:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.386655:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.386658:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.386660:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.386661:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.386662:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.386663:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.386664:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.386667:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.386668:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.386671:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.386672:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.386672:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.386674:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.386674:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.386675:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.386677:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.386677:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.386680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.386683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.386687:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.386690:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.386690:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.386691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.386691:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.386692:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.386693:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677474.386693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.386693:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.386696:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.386796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.386799:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.386802:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.386804:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.386805:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.386806:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.386807:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.386809:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.386810:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.386811:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.386812:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.386816:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.386818:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.386820:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.386821:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.386822:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.386823:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.386824:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.386825:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.386825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.386829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.386836:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.386839:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.386840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.386841:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.386842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677474.386843:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.386845:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677474.386850:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.386852:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.386853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.386855:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.386858:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.386859:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.386861:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.386862:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.386863:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.386864:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.386867:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.386868:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.386871:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.386872:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.386873:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.386875:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.386875:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.386876:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.386876:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.386881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.386888:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.386891:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.386892:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.386892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.386894:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.386896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.386912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.386920:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.386923:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.386925:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.386927:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.386927:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.386928:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.386929:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.386931:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.386932:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.386933:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.386935:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.386937:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.386940:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.386940:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.386941:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.386942:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.386943:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.386944:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.386944:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.386948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.386955:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.386958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.386958:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.386959:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.386961:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.386962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.387082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.387089:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.387092:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.387094:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677474.387096:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.387096:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.387098:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:13.0:1619677474.387100:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.387101:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.387102:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.387103:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677474.387105:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.387107:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.387109:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.387110:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677474.387110:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.387112:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.387112:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.387113:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.387114:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677474.387117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.387124:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.387126:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.387127:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.387128:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.387130:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.387131:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.387410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.387418:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.387421:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.387423:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.387425:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.387426:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.387426:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.387427:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.387428:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.387429:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.387430:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.387433:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.387435:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.387437:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.387438:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.387439:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.387440:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.387441:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.387441:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.387442:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.387446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.387453:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.387456:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.387457:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.387457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.387459:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.387461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.387675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677474.387681:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.387683:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.387686:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.387686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.387689:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.387690:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.387692:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.387693:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.387694:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.387695:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.387699:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.387700:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.387703:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.387703:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.387704:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.387705:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.387706:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.387707:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677474.387707:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677474.387712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.387719:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.387722:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.387723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.387723:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.387725:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.387727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.387730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.387737:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.387739:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.387740:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.387740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.387741:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.387742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.388340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.388346:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.388349:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.388350:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.388352:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.388354:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.388356:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.388357:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.388358:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.388359:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.388360:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.388363:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.388365:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.388368:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.388368:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.388369:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.388370:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.388371:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.388372:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.388372:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.388377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.388385:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.388387:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.388388:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.388390:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.388391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.388393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.388607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.388615:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.388617:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.388620:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.388621:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.388623:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.388624:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.388625:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.388626:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.388628:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.388629:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.388632:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.388634:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.388636:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.388637:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.388638:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.388639:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.388640:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.388641:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.388641:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.388646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.388652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.388655:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.388656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.388656:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.388658:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.388659:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.388699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.388705:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.388708:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.388709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.388710:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.388713:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.388715:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677474.388716:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.388717:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.388718:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.388719:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677474.388722:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.388724:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677474.388727:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.388728:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677474.388729:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677474.388730:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677474.388731:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.388731:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.388732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.388736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.388743:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.388746:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677474.388746:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.388747:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.388749:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.388752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.389039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.389046:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.389049:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.389051:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.389053:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.389053:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.389055:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.389056:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.389057:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.389058:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.389059:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.389063:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.389065:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.389067:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.389067:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.389068:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.389069:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.389070:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.389071:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.389073:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.389074:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.389075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.389079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.389083:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.389085:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.389086:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.389087:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.389087:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.389088:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.389088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.389089:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.389090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.389091:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.389200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.389207:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.389210:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.389212:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.389214:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.389214:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.389215:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677474.389216:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.389218:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.389219:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.389219:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677474.389222:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.389224:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677474.389227:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.389228:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677474.389228:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677474.389230:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677474.389230:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.389231:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.389232:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.389235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.389242:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.389244:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.389245:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.389245:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.389247:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.389249:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677474.389260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677474.389266:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.389268:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677474.389269:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.389269:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.389270:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.389270:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677474.389477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.389485:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.389487:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.389488:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.389489:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.389492:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.389493:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.389494:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.389496:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.389496:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.389497:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.389500:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.389502:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.389504:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.389505:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.389505:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.389506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677474.389507:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.389507:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.389508:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.389509:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.389513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677474.389513:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.389516:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.389517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.389518:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.389521:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.389521:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:24.0:1619677474.389523:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677474.389523:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:24.0:1619677474.389524:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.389524:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.389525:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.389526:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.389526:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.389527:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677474.389528:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.389529:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:24.0:1619677474.389530:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.389532:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.389534:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.389535:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.389536:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.389538:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.389538:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.389539:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.389539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.389554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.389563:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.389566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.389566:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.389567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.389569:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.389571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.389810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.389819:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.389822:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.389824:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.389826:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.389827:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.389827:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.389829:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.389830:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.389831:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.389832:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.389835:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.389837:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:4.2:1619677474.389839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:30.0:1619677474.389840:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.389841:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.389841:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.389843:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.389844:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.389844:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:30.0:1619677474.389845:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.389845:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677474.389847:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.389849:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:8.2:1619677474.389849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.389850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.389851:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.389852:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:21.0:1619677474.389853:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677474.389853:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.389854:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.389854:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.389856:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.389856:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.389857:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.389858:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.389861:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.389861:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.389862:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677474.389863:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.389865:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.389866:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.389867:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.389868:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.389869:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.389870:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.389871:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.389872:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677474.389874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.389877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677474.389881:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.389881:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.389883:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.389883:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677474.389884:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677474.389884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:22.0:1619677474.389885:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.389885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.389887:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.389890:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.390064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.390067:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.390070:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.390072:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.390073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.390074:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.390075:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.390076:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.390078:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.390079:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.390080:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.390084:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.390085:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.390088:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.390089:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.390089:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.390091:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.390092:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.390093:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.390093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.390097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.390104:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.390107:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.390108:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.390109:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.390111:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.390113:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.390501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.390507:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.390509:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.390512:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.390512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.390515:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.390516:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.390518:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.390519:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.390520:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.390521:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.390524:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.390526:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.390528:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.390529:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.390529:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.390531:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.390532:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.390532:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.390533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.390538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.390544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677474.390548:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.390550:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.390551:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.390553:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677474.390553:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.390553:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.390555:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677474.390556:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.2:1619677474.390556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:10.0:1619677474.390557:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.390559:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.390562:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.390563:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.390564:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.390565:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.390565:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:26.0:1619677474.390566:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.390567:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.390567:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.390568:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.390568:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.390569:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.390570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:26.0:1619677474.390571:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.390573:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.390576:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.390577:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.390578:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.390579:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.390580:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.390580:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.390581:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.390586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.390595:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.390598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.390598:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.390599:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.390601:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.390604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.390744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677474.390747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.390752:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.390754:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677474.390755:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677474.390756:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.390757:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.390759:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.390759:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.390761:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.390762:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.390763:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.390765:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.390765:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.390767:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.390768:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.390769:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.390769:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.390771:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.390771:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.390772:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.390773:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.390776:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.390777:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.390778:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.390779:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.390779:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.390780:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.390780:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.390781:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.390781:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.390782:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.390782:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.390783:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.390783:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.390784:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.390784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.390785:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.390786:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.390786:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.390788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:9.2:1619677474.390791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.390795:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.390797:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:8.0:1619677474.390797:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:30.0:1619677474.390798:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.390800:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.390800:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677474.390801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677474.390801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.390801:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.390802:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:8.0:1619677474.390803:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.390805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.390812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.390818:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.390820:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.390821:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.390821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.390822:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.390823:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.391140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.391147:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.391150:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.391151:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.391152:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.391155:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.391156:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.391157:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.391159:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.391160:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.391160:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.391163:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.391165:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.391168:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.391168:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.391169:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.391171:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.391171:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.391172:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.391173:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.391174:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.391177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.391180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.391184:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.391187:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.391187:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.391187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.391188:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.391189:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.391190:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677474.391190:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.391191:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.391192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.391395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.391403:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.391406:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.391407:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.391408:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.391411:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.391412:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.391413:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.391415:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.391415:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.391416:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.391419:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.391420:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.391422:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.391423:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.391423:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.391425:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.391425:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.391426:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.391426:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.391431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.391437:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.391440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.391440:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.391440:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.391442:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.391444:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.391543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.391549:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.391552:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.391553:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.391556:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.391559:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.391560:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.391561:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.391563:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.391564:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.391565:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.391568:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.391569:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.391572:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.391572:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.391573:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.391575:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.391575:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.391576:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.391576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.391581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.391589:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.391592:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.391593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.391593:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.391595:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.391597:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.391688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f1800] (20)++
00000800:00000200:21.2:1619677474.391693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.391697:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)++
00000800:00000200:30.0:1619677474.391699:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.68@o2ib3
00000800:00000200:31.0:1619677474.391701:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677474.391701:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.68@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.391703:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.391704:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.391705:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.391705:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.391706:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.391708:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.391709:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.391710:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.391712:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.391712:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.68@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.391713:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.391714:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.391714:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.391715:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.391716:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.391717:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.391720:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.391722:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.391723:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.391723:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.391724:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.391724:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.391725:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.391725:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.391726:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.391727:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.391727:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.391727:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f1800] (22)++
00000800:00000200:31.0:1619677474.391728:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.391729:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.391729:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f1800] (23)--
00000800:00000200:30.0:1619677474.391730:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)--
00000800:00000200:31.0:1619677474.391731:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)++
00000800:00000200:31.0:1619677474.391732:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.68@o2ib3
00000800:00000200:30.0:1619677474.391732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.391733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:9.2:1619677474.391733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:31.0:1619677474.391734:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.68@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.391734:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)--
00000800:00000200:30.0:1619677474.391735:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:31.0:1619677474.391736:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.391736:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.391737:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.391738:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677474.391739:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96504691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.391739:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.391739:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:31.0:1619677474.391741:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.391741:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.391741:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:31.0:1619677474.391742:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677474.391742:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.391743:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.391744:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:24.0:1619677474.391744:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.391745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:31.0:1619677474.391746:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.68@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.391747:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.391748:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677474.391750:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.391751:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677474.391751:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:25.2:1619677474.391751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.391753:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677474.391754:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f1800] (21)++
00000800:00000200:31.0:1619677474.391755:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f1800] (22)--
00000800:00000200:31.0:1619677474.391755:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)--
00000800:00000200:31.0:1619677474.391756:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.391758:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.391758:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.391759:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (21)++
00000800:00000200:9.2:1619677474.391759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.391760:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.391760:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.391764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f1800] (20)++
00000800:00000200:18.0:1619677474.391768:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:30.0:1619677474.391771:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)++
00000800:00000200:18.0:1619677474.391771:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.391772:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.391772:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677474.391775:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.68@o2ib3
00000400:00000200:18.0:1619677474.391775:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.391778:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)--
00000400:00000200:30.0:1619677474.391778:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.68@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.391779:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.391780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000400:00000200:30.0:1619677474.391781:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.391783:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.391784:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.391785:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.391786:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.391787:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.391787:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.391790:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677474.391791:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.68@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.391791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.391791:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.391792:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677474.391793:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.391793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:30.0:1619677474.391796:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.391797:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.391798:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.391799:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.391800:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f1800] (21)++
00000800:00000200:30.0:1619677474.391801:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a8677f1800] (22)++
00000800:00000200:30.0:1619677474.391803:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f1800] (23)--
00000800:00000200:30.0:1619677474.391804:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)--
00000800:00000200:25.2:1619677474.391805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.391809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f1800] (21)++
00000800:00000200:22.0:1619677474.391813:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.391816:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.391816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.391817:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)++
00000400:00000200:22.0:1619677474.391817:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.391819:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a8677f1800] (23)--
00000400:00000200:22.0:1619677474.391819:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.391820:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)--
00000800:00000200:31.0:1619677474.391821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)--
00000800:00000200:22.0:1619677474.391822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.391832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.391839:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.391842:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.391843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.391843:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.391844:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.391845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.391857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f1800] (20)++
00000800:00000200:31.0:1619677474.391864:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)++
00000800:00000200:31.0:1619677474.391867:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.68@o2ib3
00000400:00000200:31.0:1619677474.391869:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.68@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.391871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)--
00000400:00000200:31.0:1619677474.391872:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.391874:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.391875:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.391876:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.391877:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.391878:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.391881:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.68@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.391882:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.391885:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.391885:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.391886:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.391887:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.391888:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f1800] (21)++
00000800:00000200:31.0:1619677474.391889:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f1800] (22)--
00000800:00000200:31.0:1619677474.391890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)--
00000800:00000200:9.2:1619677474.391893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.391899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.391902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.391902:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.391903:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.391904:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.391907:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.391916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.391921:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.391924:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.391925:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.391925:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.391925:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.391926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.392040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f1800] (20)++
00000800:00000200:30.0:1619677474.392047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)++
00000800:00000200:30.0:1619677474.392050:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.68@o2ib3
00000400:00000200:30.0:1619677474.392052:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.68@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.392054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)--
00000400:00000200:30.0:1619677474.392054:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.392055:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677474.392056:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.392058:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.392059:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.392059:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677474.392062:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.68@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.392064:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677474.392066:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.392067:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:30.0:1619677474.392067:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:30.0:1619677474.392069:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:30.0:1619677474.392069:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f1800] (21)++
00000800:00000200:30.0:1619677474.392070:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f1800] (22)--
00000800:00000200:30.0:1619677474.392071:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)--
00000800:00000200:25.2:1619677474.392074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.2:1619677474.392077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f1800] (20)++
00000800:00000200:12.0:1619677474.392082:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.392084:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:31.0:1619677474.392085:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)++
00000400:00000200:12.0:1619677474.392085:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.392087:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.68@o2ib3
00000400:00000200:12.0:1619677474.392087:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.392089:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.68@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.392089:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.392090:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:31.0:1619677474.392091:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.392091:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)--
00000400:00000200:31.0:1619677474.392093:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677474.392094:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.392095:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.392096:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.392097:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677474.392099:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.68@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.392101:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677474.392103:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.392104:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:31.0:1619677474.392104:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:31.0:1619677474.392106:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677474.392107:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f1800] (21)++
00000800:00000200:31.0:1619677474.392108:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f1800] (22)--
00000800:00000200:31.0:1619677474.392108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)--
00000800:00000200:9.2:1619677474.392112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:5.2:1619677474.392112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:26.0:1619677474.392118:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:1.0:1619677474.392118:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:26.0:1619677474.392121:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:1.0:1619677474.392121:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000400:00000200:26.0:1619677474.392122:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.392122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:10.0:1619677474.392122:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.392125:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.392125:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.392128:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.392129:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:1.0:1619677474.392130:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677474.392131:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.392132:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.392133:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.392134:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:9.2:1619677474.392135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:1.0:1619677474.392138:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.392140:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677474.392141:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677474.392143:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.392144:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.392144:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:1.0:1619677474.392144:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677474.392144:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677474.392145:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.392146:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677474.392146:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677474.392147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:1.0:1619677474.392147:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677474.392148:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677474.392149:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:25.2:1619677474.392152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.392159:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.392162:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.392163:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.392165:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.392166:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.392168:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.392175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.392182:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.392184:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.392185:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.392186:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.392186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.392188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.392662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.392669:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.392672:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.392672:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.392675:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.392676:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.392678:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.392679:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.392680:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.392681:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.392683:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.392685:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.392687:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.392689:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.392690:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.392690:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.392692:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.392693:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.392694:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.392695:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.392695:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.392699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.392702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.392707:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.392709:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.392709:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.392709:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.392709:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.392710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.392711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.392711:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.392712:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.392713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.392720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.392728:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.392730:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.392731:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.392731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.392732:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.392733:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.392827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.392836:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.392839:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.392841:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.392843:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.392844:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.392845:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.392846:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.392847:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.392848:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.392849:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.392853:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.392854:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.392857:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.392858:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.392859:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.392860:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.392861:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.392862:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.392864:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.392865:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.392866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.392870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.392873:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.392876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.392876:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.392877:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.392878:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677474.392879:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.392880:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.392881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677474.392881:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.392882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.392884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.392888:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.392891:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.392893:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.392894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.392895:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.392896:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.392898:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.392899:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.392900:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.392901:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.392904:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.392906:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.392908:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.392909:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.392910:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.392912:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.392913:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.392913:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.392914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.392917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.392923:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.392926:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.392927:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.392927:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.392929:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.392932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.393087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.393092:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.393095:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.393096:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.393097:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.393100:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.393101:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.393103:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.393104:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.393106:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.393107:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.393110:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.393112:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.393114:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.393115:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.393115:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.393117:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.393118:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.393119:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.393119:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.393133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.393142:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.393145:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.393146:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.393146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.393148:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.393150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.393561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677474.393570:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.393573:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677474.393575:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.393577:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.393578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677474.393579:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677474.393580:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.393581:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.393583:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.393583:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677474.393587:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.393589:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677474.393592:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.393593:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677474.393593:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677474.393595:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677474.393596:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.393596:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677474.393597:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677474.393602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.393610:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.393613:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.393614:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.393614:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.393617:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.393620:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.393622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.393629:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.393631:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.393632:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.393632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.393633:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.393634:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.393690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f1800] (20)++
00000800:00000200:30.0:1619677474.393698:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)++
00000800:00000200:30.0:1619677474.393701:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.68@o2ib3
00000400:00000200:30.0:1619677474.393703:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.68@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.393704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)--
00000400:00000200:30.0:1619677474.393706:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.393708:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677474.393709:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.393710:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.393711:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.393712:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:30.0:1619677474.393715:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.68@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.393717:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.393720:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.393721:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677474.393722:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677474.393723:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677474.393724:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a8677f1800] (21)++
00000800:00000200:30.0:1619677474.393725:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a8677f1800] (22)++
00000800:00000200:30.0:1619677474.393727:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a8677f1800] (23)--
00000800:00000200:30.0:1619677474.393728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)--
00000800:00000200:25.2:1619677474.393730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.393734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a8677f1800] (21)++
00000800:00000200:31.0:1619677474.393741:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)++
00000800:00000200:21.0:1619677474.393741:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677474.393744:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a8677f1800] (23)--
00000800:00000200:21.0:1619677474.393744:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677474.393745:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (22)--
00000400:00000200:21.0:1619677474.393745:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.393746:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a8677f1800] (21)--
00000400:00000200:21.0:1619677474.393746:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.393747:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.393749:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.393750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.393750:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (23)++
00000800:00000200:20.0:1619677474.393751:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.393752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.393753:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.393820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.393825:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.393827:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.393830:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.393830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.393832:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.393833:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.393835:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.393836:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.393837:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.393838:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.393841:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.393842:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.393845:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.393846:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.393846:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.393848:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.393849:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.393850:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.393851:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.393851:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.2:1619677474.393854:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677474.393854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677474.393861:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677474.393861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677474.393862:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:31.0:1619677474.393864:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677474.393864:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677474.393864:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.393865:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.393866:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677474.393866:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:31.0:1619677474.393867:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.393867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677474.393868:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677474.393868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677474.393868:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.393870:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.393871:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:31.0:1619677474.393872:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.393873:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.393874:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.393875:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.393876:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.393879:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.393881:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.393884:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.393885:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.393885:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.393887:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.393888:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.393889:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.393890:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.393891:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.393893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.393896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.393901:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.393903:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677474.393904:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677474.393904:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.393905:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.393906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.393907:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.393908:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.393909:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.393909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.394176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.394184:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.394187:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.394188:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.394190:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.394192:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.394193:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.394195:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.394196:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.394197:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.394198:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.394201:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.394203:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.394205:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.394206:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.394207:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.394209:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.394209:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.394211:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.394212:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.394212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677474.394215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.394218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677474.394222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.394224:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.394225:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.394225:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:21.0:1619677474.394227:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.394227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.394227:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.394228:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.394229:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677474.394231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.394236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.394243:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.394246:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.394247:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.394248:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.394249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.394250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.394398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.394406:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.394409:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.394410:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.394412:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.394415:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.394416:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.394418:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.394419:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.394420:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.394421:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.394424:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.394426:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.394429:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.394429:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.394430:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.394432:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.394432:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.394433:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.394433:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.394438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.394448:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.394450:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.394451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.394452:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.394454:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.394456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.394559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.394567:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.394570:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.394572:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.394575:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.394575:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.394577:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.394578:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.394580:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.394581:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.394582:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.394585:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.394586:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.394590:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.394590:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.394591:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.394593:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.394594:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.394595:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.394595:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.394599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.394607:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.394609:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.394610:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.394612:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.394613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.394615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.394927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.394935:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.394938:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.394941:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.394942:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.394943:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.394945:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.394946:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.394947:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.394948:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.394949:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.394952:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.394954:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.394956:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.394957:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.394958:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.394959:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.394960:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.394961:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.394961:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.394965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.394971:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.394974:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.394975:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.394975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.394977:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.394979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.394985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.394991:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.394993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.394993:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.394994:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.394995:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.394995:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.395332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.395339:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.395342:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.395343:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.395345:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.395347:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.395349:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.395350:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.395351:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677474.395351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677474.395352:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.395353:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.395357:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.395359:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677474.395359:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.395361:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.395362:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677474.395362:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.395363:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677474.395363:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.395365:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.395365:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.395365:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.395366:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.395366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:26.0:1619677474.395368:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.395370:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.395371:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677474.395371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:26.0:1619677474.395372:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.395373:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.395374:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.395378:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.395379:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:26.0:1619677474.395380:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.395381:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.395382:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.395383:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.395383:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.395384:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677474.395384:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.395384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:26.0:1619677474.395386:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.395386:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677474.395386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677474.395387:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.395387:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.395392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.395401:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.395403:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.395404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.395405:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.395407:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.395410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.395478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.395486:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.395489:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.395492:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.395494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.395495:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.395496:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.395497:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.395499:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.395500:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.395501:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.395504:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.395505:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.395508:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.395509:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.395509:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.395511:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.395511:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.395512:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.395512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.395516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.395522:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.395525:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.395526:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.395526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.395528:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.395530:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.395894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.395901:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.395903:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.395904:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.395906:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.395909:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.395910:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.395911:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.395913:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.395913:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.395914:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.395917:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.395919:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.395922:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.395922:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.395923:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.395925:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.395925:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.395926:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.395927:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.395931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.395938:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.395941:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.395942:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.395942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.395944:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.395947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.395952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.395959:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.395961:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.395962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.395963:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.395963:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.395964:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.396250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.396255:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.396258:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.396259:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.396261:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.396263:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.396265:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.396266:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.396268:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.396268:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.396270:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.396273:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.396274:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.396277:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.396277:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.396278:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.396280:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.396280:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.396282:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.396283:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.396283:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.396286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.396288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.396293:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.396294:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.396298:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.396299:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.396300:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.396301:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.396302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.396302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.396304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.396307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.396386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.396395:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.396398:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.396400:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.396402:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.396402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.396403:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.396404:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.396406:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.396407:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.396407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677474.396408:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.396412:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.396413:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677474.396415:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677474.396416:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.396416:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.396417:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677474.396417:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.396418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677474.396419:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.396419:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.396420:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677474.396420:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.396421:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:26.0:1619677474.396423:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.396424:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.396425:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677474.396425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:26.0:1619677474.396426:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.396427:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.396428:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.396431:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.396432:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:26.0:1619677474.396432:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.396434:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:26.0:1619677474.396434:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677474.396435:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.396435:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.396436:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:30.0:1619677474.396437:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.396437:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.396438:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677474.396439:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.396439:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677474.396439:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.396440:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.396441:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.396443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.396446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.396449:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.396452:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.396453:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:10.0:1619677474.396453:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677474.396453:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.396455:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677474.396455:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.396456:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.396457:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.396457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.396512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.396515:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.396517:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.396519:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.396520:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.396521:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.396523:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.396524:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.396526:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.396527:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.396528:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.396531:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.396533:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.396535:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.396536:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.396536:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.396538:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.396539:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.396539:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.396540:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.396544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.396552:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.396554:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.396554:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.396556:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.396557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.396558:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.396580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.396588:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.396591:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.396593:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.396595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.396595:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.396596:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.396597:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.396599:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.396600:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.396601:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.396604:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.396605:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.396608:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.396609:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.396609:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.396611:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.396612:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.396613:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.396613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.396616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.396622:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.396625:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.396626:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.396626:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.396628:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.396630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.396773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.396780:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.396783:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.396784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.396785:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.396787:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.396789:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677474.396790:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.396791:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.396792:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.396793:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677474.396795:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.396797:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.396799:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.396799:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.396800:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.396801:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.396801:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.396802:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.396803:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.396807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.396813:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.396815:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.396815:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.396816:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.396817:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.396818:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.397275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.397279:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.397281:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.397283:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.397284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.397286:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.397287:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.397288:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.397290:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.397291:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.397292:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.397302:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.397304:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.397306:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.397307:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.397308:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.397309:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.397310:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.397310:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.397311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.397316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.397324:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.397327:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.397328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.397328:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.397329:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.397332:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.397795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.397804:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.397807:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.397809:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.397811:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.397811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.397813:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.397814:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.397815:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.397816:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.397817:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.397820:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.397821:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.397824:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.397825:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.397826:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.397827:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.397828:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.397829:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.397831:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.397831:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.397833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.397837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.397839:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.397842:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.397843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.397843:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.397845:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.397845:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.397847:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.397847:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.397848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.397849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.397856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.397864:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.397866:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.397867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.397869:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.397871:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.397872:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.397873:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.397875:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.397875:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.397876:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.397879:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.397881:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.397884:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.397885:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.397885:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.397887:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.397888:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.397888:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.397889:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.397893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.397901:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.397904:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.397905:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.397906:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.397907:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.397909:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.398419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.398426:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.398429:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.398430:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.398432:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.398434:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.398436:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.398437:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.398439:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.398440:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.398441:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.398445:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.398446:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.398449:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.398450:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.398451:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.398452:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.398453:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.398454:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.398455:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.398455:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.398459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677474.398461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.398466:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677474.398467:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:2.0:1619677474.398468:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:30.0:1619677474.398469:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:2.0:1619677474.398469:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677474.398470:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.398470:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677474.398472:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.398474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.398475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.398594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.398602:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.398605:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.398607:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.398608:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.398609:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.398611:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.398612:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.398613:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.398614:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.398615:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.398619:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.398620:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.398623:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.398624:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.398624:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.398626:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.398627:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.398628:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.398628:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.398632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.398638:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.398640:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.398641:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.398642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.398643:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.398645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.399149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.399157:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.399159:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.399161:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.399163:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.399163:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.399164:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.399165:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.399167:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.399168:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.399168:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.399171:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.399173:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.399174:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.399175:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.399176:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.399177:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.399177:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.399178:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.399179:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.399183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.399189:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.399191:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.399192:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.399194:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.399195:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.399197:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.399260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.399264:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:4.2:1619677474.399264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677474.399268:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.2:1619677474.399269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000400:00000200:20.0:1619677474.399270:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.399272:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677474.399273:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.399274:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:24.0:1619677474.399275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677474.399275:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:20.0:1619677474.399275:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.399276:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.399276:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677474.399279:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:20.0:1619677474.399279:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.399279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:8.0:1619677474.399280:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.399281:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.399282:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.399282:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677474.399283:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.399285:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.399285:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.399286:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.399287:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677474.399287:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.399288:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.399289:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.399289:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.399290:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.399290:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:8.0:1619677474.399290:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.399291:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.399293:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677474.399293:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.399294:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:22.0:1619677474.399302:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.399302:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677474.399304:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677474.399304:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.399304:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:8.0:1619677474.399304:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.399305:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.399306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677474.399306:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.399308:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.399309:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:25.2:1619677474.399310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677474.399310:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.0:1619677474.399310:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.399311:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.399311:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.399312:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:22.0:1619677474.399312:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.399313:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:8.0:1619677474.399313:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.399314:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.0:1619677474.399314:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.399315:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.399315:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677474.399318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:9.2:1619677474.399318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:25.2:1619677474.399320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.399321:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.399322:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.399324:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.399326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677474.399326:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:6.0:1619677474.399327:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.399328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:10.0:1619677474.399328:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.399329:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.399330:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.399330:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.399331:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.399331:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.399332:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.399334:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.399337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677474.399337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.399338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.399346:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.399348:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.399349:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.399349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.399350:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.399351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.399854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.399862:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.399865:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.399865:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.399867:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.399870:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.399872:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677474.399873:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.399875:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.399876:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.399877:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677474.399880:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.399881:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677474.399884:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.399884:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:4.2:1619677474.399884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:26.0:1619677474.399885:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677474.399886:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677474.399887:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.399887:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.399888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.0:1619677474.399891:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:25.2:1619677474.399892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677474.399893:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.399896:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.399896:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.399898:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.399899:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:21.0:1619677474.399900:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677474.399901:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:21.0:1619677474.399901:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.399902:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.399902:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.399904:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.399905:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.399905:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677474.399905:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.399908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:21.0:1619677474.399909:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.399911:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.399913:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.399914:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.399914:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.399916:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.399917:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.399917:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.399918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.399922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.399930:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.399932:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.399933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.399933:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.399936:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.399938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.400220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.400228:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.400230:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.400231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.400233:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.400234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:10.0:1619677474.400236:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.400237:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.400239:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.400240:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.400241:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:10.0:1619677474.400241:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.400242:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.400244:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:10.0:1619677474.400246:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.400247:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.400248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:10.0:1619677474.400248:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.400249:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.400251:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:10.0:1619677474.400251:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677474.400252:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.400252:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:30.0:1619677474.400253:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.400253:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677474.400254:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.400254:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677474.400255:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677474.400255:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.400256:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.400256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:30.0:1619677474.400258:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.400259:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:25.2:1619677474.400260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.400262:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.400263:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.400263:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.400265:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.400265:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.400266:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.400267:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.400268:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.400269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.400270:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:9.2:1619677474.400270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:22.0:1619677474.400271:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.400271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.400274:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.400274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.400277:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677474.400277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677474.400279:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.400280:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.400280:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.400282:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.400283:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.400284:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677474.400285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.400286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.400286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.400468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.400477:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.400479:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.400481:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.400484:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.400484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.400485:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.400486:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.400487:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.400488:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.400489:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.400493:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.400494:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.400496:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.400497:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.400497:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.400499:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.400499:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.400500:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.400500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.400505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.400512:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.400514:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.400515:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.400516:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.400518:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.400519:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.400522:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.400523:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.400523:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.400524:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.400525:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.400716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.400724:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.400727:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.400728:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.400729:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.400732:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.400734:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.400735:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.400736:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.400737:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.400738:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.400741:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.400743:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.400746:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.400747:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.400747:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.400749:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.400749:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.400750:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.400750:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.400755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.400763:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.400765:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.400766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.400767:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.400769:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.400771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.400843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.400846:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.400848:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.400851:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.400852:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.400853:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.400854:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.400856:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.400857:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.400858:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.400859:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.400862:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.400864:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.400866:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.400867:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.400867:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.400869:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.400870:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.400870:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.400871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.400876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.400884:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.400887:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.400888:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.400889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.400890:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.400893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677474.401047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.401053:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.401055:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.401056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.401058:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.401060:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.401061:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.401062:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.401064:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.401065:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.401066:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.401069:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.401070:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.401073:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.401074:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.401074:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.401076:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.401076:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.401077:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.401077:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.401082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.401089:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.401092:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.401092:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.401093:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.401095:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.401097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.401215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.401222:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.401225:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.401226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.401228:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.401230:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.401232:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.401233:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.401234:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.401235:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.401236:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.401240:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.401241:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.401244:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.401244:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.401245:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.401246:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.401247:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.401248:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.401248:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.401252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.401259:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.401261:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.401262:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.401262:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.401264:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.401266:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.401271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.401277:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.401280:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.401280:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.401281:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.401282:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.401282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.401554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.401560:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.401562:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.401563:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.401565:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.401568:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.401569:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.401571:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.401572:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.401573:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.401574:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.401577:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.401579:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.401582:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.401582:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.401583:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.401584:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.401585:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.401586:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.401587:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.401588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.401592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.401593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.401599:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677474.401601:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.401601:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677474.401602:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.401602:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677474.401604:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.401605:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.401607:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.401608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.401609:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.401630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:21.2:1619677474.401636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677474.401638:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.401641:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.401642:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:30.0:1619677474.401644:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677474.401644:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.401646:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677474.401646:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.401648:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.401649:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.401649:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.401650:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.401651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677474.401651:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.401652:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.401652:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.401654:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.401654:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.401655:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.401656:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.401657:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.401657:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.401660:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.401661:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.401663:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.401663:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.401664:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.401665:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.401665:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.401666:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.401666:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.401667:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.401667:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677474.401668:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.401668:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:30.0:1619677474.401669:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.401669:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.401670:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677474.401670:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.401670:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677474.401673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:9.2:1619677474.401674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.401677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.0:1619677474.401681:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.0:1619677474.401682:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677474.401683:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.401684:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677474.401684:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677474.401684:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:8.0:1619677474.401684:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.401685:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.401687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.401687:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.401687:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:8.0:1619677474.401687:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.401688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677474.401688:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.0:1619677474.401689:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677474.401691:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.401932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.401940:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.401942:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.401943:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.401945:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.401948:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.401950:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.401951:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.401952:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.401953:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.401954:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.401957:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.401959:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.401961:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.401962:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.401963:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.401964:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.401965:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.401965:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.401966:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.401970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.401978:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.401980:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.401981:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.401983:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.401985:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.401986:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.401995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677474.402001:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.402004:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.402004:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.402005:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.402006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.402006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.402271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.402278:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.402281:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.402283:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.402285:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.402285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.402287:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.402288:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.402289:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.402290:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.402291:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.402300:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.402301:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.2:1619677474.402301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677474.402304:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.402304:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.402305:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.402306:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.402307:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.402308:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.402308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:18.0:1619677474.402308:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.402312:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.402312:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.402313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:18.0:1619677474.402315:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.402318:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.402319:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:18.0:1619677474.402319:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.402320:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.402322:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:18.0:1619677474.402322:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.402322:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.402323:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.402323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:18.0:1619677474.402324:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.402326:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.402327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:18.0:1619677474.402327:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.402329:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.402330:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677474.402330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677474.402331:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.402332:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.402334:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.402334:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.402335:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.402336:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677474.402336:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.402339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.402339:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:9.2:1619677474.402340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.402340:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.402341:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.402341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677474.402347:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.402349:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.402349:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.402350:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.402351:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.402353:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.402719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677474.402725:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.402728:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.402730:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.402730:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.402732:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.402734:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677474.402735:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.402736:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.402738:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.402739:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677474.402742:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.402744:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677474.402747:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.402748:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677474.402749:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677474.402751:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677474.402751:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.402752:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677474.402753:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:9.2:1619677474.402757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.402764:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.402766:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.402767:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.402768:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.402770:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.402773:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.403160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.403164:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.403167:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.403169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.403169:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.403171:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.403173:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.403174:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.403175:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.403176:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.403177:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.403181:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.403182:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.403184:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.403185:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.403186:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.403187:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.403188:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.403190:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.403191:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.403191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677474.403194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.403201:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:4.2:1619677474.403202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677474.403204:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.403205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.403205:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677474.403206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.0:1619677474.403207:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:6.0:1619677474.403207:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.403209:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:6.0:1619677474.403209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.403210:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.403211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:18.0:1619677474.403212:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.403214:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.403215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.403217:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.403220:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.403221:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.403223:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.403224:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.403225:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.403226:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.403229:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.403231:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.403234:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.403235:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.403236:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.403237:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.403238:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.403239:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.403239:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.403243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.403250:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.403253:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.403254:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.403256:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.403258:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.403259:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.403279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.403287:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.403289:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.403292:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.403294:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.403301:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.403302:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.403303:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.403304:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.403306:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.403306:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.403310:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.403312:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.403314:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.403314:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.403315:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.403316:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.403317:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.403319:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.403320:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.403321:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.403323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.403327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.403332:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.403333:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.403334:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.403335:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.403336:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.403337:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.403337:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.403338:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.403339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.403340:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.403962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.403969:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.403972:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.403974:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.403975:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.403976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.403977:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.403978:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.403979:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.403980:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.403981:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.403984:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.403985:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.403987:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.403988:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.403989:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.403990:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.403991:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.403992:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.403992:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.403996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.404003:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.404006:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.404007:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.404007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.404009:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.404011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.404116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.404122:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.404124:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.404126:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.404127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.404129:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.404131:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.404132:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.404133:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.404134:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.404136:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.404139:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.404141:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.404143:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.404144:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.404145:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.404147:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.404147:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.404148:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.404149:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.404153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.404160:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.404163:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.404164:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.404165:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.404167:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.404168:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.404175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.404183:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.404185:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.404187:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.404189:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.404189:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.404191:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.404192:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.404193:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.404194:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.404195:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.404198:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.404199:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.404202:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.404203:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.404203:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.404205:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.404206:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.404206:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.404207:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.404210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.404217:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.404219:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.404220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.404220:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.404222:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.404224:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.404702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.404709:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.404712:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.404713:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.404714:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.404717:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.404719:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.404720:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.404721:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.404722:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.404723:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.404726:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.404728:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.404731:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.404732:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.404732:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.404734:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.404735:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.404736:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.404737:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.404738:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.404741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.404744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.404749:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.404752:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.404752:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:22.0:1619677474.404753:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.404753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.404754:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.404754:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677474.404755:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.404756:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.404758:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.404759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.404766:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.404769:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.404770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.404770:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.404771:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.404772:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.405030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.405037:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.405040:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.405041:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.405043:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.405045:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.405047:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.405048:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.405050:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.405050:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.405051:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.405055:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.405056:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.405059:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.405060:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.405060:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.405062:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.405063:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.405064:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.405064:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.405068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.405076:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.405078:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.405079:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.405079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.405082:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.405085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.405108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.405115:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.405118:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.405119:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.405120:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.405123:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.405124:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.405125:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.405127:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.405127:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.405128:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.405131:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.405133:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.405135:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.405136:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.405136:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.405138:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.405138:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.405139:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.405140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.405144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.405151:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.405154:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.405155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.405155:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.405157:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.405160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.405162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.405169:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.405172:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.405172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.405173:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.405174:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.405174:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.405326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.405335:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.405338:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.405339:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.405342:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.405342:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.405343:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.405344:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.405345:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.405347:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.405348:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.405351:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.405352:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.405354:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.405355:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.405356:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.405357:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.405358:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.405359:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.405359:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.405363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.405370:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.405373:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.405373:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.405375:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.405376:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.405378:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.405770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.405773:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.405775:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.405778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.405778:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.405781:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.405782:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.405783:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.405785:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.405786:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.405787:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.405791:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.405793:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.405795:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.405796:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.405797:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.405798:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.405799:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.405800:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.405800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.405805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677474.405811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.405816:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.405818:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.405818:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.405819:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677474.405819:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.405821:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.405823:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.405823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.405824:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.405824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677474.405825:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.405826:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.405828:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.405829:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.405830:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.405831:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.405834:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.405836:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.405839:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.405840:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.405840:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.405842:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.405842:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.405843:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.405843:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.405848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.405856:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.405858:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.405859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.405859:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.405861:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.405863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.405907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.405915:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.405917:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.405920:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.405921:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.405922:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.405923:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.405925:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.405926:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.405927:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.405928:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.405931:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.405933:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.405935:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.405936:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.405937:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.405938:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.405939:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.405940:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.405941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.405944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.405951:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.405953:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.405954:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.405954:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.405956:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.405959:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.406276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.406283:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.406286:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.406287:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.406289:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.406291:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.406293:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.406300:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.406302:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.406302:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.406303:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.406306:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.406308:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.406310:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.406311:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.406311:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.406313:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.406313:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.406314:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.406316:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.406316:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.406319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.406322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.406326:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.406328:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.406329:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.406329:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.406330:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677474.406330:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.406332:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.406333:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.406333:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.406334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677474.406625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677474.406632:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.406634:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677474.406635:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677474.406636:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.406639:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.406641:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677474.406642:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.406643:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.406644:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.406645:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677474.406648:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.406649:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677474.406651:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.406652:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677474.406653:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677474.406654:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677474.406654:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.406655:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677474.406655:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677474.406660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.406666:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.406668:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.406669:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.406669:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.406671:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.406673:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.407527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.407536:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.407539:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.2:1619677474.407540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:31.0:1619677474.407541:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.407543:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.407544:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677474.407544:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.407545:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.407545:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677474.407546:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.407547:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.407547:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.407548:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.407550:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.407552:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.407552:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.407554:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.407554:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.407555:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.407557:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.407557:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.407558:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.407561:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.407562:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.407562:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:30.0:1619677474.407562:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.407563:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.407564:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.407564:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.407566:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.407566:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.407566:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.407567:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.407568:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.407568:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.407568:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.407569:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.407570:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.407570:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.407570:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.407570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:9.2:1619677474.407573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.407574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.407579:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.407580:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.407580:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:31.0:1619677474.407581:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677474.407581:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.407581:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.407582:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.407583:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:8.0:1619677474.407583:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.407584:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:24.0:1619677474.407584:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.407585:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.0:1619677474.407586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.407587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.407587:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677474.407588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.407590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.407596:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.407599:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.407599:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.407599:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.407600:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.407601:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.408340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.408348:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.408351:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.408353:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.408355:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.408356:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677474.408356:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.408357:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.408358:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.408359:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.408360:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.408363:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.408364:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.408366:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.408367:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.408368:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.408369:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.408370:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.408370:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.408372:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.408373:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.408375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.408379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.408382:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.408385:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:30.0:1619677474.408386:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.408386:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.408386:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.408387:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.408389:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.408390:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.408390:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.408391:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.408392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.408398:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.408401:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.408402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.408404:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.408406:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.408408:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.408409:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.408411:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.408411:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.408413:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.408416:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.408418:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.408420:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.408421:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.408422:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.408423:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.408424:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.408425:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.408425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.408430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.408438:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.408441:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.408442:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.408443:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.408445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.408446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.408450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.408458:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.408461:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.408462:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.408463:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.408463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.408464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.409183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.409190:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.409193:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.409195:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.409197:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.409198:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.409199:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.409201:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.409202:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.409203:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.409204:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.409208:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.409209:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.409212:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.409212:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.409213:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.409214:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.409215:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.409216:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.409217:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.409220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.409226:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.409229:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.409230:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.409230:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.409232:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.409234:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.409513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677474.409518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:31.0:1619677474.409521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.409524:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:17.0:1619677474.409525:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:31.0:1619677474.409526:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.409528:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.409528:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.409528:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:30.0:1619677474.409529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.409530:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.409532:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.409532:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.409533:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.409534:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.409535:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.409535:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.409537:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677474.409539:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.409540:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.409540:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.409541:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.409542:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.409542:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677474.409545:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.409546:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.409546:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:17.0:1619677474.409546:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.409548:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.409548:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677474.409548:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677474.409549:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.409550:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:17.0:1619677474.409551:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.409552:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677474.409553:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:9.2:1619677474.409554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.409555:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677474.409555:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.409557:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677474.409558:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677474.409558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:24.0:1619677474.409561:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:9.2:1619677474.409561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:24.0:1619677474.409564:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.2:1619677474.409564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:24.0:1619677474.409565:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.409565:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.409567:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.409568:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:24.0:1619677474.409569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677474.409569:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:26.0:1619677474.409570:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.409571:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.409571:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:9.2:1619677474.409572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.409572:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677474.409572:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:26.0:1619677474.409573:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.409573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:26.0:1619677474.409576:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:8.0:1619677474.409578:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.409581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.409581:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:9.2:1619677474.409582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:8.0:1619677474.409582:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.409583:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.409583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677474.409588:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.409590:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.409591:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.409591:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.409592:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.409593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.409597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.409605:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.409607:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.409609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.409610:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.409613:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.409614:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.409616:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.409617:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.409618:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.409619:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.409622:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.409624:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.409627:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.409627:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.409628:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.409630:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.409630:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.409631:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.409631:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.409636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.409644:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.409647:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.409647:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.409648:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.409650:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.409653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.409848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.409851:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.409853:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.409855:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.409856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.409857:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.409859:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.409860:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.409861:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.409862:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.409863:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.409867:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.409868:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.409871:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.409872:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.409872:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.409874:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.409874:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.409876:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.409877:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.409877:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.409880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.409883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677474.409883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.0:1619677474.409888:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.409888:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.409888:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.409889:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677474.409889:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.409890:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.409890:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.409891:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677474.409892:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:26.0:1619677474.409893:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:21.0:1619677474.409893:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677474.409894:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:10.0:1619677474.409894:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.409896:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.409898:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.409899:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.409901:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.409902:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.409903:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.409904:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.409907:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.409909:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.409912:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.409913:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.409913:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.409915:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.409915:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.409916:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.409916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.409919:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:26.0:1619677474.409920:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677474.409921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.409922:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.409923:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.409924:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.409927:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.409928:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677474.409928:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:26.0:1619677474.409929:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.409930:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.409931:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.409931:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:26.0:1619677474.409932:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.409932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.409932:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.409935:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.409936:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.409938:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.409940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677474.409942:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.409943:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.409943:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.409945:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.409945:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.409946:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.409948:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.409948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.409951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.409954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.409958:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.409960:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.409961:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.409961:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.409961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.409963:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.409963:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677474.409964:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.409965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.409966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.410014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f74ac00] (20)++
00000800:00000200:12.0:1619677474.410022:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f74ac00] (21)++
00000800:00000200:12.0:1619677474.410026:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.9@o2ib3
00000400:00000200:12.0:1619677474.410028:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.6.9@o2ib3 : GET - for me
00000800:00000200:13.0:1619677474.410031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f74ac00] (22)--
00000400:00000200:12.0:1619677474.410031:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.6.9@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677474.410033:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.6.9@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677474.410034:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f74ac00] (21)++
00000800:00000200:12.0:1619677474.410035:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f74ac00] (22)--
00000400:00000200:12.0:1619677474.410037:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.6.9@o2ib3 local destination
00000800:00000200:12.0:1619677474.410039:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7efdf8080] -> 10.51.6.9@o2ib3 (2) version: 12
00000800:00000200:12.0:1619677474.410040:0:19943:0:(o2iblnd.c:1251:kiblnd_query()) peer_ni 10.51.6.9@o2ib3 ffff91a7efdf8080, alive 0 secs ago
00000800:00000200:12.0:1619677474.410041:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f74ac00] (21)--
00000800:00000200:4.2:1619677474.410044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.410051:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.410054:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.410054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.410056:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.410059:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.410061:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.410062:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.410064:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.410064:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.410065:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.410068:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.410069:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.410072:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.410073:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.410073:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.410075:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.410075:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.410076:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.410077:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.2:1619677474.410077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677474.410080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:30.0:1619677474.410084:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.410087:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:13.0:1619677474.410087:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:30.0:1619677474.410088:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.410089:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.410090:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.410091:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.410091:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.410092:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:13.0:1619677474.410092:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.410093:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677474.410093:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.410094:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:30.0:1619677474.410095:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.410095:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.410096:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.410099:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.410101:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.410103:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.410104:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.410104:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.410106:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.410106:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.410107:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.410108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.410111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.410118:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.410120:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.410121:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.410123:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.410124:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.410125:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.410245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.410250:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.410253:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.410255:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.410255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.410258:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.410259:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677474.410261:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.410262:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.410263:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.410264:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677474.410267:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.410269:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.410272:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.410273:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677474.410273:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677474.410275:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677474.410275:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.410276:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.410277:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.410278:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677474.410281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.410283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.410287:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.410289:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:2.0:1619677474.410289:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677474.410290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.410291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677474.410292:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.410293:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.410294:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.410301:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.410304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.410472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.410478:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.410481:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.410482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.410484:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.410486:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.410487:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.410488:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.410490:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.410491:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.410492:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.410495:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.410497:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.410499:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.410500:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.410501:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.410503:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.410503:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677474.410503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677474.410504:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.410505:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.410509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.410512:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.410515:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:17.0:1619677474.410518:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.410518:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.410520:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.410521:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.410521:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.410522:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:17.0:1619677474.410523:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.410524:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.410526:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.410527:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.410528:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.410531:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.410531:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.410532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.410533:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.410533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677474.410536:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.410537:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.410538:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.410539:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.410540:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.410540:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.410541:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.410546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.410555:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.410558:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.410559:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.410559:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.410561:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.410563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.410995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.411003:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.411007:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.411008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.411010:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.411013:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.411014:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.411015:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.411016:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.411018:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.411018:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.411022:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.411024:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.411027:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.411028:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.411028:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.411030:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.411030:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.411031:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.411031:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.411047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.411054:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.411057:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.411058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.411059:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.411061:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.411063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.411069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.411076:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.411079:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.411080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.411080:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.411081:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.411081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.411189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.411197:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.411200:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.411202:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.411204:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.411204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.411205:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.411206:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.411207:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.411208:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.411209:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.411212:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.411214:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.411216:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.411217:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.411217:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.411219:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.411220:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.411221:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.411221:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.411225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.411232:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.411235:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.411236:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.411237:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.411238:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.411241:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.411761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.411767:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.411770:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.411771:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.411773:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.411776:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.411777:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.411778:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.411780:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.411781:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.411782:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.411785:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.411787:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.411789:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.411790:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.411790:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.411792:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.411793:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.411794:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.411794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.411799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.411806:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.411808:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.411809:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.411810:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.411811:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.411812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.411917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.411921:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.411923:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.411925:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.411926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.411927:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.411929:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.411930:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.411931:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.411932:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.411933:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.411937:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.411939:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.411941:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.411942:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.411943:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.411944:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.411945:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.411946:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.411946:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.411951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.411960:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.411963:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.411964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.411964:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.411966:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.411968:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.411969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.411977:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.411980:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.411980:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.411980:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.411981:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.411982:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.412805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.412814:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.412816:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.412819:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.412821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.412821:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.412822:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.412824:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.412825:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.412826:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.412827:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.412830:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.412832:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.412835:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.412836:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.412837:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.412838:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.412839:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.412840:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.412840:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.412844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.412851:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.412854:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.412855:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.412855:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.412857:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.412859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.413631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.413640:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.413642:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.413645:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.413647:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.413647:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.413648:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.413649:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.413650:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.413651:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.413652:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.413655:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.413656:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.413658:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.413659:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.413659:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.413660:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.413661:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.413663:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.413664:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.413665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.413667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.413671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.413676:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.413677:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.413678:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.413679:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.413679:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.413680:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.413681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.413681:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.413682:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.413683:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.413683:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.413684:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.413684:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.413685:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.413685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.413777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.413784:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.413786:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.413788:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.413790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.413790:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.413791:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.413792:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.413793:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.413794:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.413795:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.413798:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.413800:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.413802:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.413802:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.413803:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.413805:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.413805:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.413806:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.413806:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.413810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.413817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.413820:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.413821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.413821:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.413822:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.413824:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.414016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677474.414022:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.414025:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.414026:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.414028:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.414030:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.414032:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.414033:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.414034:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.414035:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.414036:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.414039:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.414041:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.414043:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.414044:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.414045:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.414047:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.414047:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.414048:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.414048:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:25.2:1619677474.414053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.414061:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.414064:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.414065:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.414065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.414067:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.414069:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.414165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677474.414174:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.414176:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677474.414178:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.414180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677474.414181:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.414182:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.414183:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.414185:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.414186:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.414187:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.414190:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.414192:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.414194:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.414195:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.414196:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.414197:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.414198:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.414199:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.414200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.414203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.414209:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.414212:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.414213:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.414213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.414215:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.414217:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.414271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.414278:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.414281:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.414283:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.414285:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.414285:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.414286:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.414287:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.414289:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.414289:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.414290:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677474.414296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:30.0:1619677474.414297:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.414299:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.414301:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.414301:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.414302:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.414303:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.414304:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.414304:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677474.414304:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677474.414305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677474.414306:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.414307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.414308:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677474.414309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:8.0:1619677474.414312:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.414314:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.414315:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.414316:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.414317:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:8.0:1619677474.414317:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.414318:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.414319:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.414320:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.414321:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.414322:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.414323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.414324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.0:1619677474.414324:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.414326:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.414327:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.414328:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.414329:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.414330:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.414331:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.414331:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.414335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.414343:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.414346:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.414347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.414347:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.414349:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.414352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.414764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.414772:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.414774:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.414775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.414777:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.414780:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.414782:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.414783:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.414784:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.414785:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.414786:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.414789:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.414791:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.414793:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.414794:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.414795:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.414797:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.414797:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.414798:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.414798:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.414803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.414810:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.414812:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.414813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.414814:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.414816:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.414819:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.414822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.414829:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.414832:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.414832:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.414833:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.414834:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.414834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.415004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.415012:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.415014:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.415016:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.415018:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.415018:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.415019:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.415020:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.415021:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.415022:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.415023:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.415026:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.415028:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.415030:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.415031:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.415031:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.415033:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.415033:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.415034:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.415035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.415038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.415045:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.415047:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.415048:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.415048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.415050:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.415052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.415059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.415065:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.415067:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.415068:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.415068:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.415069:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.415069:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.415096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.415102:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.415104:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.415105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.415107:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.415110:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.415112:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677474.415114:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.415115:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.415116:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.415117:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.415121:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.415123:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.415125:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.415126:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.415127:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.415129:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.415129:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.415131:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677474.415132:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677474.415132:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:25.2:1619677474.415135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.415137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:30.0:1619677474.415142:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:6.0:1619677474.415142:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677474.415144:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:30.0:1619677474.415145:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:6.0:1619677474.415145:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:30.0:1619677474.415146:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.415146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:30.0:1619677474.415147:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.415148:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.415151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.415212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.415218:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.415221:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.415222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.415224:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.415227:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.415228:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.415230:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.415231:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.415232:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.415233:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:4.2:1619677474.415234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:17.0:1619677474.415236:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.415238:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.415241:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.415242:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.415242:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677474.415242:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677474.415244:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.415244:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:10.0:1619677474.415244:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.415245:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677474.415245:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.415246:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:10.0:1619677474.415247:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.415250:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677474.415250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:10.0:1619677474.415252:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.415253:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.415254:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.415255:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.415256:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.415259:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:10.0:1619677474.415260:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.415261:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.415262:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.415262:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.415264:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.415264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:5.2:1619677474.415264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677474.415265:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.415266:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.415267:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.415268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677474.415268:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.415269:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.415269:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.415270:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677474.415270:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.415273:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:25.2:1619677474.415274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.415274:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.415276:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.415278:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.415280:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.415281:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.415282:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677474.415282:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.415283:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.415284:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:1.0:1619677474.415284:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.415285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.415286:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.415288:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.415289:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.415291:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.415292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677474.415299:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.415300:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.415301:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.415303:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.415303:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.415304:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.415305:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.415306:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.415309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.415311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677474.415316:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.415317:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.415319:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677474.415319:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.415320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677474.415320:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.415321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.415322:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.415322:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.415324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.415567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.415575:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.415577:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.415580:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.415582:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.415582:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.415584:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.415585:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.415586:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.415587:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.415588:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.415592:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.415593:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.415596:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.415596:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.415597:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.415598:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.415599:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.415600:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.415602:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.415602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.415604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.415608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.415611:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.415614:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677474.415615:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.415615:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.415615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.415617:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.415618:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.415619:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677474.415619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.415620:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.415625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.415631:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.415634:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.415635:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.415635:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.415636:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.415636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.415927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.415934:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.415936:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.415938:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:4.2:1619677474.415938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:2.0:1619677474.415939:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.415941:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.415942:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.415944:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.415945:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:2.0:1619677474.415945:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.415946:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.415948:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:2.0:1619677474.415948:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.415949:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.415951:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.415952:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:24.0:1619677474.415954:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.415954:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:24.0:1619677474.415956:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.415957:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.415957:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677474.415958:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.415958:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:24.0:1619677474.415959:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.415959:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:24.0:1619677474.415960:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.415961:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.415961:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.415962:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.415963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:24.0:1619677474.415964:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.415966:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:25.2:1619677474.415967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677474.415969:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.415970:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.415970:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.415972:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.415972:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677474.415973:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:24.0:1619677474.415973:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677474.415973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:24.0:1619677474.415974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677474.415976:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677474.415976:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677474.415977:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.415978:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:8.2:1619677474.415978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677474.415979:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677474.415980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.415981:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:20.0:1619677474.415981:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.415982:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.415983:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.415984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:20.0:1619677474.415985:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.415986:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.415986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:20.0:1619677474.415987:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.415988:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.415989:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.415989:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.415990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.415990:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.415991:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:1.0:1619677474.415993:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.415994:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.415994:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677474.415996:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.415996:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.415997:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.415997:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.415997:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677474.415998:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.0:1619677474.415999:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.416000:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.416000:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677474.416000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:20.0:1619677474.416002:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.416003:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.416003:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.416004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:17.0:1619677474.416007:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.2:1619677474.416008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.416010:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.416011:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.416011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.416012:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.416013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677474.416016:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.416018:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.416019:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.416021:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.416021:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.416023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.416365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.416373:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.416376:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.416377:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.416379:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.416382:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.416383:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.416384:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.416386:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.416387:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.416388:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.416391:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.416393:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.416395:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.416396:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.416397:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.416398:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.416399:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.416400:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.416401:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.416402:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.416405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.416408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.416412:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.416414:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.416415:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.416415:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.416416:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.416417:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677474.416417:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.416418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.416419:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.416419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.416794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.416801:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.416804:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.416805:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.416806:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.416809:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.416810:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.416811:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.416812:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.416813:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.416814:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.416817:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.416818:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.416821:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.416821:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.416822:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.416824:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.416824:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.416825:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.416825:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.416830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.416838:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.416841:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.416842:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.416844:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.416845:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.416846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.416853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.416861:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.416863:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.416864:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.416865:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.416866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.416867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.417033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677474.417039:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.417042:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.417043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.417044:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.417047:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.417049:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.417050:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.417051:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.417052:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.417053:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.417056:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.417058:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.417060:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.417061:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.417061:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.417063:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.417063:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.417064:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.417064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:25.2:1619677474.417080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.417087:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.417090:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.417091:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.417091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.417093:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.417095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.417102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.417109:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.417111:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.417112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.417112:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.417113:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.417114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.417243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677474.417248:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.417251:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677474.417252:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677474.417254:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.417255:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.417257:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677474.417258:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.417259:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.417260:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.417261:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677474.417265:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.417267:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677474.417269:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.417270:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677474.417270:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677474.417272:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677474.417273:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.417273:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677474.417274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677474.417278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.417287:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.417289:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.417290:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.417290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.417293:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.417300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.417680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.417689:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.417692:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.417694:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.417696:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.417696:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.417697:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.417699:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.417700:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.417701:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.417702:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.417706:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.417707:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.417710:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.417710:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.417711:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.417713:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.417713:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.417714:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.417715:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.417719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.417726:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.417729:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.417730:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.417730:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.417732:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.417733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677474.417734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.417741:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.417743:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.417745:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.417747:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.417748:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.417749:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.417750:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.417751:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.417752:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.417753:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.417757:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.417758:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.417760:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.417761:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.417761:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.417763:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.417763:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.417764:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.417764:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.417768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.417775:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.417777:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.417778:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.417779:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.417781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.417782:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.417897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.417904:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.417907:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.417908:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.417910:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.417913:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.417914:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677474.417915:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.417916:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.417917:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.417918:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677474.417921:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.417922:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.417924:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.417925:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.417925:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.417926:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.417927:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.417928:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.417928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.417932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.417938:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.417940:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.417941:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.417942:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.417942:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.417944:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677474.418360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.418366:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.418368:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.418369:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.418371:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.418374:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.418375:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.418376:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.418378:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.418379:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.418380:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.418383:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.418385:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.418386:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.418387:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.418388:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.418389:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.418390:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.418390:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.418391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.418396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.418404:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.418405:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.418406:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.418407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.418408:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.418409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.418417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.418425:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.418427:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.418428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.418428:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.418428:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.418429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.418564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.418572:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.418575:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.418577:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.418578:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.418579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.418580:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.418581:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.418582:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.418583:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.418584:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.418587:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.418589:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.418591:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.418592:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.418593:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.418594:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.418595:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.418596:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.418598:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.418598:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.418600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.418604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.418607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.418610:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.418611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.418611:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.418612:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.418613:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.418614:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.418615:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677474.418615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.418616:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.419327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.419335:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.419338:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.419340:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.419342:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.419343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.419344:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.419345:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.419346:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.419347:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.419348:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.419351:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.419352:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.419354:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.419355:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.419355:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.419357:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.419357:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.419358:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.419359:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.419363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.419370:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.419373:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.419374:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.419374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.419375:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.419378:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.419482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.419490:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.419492:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.419493:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.419494:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.419497:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.419499:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.419500:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.419501:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.419502:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.419502:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.419505:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.419507:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.419510:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.419511:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.419511:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.419513:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.419514:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.419515:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.419516:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.419517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.419519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.419522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.419527:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.419529:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.419530:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.419531:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677474.419531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.419531:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.419531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677474.419532:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.419533:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677474.419533:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.419536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677474.419539:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.419541:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.419542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.419544:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.419546:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.419547:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.419549:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.419550:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.419551:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.419552:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.419555:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.419556:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.419559:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.419560:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.419560:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.419562:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.419563:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.419563:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.419563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.419568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.419576:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.419578:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.419579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.419580:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.419582:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.419584:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.419864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.419872:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.419875:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.419877:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.419879:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.419879:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.419881:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.419882:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.419884:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.419885:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.419886:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.419889:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.419890:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.419893:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.419894:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.419895:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.419897:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.419897:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.419898:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.419899:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.419903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.419912:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.419914:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.419915:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.419917:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.419917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.419920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.420279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.420286:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.420289:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.420289:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.420292:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.420299:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.420300:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.420302:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.420303:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.420304:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.420305:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.420308:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.420310:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.420312:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.420313:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.420314:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.420315:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.420316:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.420316:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.420317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.420322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.420330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.420333:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.420333:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.420335:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.420337:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.420338:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.420338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.420339:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.420340:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.420340:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.420340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (24)--
00000800:00000200:21.0:1619677474.420341:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.420342:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.420615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.420623:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.420626:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.420627:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.420629:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.420632:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.420632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000400:00000200:26.0:1619677474.420633:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.420635:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.420636:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.420637:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.420638:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677474.420639:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000400:00000200:26.0:1619677474.420641:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.420641:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.420642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:26.0:1619677474.420643:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:1.0:1619677474.420644:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677474.420645:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.420646:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:1.0:1619677474.420646:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.420647:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.420648:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:1.0:1619677474.420648:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:26.0:1619677474.420649:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677474.420649:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.420650:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.420650:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677474.420651:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.420652:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.420653:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:9.2:1619677474.420655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:1.0:1619677474.420656:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.420658:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.420661:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677474.420661:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.420661:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.420662:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.420663:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.420664:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677474.420664:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.420664:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000400:00000200:8.0:1619677474.420665:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.420666:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000400:00000200:8.0:1619677474.420667:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.420667:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677474.420667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:8.0:1619677474.420669:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.420672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:9.2:1619677474.420672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.420678:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677474.420678:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:24.0:1619677474.420680:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:17.0:1619677474.420680:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:9.2:1619677474.420680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.420681:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677474.420681:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.420682:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:24.0:1619677474.420683:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.420683:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677474.420683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.420685:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.420687:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.420688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.420688:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.420690:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.420693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677474.420776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.420782:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.420785:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.420786:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.420787:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.420790:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.420792:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.420793:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.420794:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.420795:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.420796:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.420799:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.420800:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.420803:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.420803:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.420804:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.420805:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.420806:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.420807:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.420807:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.420811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.420820:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.420823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.420823:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.420824:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.420826:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.420828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.420944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.420950:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.420953:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.420954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.420956:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.420958:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.420959:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.420960:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.420962:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.420963:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.420964:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.420968:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.420969:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.420972:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.420973:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.420974:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.420976:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.420977:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.420978:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.420979:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.420979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.420982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677474.420984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.420990:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677474.420992:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677474.420992:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.420992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.420993:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677474.420995:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.420997:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.420999:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.421000:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.421003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.421092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677474.421096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:30.0:1619677474.421100:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:1.0:1619677474.421102:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:30.0:1619677474.421103:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:1.0:1619677474.421105:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:30.0:1619677474.421106:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.421106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:31.0:1619677474.421108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:1.0:1619677474.421108:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.421109:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.421111:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.421111:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.421112:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.421113:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677474.421114:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.421114:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.421115:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.421116:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.421117:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.421117:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.421119:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677474.421122:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677474.421123:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.421124:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.421126:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.421127:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.421128:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.421128:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.421129:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.421129:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677474.421130:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677474.421131:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.421131:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:1.0:1619677474.421132:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677474.421132:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:30.0:1619677474.421133:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.421133:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:1.0:1619677474.421133:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.421134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677474.421138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.421147:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:9.2:1619677474.421147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677474.421149:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.421151:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.421151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:24.0:1619677474.421153:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:18.0:1619677474.421153:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.421156:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:18.0:1619677474.421156:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:24.0:1619677474.421157:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.421157:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.421159:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.421161:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.421343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.421351:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.421353:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.421354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.421356:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.421358:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.421360:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.421361:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.421362:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.421363:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.421364:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.421368:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.421369:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.421371:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.421372:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.421373:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.421374:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.421375:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.421375:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.421376:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.421380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.421387:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.421390:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.421391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.421391:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.421393:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.421394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.422412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.422420:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.422423:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.422425:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.422427:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.422427:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.422428:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.422430:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.422431:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.422432:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.422433:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.422436:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.422438:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.422439:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.422440:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.422441:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.422442:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.422443:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.422443:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.422444:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.422449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.422456:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.422458:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.422459:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.422460:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.422462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.422463:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.422466:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.422467:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.422468:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.422468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.422469:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.422549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.422556:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.422559:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.422561:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.422562:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.422563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.422563:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.422565:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.422566:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.422567:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.422568:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.422570:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.422572:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.422574:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.422575:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.422576:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.422577:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.422578:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.422578:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.422579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.422583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.422590:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.422592:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.422593:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.422593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.422595:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.422597:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.422604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.422610:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.422612:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.422613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.422613:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.422614:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.422615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.422878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.422885:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.422888:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.422889:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.422890:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.422893:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.422894:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677474.422895:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.422896:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.422897:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.422898:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677474.422901:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.422902:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.422903:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.422904:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.422905:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.422906:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.422906:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.422907:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.422907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.422912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.422918:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.422919:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.422920:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.422921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.422922:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.422923:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.423029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.423036:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.423039:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.423040:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.423041:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.423044:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.423045:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.423046:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.423047:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.423048:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.423049:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.423052:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.423053:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.423056:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.423057:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.423058:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.423059:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.423060:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.423060:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.423061:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.423076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.423085:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.423087:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.423088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.423089:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.423091:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.423093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.423133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.423141:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.423144:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.423146:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.423148:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.423148:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.423150:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.423151:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.423152:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.423153:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.423154:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.423157:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.423159:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.423161:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.423162:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.423162:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.423164:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.423165:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.423165:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.423166:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.423170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.423176:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.423179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.423179:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.423180:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.423182:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.423184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.423874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.423881:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.423884:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.423886:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.423888:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.423889:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.423889:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.423890:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.423892:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.423893:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.423893:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.423896:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.423898:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.423899:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.423900:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.423900:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.423902:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.423902:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.423904:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.423905:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.423906:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.423908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.423912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.423916:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.423918:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.423919:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.423920:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.423920:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.423921:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.423921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.423922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.423924:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.423925:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.423943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.423950:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.423953:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.423954:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.423955:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.423957:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.423959:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.423960:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.423961:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.423962:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.423963:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.423966:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.423967:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.423970:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.423971:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.423972:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.423973:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.423974:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.423974:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.423975:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.423979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.423988:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.423990:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.423991:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.423991:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.423993:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.423996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.423997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.424004:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.424007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.424007:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.424008:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.424009:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.424009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.424042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.424047:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.424050:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.424052:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.424052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.424054:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.424055:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.424057:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.424058:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.424059:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.424060:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.424063:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.424065:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.424068:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.424068:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.424069:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.424071:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.424071:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.424072:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.424073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.424078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.424086:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.424088:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.424089:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.424090:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.424091:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.424093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.424324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.424332:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.424335:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.424336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.424338:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.424341:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.424342:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.424343:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.424345:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.424346:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.424347:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.424350:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.424352:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.424354:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.424355:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.424356:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.424357:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.424358:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.424359:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.424359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.424364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.424372:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.424374:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.424375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.424376:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.424378:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.424381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.424445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.424453:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.424457:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.424459:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.424461:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.424461:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.424462:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.424464:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.424465:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.424466:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.424467:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.424470:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.424472:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.424473:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.424474:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.424475:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.424476:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.424477:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.424477:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.424478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.424482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.424489:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.424491:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.424492:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.424493:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.424495:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.424496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.424813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.424821:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.424824:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.424826:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.424828:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.424828:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.424829:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.424830:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.424832:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.424832:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.424833:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:4.2:1619677474.424834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000400:00000200:30.0:1619677474.424836:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.424837:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.424839:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.424840:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.424840:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.424841:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:30.0:1619677474.424842:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.424842:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.424844:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.0:1619677474.424844:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:30.0:1619677474.424845:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677474.424845:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:30.0:1619677474.424846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:8.0:1619677474.424847:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000800:00000200:9.2:1619677474.424848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:8.0:1619677474.424849:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.424851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:8.0:1619677474.424851:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677474.424852:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.424854:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.424855:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:8.0:1619677474.424855:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.424856:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.424856:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677474.424857:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.424858:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.424859:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677474.424859:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.424861:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.424862:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.424862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677474.424862:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.424863:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.424864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677474.424864:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677474.424864:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.424865:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.424866:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.424867:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.424869:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:8.0:1619677474.424870:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:9.2:1619677474.424871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.424871:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:4.2:1619677474.424876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.424881:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.424882:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.424883:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.424884:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.424885:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677474.424886:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.424886:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677474.424887:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677474.424887:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.424888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:4.2:1619677474.424990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.424997:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.425000:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.425001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.425003:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.425006:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.425007:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.425008:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.425010:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.425010:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.425011:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.425014:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.425016:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.425019:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.425019:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.425020:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.425022:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.425022:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.425024:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.425025:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.425025:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677474.425028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677474.425031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677474.425037:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677474.425038:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:2.0:1619677474.425039:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:26.0:1619677474.425040:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.425041:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677474.425041:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.425041:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.425043:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677474.425043:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.425047:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.425118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.425125:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.425128:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.425130:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.425132:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.425132:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.425134:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.425135:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.425136:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.425137:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.425138:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.425141:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.425142:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.425145:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.425145:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.425146:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.425148:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.425148:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.425149:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.425150:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.425153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.425159:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.425161:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.425162:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.425162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.425164:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.425166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.425365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.425372:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.425375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.425375:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.425378:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.425380:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.425381:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.425383:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.425384:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.425385:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.425385:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.425388:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.425390:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.425392:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.425392:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.425393:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.425394:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.425395:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.425395:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.425396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.425400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.425407:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.425410:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.425410:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.425411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.425412:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.425414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.425418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.425424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.425426:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.425427:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.425427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.425428:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.425429:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.425508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.425514:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.425517:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.425518:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.425521:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.425523:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.425524:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.425525:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.425527:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.425528:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.425529:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.425532:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.425533:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.425536:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.425537:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.425537:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.425539:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.425539:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.425540:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.425541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.425545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.425553:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.425556:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.425557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.425557:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.425559:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.425561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.425860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.425869:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.425872:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.425874:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.425877:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.425877:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.425878:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.425879:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.425881:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.425882:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.425883:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.425887:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.425888:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.425892:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.425893:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.425894:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.425895:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.425896:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.425897:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.425897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.425901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.425911:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.425913:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.425914:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.425916:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.425917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.425919:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.426104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.426111:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.426114:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.426114:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.426117:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.426119:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.426120:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.426121:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.426123:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.426123:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.426124:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.426127:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.426129:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.426131:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.426132:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.426132:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.426134:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.426134:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.426135:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.426137:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.426137:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.426139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.426143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677474.426146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:8.0:1619677474.426146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.426148:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.426149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.426149:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.426150:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.426151:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677474.426151:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.426152:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:10.0:1619677474.426152:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.426153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.426153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.426154:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.426155:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.426157:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.426160:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.426161:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677474.426162:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.426164:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.426165:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.426166:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.426170:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.426172:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.426175:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.426175:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.426176:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.426178:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.426179:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.426180:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.426180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677474.426184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.426191:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.426194:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.426195:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.426196:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.426198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.426200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.426255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.426262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.426265:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.426265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.426268:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.426271:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.426272:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.426273:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.426275:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.426276:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.426277:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.426280:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.426282:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.426284:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.426285:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.426285:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.426287:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.426287:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.426289:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.426289:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.426290:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.426293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.426297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.426301:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.426303:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.426303:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.426304:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.426305:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.426305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.426305:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.426306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.426307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677474.426307:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677474.427082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.427090:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.427093:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.427095:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.427098:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.427098:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.427099:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.427100:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.427102:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.427103:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.427104:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.427107:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677474.427108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.427109:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.427111:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.427112:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.427113:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.427114:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.427115:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.427116:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.427116:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.427117:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677474.427118:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.427118:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.427119:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.427120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.427121:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.427121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677474.427123:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.427124:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677474.427124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677474.427125:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677474.427125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:31.0:1619677474.427126:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.427127:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.427127:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.427128:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.427129:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.427130:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.427130:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.427131:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677474.427131:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.427132:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.427133:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.427134:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.427134:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:18.0:1619677474.427134:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:31.0:1619677474.427135:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.427135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677474.427135:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677474.427136:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.427136:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.427137:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.427137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677474.427137:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677474.427138:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.427139:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.427140:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:18.0:1619677474.427140:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.427141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000400:00000200:18.0:1619677474.427141:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.427143:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677474.427143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:18.0:1619677474.427144:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.427145:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.427147:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677474.427149:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.427150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:18.0:1619677474.427150:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.427152:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:18.0:1619677474.427152:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677474.427153:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.427153:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.427154:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.427154:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:24.0:1619677474.427154:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.427154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677474.427155:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.427156:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:24.0:1619677474.427156:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.427156:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.427157:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.427159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677474.427159:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.427160:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.427161:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.427162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.427165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.427174:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.427177:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.427178:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.427178:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.427180:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.427183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.427750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.427757:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.427760:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.427761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.427763:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.427765:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.427767:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.427768:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.427770:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.427771:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.427772:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.427775:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.427777:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.427780:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.427781:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.427782:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.427783:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.427784:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.427785:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.427785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.427790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.427798:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.427800:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.427801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.427801:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.427803:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.427806:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.428461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.428466:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.428469:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.428470:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.428471:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.428473:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.428475:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.428476:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.428478:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.428479:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.428480:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.428483:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.428485:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.428487:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.428488:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.428489:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.428490:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.428491:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.428492:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677474.428493:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.428494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.428497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.428499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677474.428505:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677474.428505:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.428506:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.428507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677474.428508:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.428508:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.428508:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677474.428510:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.428511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.428512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677474.428692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677474.428698:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.428701:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677474.428701:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677474.428703:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.428706:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.428707:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677474.428709:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.428710:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.428712:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.428712:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677474.428716:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.428717:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677474.428719:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.428720:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:17.0:1619677474.428720:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677474.428722:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677474.428722:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.428723:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677474.428724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677474.428729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.428737:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.428739:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.428740:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.428741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.2:1619677474.428742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:2.0:1619677474.428743:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.428745:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.428751:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.428754:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.428756:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.428758:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.428758:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.428760:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.428761:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.428763:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.428764:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.428765:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.428768:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.428769:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.428772:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.428773:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.428773:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.428775:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.428775:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.428776:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.428777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.428780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.428787:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.428789:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.428790:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.428790:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.428792:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.428794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.428798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.428804:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.428807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.428807:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.428808:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.428809:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.428809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.428821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.428829:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.428831:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.428832:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.428834:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.428836:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.428838:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.428839:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.428840:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.428841:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.428842:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.428845:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.428846:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.428849:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.428850:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.428850:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.428852:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.428852:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.428853:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.428855:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.428855:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.428858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.428861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.428866:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.428868:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.428869:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.428870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.428870:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.428870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.428870:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.428872:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.428872:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.428875:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.429446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.429454:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.429457:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.429459:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.429461:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.429462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.429462:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.429464:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.429465:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.429466:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.429467:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.429470:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.429471:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.429473:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.429474:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.429474:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.429476:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.429476:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.429477:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.429478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.429482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.429489:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.429491:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.429492:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.429493:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.429495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.429496:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.430050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.430057:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.430060:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.430060:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.430062:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.430065:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.430067:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.430068:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.430069:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.430070:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.430071:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.430074:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.430075:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.430078:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.430079:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.430079:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.430081:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.430081:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.430082:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.430082:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.430087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.430097:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.430100:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.430101:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.430103:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.430105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.430106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.430108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677474.430115:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.430118:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.430119:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.430119:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.430120:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.430121:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.430229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.430237:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.430240:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.430242:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.430243:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.430244:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.430244:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.430245:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.430247:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.430248:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.430249:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.430251:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.430253:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.430255:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.430255:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.430256:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.430257:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.430258:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.430259:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.430259:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.430263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.430270:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.430272:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.430273:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.430275:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.430276:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.430277:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677474.430302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.430309:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.430311:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.430312:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.430315:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.430317:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.430318:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.430320:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.430321:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.430322:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.430323:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.430326:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.430328:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.430331:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.430332:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.430333:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.430334:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.430335:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.430335:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.430336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.430341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.430349:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.430352:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.430353:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.430353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.430355:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.430357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.430428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.430435:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.430438:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.430439:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.430441:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.430444:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.430445:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.430446:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.430448:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.430449:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.430450:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.430453:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.430454:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.430457:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.430457:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.430458:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.430459:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.430460:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.430461:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.430461:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.430465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.430472:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.430474:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.430475:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.430476:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.430477:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.430479:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.430659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.430666:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.430669:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.430671:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.430673:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.430674:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.430674:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.430675:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.430676:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.430677:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.430678:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.430681:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.430682:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.430685:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.430685:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.430686:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.430687:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.430688:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.430689:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.430689:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.430693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.430699:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.430702:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.430703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.430703:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.430705:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.430707:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.430954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677474.430957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677474.430962:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.430964:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677474.430964:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677474.430966:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.430967:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:30.0:1619677474.430968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677474.430968:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:31.0:1619677474.430969:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.430969:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.430970:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.430971:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.430972:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.430973:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.430973:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.430974:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677474.430974:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.430975:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.430977:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.430978:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.430978:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.430979:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.430980:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.430982:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677474.430982:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.430983:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.430983:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.430984:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677474.430984:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.430985:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.430986:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.430986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677474.430987:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.430988:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.430988:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677474.430990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.430990:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.430991:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.430992:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.430993:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.430994:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.430996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.430997:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:4.2:1619677474.430999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677474.431000:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.431000:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.431002:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.431003:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.431004:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.431005:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.0:1619677474.431006:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.431006:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.431007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.431008:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.431009:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.431010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677474.431010:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.431011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.431013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.431190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.431197:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.431200:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.431201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.431203:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.431205:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.431206:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.431207:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.431209:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.431210:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.431210:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.431214:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.431215:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.431217:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.431218:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.431218:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.431220:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.431220:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.431221:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.431222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.431222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:9.2:1619677474.431226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677474.431228:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.431231:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677474.431231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.0:1619677474.431232:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677474.431234:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.431234:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.431235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.431235:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.431236:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.431238:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677474.431238:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.431239:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.431240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:17.0:1619677474.431241:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.431242:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.431243:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677474.431246:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.431248:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677474.431250:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.431250:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677474.431251:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677474.431253:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677474.431253:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.431254:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677474.431255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677474.431259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.431268:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.431270:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.431271:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.431271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.431273:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.431276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.431282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677474.431288:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.431291:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.431292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.431292:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.431298:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.431298:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.431534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677474.431541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:31.0:1619677474.431542:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.431545:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.431547:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.431549:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.431549:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677474.431549:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:31.0:1619677474.431550:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.431552:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.431552:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.431553:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:31.0:1619677474.431554:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.431555:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.431555:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.431556:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.431558:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.431559:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.431560:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.431560:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.431562:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.431562:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.431563:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.431563:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.431564:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.431565:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.431566:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.431567:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:10.0:1619677474.431567:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.431568:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.431569:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.431570:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677474.431571:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.431572:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.431572:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677474.431572:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.431573:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677474.431574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.431574:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.431575:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.431575:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.431576:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.2:1619677474.431578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677474.431580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.431582:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.431584:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.431585:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.431586:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.431586:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.431587:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.431587:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.431588:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.431589:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.431589:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.431590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.431591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.431591:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.431593:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.431593:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.431595:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.431598:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.431598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.431599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.431601:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.431602:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.431602:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.431603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.431604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677474.431606:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.431609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.431609:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.431610:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.431611:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.431612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.431797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.431802:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.431805:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.431807:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.431808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.431809:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.431811:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.431812:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.431814:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.431815:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.431816:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.431820:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.431821:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.431824:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.431824:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.431825:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.431827:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.431827:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.431828:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.431829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.431834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.431842:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.431844:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.431845:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.431846:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.431847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.431848:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.432296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.432304:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.432307:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.432309:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.432311:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.432311:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.432312:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.432313:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.432315:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.432316:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.432317:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.432319:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.432321:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.432324:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.432324:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.432325:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.432327:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.432327:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.432329:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.432330:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.432331:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.432333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.432337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.432340:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.432343:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.432344:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.432344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.432346:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.432346:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.432348:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677474.432348:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.432349:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.432350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677474.433452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.433459:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.433462:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.433463:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.433464:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.433467:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.433468:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.433469:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.433471:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.433472:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.433473:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.433477:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.433479:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.433482:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.433483:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.433483:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.433485:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.433486:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.433487:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.433487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.433492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.433499:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.433502:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.433503:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.433504:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.433506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.433508:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.433672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.433679:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.433682:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.433682:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.433685:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.433688:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.433689:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.433690:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.433692:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.433693:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.433694:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.433697:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.433699:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.433702:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.433703:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.433703:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.433705:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.433706:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.433706:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.433707:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.433711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.433719:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.433722:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.433723:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.433723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.433725:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.433728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.433732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.433739:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.433742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.433742:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.433743:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.433744:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.433744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.434236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.434243:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.434246:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.434248:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.434250:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.434250:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.434252:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.434253:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.434254:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.434255:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.434256:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.434259:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.434260:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.434262:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.434263:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.434264:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.434265:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.434266:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.434267:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.434267:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.434271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.434277:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.434280:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.434280:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.434282:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.434284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.434284:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.434312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.434320:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.434322:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.434324:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.434326:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.434326:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.434327:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.434328:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.434329:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.434330:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.434331:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.434334:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.434335:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.434338:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.434338:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.434339:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.434340:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.434341:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.434342:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.434342:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.434347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.434354:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.434356:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.434357:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.434357:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.434359:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.434361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.434748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.434754:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.434757:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.434758:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.434760:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.434763:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.434764:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.434765:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.434767:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.434768:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.434769:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.434772:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.434773:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.434775:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.434776:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.434776:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.434778:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.434779:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.434780:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.434780:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.434785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.434792:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.434794:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.434794:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.434795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.434796:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.434797:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.434820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.434826:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.434829:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.434830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.434831:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.434834:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.434835:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.434837:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.434838:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.434839:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.434840:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.434843:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.434844:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.434847:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.434848:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.434848:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.434850:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.434850:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.434851:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.434852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.434856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.434865:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.434867:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.434868:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.434869:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.434870:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.434872:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.434874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.434882:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.434884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677474.434886:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.434887:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.434888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.434888:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.434890:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677474.434891:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.434894:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.434895:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.434897:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.434900:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.434901:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.434902:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.434903:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.434904:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.434905:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.434908:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.434910:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.434912:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.434913:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.434914:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.434915:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.434916:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.434916:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.434917:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.434921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.434927:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.434929:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.434930:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.434930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.434932:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.434934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.434940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.434946:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.434948:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.434949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.434949:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.434950:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.434951:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.435060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.435067:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.435069:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.435070:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.435071:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.435074:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.435075:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.435076:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.435078:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.435078:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.435079:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.435082:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.435084:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.435086:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.435087:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.435088:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.435089:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.435090:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.435090:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.435091:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.435095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.435103:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.435106:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.435107:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.435107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.435109:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.435112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.435466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.435474:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.435477:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.435479:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.435481:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.435482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.435483:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.435484:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.435485:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.435486:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.435487:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.435490:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.435492:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.435494:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.435494:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.435495:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.435496:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.435497:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.435498:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.435498:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.435503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.435509:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.435512:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.435512:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.435514:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.435516:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.435516:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677474.435550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.435557:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.435560:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.435561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.435563:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.435565:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.435567:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.435568:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.435569:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.435570:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.435571:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.435574:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.435576:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.435578:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.435579:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.435580:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.435581:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.435582:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.435583:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.435584:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.435584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.435587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.435589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677474.435595:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.435596:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.435597:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.435597:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.435598:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677474.435599:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.435599:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.435599:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.435601:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.435603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.435655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.435663:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.435665:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.435666:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.435668:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.435671:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.435672:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.435674:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.435675:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.435676:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.435677:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.435680:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.435682:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.435685:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.435686:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.435686:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.435688:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.435688:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.435690:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.435691:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.435691:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.435694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.435697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.435701:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.435704:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677474.435704:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677474.435705:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.435705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.435706:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.435707:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677474.435707:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.435707:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677474.435710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.435975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.435983:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.435985:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.435987:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.435989:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.435989:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.435990:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.435991:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.435993:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.435994:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.435994:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.435997:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.435999:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.436001:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.436002:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.436003:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.436005:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.436005:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.436006:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.436007:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.436010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.436018:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.436020:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.436021:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.436021:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.436023:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.436025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.436033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.436038:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.436041:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.436042:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.436042:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.436042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.436043:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.436181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.436188:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.436191:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.436193:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677474.436193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:30.0:1619677474.436195:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.436196:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.436196:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677474.436197:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000400:00000200:30.0:1619677474.436197:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.436198:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677474.436199:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.436200:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.436200:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.436204:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.436205:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.436206:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.436207:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.436208:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.436208:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.436208:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677474.436210:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:30.0:1619677474.436210:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677474.436211:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.436212:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.436213:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.436214:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:30.0:1619677474.436214:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.436215:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.436215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.436216:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677474.436217:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.436218:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677474.436219:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677474.436219:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.436220:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.436220:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.436221:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.436222:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:30.0:1619677474.436222:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.436223:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:30.0:1619677474.436223:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.436224:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:30.0:1619677474.436224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.436225:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.436226:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:31.0:1619677474.436228:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:31.0:1619677474.436228:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:9.2:1619677474.436230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677474.436234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:10.0:1619677474.436236:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.436239:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.436240:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.436240:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.436241:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.436242:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:30.0:1619677474.436244:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:10.0:1619677474.436244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:30.0:1619677474.436245:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.436246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:21.2:1619677474.437014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.437022:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.437025:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.437027:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.437029:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.437030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.437030:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.437031:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.437032:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.437033:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.437034:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.437037:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.437038:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.437041:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.437041:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.437042:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.437043:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.437044:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.437045:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.437046:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.437047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.437050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.437053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.437057:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.437059:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.437060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.437060:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.437062:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.437062:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.437064:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.437064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.437065:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.437066:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.437366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.437372:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.437375:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.437377:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.437378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.437379:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.437381:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.437382:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.437384:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.437384:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.437386:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.437389:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.437391:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.437393:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.437394:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.437395:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.437397:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.437397:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.437398:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.437399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.437404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.437412:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.437415:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.437416:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.437416:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.437418:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.437421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.437731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.437739:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.437741:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.437742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.437744:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.437746:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.437748:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.437749:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.437750:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.437751:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.437752:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.437756:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.437757:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.437760:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.437761:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.437762:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.437764:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.437764:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.437765:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.437765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677474.437770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.437779:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.437782:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.437782:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.437783:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.437785:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.437788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.437906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.437913:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.437916:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.437917:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.437919:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.437922:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.437923:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677474.437925:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.437926:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.437927:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.437928:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677474.437931:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677474.437931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:26.0:1619677474.437932:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677474.437935:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.437936:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677474.437936:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677474.437938:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:31.0:1619677474.437939:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:26.0:1619677474.437939:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.437940:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.437941:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.437941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:31.0:1619677474.437942:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.437944:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677474.437944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:31.0:1619677474.437946:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.437946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.437947:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.437948:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677474.437949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000400:00000200:31.0:1619677474.437950:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.437950:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.437951:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.2:1619677474.437952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:1.0:1619677474.437953:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:31.0:1619677474.437954:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.437955:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:31.0:1619677474.437956:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.437957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677474.437957:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:1.0:1619677474.437957:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.437958:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:31.0:1619677474.437959:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.437959:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.437959:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:1.0:1619677474.437959:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.437960:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.437960:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677474.437961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:2.0:1619677474.437961:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:31.0:1619677474.437962:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677474.437962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:1.0:1619677474.437962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677474.437963:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.437964:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.437964:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:2.0:1619677474.437964:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.437967:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.437968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677474.437968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000400:00000200:2.0:1619677474.437968:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677474.437969:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.437971:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.437972:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.437973:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:8.0:1619677474.437975:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:17.0:1619677474.437976:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000400:00000200:2.0:1619677474.437976:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.437977:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677474.437978:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.437979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677474.437979:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.437979:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000400:00000200:8.0:1619677474.437979:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.437979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:1.0:1619677474.437979:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.437980:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.437981:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.437982:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:8.0:1619677474.437982:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.437982:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677474.437982:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677474.437984:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677474.437984:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:8.0:1619677474.437985:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677474.437985:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:26.0:1619677474.437986:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677474.437986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:26.0:1619677474.437989:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.437990:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677474.437990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:26.0:1619677474.437992:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.437994:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.437996:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.437997:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.437998:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.437999:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.437999:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:26.0:1619677474.438000:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:20.0:1619677474.438001:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.438002:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.438004:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677474.438004:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.438005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.438006:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677474.438007:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677474.438009:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.438010:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.438011:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.438013:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.438013:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.438014:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.438015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.438019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.438027:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.438029:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.438030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.438031:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.438033:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.438036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.438190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.438195:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.438198:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.438200:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.438200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.438202:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.438203:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677474.438205:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.438206:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.438207:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.438208:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677474.438211:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.438213:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.438216:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.438217:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677474.438217:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677474.438219:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677474.438219:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.438221:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.438222:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.438222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677474.438225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.438228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.438231:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:2.0:1619677474.438232:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677474.438234:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.438234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.438235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677474.438235:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.438236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.438236:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.438238:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.438241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.438245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.438252:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.438255:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.438256:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.438256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.438257:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.438258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.438450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.438458:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.438461:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.438462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.438464:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.438467:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.438469:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.438470:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.438472:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.438473:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.438474:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.438477:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.438479:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.438482:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.438483:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.438483:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.438485:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.438485:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.438486:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.438487:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.438492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.438500:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.438503:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.438504:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.438504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.438506:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.438508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.438667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.438675:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.438678:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.438679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.438680:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.438684:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.438685:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.438686:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.438688:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.438689:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.438689:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.438693:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.438694:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.438697:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.438697:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.438698:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.438700:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.438700:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.438701:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.438701:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.438705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.438713:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.438715:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.438716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.438717:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.438719:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.438721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.438726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.438734:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.438736:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.438737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.438738:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.438739:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.438739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.438900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb0000] (20)++
00000800:00000200:1.0:1619677474.438907:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb0000] (21)++
00000800:00000200:1.0:1619677474.438910:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.36@o2ib3
00000800:00000200:20.2:1619677474.438911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677474.438911:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb0000] (22)--
00000400:00000200:1.0:1619677474.438913:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.36@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.438915:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.438916:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:2.0:1619677474.438917:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:1.0:1619677474.438918:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.438919:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.438920:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.438921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677474.438921:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.438922:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677474.438923:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.438926:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.438926:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.36@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.438928:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677474.438928:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:2.0:1619677474.438929:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.438931:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.438931:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677474.438932:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677474.438932:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:2.0:1619677474.438933:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677474.438933:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677474.438934:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677474.438935:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb0000] (21)++
00000800:00000200:1.0:1619677474.438936:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb0000] (22)++
00000400:00000200:2.0:1619677474.438937:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.438937:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb0000] (23)--
00000800:00000200:2.0:1619677474.438938:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677474.438938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb0000] (22)--
00000800:00000200:9.2:1619677474.438940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.438942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb0000] (21)++
00000800:00000200:2.0:1619677474.438942:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.438943:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.438943:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.438945:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.438946:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.438946:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677474.438947:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:2.0:1619677474.438947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677474.438948:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb0000] (22)++
00000800:00000200:8.0:1619677474.438949:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.438950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677474.438950:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb0000] (23)--
00000400:00000200:8.0:1619677474.438950:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.438951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677474.438951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb0000] (22)--
00000800:00000200:17.0:1619677474.438951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb0000] (21)--
00000400:00000200:8.0:1619677474.438953:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.438955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677474.438958:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.438961:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.438962:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.438964:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.438966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.438967:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.439094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.439102:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.439105:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.439107:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.439109:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.439109:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.439110:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.439112:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.439113:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.439114:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.439115:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.439118:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.439120:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.439122:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.439123:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.439123:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.439124:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.439125:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.439126:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.439126:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.439130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.439137:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.439139:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.439140:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.439141:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.439143:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.439144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.439151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.439157:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.439160:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.439161:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.439162:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.439162:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.439163:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.439567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677474.439573:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.439576:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677474.439577:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677474.439579:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.439581:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.439583:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677474.439584:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.439586:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.439587:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.439588:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677474.439592:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.439594:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677474.439596:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.439597:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677474.439598:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677474.439600:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677474.439600:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.439602:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677474.439603:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677474.439603:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:25.2:1619677474.439607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.2:1619677474.439609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:21.2:1619677474.439614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:17.0:1619677474.439615:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:13.0:1619677474.439616:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:17.0:1619677474.439617:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677474.439617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677474.439618:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:13.0:1619677474.439619:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677474.439621:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.439622:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:13.0:1619677474.439623:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.439624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:30.0:1619677474.439626:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:13.0:1619677474.439627:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:30.0:1619677474.439628:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.439630:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.439630:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.439631:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.439632:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.439634:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.439635:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.439636:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.439639:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.439641:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.439644:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.439645:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.439646:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.439647:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.439648:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.439649:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.439651:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.439652:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.439653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.439657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.439660:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.439662:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.439663:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.439663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.439665:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.439665:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.439667:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.439668:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.439668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.439669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.439911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677474.439918:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.439920:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677474.439921:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677474.439923:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.439925:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.439927:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677474.439928:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.439930:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.439930:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.439931:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677474.439934:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.439936:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677474.439939:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.439940:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677474.439940:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677474.439942:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677474.439943:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.439943:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.439944:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677474.439948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.439956:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.439958:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.439959:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.439961:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.439962:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.439964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.440126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.440134:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.440136:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.440137:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.440139:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.440142:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.440144:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.440145:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.440146:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.440147:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.440148:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.440151:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.440152:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.440154:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.440155:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.440156:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.440157:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.440158:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.440159:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.440159:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.440163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.440169:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.440171:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.440172:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.440173:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.440174:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.440176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.440316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.440322:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.440325:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.440326:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.440328:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.440331:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.440332:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.440333:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.440335:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.440336:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.440337:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.440340:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.440342:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.440344:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.440345:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.440346:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.440347:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.440348:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.440349:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.440349:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.440354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.440362:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.440365:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.440366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.440366:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.440368:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.440370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.440495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.440503:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.440506:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.440508:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.440510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.440511:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.440512:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.440513:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.440515:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.440516:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.440517:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.440520:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.440521:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.440523:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.440524:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.440525:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.440526:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.440527:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.440528:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.440528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.440532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.440538:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.440541:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.440542:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.440544:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.440545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.440546:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.440989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:4.2:1619677474.440992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.0:1619677474.440995:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.440997:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.440999:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.441000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:10.0:1619677474.441000:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:21.0:1619677474.441002:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.441003:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.441004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:21.0:1619677474.441004:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.441005:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.441006:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.441007:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.441008:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.441009:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.441009:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.441010:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.441012:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.441013:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.441014:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.441015:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.441016:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.441017:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.441019:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.441019:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.441020:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.441020:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.441022:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.441022:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.441023:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:10.0:1619677474.441023:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.441024:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:10.0:1619677474.441026:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.441026:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.441027:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.441029:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677474.441029:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.441030:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.441031:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.441032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.441039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677474.441042:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677474.441042:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000400:00000200:22.0:1619677474.441043:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.441043:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677474.441044:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.441046:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.441046:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.441049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.441050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.441050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.441224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.441229:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.441232:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.441232:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.441234:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.441237:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.441238:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677474.441239:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.441241:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.441242:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.441243:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.441247:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.441249:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.441251:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.441252:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.441253:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.441255:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.441256:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.441256:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.441257:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677474.441261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.441268:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.441271:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.441272:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.441274:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.441275:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.441277:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.441640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.441648:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.441651:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.441653:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.441655:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.441655:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.441656:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.441657:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.441659:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.441659:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.441661:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.441664:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.441666:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.441668:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.441669:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.441670:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.441671:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.441672:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.441673:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.441673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.441678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.441685:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.441687:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.441688:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.441689:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.441690:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.441692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.441729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.441737:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.441740:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.441741:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.441744:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.441744:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.441745:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.441746:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.441747:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.441748:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.441749:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.441752:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.441753:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.441756:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.441756:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.441757:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.441758:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.441759:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.441760:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.441760:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.441764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.441770:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.441772:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.441773:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.441773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.441775:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.441777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.441784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.441790:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.441793:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.441794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.441794:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.441795:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.441796:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.442063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.442069:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.442072:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.442073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.442075:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.442078:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.442079:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677474.442080:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.442082:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.442083:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.442084:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.442087:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.442089:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.442091:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.442092:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677474.442093:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.442095:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677474.442095:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.442096:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.442096:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.442100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.442107:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.442109:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.442110:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.442110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.442113:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.442115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.442120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.442126:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.442129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.442129:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.442130:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.442131:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.442131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.442466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.442473:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.442476:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.442477:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.442479:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.442481:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.442483:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.442484:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.442485:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.442486:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.442487:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.442490:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.442492:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.442495:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.442495:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.442496:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.442498:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.442498:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.442499:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.442499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.442504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.442513:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.442516:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.442516:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.442517:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.442519:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.442522:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.442800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.442807:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.442810:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.442811:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.442812:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.442816:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.442817:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.442818:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.442819:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.442820:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.442821:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.442824:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.442825:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.442828:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.442829:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.442829:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.442831:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.442831:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.442832:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.442832:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.442837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.442844:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.442847:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.442848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.442848:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.442850:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.442854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.442856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.442863:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.442865:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.442866:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.442866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.442867:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.442868:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.443112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:21.2:1619677474.443116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677474.443117:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.443119:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677474.443120:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677474.443122:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.443124:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677474.443124:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.443125:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.443127:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677474.443127:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.443129:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.443130:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.443131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.443132:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.443132:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677474.443135:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.443137:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.443139:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.443141:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:6.0:1619677474.443141:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.443142:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:30.0:1619677474.443143:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.443143:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677474.443144:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.443144:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677474.443145:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.443145:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.443145:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000400:00000200:30.0:1619677474.443146:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:6.0:1619677474.443146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:30.0:1619677474.443149:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677474.443150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.443151:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.443154:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.443155:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.443155:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.443157:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.443157:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.443157:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.443158:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.443159:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.443159:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677474.443161:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.443161:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.443161:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.443162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.443163:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.443164:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.443167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.443169:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.443172:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.443173:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.443173:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.443174:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.443175:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.443176:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.443177:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.443177:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.443178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.443239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.443245:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.443248:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.443249:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.443251:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.443254:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.443255:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677474.443257:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.443258:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.443259:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.443260:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677474.443264:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.443265:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.443268:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.443269:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677474.443270:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.443271:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677474.443272:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.443273:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.443273:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:25.2:1619677474.443277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.443285:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.443288:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.443289:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.443289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.443291:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.443297:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.443540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677474.443547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.443550:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677474.443551:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677474.443553:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677474.443556:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.443557:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677474.443558:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.443559:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.443560:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.443561:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677474.443564:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.443566:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.443567:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.443568:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.443568:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.443570:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.443571:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.443571:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677474.443572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677474.443576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.443582:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.443584:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.443584:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.443585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.443586:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.443588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.443887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.443895:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.443898:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.443900:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.443902:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.443902:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.443903:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.443904:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.443905:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.443906:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.443907:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.443910:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.443912:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.443914:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.443915:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.443915:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.443917:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.443917:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.443918:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.443919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.443923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.443929:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.443932:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.443932:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.443934:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.443936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.443937:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.443946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.443953:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.443955:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.443956:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.443957:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.443957:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.443958:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.444273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.444277:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.444280:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.444282:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.444283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.444284:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.444286:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.444287:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.444289:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.444289:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.444291:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.444299:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.444301:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.444304:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.444304:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.444305:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.444307:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.444308:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.444309:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.444309:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.444314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.444322:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.444324:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.444325:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.444327:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.444329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.444330:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.444545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.444552:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.444555:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.444555:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.444558:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.444560:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.444561:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.444563:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.444564:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.444565:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.444566:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.444569:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.444571:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.444573:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.444574:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.444574:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.444576:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.444577:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.444577:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.444578:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.444583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.444590:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.444592:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.444592:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.444593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.444594:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.444595:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.444709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.444714:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.444716:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.444717:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.444718:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.444721:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.444722:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.444723:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.444724:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.444725:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.444726:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.444729:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.444730:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.444732:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.444733:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.444733:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.444735:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.444735:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.444736:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.444737:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.444738:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.444742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.444743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.444748:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.444750:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677474.444750:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677474.444751:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.444752:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677474.444753:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.444754:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.444755:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.444756:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.444758:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.444961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.444968:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.444971:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.444973:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.444975:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.444975:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.444976:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.444978:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.444979:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.444980:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.444981:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.444983:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.444985:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.444987:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.444988:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.444989:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.444990:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.444991:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.444992:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.444993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.444996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.445003:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.445005:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.445006:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.445007:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.445009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.445010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677474.445069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.445075:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.445077:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.445078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.445080:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.445082:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.445084:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.445085:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.445087:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.445087:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.445089:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.445092:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.445094:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.445096:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.445097:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.445098:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.445099:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.445100:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.445101:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.445101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.445106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.445113:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.445116:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.445117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.445117:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.445119:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.445121:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.445325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.445333:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.445336:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.445337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.445339:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.445342:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.445344:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.445345:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.445346:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.445347:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.445348:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.445352:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.445354:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.445356:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.445357:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.445358:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.445359:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.445360:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.445361:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.445361:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.445366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.445374:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.445377:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.445378:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.445378:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.445380:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.445383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.445385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.445393:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.445395:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677474.445395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677474.445396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.445397:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.445398:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.445398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.445403:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.445406:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.445407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.445409:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.445411:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.445412:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.445414:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.445415:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.445416:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.445417:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.445420:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.445422:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.445424:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.445425:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.445426:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.445427:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.445428:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.445428:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.445429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.445433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.445440:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.445443:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.445444:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.445444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.445446:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.445449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.445855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.445863:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.445866:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.445868:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.445870:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.445871:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.445871:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.445872:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.445873:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.445874:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.445875:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.445878:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.445880:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.445881:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.445882:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.445883:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.445884:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.445884:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.445885:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.445886:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.445890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.445896:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.445899:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.445900:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.445901:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.445903:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.445904:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.446096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.446103:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.446106:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.446106:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.446108:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.446111:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.446112:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.446114:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.446115:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.446116:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.446116:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.446120:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.446121:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.446124:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.446125:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.446125:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.446127:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.446127:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.446128:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.446130:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.446130:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.446133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.446136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.446140:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.446143:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.446143:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.446144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.446144:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.446145:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.446146:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.446146:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.446147:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.446149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.446260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.446267:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.446270:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.446272:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.446274:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.446274:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.446275:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.446277:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.446279:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.446280:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.446281:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.446284:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.446286:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.446288:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.446289:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.446289:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.446291:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.446291:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.446292:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.446293:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.446298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.446305:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.446307:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.446308:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.446309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.446310:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.446312:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.446407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.446413:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.446415:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.446418:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.446418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.446421:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.446422:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.446423:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.446425:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.446426:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.446427:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.446430:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.446432:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.446434:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.446435:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.446436:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.446437:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.446438:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.446439:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.446440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.446444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.446451:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.446454:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.446455:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.446455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.446457:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.446459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.446934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.446942:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.446944:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.446945:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.446947:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.446950:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.446951:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.446952:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.446954:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.446955:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.446956:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.446958:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.446960:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.446963:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.446964:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.446965:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.446967:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.446967:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.446968:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.446969:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.446969:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677474.446973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.446975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677474.446982:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677474.446983:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677474.446984:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677474.446985:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.446985:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677474.446985:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.446986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677474.446987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.446987:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.446990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.446993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.447001:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.447003:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.447004:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.447005:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.447006:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.447007:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.447224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.447230:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.447232:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.447234:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.447235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.447236:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.447238:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.447239:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.447240:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.447241:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.447242:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.447245:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.447246:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.447249:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.447250:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.447251:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.447252:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.447253:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.447253:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.447254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.447258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.447265:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.447268:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.447268:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.447270:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.447271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.447272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.447281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677474.447287:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.447289:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677474.447290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677474.447292:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.447301:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.447303:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677474.447304:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.447305:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.447306:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.447307:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677474.447310:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.447312:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677474.447315:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.447316:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677474.447317:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.447319:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677474.447319:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.447321:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677474.447322:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677474.447322:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677474.447325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.447328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:8.0:1619677474.447333:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677474.447334:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:24.0:1619677474.447336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:22.0:1619677474.447336:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:8.0:1619677474.447336:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:22.0:1619677474.447337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:8.0:1619677474.447337:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.447338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:8.0:1619677474.447339:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.447342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.447433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677474.447438:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.447441:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677474.447443:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.447443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677474.447445:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.447447:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677474.447448:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.447449:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.447450:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.447451:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677474.447454:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.447456:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.447458:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.447459:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.447460:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.447461:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.447462:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.447463:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677474.447463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.447468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.447475:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.447478:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.447479:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.447480:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.447481:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.447482:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.447629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.447637:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.447640:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.447641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.447643:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.447646:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.447647:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.447649:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.447650:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.447651:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.447652:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.447655:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.447657:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.447660:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.447660:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.447661:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.447663:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.447663:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.447665:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.447666:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.447666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.447669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.447672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.447677:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.447679:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.447679:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.447680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.447681:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.447681:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.447681:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.447682:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.447683:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.447686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.447946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.447951:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.447953:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.447955:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.447956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.447957:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.447959:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.447960:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.447961:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.447962:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.447963:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.447967:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.447969:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.447971:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.447971:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.447972:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.447973:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.447974:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.447975:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.447975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.447979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.447985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677474.447987:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.447989:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.447990:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.447990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.447992:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.447993:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.447994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.447996:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.447998:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.448000:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.448001:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.448002:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.448003:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.448004:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.448005:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677474.448005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000400:00000200:30.0:1619677474.448006:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.448009:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.448010:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677474.448012:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.448013:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.448014:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.448014:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.448015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.448015:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.448016:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.448016:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677474.448016:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.448017:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.448017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.448018:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.448019:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.448020:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.448021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.448026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.448028:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.448031:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.448032:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.448032:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.448034:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.448035:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.448036:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.448036:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.448037:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.448038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.448231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.448238:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.448241:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.448243:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.448244:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.448245:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.448246:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.448248:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.448249:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.448250:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.448251:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.448253:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.448255:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.448257:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.448258:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.448258:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.448260:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.448261:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.448261:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.448262:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.448276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.448282:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.448284:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.448285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.448285:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.448287:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.448289:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.448839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.448847:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.448850:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.448852:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.448854:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.448855:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.448855:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.448856:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.448858:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.448859:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.448859:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.448862:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.448864:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.448866:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.448867:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.448867:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.448869:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.448869:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.448870:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.448870:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.448875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.448882:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.448884:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.448885:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.448886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.448887:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.448889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.448891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:9.2:1619677474.448893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.448898:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.0:1619677474.448899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.448901:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:8.0:1619677474.448901:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.448902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.448902:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.0:1619677474.448902:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.448903:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:26.0:1619677474.448904:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.448904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:26.0:1619677474.448907:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.448908:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677474.448909:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.448911:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.448912:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.448913:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677474.448916:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.448917:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677474.448920:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.448921:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677474.448922:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.448923:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.448924:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.448924:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.448925:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.448930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.448939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.448942:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.448943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.448943:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.448945:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.448948:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.449103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.449109:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.449112:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.449113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.449115:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.449117:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.449118:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.449119:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.449121:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.449122:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.449123:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.449126:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.449128:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.449131:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.449132:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.449132:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.449134:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.449135:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.449136:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.449136:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677474.449140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.449148:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.449150:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.449151:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.449153:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.449155:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.449156:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.449571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.449579:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.449582:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.449585:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.449587:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.449587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.449588:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.449589:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.449591:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.449592:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.449593:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.449596:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.449597:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.449599:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.449600:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.449601:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.449602:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.449603:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.449604:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.449604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.449609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.449615:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.449618:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.449618:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.449620:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.449622:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.449623:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.449855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.449862:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.449865:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.449867:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.449868:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.449869:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.449870:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.449871:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.449872:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.449873:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.449874:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.449877:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.449878:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.449880:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.449881:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.449882:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.449883:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.449884:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.449885:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.449887:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.449888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.449889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.449893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.449896:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.449898:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.449899:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.449899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.449901:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.449901:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.449903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.449904:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.449904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.449905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.449908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.449914:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.449917:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.449918:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.449918:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.449919:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.449919:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.450163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.450171:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.450173:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.450174:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.450176:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.450179:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.450180:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.450182:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.450183:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.450184:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.450184:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.450188:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.450189:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.450192:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.450193:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.450194:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.450195:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.450196:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.450196:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.450197:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.450201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.450203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677474.450209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.450210:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.450212:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.450213:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:22.0:1619677474.450213:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.450213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677474.450216:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.450216:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.450217:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.450218:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.450219:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:22.0:1619677474.450219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677474.450221:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.450222:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.450223:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.450224:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:5.2:1619677474.450224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677474.450227:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.450229:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.450231:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.450231:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677474.450232:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.450233:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677474.450233:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:31.0:1619677474.450234:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677474.450234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677474.450235:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.450235:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.450236:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677474.450236:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.450238:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.450240:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:9.2:1619677474.450240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677474.450241:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.450242:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.450243:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.450244:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677474.450246:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.450248:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:17.0:1619677474.450248:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.450249:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.450249:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677474.450250:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677474.450251:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.450252:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.450253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677474.450253:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.450254:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.450255:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.450256:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.450257:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.450257:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.450262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.450269:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.450272:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.450273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.450273:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.450275:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.450277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.450306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.450313:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.450316:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.450317:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.450319:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.450322:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.450323:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.450325:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.450326:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.450327:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.450328:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.450332:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.450333:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.450336:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.450337:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.450338:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.450339:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.450340:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.450341:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.450342:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.450343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.450345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.450349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.450353:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.450355:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.450356:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.450356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.450357:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.450358:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.450359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677474.450359:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.450361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.450362:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.450910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.450917:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.450920:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.450921:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.450923:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.450926:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.450928:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.450929:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.450930:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.450931:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.450932:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.450935:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.450937:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.450939:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.450940:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.450941:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.450942:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.450943:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.450944:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.450944:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.450949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.450956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.450959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.450959:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.450960:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.450962:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.450965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.451857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.451865:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.451868:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.451870:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.451872:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.451873:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.451874:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.451875:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.451877:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.451878:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.451879:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.451882:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.451883:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.2:1619677474.451883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677474.451886:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.451887:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.451887:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.451889:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.451889:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:2.0:1619677474.451889:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.451890:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.451891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:2.0:1619677474.451892:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.451893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:9.2:1619677474.451895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:2.0:1619677474.451895:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.451898:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.451899:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.451901:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.451902:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.451903:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:2.0:1619677474.451903:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.451904:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.451905:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.451906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.451906:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.451908:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677474.451909:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.451910:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.451912:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677474.451913:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.451914:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.451915:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.451917:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.451918:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.451918:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.451919:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.451923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.451930:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.451932:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.451933:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.451935:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.451936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.451938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.452485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.452493:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.452495:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.452496:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.452499:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.452502:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.452503:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.452504:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.452506:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.452507:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.452508:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.452510:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.452512:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.452515:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.452516:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.452516:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.452518:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.452519:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.452519:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.452520:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.452525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.452533:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.452536:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.452537:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.452537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.452540:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.452542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.452547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.452554:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.452556:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.452557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.452557:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.452558:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.452559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.453122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.453126:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.453129:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.453131:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.453132:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.453133:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.453134:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.453136:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.453137:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.453138:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.453139:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.453142:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.453144:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.453145:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.453146:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.2:1619677474.453147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677474.453147:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.453148:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.453149:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.453150:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.453150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677474.453155:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677474.453155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677474.453158:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.453161:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.453163:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.453163:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.453163:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:30.0:1619677474.453165:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.453166:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.453166:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.453166:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.453168:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.453169:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.453169:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.453169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:30.0:1619677474.453171:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677474.453171:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:30.0:1619677474.453175:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.453177:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.453179:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.453180:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.453181:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.453182:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.453183:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.453184:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.453186:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.453187:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.453188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.453192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.453197:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.453198:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.453199:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.453200:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.453200:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.453201:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.453201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.453202:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.453204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.453205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.453207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.453213:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.453216:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.453216:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.453217:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.453218:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.453218:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.453264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.453272:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.453274:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.453276:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.453278:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.453279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.453279:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.453280:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.453282:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.453282:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.453283:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.453286:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.453287:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.453289:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.453290:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.453290:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.453292:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.453293:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.453298:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.453298:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.453298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.453305:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.453307:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.453308:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.453309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.453310:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.453312:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.453417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:20.2:1619677474.453423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677474.453425:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.453428:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677474.453429:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.453429:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677474.453432:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.453432:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:18.0:1619677474.453432:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:31.0:1619677474.453433:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:2.0:1619677474.453433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677474.453436:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.453436:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.453437:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.453438:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.453439:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.453439:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.453440:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.453442:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.453444:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.453446:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677474.453446:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.453447:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.453449:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.453450:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.453451:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.453452:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.453452:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.453453:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.453454:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.453454:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.453455:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.453455:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:18.0:1619677474.453455:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677474.453456:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:18.0:1619677474.453456:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.453457:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.453458:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:25.2:1619677474.453459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.453459:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.453459:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.453464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.453466:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.453469:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.453470:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.453471:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.453471:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:31.0:1619677474.453473:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.453473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:2.0:1619677474.453473:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.453474:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.453475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.453476:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.453478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.453478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.453485:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.453488:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.453489:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.453489:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.453490:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.453491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.453668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.453671:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.453673:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.453676:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.453677:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.453678:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.453679:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.453681:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.453682:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.453683:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.453684:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.453687:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.453689:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.453691:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.453692:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.453693:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.453695:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.453695:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.453696:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.453698:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.453698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.453701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.453704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.453707:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.453709:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.453709:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.453709:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.453710:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.453711:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.453711:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.453712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.453713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677474.453714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.453722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.453730:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.453732:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.453733:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.453734:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.453735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.453736:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.454040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.454047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.454050:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.454052:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.454054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.454054:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.454055:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.454057:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.454058:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.454059:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.454060:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.454064:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.454066:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.454067:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.454068:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.454069:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.454070:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.454071:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.454072:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.454073:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.454076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.454083:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.454085:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.454086:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.454088:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.454089:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.454090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.454896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.454904:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.454907:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.454909:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.454911:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.454911:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.454912:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:31.0:1619677474.454913:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.454913:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:5.2:1619677474.454913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:30.0:1619677474.454915:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.454916:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.454917:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.454919:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.454919:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677474.454920:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.454921:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.454921:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677474.454922:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677474.454922:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677474.454923:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.454924:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.454925:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.454925:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.454926:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.454927:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:1.0:1619677474.454927:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.454929:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.454931:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96504436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.454932:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677474.454932:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.454933:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.454935:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.454935:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677474.454935:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.454936:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.454936:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.454936:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.454937:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.454938:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.454938:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.454938:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)++
00000800:00000200:30.0:1619677474.454939:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (24)--
00000400:00000200:1.0:1619677474.454939:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.454940:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.454940:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.454940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.454941:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.454941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:1.0:1619677474.454941:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.454942:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.454944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677474.454944:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.454944:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.454945:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:25.2:1619677474.454946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:1.0:1619677474.454947:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.454947:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.454948:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.454949:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.454950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677474.454952:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:30.0:1619677474.454953:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:25.2:1619677474.454953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677474.454954:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:30.0:1619677474.454955:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:24.0:1619677474.454955:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.454955:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677474.454955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677474.454956:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.454958:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.454958:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.454959:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677474.454960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.454961:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.454961:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:30.0:1619677474.454962:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:17.0:1619677474.454963:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677474.454964:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.454964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677474.454965:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.454965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.454965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.454967:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.454970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.454971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.454979:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.454982:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.454983:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.454985:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.454988:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.454989:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.454991:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.454992:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.454993:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.454994:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.454997:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.454999:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.455002:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.455002:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.455003:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.455005:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.455005:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.455006:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.455006:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.455010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.455016:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.455019:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.455020:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.455020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.455022:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.455024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.455181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.455188:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.455191:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.455192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.455194:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.455197:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.455199:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.455201:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.455203:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.455205:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.455207:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.455212:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.455216:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.455219:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677474.455221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:17.0:1619677474.455221:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.455222:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.455225:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.455225:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.455226:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.455227:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:31.0:1619677474.455229:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.455232:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.455234:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.455235:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.455237:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.455238:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.455239:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.455240:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.455241:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677474.455241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677474.455242:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.455245:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.455247:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.455250:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.455250:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.455251:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677474.455252:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.455253:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.455253:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.455255:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.455255:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.455256:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.455258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.455258:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.455258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:9.2:1619677474.455261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677474.455262:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.455266:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.455267:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677474.455269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.455269:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.455269:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.455270:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.455271:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.455272:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.455273:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.455275:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.455277:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.455278:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.455279:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:8.0:1619677474.455279:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.455281:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.455282:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.455283:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.455284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.455284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677474.455285:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.455286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.455287:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:8.0:1619677474.455288:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.455289:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.455290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.455290:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.455293:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.455294:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.455299:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.455301:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.455302:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.455303:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.455304:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677474.455305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.455312:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.455315:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.455316:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677474.455318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.455318:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.455320:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.455321:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.455322:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.455324:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.455325:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.455325:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.455328:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.455329:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.455330:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.455333:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.455333:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677474.455333:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677474.455334:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677474.455335:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.455335:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.455336:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.455336:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.455337:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.455337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:6.0:1619677474.455339:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.455342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677474.455343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677474.455351:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:4.2:1619677474.455351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:18.0:1619677474.455354:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.455355:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.455355:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:21.0:1619677474.455357:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:18.0:1619677474.455358:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.455360:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.455362:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:18.0:1619677474.455362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.0:1619677474.455363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.455365:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.455366:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.455368:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.455370:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.455371:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.455372:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.455376:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.455378:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.455381:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.455382:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.455382:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.455384:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.455385:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.455386:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.455387:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.455388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677474.455390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.455394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.455398:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677474.455399:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.455400:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.455401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.455402:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677474.455403:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.455405:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.455405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.455408:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.455411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.455650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.455658:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.455661:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.455662:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.455665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.455665:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.455667:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.455668:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.455669:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.455670:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.455671:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.455674:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.455676:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.455678:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.455679:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.455679:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.455681:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.455681:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.455683:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.455684:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.455685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.455686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.455690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.455695:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.455696:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.455697:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.455699:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.455699:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.455699:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.455700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.455700:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.455701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.455702:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.455718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.455726:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.455728:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.455729:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.455731:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.455734:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.455735:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.455736:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.455737:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.455738:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.455739:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.455742:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.455744:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.455746:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.455746:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.455747:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.455748:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.455749:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.455750:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.455751:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.455752:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.455754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.455757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.455761:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.455763:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.455764:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.455764:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.455765:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.455766:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.455766:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.455767:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.455768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.455768:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.455774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.455780:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.455782:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.455783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.455783:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.455784:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.455784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.455985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677474.455992:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.455994:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677474.455995:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677474.455997:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.455999:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.456001:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677474.456002:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.456003:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.456004:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.456005:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677474.456008:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.456009:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.456011:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.456012:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677474.456012:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677474.456014:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677474.456014:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.456015:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677474.456015:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.2:1619677474.456020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.456027:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.456029:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.456030:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.456031:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.456033:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.456034:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.456821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.456827:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.456830:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.456831:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.456833:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.456835:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.456837:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.456838:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.456840:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.456841:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.456842:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.456845:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.456847:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.456850:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.456851:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.456851:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.456853:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.456854:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.456855:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.456855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.456860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.456868:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.456870:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.456871:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.456873:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.456875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.456876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.456942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.456950:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.456953:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.456955:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.456957:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.456957:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.456958:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.456959:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.456960:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.456961:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.456962:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.456965:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.456967:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.456969:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.456970:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.456971:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.456972:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.456973:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.456974:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.456976:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.456977:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.456978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.456982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.456985:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.456987:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.456988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.456988:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.456990:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677474.456990:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.456992:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677474.456992:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.456993:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.456994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.456999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.457005:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.457007:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.457008:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.457008:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.457008:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.457009:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.457281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.457289:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.457292:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.457293:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.457301:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.457304:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.457306:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.457307:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.457308:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.457309:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.457310:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.457313:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.457315:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.457318:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.457318:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.457319:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.457321:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.457321:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.457322:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.457322:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.457327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.457336:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.457338:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.457339:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.457340:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.457342:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.457345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.458132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.458140:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.458143:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.458145:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.458147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.458147:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.458148:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.458150:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.458151:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.458152:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.458153:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.458156:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.458157:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.458159:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.458160:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.458161:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.458162:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.458163:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.458163:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.458164:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.458168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.458174:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.458177:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.458178:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.458179:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.458181:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.458182:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.458185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.458192:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.458195:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.458196:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.458197:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.458197:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.458199:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.459076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.459084:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.459086:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.459088:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.459090:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.459091:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.459091:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.459092:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.459094:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.459095:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.459095:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.459098:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.459099:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.459101:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.459102:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.459102:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.459104:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.459104:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.459105:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.459105:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.459110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.459118:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.2:1619677474.459118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:31.0:1619677474.459120:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.459120:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.459122:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.459124:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.459124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.459124:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.459127:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.459128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.459129:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.459133:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.459134:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677474.459135:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.459137:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.459138:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.459139:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677474.459142:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.459144:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.459147:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.459148:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677474.459149:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.459151:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.459151:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.459152:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.459153:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677474.459167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.459175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.459178:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.459179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:21.2:1619677474.459179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:8.0:1619677474.459179:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.459181:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.459184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.459186:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.459189:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.459191:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.459193:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.459193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.459194:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.459195:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.459197:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.459198:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.459199:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.459201:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.459203:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.459205:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.459206:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.459207:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.459208:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.459209:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.459210:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.459211:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.459214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.459220:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.459222:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.459223:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.459224:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.459225:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.459227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.459250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.459257:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.459261:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.459264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.459266:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.459268:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.459270:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.459272:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.459275:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.459276:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.459278:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.459283:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.459286:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.459289:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.459290:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.459291:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.459298:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.459299:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:25.2:1619677474.459300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.459301:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.459302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.459308:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.459310:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.459311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.459311:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.459314:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.459315:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.459386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.459392:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.459394:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.459397:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.459399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.459399:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.459401:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.459402:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.459403:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.459404:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.459405:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.459409:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.459410:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.459412:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.459413:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.459414:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.459415:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.459416:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.459417:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.459418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.459433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.459442:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.459445:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.459447:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.459449:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.459449:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.459452:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677474.459626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.459633:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.459636:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.459637:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.459639:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.459641:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.459642:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.459644:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.459645:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.459646:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.459647:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.459650:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.459652:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.459655:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.459655:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.459656:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.459657:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.459658:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.459659:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.459659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.459664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.459671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.459674:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.459675:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.459675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.459677:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.459679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.459845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.459853:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.459856:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.459858:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.459860:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.459860:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.459862:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.459863:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.459864:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.459865:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.459866:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.459869:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.459871:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.459873:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.459874:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.459875:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.459876:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.2:1619677474.459876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677474.459877:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.459878:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.459878:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.459882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677474.459885:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677474.459888:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:18.0:1619677474.459890:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:8.0:1619677474.459891:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:2.0:1619677474.459891:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677474.459892:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.459892:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.459896:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:18.0:1619677474.459897:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.459898:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:18.0:1619677474.459901:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.459903:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.459905:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.459907:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.459909:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.459910:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.459916:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.459919:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.459922:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.459923:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.459924:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.459926:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.459927:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.459929:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.459930:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.459932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.459933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.459937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.459943:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.459945:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.459946:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677474.459946:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.459948:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.459949:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.459950:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.459951:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.459952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677474.459953:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677474.460038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.460048:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.460051:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.460052:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.460054:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.460058:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.460059:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.460061:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.460062:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.460063:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.460064:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.460067:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.460068:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.460071:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.460072:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.460073:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.460075:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.460075:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.460076:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.460076:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.460081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.460089:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.460091:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.460092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.460093:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.460095:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.460098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.460102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.460110:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.460112:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.460113:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.460113:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.460114:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.460115:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.460558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.460566:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.460568:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.460570:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.460571:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.460573:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.460575:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.460576:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.460578:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.460579:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.460580:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.460583:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.460584:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.460587:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.460588:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.460588:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.460590:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.460590:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.460591:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.460591:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.460596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.460604:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.460607:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.460608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.460608:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.460611:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.460613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.460879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.460887:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.460890:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.460892:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.460894:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.460894:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.460895:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.460897:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.460898:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.460899:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.460900:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.460903:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.460904:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.460907:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.460907:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.460908:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.460908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677474.460909:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.460910:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.460911:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.460911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677474.460914:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:9.2:1619677474.460915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.460916:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.460919:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.460919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.460921:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.460922:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.0:1619677474.460922:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:21.0:1619677474.460924:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.460924:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.460925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677474.460925:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.460925:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.460926:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.460927:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.460928:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.460931:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.460931:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677474.460933:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.460935:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.460936:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677474.460937:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677474.460938:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677474.460939:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.460940:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.460940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677474.460945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.460952:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.460955:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.460956:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.460956:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.460958:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.460961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.460967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.460974:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.460977:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.460978:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.460978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.460979:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.460980:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.461167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.461176:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.461178:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.461179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.461181:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.461184:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.461186:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.461187:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.461188:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.461189:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.461190:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.461193:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.461195:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.461197:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.461198:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.461199:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.461200:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.461201:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.461201:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.461202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.461206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.461212:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.461214:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.461215:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.461215:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.461217:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.461219:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.461630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.461638:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.461641:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677474.461641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677474.461643:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.461645:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.461646:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.461646:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.461647:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.461648:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.461648:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677474.461649:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.461650:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.461651:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.461652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677474.461653:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.461654:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.461655:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.461657:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.461658:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.461658:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:10.0:1619677474.461658:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677474.461659:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677474.461659:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.461660:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.461661:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677474.461661:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.461661:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.461662:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677474.461662:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677474.461663:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.461664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677474.461665:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.461667:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677474.461667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.461669:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677474.461670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.461670:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.461671:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.461673:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.461673:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.461673:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.461674:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677474.461674:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.461676:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.461677:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.461677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.461678:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.461679:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.461680:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:25.2:1619677474.461680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.461681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677474.461681:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.461682:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.461685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677474.461688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:24.0:1619677474.461691:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.461691:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.461692:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.461692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677474.461695:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677474.461695:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.461695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.461696:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.461697:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.461698:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.461698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.461704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.461711:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.461713:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.461714:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.461715:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.461716:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.461716:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.461953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.461958:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.461961:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.461962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.461964:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.461966:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.461968:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677474.461969:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.461970:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.461971:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.461972:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.461976:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.461977:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.461980:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.461981:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.461982:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.461983:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.461984:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.461984:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.461985:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677474.461990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.461996:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.461999:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.462000:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.462002:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.462003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.462005:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.462119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.462125:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.462128:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.462129:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.462131:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.462134:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.462135:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.462137:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.462138:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.462139:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.462140:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.462143:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.462144:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.462147:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.462148:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.462148:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.462150:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.462151:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.462152:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.462152:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.462156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.462165:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.462167:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.462168:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.462168:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.462171:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.462173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.462693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.462701:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.462704:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.462707:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.462709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.462709:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.462711:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.462712:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.462713:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.462715:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.462716:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.462719:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.462721:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.462724:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.462725:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.462725:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.462727:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.462728:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.462729:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.462729:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677474.462734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.462742:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.462745:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.462746:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.462748:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.462748:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.462751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.462790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.462798:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.462801:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.462801:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.462804:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.462807:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.462808:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.462809:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.462811:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.462812:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.462813:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.462816:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.462817:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.462820:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.462820:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.462821:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.462823:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.462823:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.462824:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.462824:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.462829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.462839:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.462843:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.462844:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.462846:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.462849:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.462851:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.462855:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.462856:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.462857:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.462858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.462860:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.463138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.463143:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.463145:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.463148:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.463148:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.463150:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.463152:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.463153:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.463154:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.463155:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.463156:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.463159:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.463161:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.463164:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.463164:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.463165:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.463167:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.463167:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.463168:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.463169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.463173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.463179:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.463182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.463182:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.463183:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.463185:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.2:1619677474.463187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677474.463187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.463195:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.463198:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.463200:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.463201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.463203:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.463204:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.463205:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.463207:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.463208:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.463209:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.463212:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.463214:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.463216:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.463217:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.463218:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.463219:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.463220:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.463221:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.463222:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.463225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.463231:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.463233:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.463234:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.463235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.463236:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.463239:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.463805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.463813:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.463816:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.463818:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.463820:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.463821:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.463822:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.463823:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.463824:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.463825:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.463826:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.463829:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.463831:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.463833:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.463834:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.463834:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.463836:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.463836:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.463838:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.463839:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.463840:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.463842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.463846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.463849:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.463851:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.463852:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.463853:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.463854:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.463855:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.463857:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677474.463857:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.463858:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.463859:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.463859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.463865:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.463867:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.463868:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.463868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.463869:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.463870:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.463886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.463895:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.463897:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.463899:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.463901:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.463901:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.463902:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.463904:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.463905:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.463906:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.463906:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.463910:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.463911:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.463913:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.463914:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.463914:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.463916:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.463916:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.463917:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.463917:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.463921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.463928:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:20.2:1619677474.463930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:30.0:1619677474.463931:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.463932:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.463933:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.463935:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.463936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:18.0:1619677474.463937:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.463939:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.463940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.463943:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.463945:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.463946:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677474.463948:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.463949:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.463950:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.463951:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677474.463954:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.463955:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677474.463959:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.463960:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677474.463960:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677474.463962:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677474.463963:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.463964:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677474.463965:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677474.463965:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677474.463968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677474.463970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677474.463975:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:2.0:1619677474.463976:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:10.0:1619677474.463978:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:2.0:1619677474.463978:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:26.0:1619677474.463979:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:2.0:1619677474.463979:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:10.0:1619677474.463980:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.463981:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:10.0:1619677474.463982:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.463986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677474.463991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.463997:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.464000:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.464000:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.464001:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.464001:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.464002:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677474.464283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677474.464286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.464288:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677474.464290:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.464291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677474.464296:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.464298:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677474.464299:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.464300:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.464301:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.464303:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677474.464306:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.464308:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677474.464311:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.464312:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677474.464313:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677474.464315:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677474.464315:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.464316:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677474.464317:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677474.464321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677474.464329:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.464331:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.464333:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.464334:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.464336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.464338:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:5.2:1619677474.464508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.464514:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.464517:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.464518:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.464520:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.464522:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.464524:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.464525:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.464526:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.464527:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.464528:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.464532:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.464533:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.464536:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.464536:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.464537:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.464539:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.464539:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.464540:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.464540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.464545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.464553:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.464556:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.464557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.464557:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.464559:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.464561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.464699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.464705:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.464708:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.464710:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.464710:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.464712:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.464714:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.464715:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.464716:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.464717:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.464719:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.464722:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.464724:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.464726:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.464727:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.464727:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.464729:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.464729:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.464730:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.464731:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.464735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.464743:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.464746:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.464746:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.464748:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.464749:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.464750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.464796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.464803:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.464806:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.464807:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.464809:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.464811:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.464814:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.464815:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.464816:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.464817:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.464818:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.464821:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.464823:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.464826:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.464827:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.464827:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.464829:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.464830:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.464831:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.464831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.464835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.464843:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.464846:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.464846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.464847:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.464849:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.464852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.464878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.464886:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.464889:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.464890:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.464891:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.464894:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.464895:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.464896:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.464898:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.464899:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.464899:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.464903:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.464904:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.464907:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.464908:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.464908:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.464910:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.464911:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.464911:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.464912:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.464916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.464924:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.464927:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.464927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.464928:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.464930:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.464933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.465201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.465207:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.465210:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.465212:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.465212:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.465214:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.465216:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677474.465217:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.465218:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.465219:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.465220:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677474.465224:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.465226:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.465228:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.465229:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.2:1619677474.465230:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:21.0:1619677474.465233:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677474.465235:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677474.465236:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677474.465236:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:21.0:1619677474.465237:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.465238:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.465239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:1.0:1619677474.465239:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.465240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:9.2:1619677474.465241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:1.0:1619677474.465242:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.465244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677474.465244:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.465246:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.465247:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.465248:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:1.0:1619677474.465248:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.465249:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.465250:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:18.0:1619677474.465250:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:1.0:1619677474.465250:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.465251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.465251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677474.465252:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.465253:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.465253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:1.0:1619677474.465253:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.465255:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:18.0:1619677474.465256:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.465257:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.465258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677474.465258:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.465259:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.465261:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.465261:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.465262:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.465263:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.465267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.465275:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.465277:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.465278:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.465280:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.465281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.465282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.465596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.465604:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.465607:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.465609:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.465611:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.465612:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.465613:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.465614:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.465615:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.465616:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.465617:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.465621:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.465623:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.465625:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.465626:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.465626:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.465628:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.465629:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.465630:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.465630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.465634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.465641:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.465644:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.465645:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.465645:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.465647:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.465649:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.465933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.465941:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.465943:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.465944:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.465945:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.465948:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.465949:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.465950:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.465952:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.465953:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.465953:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.465957:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.465958:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.465961:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.465962:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.465962:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.465964:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.465964:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.465966:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.465967:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.465967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.465971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.465973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.465979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.465980:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.465982:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677474.465982:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.465983:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677474.465983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.465983:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.465985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677474.465985:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.465988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.466045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.466053:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.466056:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.466058:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.466060:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.466060:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.466061:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.466063:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.466064:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.466065:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.466066:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.466069:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.466070:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.466073:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.466073:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.466074:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.466075:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.466076:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.466077:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.466077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.466081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.466087:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.466090:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.466091:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.466091:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.466093:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.466095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.466298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.466304:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.466306:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.466307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.466309:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.466312:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.466314:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677474.466315:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.466317:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.466318:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.466319:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677474.466322:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.466324:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.466327:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.466328:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677474.466328:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.466330:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.466330:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.466331:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.466332:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677474.466336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.466343:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.466345:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.466346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.466347:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.466349:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.466352:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.466596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.466604:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.466607:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.466608:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.466610:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.466613:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.466614:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.466615:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.466617:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.466618:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.466619:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.466622:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.466624:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.466627:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.466628:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.466628:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.466630:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.466630:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.466632:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.466633:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.466633:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.466636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.466639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.466643:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.466646:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.466646:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.466647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.466647:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.466648:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.466649:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.466649:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.466650:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.466653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.467219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.467227:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.467230:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.467232:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.467234:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.467234:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.467236:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.467237:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.467238:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.467239:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.467240:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.467243:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.467245:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.467247:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.467248:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.467248:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.467250:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.467250:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.467251:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.467252:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.467256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.467262:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.467264:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.467265:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.467265:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.467267:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.467269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.467595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.467602:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.467605:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.467606:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.467608:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.467610:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.467612:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.467613:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.467614:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.467615:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.467616:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.467619:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.467620:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.467622:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.467623:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.467624:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.467625:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.467626:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.467627:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.467628:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.467629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.467632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.467635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.467639:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.467641:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.467642:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.467642:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.467642:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.467644:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.467644:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.467645:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.467646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.467646:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.467649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.467655:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.467658:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.467658:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.467659:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.467660:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.467660:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.467775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.467784:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.467786:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.467788:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.467790:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.467791:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.467792:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.467793:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.467794:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.467795:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.467796:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.467799:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.467800:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.467803:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.467803:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.467804:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.467805:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.467806:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.467807:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.467809:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.467809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.467810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.467815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.467817:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.467819:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.467820:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.467820:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.467822:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.467822:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.467824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.467825:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.467826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.467827:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.468141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.468147:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.468150:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.468151:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.468153:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.468156:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.468158:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.468159:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.468160:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.468161:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.468162:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.468165:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.468167:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.468169:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.468170:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.468170:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.468172:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.468173:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.468174:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.468174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.468180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.468187:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.468189:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.468190:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.468191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.468192:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.468193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.468331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.468339:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.468342:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.468345:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.468347:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.468347:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.468348:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.468350:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.468351:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.468352:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.468353:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.468356:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.468357:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.468359:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.468360:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.468361:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.468362:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.468363:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.468364:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.468364:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.468368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.468375:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.468378:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.468378:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.468380:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.468382:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.468383:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.468385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.468391:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.468394:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.468394:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.468395:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.468396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.468396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.468724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677474.468732:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.468734:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677474.468736:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.468738:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.468739:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677474.468739:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677474.468741:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.468742:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.468743:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.468743:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677474.468746:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.468747:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677474.468750:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.468751:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:30.0:1619677474.468751:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:30.0:1619677474.468753:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:30.0:1619677474.468753:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.468754:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.468754:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:25.2:1619677474.468758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677474.468766:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677474.468768:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677474.468769:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.468770:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.468772:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677474.468773:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:25.2:1619677474.468777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:13.0:1619677474.468783:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677474.468786:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677474.468787:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677474.468787:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677474.468788:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677474.468788:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677474.468866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.468875:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677474.468876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:31.0:1619677474.468877:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.468879:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.468881:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.468881:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677474.468882:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677474.468883:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.468884:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.468885:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.468885:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.468886:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677474.468887:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.468888:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677474.468889:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677474.468891:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.468891:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.468892:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:31.0:1619677474.468893:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:2.0:1619677474.468894:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.468895:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.468896:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.468897:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.468897:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:2.0:1619677474.468897:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.468898:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.468899:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:2.0:1619677474.468901:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677474.468902:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.468903:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677474.468903:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.468904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.468905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677474.468906:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.468906:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.468907:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.468909:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.468910:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.468911:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:2.0:1619677474.468911:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.468912:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.468913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677474.468914:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.468915:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.468915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:30.0:1619677474.468917:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.468918:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.2:1619677474.468918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.468919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.468920:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.468922:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.468923:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.468924:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:31.0:1619677474.468925:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.468926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.468926:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.468927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.468927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677474.468928:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.469184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.469192:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.469195:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.469196:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.469198:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.469200:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.469202:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.469203:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.469204:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.469206:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.469207:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.469210:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.469212:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.469215:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.469216:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.469216:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.469218:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.469219:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.469219:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.469220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.469224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.469233:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.469236:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.469237:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.469237:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.469239:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.469243:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.469244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.469252:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.469254:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.469255:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.469255:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.469256:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.469257:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677474.469258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.469264:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.469266:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.469267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.469269:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.469270:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.469272:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.469273:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.469275:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.469275:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.469276:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.469279:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.469281:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.469284:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.469284:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.469285:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.469286:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.469287:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.469287:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.469288:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.469293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.469302:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.469304:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.469305:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.469305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.469307:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.469310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.469435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.469443:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.469446:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.469448:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.469450:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.469451:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.469451:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.469453:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.469454:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.469455:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.469456:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.469459:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.469461:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.469464:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.469464:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.469465:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.469467:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.469467:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.469468:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.469469:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.469473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.469480:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.469482:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.469483:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.469484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.469485:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.469488:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.469673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.469681:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.469684:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.469685:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.469687:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.469688:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.469690:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.469691:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.469692:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.469693:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.469694:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.469697:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.469698:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.469701:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.469702:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.469703:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.469704:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.469705:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.469705:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.469706:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.469710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.469718:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.469720:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.469721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.469722:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.469724:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.469726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.469844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677474.469851:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.469854:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677474.469855:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677474.469857:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.469860:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.469861:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677474.469862:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.469864:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.469865:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.469865:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677474.469868:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.469870:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677474.469872:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.469873:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677474.469873:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677474.469875:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677474.469875:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.469876:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.469876:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677474.469880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677474.469883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.469888:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.469890:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.469891:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:10.0:1619677474.469891:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.469892:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.469893:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.469895:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677474.469896:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677474.469897:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.469897:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677474.469898:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.469899:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.469900:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.469902:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.469902:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.469903:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.469907:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.469908:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.469911:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.469912:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.469912:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.469914:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.469914:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.469915:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.469916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.469920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.469929:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.469931:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.469932:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.469934:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.469934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.469937:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:25.2:1619677474.470172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.470180:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.470182:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.470183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.470184:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.470185:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.470185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.470283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.470286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.470289:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.470291:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.470292:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.470297:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.470299:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.470300:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.470301:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.470302:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.470303:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.470306:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.470308:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.470311:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.470311:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.470312:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.470314:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.470314:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.470315:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.470316:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.470320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.470328:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.470331:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.470331:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.470333:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.470334:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.470335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.470340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.470348:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.470350:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.470351:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.470352:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.470353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677474.470353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:4.2:1619677474.470740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.470747:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.470750:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.470751:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.470752:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.470753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:24.0:1619677474.470755:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.470757:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.470758:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.470759:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.470760:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.470761:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:24.0:1619677474.470761:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.470763:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:24.0:1619677474.470765:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.470766:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.470767:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.470768:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.470768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.470770:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:24.0:1619677474.470770:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.470771:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.470771:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:31.0:1619677474.470772:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.470772:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677474.470773:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.470773:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677474.470774:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677474.470774:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.470775:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.470775:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677474.470778:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.470780:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677474.470780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.470783:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.470784:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.470784:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.470786:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.470786:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.470788:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677474.470788:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.470789:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.470790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.470791:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.470792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.2:1619677474.470792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:6.0:1619677474.470792:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.470794:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.470796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.470797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677474.470800:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.470802:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677474.470803:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:20.0:1619677474.470803:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.470805:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.470806:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:21.0:1619677474.470806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.470807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.470808:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.0:1619677474.470808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.470812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.470819:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.470822:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.470823:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.470824:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.470825:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.470826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.471452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.471460:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.471463:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.471463:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677474.471466:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.471469:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.471470:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.471472:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.471473:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.471474:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.471475:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.471478:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.471481:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.471483:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.471484:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.471485:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.471486:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.471487:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.471487:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.471488:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.471493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.471501:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.471504:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.471505:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.471505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.471507:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.471510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.471516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.471523:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.471525:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.471526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.471526:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.471527:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.471528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.471697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.471706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.471708:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.471711:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.471713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.471714:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.471715:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.471717:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.471718:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.471719:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.471720:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.471723:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.471724:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.471727:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.471727:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.471728:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.471730:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.471730:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.471731:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.471732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.471735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.471743:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.471745:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.471746:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.471746:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.471748:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.471751:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.471898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.471905:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.471908:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.471909:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.471910:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.471914:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.471915:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.471916:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.471918:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.471918:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.471919:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.471922:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.471924:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.471927:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.471928:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.471928:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.471930:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.471930:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.471931:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.471931:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.471941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.471948:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.471951:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.471952:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.471952:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.471955:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.471958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.471978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.471984:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.471986:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.471988:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.471989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.471990:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.471992:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.471993:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.471995:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.471996:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.471997:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.472000:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.472001:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.472003:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.472004:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.472005:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.472006:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.472007:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.472008:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.472008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.472012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.472020:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.472022:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.472023:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.472023:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.472025:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.472028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.472105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.472111:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.472114:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.472115:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.472117:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.472120:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.472122:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.472123:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.472124:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.472125:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.472126:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.472129:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.472131:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.472133:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.472134:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.472135:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.472136:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.472137:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.472137:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.472138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.472142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.472150:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.472152:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.472153:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.472155:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.472156:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.472157:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.472624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.472632:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.472635:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.472637:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.472639:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.472640:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.472641:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.472642:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.472644:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.472645:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.472646:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.472649:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.472651:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.472653:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.472654:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.472655:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.472657:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.472657:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.472658:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.472659:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.472663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.472670:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.472673:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.472674:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.472674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.472676:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.472678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.472683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.472690:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.472692:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.472693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.472693:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.472694:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.472695:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.472724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.472732:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.472734:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.472737:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.472739:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.472739:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.472740:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.472742:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.472743:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.472744:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.472745:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.472748:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.472750:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.472754:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.472754:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.472755:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.472757:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.472757:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.472758:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.472760:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.472761:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.472763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.472767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.472775:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.472775:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.472777:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677474.472778:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.472779:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677474.472779:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.472780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677474.472781:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.472781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.472784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.472912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.472915:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.472917:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.472920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.472920:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.472922:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.472923:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.472925:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.472926:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.472927:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.472928:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.472931:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.472933:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.472935:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.472936:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.472937:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.472939:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.472939:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.472941:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.472942:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.472942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.472945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.472947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.472953:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677474.472953:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677474.472955:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.472956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677474.472956:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677474.472957:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677474.472957:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.472957:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.472959:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.472962:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.473156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.473163:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.473166:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.473169:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.473170:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.473172:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.473173:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.473174:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.473176:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.473177:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.473178:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.473181:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.473183:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.473185:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.473186:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.473186:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.473188:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.473188:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.473189:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.473190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.473193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.473200:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.473202:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.473203:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.473205:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.473207:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.473208:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.473227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.473233:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.473235:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.473237:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.473238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.473240:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.473242:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.473243:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.473245:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.473246:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.473247:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.473250:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.473252:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.473255:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.473255:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.473256:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.473258:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.473259:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.473259:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.473260:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.473264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.473271:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.473273:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.473274:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.473276:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.473277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.473279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.473478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.473485:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.473488:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.473489:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.473491:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.473494:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.473495:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.473497:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.473498:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.473499:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.473500:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.473502:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.473504:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.473506:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.473507:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.473507:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.473509:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.473509:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.473510:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.473511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.473515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.473522:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.473525:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.473525:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.473526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.473527:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.473530:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.473650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.473658:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.473661:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.473663:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.473664:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.473665:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677474.473666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.473667:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.473668:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.473669:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.473669:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.473672:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.473674:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.473676:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.473676:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.473677:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.473678:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.473679:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.473680:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.473680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.473684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.473692:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.473694:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.473695:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.473696:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.473698:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.473699:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.473758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677474.473764:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.473767:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677474.473768:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677474.473770:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.473772:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.473773:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677474.473775:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.473776:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.473777:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.473778:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677474.473781:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.473783:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677474.473784:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.473785:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.473786:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.473787:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.473787:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.473788:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.473789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677474.473793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.473801:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.473803:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.473803:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.473804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.473805:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.473807:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.473813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.473820:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.473822:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677474.473823:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.473824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.473824:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.473824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.474260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.474267:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.474270:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.474272:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.474274:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.474274:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.474275:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.474276:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.474278:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.474279:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.474280:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.474282:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.474284:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.474287:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.474287:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.474288:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.474290:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.474290:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.474291:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.474292:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.474296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.474303:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.474306:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.474307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.474307:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.474309:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.474310:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.474335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.474343:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.474346:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.474347:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.474350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.474350:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.474351:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.474352:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.474353:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.474354:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.474355:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.474358:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.474360:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.474362:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.474363:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.474363:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.474365:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.474365:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.474366:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.474367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.474371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.474377:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.474379:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.474380:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.474381:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.474382:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.474384:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.474388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.474394:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.474397:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.474398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.474398:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.474399:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.474400:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.474478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.474484:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.474487:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.474488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.474490:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.474492:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.474493:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.474494:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.474496:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.474497:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.474498:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.474501:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.474503:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.474506:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.474507:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.474507:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.474509:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.474509:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.474511:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.474512:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.474512:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.474515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.474517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.474522:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.474523:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.474525:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.474525:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677474.474526:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.474526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.474527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.474527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.474529:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.474531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.474584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.474591:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.474594:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.474595:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677474.474596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000400:00000200:18.0:1619677474.474597:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.474601:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.474602:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000400:00000200:18.0:1619677474.474603:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677474.474603:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000400:00000200:18.0:1619677474.474604:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.474606:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.474607:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.474608:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.474611:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.474612:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677474.474613:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.474614:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.474615:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.474616:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.474617:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.474618:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.474619:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.474622:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.474623:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:2.0:1619677474.474623:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.474624:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.474625:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.474627:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.474628:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.474628:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.474629:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.474629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677474.474629:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.474630:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.474630:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.474632:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:25.2:1619677474.474633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.474633:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.474634:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677474.474635:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.474635:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:25.2:1619677474.474638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677474.474640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:13.0:1619677474.474640:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.474643:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.474644:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.474645:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:13.0:1619677474.474645:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.474646:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:12.0:1619677474.474647:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677474.474648:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:18.0:1619677474.474648:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:13.0:1619677474.474648:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:31.0:1619677474.474649:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.474649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.474650:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:31.0:1619677474.474651:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.474651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.474655:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.474657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.474664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.474667:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.474668:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.474669:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.474669:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.474671:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.475247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.475255:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.475257:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.475258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.475260:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.475262:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.475264:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.475265:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.475266:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.475267:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.475268:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.475271:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.475272:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.475275:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.475275:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.475276:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.475278:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.475278:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.475279:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.475281:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.475281:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677474.475283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.475287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677474.475290:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.475293:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677474.475296:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:13.0:1619677474.475300:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:26.0:1619677474.475301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:12.0:1619677474.475302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.475302:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.475303:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:12.0:1619677474.475304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.475306:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.475734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.475738:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.475741:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.475743:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.475744:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.475745:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.475747:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.475748:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.475750:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.475751:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.475752:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.475755:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.475756:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.475759:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.475759:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.475760:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.475762:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.475762:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.475764:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.475765:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.475766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677474.475768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.475775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677474.475776:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.475779:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.475780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.475780:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.475781:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:6.0:1619677474.475782:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.475783:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.475784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677474.475784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.475785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.475788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.475795:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.475798:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.475799:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.475799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.475800:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.475800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.475878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677474.475886:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.475888:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.475889:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.475891:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.475893:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.475895:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677474.475896:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.475897:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.475899:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.475899:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677474.475903:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.475905:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677474.475907:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.475908:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.475908:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677474.475910:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.475910:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.475911:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.475911:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.475917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.475920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.475922:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677474.475923:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.475924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677474.475925:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.475927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677474.475955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.475963:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.475965:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.475966:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.475968:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.475970:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.475972:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.475973:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.475974:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.475975:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.475976:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.475979:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.475980:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.475983:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.475984:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.475984:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.475986:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.475986:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.475987:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.475988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.475992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.475999:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.476002:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.476003:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.476003:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.476006:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.476009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.476052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.476059:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.476061:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.476062:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.476064:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.476067:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.476068:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677474.476069:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.476071:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.476072:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.476073:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677474.476076:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.476077:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.476079:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677474.476080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677474.476080:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.476080:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.476082:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.476082:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.476083:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.476083:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:18.0:1619677474.476087:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677474.476088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.476089:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.476090:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.476092:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.476095:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.476096:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677474.476096:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:18.0:1619677474.476097:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.476098:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:18.0:1619677474.476099:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.476099:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.476099:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.476100:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:18.0:1619677474.476101:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677474.476101:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.476103:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:18.0:1619677474.476104:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.476106:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.476108:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.476109:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.476109:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.476111:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.476111:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.476112:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.476113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.476117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.476124:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.476126:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.476127:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.476128:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.476129:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.476130:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.476200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.476207:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.476210:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.476211:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.476213:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.476216:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.476217:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.476218:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.476219:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.476220:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.476221:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.476225:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.476226:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.476229:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.476230:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.476231:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.476233:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.476234:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.476234:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.476235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.476239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.476246:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.476249:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.476250:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.476250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.476253:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.476255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.476291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.476301:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.476304:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.476307:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.476308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.476309:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.476310:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.476312:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.476313:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.476314:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.476315:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.476318:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.476320:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.476323:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.476323:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.476324:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.476326:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.476326:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.476327:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.476328:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.476332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.476339:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.476341:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.476342:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.476343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:4.2:1619677474.476343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677474.476345:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.476347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677474.476350:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.476353:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.476354:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.476356:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.476359:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.476360:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.476362:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.476363:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.476364:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.476365:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.476368:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.476369:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.476372:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.476373:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.476373:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.476375:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.476376:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.476376:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.476377:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.476383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.476390:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.476393:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.476394:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.476394:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.476396:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.2:1619677474.476398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:6.0:1619677474.476399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677474.476404:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.476407:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677474.476408:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677474.476410:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677474.476412:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.476414:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677474.476415:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.476416:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.476417:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.476418:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677474.476422:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677474.476423:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677474.476426:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.476427:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:17.0:1619677474.476428:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677474.476430:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677474.476430:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.476431:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677474.476432:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677474.476432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:25.2:1619677474.476435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:5.2:1619677474.476438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:30.0:1619677474.476442:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:1.0:1619677474.476443:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:30.0:1619677474.476445:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:1.0:1619677474.476445:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000400:00000200:30.0:1619677474.476446:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.476446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677474.476447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000400:00000200:30.0:1619677474.476448:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.476449:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.476452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.476659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.476665:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.476667:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.476669:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.476670:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.476672:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.476673:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.476675:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.476676:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.476677:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.476678:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.476681:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.476683:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.476685:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.476686:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.476686:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.476688:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.476688:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.476689:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.476690:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.476694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.476703:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.476705:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.476706:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.476707:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.476708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.476710:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.476949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.476957:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.476960:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.476962:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.476964:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.476965:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.476966:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677474.476968:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.476969:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.476970:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.476971:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677474.476975:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.476976:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677474.476979:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.476980:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677474.476980:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677474.476982:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677474.476983:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.476984:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677474.476986:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677474.476987:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:25.2:1619677474.476988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.476992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:31.0:1619677474.476995:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.476997:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.476998:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.476999:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.477000:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:30.0:1619677474.477001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.477002:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:12.0:1619677474.477002:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677474.477003:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.477004:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:21.2:1619677474.477545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.477553:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.477556:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.477558:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.477560:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.477561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.477561:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.477562:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.477563:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.477565:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.477565:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.477568:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.477570:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.477572:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.477572:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.477573:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.477574:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.477575:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.477576:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.477578:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.477578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.477581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.477585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.477589:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.477590:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.477591:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.477593:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.477593:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.477594:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.477594:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.477595:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.477596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.477597:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.477600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.477606:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.477609:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.477610:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.477610:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.477611:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.477612:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.477938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.477945:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.477948:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.477949:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.477951:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.477952:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677474.477952:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.477953:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.477955:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.477955:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.477956:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.477959:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.477961:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.477963:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.477964:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.477964:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.477966:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.477966:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.477968:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.477969:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.477970:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.477972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.477976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.477979:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.477981:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.477982:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.477982:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.477982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:30.0:1619677474.477984:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677474.477984:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.477986:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677474.477986:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.477987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677474.477987:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677474.477988:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677474.477989:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.477991:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.477992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.477994:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.477996:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.477997:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.477998:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.477999:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.478000:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.478004:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.478006:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.478009:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.478010:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.478010:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.478012:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.478013:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.478014:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.478015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.478018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.478025:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.478028:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.478029:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.478030:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.478032:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.478033:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.478094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.478102:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.478105:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.478107:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.478109:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.478109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.478111:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.478112:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.478113:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.478114:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.478115:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.478118:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.478120:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.478122:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.478123:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.478123:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.478125:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.478125:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.478126:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.478127:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.478130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.478137:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.478139:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.478140:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.478141:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.478143:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.478144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.478476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.478480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.478482:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.478484:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.478485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.478487:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.478489:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.478490:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.478491:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.478492:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.478493:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.478497:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.478499:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.478501:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.478502:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.478502:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.478504:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.478505:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.478506:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.478506:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.478511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.478519:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.478522:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.478523:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.478523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.478525:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.478527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.478580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.478588:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.478591:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.478593:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.478595:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.478596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.478596:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.478598:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.478599:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.478600:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.478601:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.478604:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.478606:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.478609:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.478609:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.478610:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.478612:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.478612:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.478613:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.478614:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.478617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.478624:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.478627:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.478628:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.478628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.478630:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.478632:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.478654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.478660:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.478662:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.478664:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.478665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.478667:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.478668:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.478670:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.478671:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.478672:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.478673:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.478677:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.478678:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.478681:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.478681:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.478682:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.478684:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.478684:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.478685:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.478686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.478690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.478699:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.478701:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.478702:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.478702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.478704:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.478707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.478755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.478763:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.478765:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.478766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.478768:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.478771:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.478772:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.478774:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.478775:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.478776:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.478777:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.478780:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.478781:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.478783:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.478784:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.478785:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.478786:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.478787:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.478788:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.478789:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.478790:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.478792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.478795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.478798:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.478800:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.478801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.478801:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.478803:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677474.478803:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.478805:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.478805:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.478805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.478807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.478904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.478910:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.478912:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.478913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.478915:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.478917:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.478919:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.478920:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.478921:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.478922:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.478923:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.478927:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.478929:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.478931:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.478932:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.478932:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.478934:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.478934:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.478936:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.478937:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.478937:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.478940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.478942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.478947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677474.478948:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.478949:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677474.478950:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677474.478950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677474.478951:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.478951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677474.478953:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.478953:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.478955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.479116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.479124:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.479126:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.479127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.479129:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.479131:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.479133:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677474.479134:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.479136:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.479137:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.479138:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677474.479141:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.479143:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677474.479146:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.479147:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677474.479148:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677474.479150:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677474.479150:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.479151:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.479152:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677474.479155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.479161:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.479164:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.479165:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.479165:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.479167:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.479170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.479230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.479236:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.479239:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.479240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.479241:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.479244:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.479245:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.479247:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.479248:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.479249:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.479250:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.479253:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.479254:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.479257:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.479258:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.479258:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.479260:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.479260:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.479261:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.479261:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.479266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.479274:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.479276:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.479277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.479277:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.479279:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.479281:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.479417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.479425:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.479428:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.479430:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.479433:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.479433:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.479434:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.479436:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.479437:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.479438:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.479439:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.479443:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.479444:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.479447:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.479448:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.479448:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.479450:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.479450:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.479451:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.479452:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.479456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.479463:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.479465:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.479466:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.479467:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.479468:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.479470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.479474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.479480:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.479483:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.479484:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.479484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.479485:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.479486:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.479936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.479944:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.479946:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.479949:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.479950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.479951:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.479952:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.479954:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.479955:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.479956:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.479957:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.2:1619677474.479958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:30.0:1619677474.479959:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.479961:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.479963:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.479964:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677474.479964:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.479965:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.479966:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.479967:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:2.0:1619677474.479967:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:30.0:1619677474.479968:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.479968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:18.0:1619677474.479968:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.479970:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:9.2:1619677474.479972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:2.0:1619677474.479973:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.479974:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.479976:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.479977:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.479978:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:2.0:1619677474.479978:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.479979:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.479981:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.479982:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.479982:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.479982:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.479984:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.479985:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.479987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677474.479987:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.479987:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.479988:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:9.2:1619677474.479989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677474.479990:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.479990:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.479991:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.479992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677474.479996:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677474.479996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.479999:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.480000:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.480001:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.480002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677474.480002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:17.0:1619677474.480005:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.480007:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.480008:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.480008:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.480010:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.480012:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.480035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.480040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.480043:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.480045:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.480045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.480047:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.480049:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.480050:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.480052:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.480052:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.480054:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.480057:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.480058:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.480061:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.480062:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.480063:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.480064:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.480065:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.480066:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.480066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.480070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.480076:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.480079:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.480080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.480080:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.480082:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.480085:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.480090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.480095:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.480098:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.480099:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.480101:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.480103:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.480104:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.480105:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.480107:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.480107:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.480108:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.480111:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.480113:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.480115:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.480116:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.480117:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.480118:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.480119:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.480120:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677474.480121:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677474.480121:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:25.2:1619677474.480124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.480128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:22.0:1619677474.480132:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.480134:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677474.480134:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:22.0:1619677474.480136:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.480136:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:6.0:1619677474.480136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.480137:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.480137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.480138:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677474.480140:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.480144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.480151:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.480154:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.480155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.480155:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.480156:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.480156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.480563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.480571:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.480574:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.480576:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.480578:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.480578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.480580:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.480581:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.480582:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.480584:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.480585:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.480588:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.480589:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.480592:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.480593:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.480593:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.480595:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.480596:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.480597:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.480599:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.480599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.480601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.480605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.480608:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.480611:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.480612:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.480612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.480613:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.480614:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.480616:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677474.480616:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.480617:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.480617:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.480711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.480719:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.480722:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.480723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.480725:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.480727:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.480729:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.480730:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.480731:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.480732:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.480733:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.480736:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.480737:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.480740:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.480741:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.480742:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.480743:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.480744:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.480745:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.480745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.480749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.480757:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.480760:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.480761:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.480761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.480764:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.480766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.481132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.481139:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.481142:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.481143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.481145:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.481147:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.481149:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.481150:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.481151:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.481153:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.481154:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.481157:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.481158:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.481161:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.481162:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.481163:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.481164:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.481165:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.481166:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.481167:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.481168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.481170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.481171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677474.481175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677474.481178:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.481178:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.481181:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677474.481181:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.481182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.481182:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.481182:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.481183:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:6.0:1619677474.481184:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.481185:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.481185:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.481185:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.481186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.481187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677474.481188:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.481189:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.481191:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.481192:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.481193:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.481194:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.481197:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.481199:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.481202:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.481203:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.481203:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.481205:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.481205:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.481206:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.481207:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.481211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.481218:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.481221:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.481221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.481222:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.481224:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.481226:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.481231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.481238:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.481241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.481241:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.481242:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.481243:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.481243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.481482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd9200] (20)++
00000800:00000200:20.0:1619677474.481486:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd9200] (21)++
00000800:00000200:20.0:1619677474.481489:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.25@o2ib3
00000400:00000200:20.0:1619677474.481491:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.4.25@o2ib3 : GET - for me
00000800:00000200:21.0:1619677474.481493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd9200] (22)--
00000400:00000200:20.0:1619677474.481494:0:19934:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.4.25@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:20.0:1619677474.481497:0:19934:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.4.25@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:20.0:1619677474.481498:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd9200] (21)++
00000800:00000200:20.0:1619677474.481499:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd9200] (22)--
00000400:00000200:20.0:1619677474.481500:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.4.25@o2ib3 local destination
00000400:00000200:20.0:1619677474.481504:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.4.25@o2ib3(10.51.4.25@o2ib3:10.51.4.25@o2ib3) : REPLY try# 0
00000800:00000200:20.0:1619677474.481506:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.4.25@o2ib3
00000800:00000200:20.0:1619677474.481508:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9197c81a8600] -> 10.51.4.25@o2ib3 (2) version: 12
00000800:00000200:20.0:1619677474.481509:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919460fd9200] (21)++
00000800:00000200:20.0:1619677474.481509:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fd9200] (22)++
00000800:00000200:20.0:1619677474.481510:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919460fd9200] (23)--
00000800:00000200:20.0:1619677474.481511:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd9200] (22)--
00000800:00000200:4.2:1619677474.481516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd9200] (21)++
00000800:00000200:21.0:1619677474.481522:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd9200] (22)++
00000800:00000200:21.0:1619677474.481524:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fd9200] (23)--
00000400:00000200:21.0:1619677474.481525:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.481526:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.4.25@o2ib3: REPLY: OK
00000800:00000200:20.0:1619677474.481527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd9200] (22)--
00000800:00000200:21.0:1619677474.481529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd9200] (21)--
00000800:00000200:21.2:1619677474.481837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.481845:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.481849:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.481851:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.481853:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.481853:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.481855:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.481856:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.481857:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.481858:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.481859:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.481862:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.481864:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.481866:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.481867:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.481868:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.481869:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.481870:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.481871:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.481871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.481876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.481883:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.2:1619677474.481884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.481886:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677474.481886:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.481887:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.481887:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.481889:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:8.0:1619677474.481890:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677474.481891:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.481892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677474.481892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:20.0:1619677474.481894:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.481895:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.481897:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.481898:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.481899:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.481900:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.481903:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.481905:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.481907:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.481908:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.481908:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.481910:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.481910:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.481911:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.481912:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.481916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.481924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.481926:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.481927:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.481929:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.481929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.481931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.481941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.481949:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.481952:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.481952:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.481953:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.481953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.481955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.482042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677474.482049:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.482052:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677474.482053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677474.482055:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.482058:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.482059:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.482060:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.482062:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.482063:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.482064:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.482067:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.482069:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.482072:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.482073:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.482073:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.482075:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.482076:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.482077:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677474.482078:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677474.482079:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.2:1619677474.482081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.482084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:18.0:1619677474.482090:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.482092:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677474.482092:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:2.0:1619677474.482093:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.482094:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.482094:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677474.482095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:18.0:1619677474.482096:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.482096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:18.0:1619677474.482099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.482105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.482112:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.482114:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.482115:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.482115:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.482116:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.482117:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.482177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.482185:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.482187:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.482188:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.482190:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.482194:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.482195:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677474.482196:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.482197:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.482198:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.482199:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677474.482202:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.482203:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.482205:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.482205:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.482206:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.482207:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.482207:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.482208:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.482209:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.482213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.482219:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.482221:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.482222:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.482222:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.482223:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.482225:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.482772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.482780:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.482782:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.482785:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.482787:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.482787:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.482788:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.482789:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.482790:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.482791:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.482792:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.482795:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.482797:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.482799:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.482799:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.482800:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.482801:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.482802:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.482803:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.482803:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.482817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.482825:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.482827:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.482828:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.482829:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.482831:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.482832:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.482990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.482996:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.482999:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.483000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.483002:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.483005:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.483006:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.483007:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.483009:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.483010:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.483010:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.483013:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.483015:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.483017:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.483018:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.483018:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.483020:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.483021:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.483022:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.483022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.483026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.483033:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.483036:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.483036:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.483037:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.483038:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.483039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.483261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.483269:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.483271:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.483272:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.483274:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.483276:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.483277:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.483278:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.483280:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.483281:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.483282:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.483285:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.483286:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.483289:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.483290:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.483290:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.483292:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.483293:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.483298:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.483298:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.483299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.483307:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.483310:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.483310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.483311:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.483313:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.483317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.483320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677474.483328:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.483330:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.483331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.483331:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.483333:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.483333:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.483501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677474.483504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677474.483508:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.483511:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677474.483512:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.483512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677474.483514:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677474.483514:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.483516:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.483517:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.483517:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.483519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.483519:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677474.483519:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.483520:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.483520:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.483522:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.483522:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.483524:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.483524:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.483526:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677474.483526:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.483530:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.483530:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.483533:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677474.483533:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.483535:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.483535:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.483536:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677474.483536:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.483536:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.483537:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.483538:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677474.483538:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.483539:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677474.483539:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677474.483540:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677474.483540:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677474.483541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677474.483541:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.483544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:9.2:1619677474.483544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.483552:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:31.0:1619677474.483553:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:24.0:1619677474.483554:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:31.0:1619677474.483555:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:24.0:1619677474.483555:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.483555:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:31.0:1619677474.483556:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.483557:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.483557:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.483559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677474.483559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.483560:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.483761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.483767:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.483770:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.483771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.483773:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.483776:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.483777:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.483778:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.483780:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.483781:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.483782:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.483785:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.483787:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.483789:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.483790:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.483791:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.483792:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.483793:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.483794:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.483794:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.483799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.483807:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.483809:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.483810:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.483812:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.483813:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.483814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.483835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.483843:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.483845:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.483847:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.483849:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.483850:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.483851:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.483852:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.483854:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.483855:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.483856:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.483859:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.483861:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.483864:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.483864:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.483865:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.483867:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.483867:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.483868:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.483869:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.483872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.483879:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.483881:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.483882:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.483882:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.483884:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.483886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677474.483964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.483969:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.483972:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.483973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.483975:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.483977:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.483979:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.483980:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.483981:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.483982:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.483983:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.483986:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.483988:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.483990:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.483991:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.483992:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.483993:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.483994:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.483994:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.483995:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.483999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.484007:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.484010:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.484011:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.484011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.484013:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.484015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.484038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (20)++
00000800:00000200:10.0:1619677474.484046:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)++
00000800:00000200:10.0:1619677474.484049:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.56@o2ib3
00000800:00000200:26.0:1619677474.484050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000400:00000200:10.0:1619677474.484051:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.56@o2ib3 : GET - for me
00000400:00000200:10.0:1619677474.484054:0:19940:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.56@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:10.0:1619677474.484056:0:19940:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.56@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:10.0:1619677474.484058:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37e800] (21)++
00000800:00000200:10.0:1619677474.484059:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37e800] (22)--
00000400:00000200:10.0:1619677474.484061:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.56@o2ib3 local destination
00000400:00000200:10.0:1619677474.484065:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.56@o2ib3(10.51.5.56@o2ib3:10.51.5.56@o2ib3) : REPLY try# 0
00000800:00000200:10.0:1619677474.484066:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.56@o2ib3
00000800:00000200:10.0:1619677474.484068:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3c4599b00] -> 10.51.5.56@o2ib3 (2) version: 12
00000800:00000200:10.0:1619677474.484069:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a07e37e800] (21)++
00000800:00000200:10.0:1619677474.484069:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37e800] (22)++
00000800:00000200:10.0:1619677474.484071:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a07e37e800] (23)--
00000800:00000200:10.0:1619677474.484071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000800:00000200:4.2:1619677474.484077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37e800] (21)++
00000800:00000200:26.0:1619677474.484084:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)++
00000800:00000200:26.0:1619677474.484086:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37e800] (23)--
00000400:00000200:26.0:1619677474.484087:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.484087:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (22)--
00000400:00000200:26.0:1619677474.484089:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.56@o2ib3: REPLY: OK
00000800:00000200:26.0:1619677474.484090:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37e800] (21)--
00000800:00000200:20.2:1619677474.484200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.484203:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.484205:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.484208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.484208:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.484210:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.484211:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.484213:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.484214:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.484215:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.484216:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.484219:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.484221:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.484223:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.484224:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.484225:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.484226:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.484227:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.484228:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.484229:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.484230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.484233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.484235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.484240:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677474.484241:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677474.484243:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.484243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.484244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677474.484244:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.484245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.484245:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.484247:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.484250:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.484271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.484276:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.484278:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.484281:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.484281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.484283:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.484284:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.484286:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.484287:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.484288:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.484289:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.484292:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.484299:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.484301:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.484302:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.484303:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.484304:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.484305:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.484305:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.484306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.484311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.484319:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.484322:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.484323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.484323:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.484325:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.484327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.484756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.484764:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.484767:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.484769:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.484771:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.484771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.484773:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.484774:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.484775:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.484776:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.484777:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.484781:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.484782:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.484785:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.484786:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.484786:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.484788:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.484788:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.484789:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.484790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.484793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.484800:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.484803:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.484804:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.484804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.484806:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.484808:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.485491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.485499:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.485502:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.485504:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.485506:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.485506:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.485507:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.485508:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.485510:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.485510:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.485511:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.485514:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.485516:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.485518:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.485519:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.485519:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.485521:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.485521:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.485523:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.485524:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.485525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.485527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.485531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.485534:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.485536:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.485537:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.485538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.485539:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677474.485539:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.485541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.485542:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.485542:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.485543:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.485545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.485551:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.485553:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.485554:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.485554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.485555:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.485556:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.485568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.485576:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.485579:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.485580:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.485581:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.485583:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.485583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:10.0:1619677474.485585:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.485586:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.485588:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.485589:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.485590:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.485591:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.485592:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677474.485594:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.485595:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.485596:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.485597:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.485599:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677474.485599:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.485600:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.485600:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.485601:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677474.485602:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.485603:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.485603:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677474.485604:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.485605:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.485605:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.485606:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:26.0:1619677474.485607:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.485607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.485609:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:25.2:1619677474.485609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.485611:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.485612:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.485612:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.485614:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.485615:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.485616:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.485617:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.485617:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677474.485617:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:25.2:1619677474.485620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.485620:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.485621:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.485621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.485623:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.485623:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:4.2:1619677474.485623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.485624:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.485625:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.485626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:6.0:1619677474.485626:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.485628:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:25.2:1619677474.485629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.485630:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.485632:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677474.485632:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677474.485633:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677474.485633:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.485634:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.485635:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.485637:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.485637:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.485638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.485638:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.485960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.485968:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.485971:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.485973:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.485975:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.485975:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.485977:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.485978:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.485979:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.485980:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.485981:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.485984:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.485985:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.485988:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.485988:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.485989:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.485990:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.485991:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.485992:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.485994:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.485994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.485996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.486000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.486002:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.486005:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.486006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.486006:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.486008:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.486008:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.486010:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.486010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.486011:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.486012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.486018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.486024:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.486027:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.486028:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.486028:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.486028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.486029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.486108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.486116:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.486118:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.486120:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.486121:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.486124:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.486125:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.486126:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.486128:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.486129:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.486129:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.486132:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.486134:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.486137:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.486137:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.486138:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.486140:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.486140:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.486141:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.486141:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.486146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.486153:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.486156:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.486157:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.486157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.486159:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.486162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.486422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.486429:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.486432:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.486433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.486435:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.486437:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.486438:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.486440:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.486441:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.486442:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.486443:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.486447:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.486449:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.486452:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.486453:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.486453:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.486455:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.486456:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.486457:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.486457:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.486462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.486469:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.486472:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.486473:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.486475:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.486477:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.486478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.486482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.486489:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.486492:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.486493:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.486494:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.486495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.486496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.486765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.486773:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.486776:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.486778:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.486780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.486780:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.486781:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.486783:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.486784:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.486785:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.486786:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.486789:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.486791:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.486794:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.486795:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.486795:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.486797:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.486798:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.486799:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.486799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.486804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.486812:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.486815:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.486816:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.486818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.486818:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.486821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.486947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.486952:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.486954:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.486956:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.486957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.486959:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.486960:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.486961:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.486962:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.486964:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.486965:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.486968:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.486969:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.486972:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.486973:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.486973:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.486975:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.486976:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.486976:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.486977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.486981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.486988:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.486990:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.486991:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.486992:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.486993:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.486994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.487351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.487359:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.487362:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.487364:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.487366:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.487367:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.487369:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.487370:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.487371:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.487372:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.487373:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.487376:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.487378:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.487380:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.487381:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.487382:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.487383:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.487384:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.487385:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.487385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.487389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.487396:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.487398:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.487399:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.487400:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.487401:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.487404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.487603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677474.487609:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.487612:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677474.487613:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677474.487615:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.487618:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.487619:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:4.2:1619677474.487619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000400:00000200:6.0:1619677474.487620:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.487622:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.487623:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.487623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:6.0:1619677474.487624:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.487627:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:6.0:1619677474.487627:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.487628:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677474.487629:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.487630:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:21.0:1619677474.487631:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677474.487631:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:6.0:1619677474.487632:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677474.487633:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.487633:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:6.0:1619677474.487633:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677474.487634:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:26.0:1619677474.487635:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.487636:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.487636:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:26.0:1619677474.487637:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.487637:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000400:00000200:21.0:1619677474.487638:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:6.0:1619677474.487638:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677474.487639:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000400:00000200:26.0:1619677474.487640:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:21.0:1619677474.487640:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.487640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:26.0:1619677474.487642:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.487642:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677474.487642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:21.0:1619677474.487643:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.487644:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677474.487644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:26.0:1619677474.487645:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677474.487645:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000400:00000200:26.0:1619677474.487646:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677474.487648:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.487648:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677474.487650:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:8.0:1619677474.487651:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.487652:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.487652:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:10.0:1619677474.487652:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:8.0:1619677474.487652:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.487653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.487653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:21.0:1619677474.487654:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.487654:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:8.0:1619677474.487654:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.487655:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.487657:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677474.487657:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677474.487658:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677474.487658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:10.0:1619677474.487659:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.487660:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.487661:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:10.0:1619677474.487661:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677474.487662:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.487662:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677474.487663:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000400:00000200:10.0:1619677474.487663:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96504292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.487664:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677474.487664:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:10.0:1619677474.487664:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.487665:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.487665:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:10.0:1619677474.487665:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.487666:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677474.487666:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:26.0:1619677474.487667:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:10.0:1619677474.487667:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.487668:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677474.487669:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:26.0:1619677474.487670:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677474.487670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:10.0:1619677474.487670:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.487672:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.2:1619677474.487674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:10.0:1619677474.487675:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.487675:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.487676:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:30.0:1619677474.487677:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:10.0:1619677474.487677:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.487678:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.487678:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677474.487679:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:10.0:1619677474.487679:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:30.0:1619677474.487680:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.487682:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.487682:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:31.0:1619677474.487683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.487685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.487685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.487686:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.487687:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.487690:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.487691:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.487693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:13.0:1619677474.487694:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.487696:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.487697:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.487699:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.487700:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.487702:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.488188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.488195:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.488197:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.488198:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.488200:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.488202:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.488203:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.488205:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.488206:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.488207:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.488208:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.488211:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.488212:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.488214:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.488215:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.488215:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.488217:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.488217:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.488218:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.488220:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.488220:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.488222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.488225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.488228:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.488231:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.488231:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.488232:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.488233:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.488233:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.488235:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.488235:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.488236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.488237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.488376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.488384:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.488387:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.488388:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.488390:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.488391:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.488392:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.488393:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.488394:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.488395:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.488396:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.488399:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.488400:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.488402:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.488403:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.488403:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.488405:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.488405:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.488406:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.488407:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.488411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.488418:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.488420:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.488421:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.488422:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.488424:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.488425:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.488428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.488435:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.488437:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.488438:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.488439:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.488440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.488440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.488482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.488486:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.488489:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.488491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.488491:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.488493:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.488495:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.488496:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.488497:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.488498:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.488499:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.488502:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.488504:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.488506:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.488507:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.488508:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.488509:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.488510:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.488511:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.488513:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.488513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677474.488518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.488523:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.488525:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:25.2:1619677474.488526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.488526:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.488527:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.488534:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.488536:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.488537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.488537:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.488539:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.488541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.488697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.488703:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.488706:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.488706:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.488708:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.488710:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.488712:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.488713:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.488714:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.488715:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.488717:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.488720:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.488721:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.488724:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.488725:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.488725:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.488727:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.488727:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.488728:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.488728:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.488733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.488740:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.488743:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.488744:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.488744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.488746:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.488748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.489047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666c00] (20)++
00000800:00000200:8.0:1619677474.489056:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666c00] (21)++
00000800:00000200:8.0:1619677474.489059:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.6.23@o2ib3
00000800:00000200:24.0:1619677474.489060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (22)--
00000400:00000200:8.0:1619677474.489062:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.23@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.489065:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.489066:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.489068:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.489069:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.489070:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.489071:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.489075:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.489077:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.489079:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.489080:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.489081:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.489083:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.489083:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666c00] (21)++
00000800:00000200:4.2:1619677474.489083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666c00] (21)++
00000800:00000200:8.0:1619677474.489085:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666c00] (23)--
00000800:00000200:8.0:1619677474.489086:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (22)--
00000800:00000200:8.0:1619677474.489087:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666c00] (21)++
00000800:00000200:25.2:1619677474.489088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.489089:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.23@o2ib3
00000800:00000200:24.0:1619677474.489090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (22)--
00000400:00000200:8.0:1619677474.489091:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.23@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.489094:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.489095:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677474.489096:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.489096:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:8.0:1619677474.489097:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.489098:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.489098:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.489099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:8.0:1619677474.489099:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677474.489100:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.489102:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.489103:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.489105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.489106:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677474.489108:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.489109:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677474.489110:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677474.489112:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677474.489112:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666c00] (21)++
00000800:00000200:8.0:1619677474.489113:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666c00] (22)--
00000800:00000200:8.0:1619677474.489114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (21)--
00000800:00000200:8.2:1619677474.489118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.489126:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.489129:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.489130:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.489130:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.489132:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.489135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666c00] (20)++
00000800:00000200:1.0:1619677474.489135:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677474.489142:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666c00] (21)++
00000800:00000200:24.0:1619677474.489145:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.23@o2ib3
00000800:00000200:8.0:1619677474.489145:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666c00] (22)++
00000800:00000200:8.0:1619677474.489146:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.23@o2ib3
00000400:00000200:24.0:1619677474.489148:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.23@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.489149:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.489151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.489153:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:24.0:1619677474.489154:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.489156:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.489157:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.489158:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:8.0:1619677474.489159:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.489160:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677474.489161:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.489162:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.489163:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677474.489163:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.489164:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.489165:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677474.489167:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.489168:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:24.0:1619677474.489169:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:8.0:1619677474.489169:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.489170:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677474.489171:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677474.489172:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666c00] (23)++
00000800:00000200:24.0:1619677474.489173:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666c00] (24)--
00000800:00000200:24.0:1619677474.489173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (23)--
00000800:00000200:24.0:1619677474.489174:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666c00] (22)++
00000800:00000200:8.0:1619677474.489174:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.489175:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677474.489175:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:25.2:1619677474.489176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.489176:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.23@o2ib3
00000800:00000200:8.0:1619677474.489177:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:24.0:1619677474.489178:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.489178:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666c00] (23)++
00000800:00000200:8.0:1619677474.489179:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8666c00] (24)++
00000400:00000200:24.0:1619677474.489180:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.489181:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677474.489181:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666c00] (25)--
00000800:00000200:30.0:1619677474.489183:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:24.0:1619677474.489183:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96504283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677474.489183:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677474.489185:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.489185:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.489186:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.489186:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.489186:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.489187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (24)--
00000400:00000200:30.0:1619677474.489188:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.489188:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666c00] (23)++
00000800:00000200:8.0:1619677474.489189:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8666c00] (24)--
00000800:00000200:31.0:1619677474.489190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677474.489190:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (23)--
00000400:00000200:24.0:1619677474.489191:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.489191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (22)--
00000800:00000200:30.0:1619677474.489192:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.0:1619677474.489192:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:24.0:1619677474.489193:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.0:1619677474.489194:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.489195:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.489195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666c00] (21)++
00000800:00000200:24.0:1619677474.489196:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.489197:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000400:00000200:21.0:1619677474.489197:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.489198:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677474.489199:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677474.489200:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.0:1619677474.489200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677474.489201:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666c00] (22)++
00000800:00000200:24.0:1619677474.489202:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666c00] (23)--
00000800:00000200:24.0:1619677474.489203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (22)--
00000800:00000200:8.0:1619677474.489203:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666c00] (21)++
00000800:00000200:24.0:1619677474.489205:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666c00] (22)++
00000800:00000200:8.2:1619677474.489206:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.489207:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.23@o2ib3
00000400:00000200:24.0:1619677474.489210:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.489210:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.23@o2ib3
00000400:00000200:8.0:1619677474.489212:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.489213:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.489214:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677474.489214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:24.0:1619677474.489215:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.489217:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.489217:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:24.0:1619677474.489218:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.489218:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.489218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:24.0:1619677474.489219:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.489220:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.489221:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.489222:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677474.489223:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96504281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.489224:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.489225:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.489226:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.489226:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.489227:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677474.489231:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.489232:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.489233:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.489233:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677474.489233:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.489234:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.489236:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.489236:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.489237:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666c00] (23)++
00000800:00000200:24.0:1619677474.489237:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666c00] (24)--
00000800:00000200:8.0:1619677474.489237:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677474.489237:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677474.489238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (23)--
00000800:00000200:24.0:1619677474.489239:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666c00] (22)++
00000800:00000200:8.0:1619677474.489239:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677474.489240:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.23@o2ib3
00000800:00000200:8.0:1619677474.489240:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666c00] (23)++
00000800:00000200:8.0:1619677474.489240:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666c00] (24)--
00000800:00000200:8.0:1619677474.489241:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (23)--
00000800:00000200:25.2:1619677474.489242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:24.0:1619677474.489242:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.489242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (22)--
00000800:00000200:8.2:1619677474.489245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:24.0:1619677474.489246:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.489248:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677474.489250:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96504280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.489250:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:24.0:1619677474.489251:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.489252:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.489252:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.489252:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.489253:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.489253:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.489253:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:22.0:1619677474.489256:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.489256:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:24.0:1619677474.489257:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.489258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:4.2:1619677474.489258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666c00] (21)++
00000400:00000200:1.0:1619677474.489258:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.489259:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.489261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:1.0:1619677474.489261:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.489262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.489263:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.489264:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:24.0:1619677474.489264:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.489265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677474.489266:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677474.489267:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666c00] (22)++
00000800:00000200:8.2:1619677474.489267:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677474.489269:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fe8666c00] (23)++
00000800:00000200:24.0:1619677474.489270:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666c00] (24)--
00000800:00000200:24.0:1619677474.489270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (23)--
00000800:00000200:6.0:1619677474.489270:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677474.489272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677474.489272:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666c00] (22)++
00000800:00000200:6.0:1619677474.489272:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.489274:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.23@o2ib3
00000800:00000200:8.0:1619677474.489274:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666c00] (23)++
00000800:00000200:6.0:1619677474.489274:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.489275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677474.489275:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677474.489275:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fe8666c00] (24)--
00000800:00000200:6.0:1619677474.489275:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000400:00000200:24.0:1619677474.489276:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.489276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (23)--
00000800:00000200:6.0:1619677474.489276:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.489277:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.489278:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677474.489278:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (22)--
00000400:00000200:24.0:1619677474.489279:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.489279:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.489280:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.489280:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:6.0:1619677474.489280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.489280:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:24.0:1619677474.489281:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677474.489282:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.489282:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.489282:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:22.0:1619677474.489283:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.489284:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.489285:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.489286:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677474.489290:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.489292:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.489292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.489297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:24.0:1619677474.489302:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.489303:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677474.489304:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.489305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:24.0:1619677474.489306:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (23)--
00000800:00000200:24.0:1619677474.489307:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666c00] (21)++
00000800:00000200:24.0:1619677474.489307:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666c00] (22)--
00000800:00000200:24.0:1619677474.489308:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666c00] (21)--
00000800:00000200:22.0:1619677474.489309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.489309:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.489310:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (23)++
00000800:00000200:6.0:1619677474.489311:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:25.2:1619677474.489312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (23)++
00000800:00000200:6.0:1619677474.489315:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.489317:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.489319:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.489320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.489320:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.489322:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.489324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.489447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.489455:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.489459:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.489461:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.489463:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.489463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.489464:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.489466:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.489467:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.489468:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.489469:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.489472:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.489473:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.489476:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.489477:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.489477:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.489479:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.489479:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.489480:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.489481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.489495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.489501:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.489504:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.489505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.489505:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.489507:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.489509:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.489554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677474.489562:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.489565:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.489567:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.489568:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.489571:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.489572:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677474.489573:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.489574:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.489575:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.489576:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677474.489579:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.489581:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677474.489583:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.489583:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.489584:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677474.489585:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.489586:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.489586:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.489587:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.489591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:21.2:1619677474.489591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:24.0:1619677474.489595:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.489597:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:1.0:1619677474.489597:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:24.0:1619677474.489598:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.489598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677474.489600:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.489600:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.489601:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:24.0:1619677474.489602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000400:00000200:1.0:1619677474.489603:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.489605:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.489607:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.489608:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.489609:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.489610:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.489611:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.489615:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.489616:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.489619:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.489620:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.489621:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.489622:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.489623:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.489623:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.489624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.489629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.489637:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.489639:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.489640:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.489642:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.489643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.489644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.489648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.489656:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.489658:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.489659:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.489660:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.489660:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.489661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.489984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677474.489992:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.489995:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677474.489997:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.489998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677474.490000:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.490002:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.490003:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.490004:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.490005:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.490006:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.490010:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.490012:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.490015:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.490016:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.490017:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.490019:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.490019:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.490020:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.490021:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677474.490025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.490032:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.490035:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.490036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.490036:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.490038:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.490042:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.490071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.490078:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.490081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.490081:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.490083:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.490086:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.490088:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.490088:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.490090:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.490091:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.490092:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.490095:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.490096:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.490099:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.490100:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.490100:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.490102:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.490102:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.490103:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.490104:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677474.490104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:8.2:1619677474.490108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677474.490109:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.490112:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.490114:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.490114:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.490116:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.490116:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:21.0:1619677474.490118:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.490119:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.490119:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.490120:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.490120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:21.0:1619677474.490121:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.490121:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.490123:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.490123:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.490126:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.490126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.0:1619677474.490128:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.2:1619677474.490129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:21.0:1619677474.490131:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.490132:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.490132:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.490134:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.490135:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.490135:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.490136:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:1.0:1619677474.490137:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.490139:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:25.2:1619677474.490140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.490140:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.490141:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.490142:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.490142:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677474.490148:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.490150:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.490151:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.490151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.490154:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.490156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.490325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.490333:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.490336:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.490336:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677474.490339:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.490341:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.490342:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.490344:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.490345:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.490346:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.490347:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.490351:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.490352:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.490355:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.490356:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.490357:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.490358:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.490359:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.490360:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.490360:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.490365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.490373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.490375:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.490376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.490377:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.490379:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.490382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.490466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.490474:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.490476:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.490477:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.490479:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.490482:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.490483:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.490485:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.490486:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.490487:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.490488:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.490491:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.490493:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.490495:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.490496:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.490496:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.490498:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.490498:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.490499:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.490500:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.490504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.490511:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.490514:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.490515:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.490515:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.490517:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.490520:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.491079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.491087:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.491091:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.491093:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.491095:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.491096:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.491097:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.491098:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.491100:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.491101:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.491102:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.491105:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.491106:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.491109:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.491110:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.491110:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.491112:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.491112:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.491113:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.491114:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.491117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.491125:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.491127:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.491128:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.491128:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.491130:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.491132:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.491269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.491276:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.491278:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.491279:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.491281:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.491284:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.491285:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677474.491287:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.491288:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.491289:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.491290:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677474.491293:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.491299:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677474.491301:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.491302:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677474.491302:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677474.491304:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677474.491304:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.491306:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.491307:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.491307:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677474.491310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.491314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677474.491318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.491320:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.491321:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.491322:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.491322:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.491324:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.491324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:30.0:1619677474.491325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.491326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:26.0:1619677474.491326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.2:1619677474.491362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.491370:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.491372:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.491374:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.491376:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.491377:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.491378:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.491379:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.491380:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.491381:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.491382:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.491385:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.491386:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.491389:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.491390:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.491391:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.491392:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.491393:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.491394:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.491396:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.491396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.491399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.491402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.491408:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677474.491410:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677474.491410:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.491411:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.491412:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.491413:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.491413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.491413:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.491414:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.491416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.491469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.491476:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.491479:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.491480:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.491482:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.491485:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.491486:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.491487:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.491488:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.491490:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.491491:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.491494:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.491495:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.491497:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.491498:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.491499:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.491500:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.491501:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.491502:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.491504:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.491504:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.491506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.491509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.491515:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.491516:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.491517:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.491517:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677474.491517:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.491518:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.491519:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.491519:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.491520:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.491520:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.491752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.491759:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.491762:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.491763:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.491765:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.491768:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.491770:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.491771:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.491772:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.491773:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.491774:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.491777:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.491779:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.491782:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.491782:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.491783:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.491785:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.491785:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.491786:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.491786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.491791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.491798:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.491801:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.491802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.491802:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.491804:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.491807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.492184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.492189:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.492192:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.492194:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.492194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.492196:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.492198:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.492199:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.492200:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.492202:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.492202:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.492206:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.492207:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.492209:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.492210:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.492211:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.492212:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.492213:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.492214:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.492214:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.492219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.492226:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.492229:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.492229:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.492231:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.492232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.492233:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.492286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.492293:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.492299:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.492300:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:21.2:1619677474.492302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677474.492302:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.492305:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.492306:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.492307:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.492308:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.492309:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.492310:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677474.492311:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.492313:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677474.492314:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.492316:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.492316:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.492318:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.492319:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.492319:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.492320:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677474.492320:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:31.0:1619677474.492321:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.492321:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:31.0:1619677474.492322:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.492322:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:31.0:1619677474.492323:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.492323:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:31.0:1619677474.492324:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.492324:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.492324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:31.0:1619677474.492327:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.492329:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:9.2:1619677474.492329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.492331:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.492332:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.492332:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.492334:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.492334:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.492335:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.492336:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677474.492338:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:25.2:1619677474.492339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677474.492340:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.492342:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.492342:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.492344:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.492346:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:2.0:1619677474.492348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:30.0:1619677474.492349:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.492350:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.492352:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.492353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.492354:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.492898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.492904:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.492907:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.492907:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.492910:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.492911:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.492913:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.492914:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.492915:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.492916:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.492917:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.492921:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.492923:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.492926:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.492927:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.492927:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.492929:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.492929:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.492931:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.492932:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.492932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.492935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677474.492937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677474.492943:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677474.492943:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677474.492945:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:2.0:1619677474.492945:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.492946:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:12.0:1619677474.492947:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.492947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677474.492949:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.492950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.492951:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.492953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677474.492960:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.492962:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.492963:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.492964:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.492964:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.492966:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.493174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.493181:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.493184:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.493186:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.493188:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.493188:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.493189:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.493190:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.493192:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.493193:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.493194:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.493196:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.493198:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.493200:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.493201:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.493202:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.493203:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.493204:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.493204:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.493205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.493209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.493216:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.493218:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.493219:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.493220:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.493222:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.493223:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.493253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.493261:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.493263:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.493265:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.493267:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.493268:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.493268:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.493269:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.493270:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.493271:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.493272:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.493275:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.493277:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.493279:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.493280:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.493280:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.493282:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.493282:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.493283:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.493283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.493287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.493294:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.493300:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.493300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.493301:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.493303:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.493305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.493426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.493434:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.493437:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.493438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.493439:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.493442:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.493443:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.493444:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.493445:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.493446:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.493447:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.493450:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.493451:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.493453:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.493454:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.493455:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.493456:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.493457:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.493457:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.493458:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.493462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.493469:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.493472:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.493472:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.493473:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.493474:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.493477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.493483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:5.2:1619677474.493486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677474.493489:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.493492:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:17.0:1619677474.493492:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677474.493493:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.493493:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.493493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.493494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677474.493495:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.493496:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.493498:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.493500:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.493502:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.493503:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.493505:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.493506:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.493507:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.493510:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.493511:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.493514:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.493515:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.493515:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.493517:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.493517:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.493519:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.493520:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.493520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.493523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.493525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.493531:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.493531:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.493533:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.493533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677474.493534:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.493534:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.493535:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.493535:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.493537:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.493539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.493823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.493831:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.493834:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.493836:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.493839:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.493839:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.493840:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677474.493842:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.493843:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.493844:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.493845:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677474.493848:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.493850:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677474.493853:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.493854:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677474.493854:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677474.493856:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677474.493856:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.493857:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.493858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677474.493862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.493869:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.493872:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.493872:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.493874:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.493876:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.493877:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.494654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:4.2:1619677474.494658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:31.0:1619677474.494663:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677474.494664:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:31.0:1619677474.494666:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.0:1619677474.494666:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:20.0:1619677474.494669:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:31.0:1619677474.494670:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.494670:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:21.0:1619677474.494670:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.494672:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.494673:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.494673:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.494674:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.494674:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.494677:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.494677:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.494679:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.494679:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.494680:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.494682:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.494682:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.494685:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.494687:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.494689:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.494690:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.494691:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.494691:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.494691:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.494693:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.494694:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.494695:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:21.0:1619677474.494695:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.494695:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.494696:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:21.0:1619677474.494696:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (22)++
00000800:00000200:31.0:1619677474.494697:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677474.494698:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677474.494699:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.494699:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.494700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.494700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.2:1619677474.494703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.494707:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:8.0:1619677474.494709:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677474.494710:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)++
00000800:00000200:30.0:1619677474.494711:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677474.494711:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:8.0:1619677474.494711:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.494712:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.494713:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:24.0:1619677474.494713:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.494713:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.494714:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.494715:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.0:1619677474.494715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.494716:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.494717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.494719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.494725:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.494727:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.494728:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.494728:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.494729:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.494730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.494920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.494927:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.494930:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.494931:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.494933:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.494935:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.494937:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.494938:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.494940:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.494941:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.494942:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.494945:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.494947:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.494950:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.494951:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.494951:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.494953:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.494953:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.494954:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.494955:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677474.494959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.494968:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.494970:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.494971:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.494971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.494973:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.494976:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.495222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.495230:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.495233:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.495234:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.495235:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.495238:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.495240:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.495241:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.495243:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.495244:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.495245:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.495248:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.495249:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.495252:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.495253:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.495254:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.495255:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.495256:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.495256:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.495257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.495261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.495269:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.495272:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.495273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.495273:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.495275:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.495278:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.495283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.495290:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.495292:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.495293:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.495297:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.495298:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.495298:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.495321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677474.495324:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.495327:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677474.495329:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.495330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677474.495332:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.495333:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.495334:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.495336:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.495337:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.495338:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.495341:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.495343:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.495346:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.495347:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.495347:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.495349:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.495350:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.495351:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677474.495352:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677474.495353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:25.2:1619677474.495356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.2:1619677474.495358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.495363:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:6.0:1619677474.495364:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.495365:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677474.495366:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:6.0:1619677474.495366:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677474.495367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:6.0:1619677474.495367:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.495368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.495369:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.495371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.495372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677474.495376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.495379:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.495382:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:22.0:1619677474.495383:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.495383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.495385:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.495386:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.495387:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.495387:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677474.495388:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.495388:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.495389:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677474.495390:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.495391:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.495393:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.495394:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.495395:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.495398:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.495400:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.495403:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.495404:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.495404:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.495406:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.495406:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.495407:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.495408:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.495412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.495418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677474.495419:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.495422:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.495423:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.495423:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.495425:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.495426:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.495428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677474.495429:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.495429:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.495431:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.495434:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.495436:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.495437:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.495438:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.495439:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.495440:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.495443:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.495444:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.495447:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.495448:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.495448:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.495450:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.495450:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.495452:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.495453:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.495454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677474.495456:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677474.495459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677474.495464:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677474.495466:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677474.495467:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677474.495468:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677474.495468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.495468:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.495469:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.495470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677474.495470:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.495474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.495547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.495552:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.495555:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.495555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.495558:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.495560:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.495561:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.495562:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.495563:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.495564:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.495565:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.495569:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.495571:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.495573:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.495574:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.495574:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.495576:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.495576:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.495577:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.495577:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.495582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.495591:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.495593:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.495594:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.495595:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.495596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.495598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:25.2:1619677474.495951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677474.495958:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.495961:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677474.495963:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.111@o2ib7 : GET - for me
00000800:00000200:12.0:1619677474.495965:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:13.0:1619677474.495966:0:19942:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.111@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:13.0:1619677474.495968:0:19942:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.111@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:13.0:1619677474.495969:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.495970:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:13.0:1619677474.495972:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:13.0:1619677474.495976:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : REPLY try# 0
00000800:00000200:13.0:1619677474.495977:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.495980:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.495980:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.495981:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.495982:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.495983:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:25.2:1619677474.495988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.495995:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.495997:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.495998:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.495999:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: REPLY: OK
00000800:00000200:12.0:1619677474.496001:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.496002:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.496335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.496343:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.496345:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.496347:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.496350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.496350:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.496351:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.496352:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.496353:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.496354:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.496355:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.496359:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.496360:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.496362:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.496363:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.496363:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.496365:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.496365:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.496366:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.496367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.496371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.496378:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.496381:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.496381:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.496383:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.496384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.496385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.496391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.496398:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.496400:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.496401:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.496402:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.496402:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.496403:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.496613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.496621:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.496623:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.496624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.496626:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.496629:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.496631:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.496632:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.496633:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.496634:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.496635:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.496638:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.496640:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.496643:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.496643:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.496644:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.496646:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.496646:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.496647:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.496647:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.496652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.496660:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.496663:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.496664:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.496664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.496666:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.496669:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.496693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.496701:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.496704:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.496706:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.496708:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.496708:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.496709:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.496710:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.496711:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.496712:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.496713:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.496716:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.496718:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.496720:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.496721:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.496722:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.496723:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.496724:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.496725:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.496725:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.496729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.496736:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.496738:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.496739:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.496739:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.496741:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.496743:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.496765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.496772:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.496775:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.496776:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.496777:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.496780:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.496781:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.496783:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.496784:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.496785:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.496785:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.496788:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.496790:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.496793:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.496793:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.496794:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.496796:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.496796:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.496797:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.496798:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.496799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677474.496802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.496805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677474.496811:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677474.496812:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677474.496813:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677474.496814:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.496815:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677474.496815:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.496816:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677474.496816:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.496817:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.496820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.496822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.496830:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.496833:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.496833:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.496834:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.496835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.496836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.496849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.496853:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.496855:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677474.496858:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.496858:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.496860:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.496861:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.496862:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.496864:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.496865:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.496866:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.496869:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.496871:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.496874:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.496875:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.496876:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.496878:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.496878:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.496879:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.496881:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.496882:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677474.496884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.496887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677474.496891:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:21.0:1619677474.496892:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677474.496893:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.496894:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.496894:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.496895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677474.496896:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.496896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677474.496898:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.496899:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.496901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.496908:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.496911:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.496911:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.496912:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.496912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.496913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.497029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.497034:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.497037:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.497039:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.497039:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.497042:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.497043:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.497044:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.497046:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.497047:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.497048:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.497051:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.497052:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.497055:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.497056:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.497057:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.497058:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.497059:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.497060:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.497060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.497064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.497071:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.497073:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.497074:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.497076:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.497077:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.497078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.497170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.497177:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.497180:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.497181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.497183:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.497185:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.497187:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677474.497188:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.497190:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.497190:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.497192:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677474.497195:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.497197:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677474.497200:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.497201:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677474.497201:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677474.497203:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677474.497204:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.497205:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.497205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.497209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.497216:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.497219:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677474.497219:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.497220:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.497222:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.497225:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.497434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.497442:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.497445:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.497447:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.497450:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.497450:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.497451:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.497452:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.497454:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.497454:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.497456:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.497459:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.497461:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.497462:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.497463:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.497464:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.497465:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.497466:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.497467:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.497469:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.497469:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.497472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.497476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.497480:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.497482:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.497482:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.497484:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.497484:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.497485:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.497485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.497486:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.497487:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.497487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.497690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677474.497697:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.497700:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.497701:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.497702:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.497705:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.497706:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677474.497707:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.497709:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.497710:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.497711:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677474.497714:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.497716:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677474.497718:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.497718:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.497719:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677474.497720:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.497721:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.497722:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.497722:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.497727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.497730:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.497732:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677474.497733:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.497734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677474.497735:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.497737:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:20.2:1619677474.497911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.497917:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.497920:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.497921:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.497923:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.497926:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.497928:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.497929:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.497930:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.497932:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.497933:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.497936:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.497938:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.497940:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.497941:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.497942:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.497943:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.497944:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.497945:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.497945:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.497949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.497958:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.497960:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.497961:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.497961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.497963:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.497966:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.498161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.498168:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.498171:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.498172:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.498174:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.498176:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677474.498176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677474.498177:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.498179:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.498180:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.498181:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.498182:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677474.498182:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.498184:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.498185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677474.498185:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677474.498187:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.498188:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.498190:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.498190:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:1.0:1619677474.498190:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.498191:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:1.0:1619677474.498191:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677474.498192:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:1.0:1619677474.498192:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.498193:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.498194:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:1.0:1619677474.498194:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.498195:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:1.0:1619677474.498195:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.498196:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:1.0:1619677474.498196:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677474.498198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:1.0:1619677474.498199:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.498201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677474.498201:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.498203:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.498204:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677474.498204:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.498205:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677474.498207:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677474.498207:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.498207:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677474.498208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.498208:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.498208:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.498208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677474.498209:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677474.498210:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.498211:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.498212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.498212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.498213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.498213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.498220:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.498223:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.498224:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.498224:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.498226:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.498228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.498447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.498453:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.498456:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.498456:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.498458:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.498461:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.498462:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677474.498464:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.498465:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.498466:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.498467:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677474.498470:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.498471:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677474.498474:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.498475:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677474.498476:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677474.498478:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677474.498478:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.498479:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.498479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677474.498484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.498491:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.498494:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.498495:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.498496:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.498499:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.498500:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.498600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.498606:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.498609:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.498611:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.498611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.498613:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.498615:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.498616:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.498618:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.498619:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.498620:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.498623:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.498625:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.498628:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.498629:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.498629:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.2:1619677474.498629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.498631:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.498632:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.498632:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.498633:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.498633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.0:1619677474.498633:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.498634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.498635:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.498637:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.498637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677474.498639:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.498640:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.498641:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.498642:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.498643:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.498644:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.498647:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:20.0:1619677474.498647:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.498648:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.498648:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.498649:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677474.498650:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.498652:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.498652:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677474.498653:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677474.498653:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.498655:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.498655:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.498657:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.498658:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.498659:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.498661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.498664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.498669:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677474.498669:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677474.498671:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.498672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677474.498672:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677474.498673:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.498673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.498673:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.498675:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.498678:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.499230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.499238:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.499240:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.499243:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.499245:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.499245:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.499247:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.499248:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.499249:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.499250:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.499251:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.499255:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.499257:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.499259:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.499260:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.499261:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.499262:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.499263:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.499264:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.499265:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.499268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.499274:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.499277:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.499278:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.499278:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.499280:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.499282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.499287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.499292:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.499297:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.499299:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.499300:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.499301:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.499301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.500214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.500222:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.500225:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.500226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.500227:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.500229:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.500231:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.500232:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.500233:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.500234:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.500235:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.500238:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.500239:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.500242:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.500243:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.500244:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.500245:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.500246:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.500246:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.500247:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.500252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.500261:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.500263:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.500264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.500265:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.500267:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.500270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.500438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.500447:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.500449:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.500452:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.500454:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.500454:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.500455:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.500457:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.500458:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.500459:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.500460:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.500463:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.500465:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.500468:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.500469:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.500470:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.500471:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.500472:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.500473:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.500474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.500478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.500487:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.500490:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.500491:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.500492:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.500493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.500496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.500827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.500834:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.500837:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.500838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.500840:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.500842:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.500844:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.500845:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.500846:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.500848:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.500848:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.500852:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.500853:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.500856:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.500857:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.500858:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.500859:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.500860:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.500861:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.500861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.500865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.500874:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.500877:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.500878:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.500878:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.500881:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.500884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.500885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.500893:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.500895:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.500896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.500896:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.500897:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.500898:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677474.500955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.500961:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.500964:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.500965:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.500967:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.500969:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.500970:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.500972:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.500973:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.500974:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.500975:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.500978:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.500980:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.500982:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.500983:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.500983:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.500985:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.500985:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.500986:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.500987:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.500992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.500999:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.501001:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.501002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.501002:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.501003:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.501005:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.501015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.501022:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.501024:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.501025:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.501026:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.501026:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677474.501026:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:20.2:1619677474.501443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677474.501451:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.501454:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677474.501456:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.501458:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677474.501460:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.501461:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677474.501462:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.501464:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.501465:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.501466:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677474.501470:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.501471:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.501475:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.501475:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677474.501476:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677474.501478:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677474.501479:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.501480:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.501480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677474.501485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.501493:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.501496:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.501497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.501497:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.501499:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.501502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.501517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.501525:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.501527:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.501530:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.501531:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.501533:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.501534:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.501535:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.501537:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.501538:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.501539:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.501542:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.501544:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.501547:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.501548:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.501549:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.501550:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.501551:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.501552:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.501552:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.501557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.501566:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.501568:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.501569:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.501571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.501571:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.501574:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.501631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.501639:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.501641:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.501644:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.501645:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.501647:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.501648:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.501649:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.501651:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.501652:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.501653:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.501656:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.501657:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.501659:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.501660:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.501660:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.501662:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.501663:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.501663:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.501664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.501668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.501674:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.501677:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.501678:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.501678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.501680:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.501682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.501682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.501690:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.501692:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.501693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.501695:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.501697:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.501699:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.501700:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.501701:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.501702:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.501703:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.501706:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.501707:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.501710:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.501711:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.501712:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.501714:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.501714:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.501715:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.501715:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.501719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.501728:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.501730:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.501731:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.501733:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.501733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.501736:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.501985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.501992:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.501995:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.501996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.501997:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.502000:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.502002:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.502003:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.502004:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.502005:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.502006:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.502009:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.502011:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.502014:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.502015:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.502015:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.502017:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.502017:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.502018:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.502018:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.502034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.502043:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.502045:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.502046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.502046:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.502049:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.502051:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.502055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.502062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.502065:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.502066:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.502066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.502067:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.502067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.502491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.502499:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.502502:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.502504:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.502506:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.502506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.502507:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.502509:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.502510:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.502511:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.502512:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.502515:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.502517:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.502518:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.502519:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.502520:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.502521:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.502522:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.502523:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.502523:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.502528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.502535:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.502537:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.502538:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.502539:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.502541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.502542:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.502806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677474.502812:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.502815:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677474.502816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677474.502818:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677474.502821:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.502822:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677474.502823:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677474.502824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:17.0:1619677474.502825:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.502826:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.502828:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.502830:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:17.0:1619677474.502832:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.502832:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677474.502834:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.502835:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.502837:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.502838:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.502839:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:1.0:1619677474.502839:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677474.502840:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:1.0:1619677474.502840:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.502841:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.502842:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:1.0:1619677474.502842:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.502843:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.502844:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.502845:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677474.502846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:17.0:1619677474.502847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.502847:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677474.502848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.502849:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.502851:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.502852:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.502853:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.502854:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677474.502855:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:1.0:1619677474.502855:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.502856:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:24.0:1619677474.502857:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677474.502857:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.502857:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:24.0:1619677474.502858:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.502858:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.502860:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.2:1619677474.502860:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.502862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:24.0:1619677474.502864:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677474.502868:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677474.502869:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.502870:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677474.502871:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677474.502871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677474.502872:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.502872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677474.502874:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.502875:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.502876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.502882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.502890:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.502892:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.502893:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.502894:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.502894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.502895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.503020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.503027:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.503030:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.503031:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.503033:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.503036:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.503037:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.503038:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.503039:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.503041:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.503042:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.503045:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.503047:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.503049:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.503050:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.503051:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.503052:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.503053:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.503053:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.503054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.503058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.503064:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.503066:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.503067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.503067:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.503069:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.503071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.503374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.503379:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.503381:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.503383:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.503384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.503386:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.503387:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.503389:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.503390:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.503391:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.503392:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.503395:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.503397:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.503399:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.503400:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.503400:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.503404:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:21.0:1619677474.503406:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.503407:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.503407:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.503408:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.503409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.503410:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.503412:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677474.503413:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677474.503415:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.503417:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:17.0:1619677474.503418:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.503419:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:17.0:1619677474.503419:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.503420:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.503420:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.503422:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.503422:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.503423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:17.0:1619677474.503423:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.503424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:17.0:1619677474.503424:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.503428:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.2:1619677474.503429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677474.503430:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.503433:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.503434:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677474.503434:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677474.503436:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677474.503436:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677474.503437:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.503437:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:18.0:1619677474.503438:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677474.503438:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:2.0:1619677474.503439:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.503441:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:9.2:1619677474.503442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:18.0:1619677474.503443:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.503444:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.503446:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.503447:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.503448:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:18.0:1619677474.503448:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.503449:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.503451:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.503452:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.503452:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:18.0:1619677474.503453:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:24.0:1619677474.503454:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.503455:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.503457:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677474.503458:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.503459:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.503459:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.503462:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.503462:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.503463:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.503464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.503467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.503474:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.503477:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.503478:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.503480:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.503482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.503483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.503898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.503906:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.503909:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.503911:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.503913:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.503913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.503914:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.503915:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.503917:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.503918:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.503919:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.503922:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.503923:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.503926:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.503927:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.503928:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.503929:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.503930:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.503932:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.503933:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.503934:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.503935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.503940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.503941:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.503944:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.503945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.503945:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.503947:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.503948:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.0:1619677474.503948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.503950:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.503951:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.503952:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.503961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.503967:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.503969:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.503970:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.503970:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.503971:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.503972:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.504306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.504313:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.504316:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.504317:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.504319:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.504321:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.504323:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.504324:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.504326:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.504326:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.504328:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.504331:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.504333:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.504336:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.504336:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.504337:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.504339:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.504339:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.504340:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.504340:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.504345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.504353:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.504356:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.504357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.504357:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.504359:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.504362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.504645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.504650:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.504653:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.504655:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.504655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.504657:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.504659:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.504660:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.504662:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.504662:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.504664:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.504667:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.504669:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.504671:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.504672:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.504672:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.504674:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.504674:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.504675:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.504676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677474.504678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677474.504681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.504685:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.504687:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:20.0:1619677474.504688:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.504689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677474.504690:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:1.0:1619677474.504690:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.504691:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.504693:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.504693:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.504694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.504695:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:1.0:1619677474.504695:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.504696:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.504697:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.504698:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.504699:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.504703:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.504705:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.504707:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.504708:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.504709:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.504710:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.504710:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.504711:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.504712:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.504716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.504723:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.504726:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.504727:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.504727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.504729:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.504731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.505114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.505121:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.505124:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.505125:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.505127:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.505129:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.505131:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.505132:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.505134:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.505134:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.505136:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.505139:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.505141:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.505144:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.505144:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.505145:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.505147:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.505147:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.505148:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.505148:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.505153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.505160:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.505163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.505163:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.505164:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.505166:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.505169:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.505588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.505596:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.505599:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.505601:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.505603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.505603:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.505605:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.505606:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.505607:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.505608:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.505609:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.505612:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.505613:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.505617:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.505617:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.505618:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.505620:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.505620:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.505622:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.505623:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.505624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.505627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.505630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.505639:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.505639:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.505641:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677474.505642:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.505643:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677474.505643:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.505644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677474.505644:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.505644:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.505646:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.505647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (24)--
00000800:00000200:20.0:1619677474.505647:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:20.0:1619677474.505648:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (24)--
00000800:00000200:20.0:1619677474.505648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.505649:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.505654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.505661:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.505664:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.505665:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.505667:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.505670:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.505671:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.505673:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.505674:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.505675:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.505676:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.505679:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.505681:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.505683:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.505684:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.505685:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.505686:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.505687:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.505688:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.505689:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.505690:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677474.505693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677474.505695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677474.505701:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677474.505702:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677474.505704:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.505704:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.505705:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.505705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677474.505705:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.505707:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677474.505707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000400:00000200:17.0:1619677474.505707:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.505710:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:17.0:1619677474.505710:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677474.505713:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:4.2:1619677474.505713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.0:1619677474.505715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677474.505715:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.505718:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.505719:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:26.0:1619677474.505721:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:20.0:1619677474.505721:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.505722:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.505723:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.505724:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:20.0:1619677474.505724:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677474.505725:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.505729:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.505729:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.505731:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:26.0:1619677474.505732:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.505734:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.505735:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677474.505735:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:26.0:1619677474.505736:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.505736:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:26.0:1619677474.505737:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.505737:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.505738:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000400:00000200:26.0:1619677474.505739:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.505739:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.505739:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:20.0:1619677474.505740:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677474.505741:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:26.0:1619677474.505743:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:8.2:1619677474.505743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677474.505745:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.2:1619677474.505746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:26.0:1619677474.505748:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.505748:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.505749:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.505750:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:26.0:1619677474.505751:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.505751:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677474.505751:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:26.0:1619677474.505752:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:21.0:1619677474.505752:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:20.0:1619677474.505752:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.505753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677474.505753:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677474.505754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677474.505754:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.505755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.505756:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.505757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.505758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.505766:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.505768:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.505769:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.505771:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.505772:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.505773:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.506129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.506136:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.506139:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.506142:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.506143:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.506144:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.506146:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.506147:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.506148:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.506149:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.506150:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.506153:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.506155:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.506157:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.506158:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.506159:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.506160:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.506161:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.506162:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.506162:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.506167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.506173:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.506176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.506176:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.506177:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.506179:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.506181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.506635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.506643:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.506646:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.506648:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.506650:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.506650:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.506652:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.506653:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.506655:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.506655:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.506656:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.506660:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.506661:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.506665:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.506666:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.506666:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.506668:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.506669:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.506669:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.506670:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.506675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.506684:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.506686:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.506687:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.506689:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.506690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.506692:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.507067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.507074:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.507076:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.507077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.507079:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.507082:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.507083:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.507085:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.507086:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.507087:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.507088:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.507091:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.507093:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.507096:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.507097:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.507097:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.507099:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.507099:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.507100:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.507101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.507106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.507114:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.507117:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.507118:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.507118:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.507120:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.507123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.507501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.507509:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.507512:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.507514:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.507516:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.507517:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.507519:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677474.507520:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.507522:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.507523:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.507524:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677474.507527:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.507529:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677474.507531:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.507532:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677474.507533:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677474.507534:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677474.507535:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.507536:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.507536:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677474.507541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.507550:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.507553:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.507554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.507554:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.507557:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.507559:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.507632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.507640:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.507643:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.507645:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.507647:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.507648:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.507649:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.507650:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.507652:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.507653:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.507653:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.507657:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.507658:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.507660:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.507661:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.507661:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.507663:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.507663:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.507664:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.507665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.507669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.507676:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.507678:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.507679:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.507681:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.507682:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.507683:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.507708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.507716:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.507718:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.507720:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.507722:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.507722:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.507723:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.507725:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.507726:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.507727:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.507727:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.507730:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.507732:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.507734:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.507734:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.507735:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.507736:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.507737:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.507738:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.507738:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.507742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.507749:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.507752:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.507753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.507753:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.507755:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.507757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.507929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.507935:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.507938:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.507939:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.507941:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.507944:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.507945:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.507947:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.507948:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.507949:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.507950:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.507953:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.507955:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.507957:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.507957:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.507958:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.507959:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.507960:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.507961:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.507961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.507966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.507973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.507975:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.507976:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.507976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.507977:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.507979:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.508244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.508252:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.508255:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.508256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.508258:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.508261:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.508262:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.508263:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.508264:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.508265:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.508266:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.508269:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.508271:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.508273:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.508274:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.508275:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.508276:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.508277:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.508278:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.508279:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.508280:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.508282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.508285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.508288:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.508290:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.508291:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.508292:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.508292:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.508298:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.508299:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.508301:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.508301:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.508302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677474.509141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.509148:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.509151:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.509152:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.509153:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.509155:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.509157:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.509158:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.509159:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.509160:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.509161:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.509164:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.509165:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.509168:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.509169:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.509169:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.509171:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.509171:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.509172:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.509173:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.509174:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.509178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.509179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.509186:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.509186:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.509188:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.509189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677474.509189:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677474.509190:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.509190:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.509191:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.509193:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.509195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.509198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.509205:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.509207:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.509208:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.509208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.509209:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.509210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.509449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.509455:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.509458:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.509459:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.509460:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.509463:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.509464:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.509465:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.509467:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.509468:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.509469:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.509472:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.509473:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.509476:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.509476:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.509477:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.509478:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.509479:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.509480:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.509480:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.509485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.509493:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.509496:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.509497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.509497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.509499:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.509501:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.509678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.509687:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.509690:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.509692:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.509694:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.509694:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.509696:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.509697:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.509699:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.509700:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.509701:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.509704:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.509706:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.509709:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.509709:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.509710:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.509712:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.509712:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.509713:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.509714:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.509717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.509725:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.509727:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.509728:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.509728:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.509730:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.509732:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.509975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.509983:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.509985:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.509986:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.509988:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.509991:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.509993:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.509994:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.509995:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.509996:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.509997:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.510000:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.510002:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.510005:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.510006:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.510006:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.510008:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.510008:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.510009:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.510009:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.510014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.510021:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.510024:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.510025:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.510025:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.510027:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.510030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.510062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.510069:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.510071:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.510072:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.510074:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.510077:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.510079:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.510080:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.510082:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.510083:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.510083:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.510087:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.510089:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.510091:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.510092:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.510092:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.510094:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.510095:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.510095:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.510096:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.2:1619677474.510096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.510099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677474.510101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.510102:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.510104:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.510105:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.510106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.510107:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.0:1619677474.510108:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:20.0:1619677474.510108:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.510109:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.510109:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.510110:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.510110:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.510112:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000400:00000200:20.0:1619677474.510112:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.510115:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.510117:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.510118:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.510119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677474.510120:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.510121:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.510121:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.510123:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.510123:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.510124:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.510124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.510129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.510136:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.510138:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.510139:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.510140:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.510142:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.510143:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.510149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.510158:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.510160:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.510161:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.510162:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.510162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.510164:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.510452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.510460:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.510463:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.510465:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.510468:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.510468:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.510469:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.510470:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.510471:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.510472:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.510473:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.510477:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.510479:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.510481:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.510482:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.510482:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.510484:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.510484:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.510485:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.510486:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.510490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.510497:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.510499:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.510500:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.510501:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.510503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.510504:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.510508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.510515:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.510517:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.510518:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.510519:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.510519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.510520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.510653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.510659:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.510662:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.510663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.510664:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.510667:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.510669:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.510670:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.510671:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.510672:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.510673:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.510676:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.510678:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.510681:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.510682:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.510682:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.510684:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.510685:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.510685:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.510686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.510690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.510697:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.510700:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.510701:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.510702:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.510705:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.510705:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.511001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677474.511009:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.511011:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677474.511012:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677474.511014:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.511017:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.511018:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.511019:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.511021:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.511022:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.511023:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.511026:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.511028:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.511030:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.511031:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.511031:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.511033:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.511033:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.511034:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677474.511035:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677474.511036:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677474.511039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:4.2:1619677474.511041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:24.0:1619677474.511042:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.511044:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677474.511045:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.511046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677474.511047:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.511048:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:24.0:1619677474.511049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:26.0:1619677474.511050:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677474.511051:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.511052:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.511055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677474.511058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.511061:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677474.511062:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.511062:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (23)--
00000800:00000200:8.0:1619677474.511062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677474.511063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677474.511301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.511309:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.511311:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.511313:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.511315:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.511315:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.511316:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.511317:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.511319:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.511320:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.511320:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.511323:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.511325:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.511327:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.511328:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.511328:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.511328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677474.511330:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.511330:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.511332:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.511333:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:21.0:1619677474.511333:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677474.511334:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677474.511336:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:9.2:1619677474.511336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.511339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.2:1619677474.511340:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:21.0:1619677474.511343:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.511343:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:21.0:1619677474.511345:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.511345:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:21.0:1619677474.511346:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677474.511346:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.511347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.511348:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:21.0:1619677474.511348:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.511349:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.511350:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.511350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:21.0:1619677474.511350:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.511351:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:21.0:1619677474.511351:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.511352:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.511352:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.511353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:21.0:1619677474.511356:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.511358:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.511359:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.511361:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.511362:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.511362:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677474.511363:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677474.511363:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677474.511363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.511364:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.511365:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677474.511365:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.511366:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.511367:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.511367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.511371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.511378:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.511381:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.511382:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.511383:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.511384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.511386:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.511728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.511735:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.511738:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.511739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.511741:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.511743:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.511745:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.511746:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.511748:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.511749:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.511750:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.511753:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.511755:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.511758:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.511758:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.511759:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.511761:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.511761:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.511762:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.511762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.511767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.511775:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.511778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.511778:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.511779:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.511781:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.511783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.511787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.511794:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.511797:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.511798:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.511798:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.511799:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.511799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.511888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.511896:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.511899:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.511901:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.511903:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.511903:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.511905:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.511906:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.511907:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.511908:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.511909:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.511912:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.511914:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.511917:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.511918:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.511919:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.511920:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.511921:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.511922:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.511924:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.511924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.511926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.511930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.511934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.511937:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677474.511937:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.511938:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.511939:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.511940:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.511940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.511941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.511942:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.511943:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.511943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.511951:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.511954:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.511955:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.511955:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.511956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.511957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.512505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.512511:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.512513:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.512515:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.512516:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.512518:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.512519:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.512521:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.512522:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.512523:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.512524:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.512527:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.512529:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.512531:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.512532:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.512532:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.512534:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.512535:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.512536:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.512538:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.512538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.512541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.512543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.512546:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.512548:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.512549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.512549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677474.512550:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.512553:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.512554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.512554:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.512556:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.512558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.512848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.512857:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.512860:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.512862:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.512864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.512865:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.512866:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.512868:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.512869:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.512870:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.512871:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.512874:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.512876:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.512879:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.512880:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.512880:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.512882:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.512883:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.512884:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.512884:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.512888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.512896:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.512899:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.512900:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.512901:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.512902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.512904:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.513327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.513334:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.513338:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.513340:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.513342:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.513342:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.513344:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.513345:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.513346:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.513347:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.513348:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.513351:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.513352:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.513355:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.513355:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.513356:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.513357:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.513358:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.513359:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.513359:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.513364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.513371:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.513374:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.513375:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.513375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.513377:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.513379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.513481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677474.513489:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.513491:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677474.513492:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677474.513494:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.513496:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.513497:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.513498:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.513500:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.513500:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.513501:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.513504:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.513505:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.513508:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.513508:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.513509:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.513510:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.513511:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.513511:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.513512:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.513516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.513519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.513522:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.513523:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.513523:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.513524:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.513526:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677474.513631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.513638:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.513641:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.513642:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.513643:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.513646:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.513647:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.513648:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.513649:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.513650:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.513651:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.513654:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.513655:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.513657:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.513658:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.513658:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.513660:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.513660:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.513661:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.513661:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.513665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.513671:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.513674:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.513674:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.513675:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.513676:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.513679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.513684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.513691:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.513693:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.513694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.513695:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.513695:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.513696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.514080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.514086:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.514089:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:4.2:1619677474.514090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:1.0:1619677474.514090:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.514092:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.514094:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.514095:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.514096:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.514097:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:17.0:1619677474.514098:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.514099:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.514100:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677474.514100:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.514100:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677474.514103:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.514104:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.514106:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.514106:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.514108:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:17.0:1619677474.514109:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.514109:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.514110:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.514110:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677474.514110:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.514111:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677474.514112:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:10.0:1619677474.514112:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.514113:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.514114:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.514115:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.514116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677474.514116:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677474.514118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.514118:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.2:1619677474.514120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677474.514121:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.514121:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.514122:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (23)++
00000800:00000200:10.0:1619677474.514124:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (24)--
00000800:00000200:10.0:1619677474.514124:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.514125:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.514126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.514126:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:1.0:1619677474.514126:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.514129:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000800:00000200:1.0:1619677474.514129:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677474.514130:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (24)++
00000800:00000200:1.0:1619677474.514130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677474.514131:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000400:00000200:6.0:1619677474.514131:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.514131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.514132:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.514133:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.514134:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.514136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.514137:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.514137:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.514449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.514456:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.514459:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.514460:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.514462:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.514465:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.514466:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.514468:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.514469:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.514470:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.514471:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.514473:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.514475:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.514477:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.514478:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.514479:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.514481:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.514481:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.514482:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.514484:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.514484:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677474.514488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.514490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677474.514496:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677474.514497:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:21.0:1619677474.514499:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.514499:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.514500:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.514500:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677474.514501:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:10.0:1619677474.514501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677474.514502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.514504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.514687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.514695:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.514700:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.514701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.514705:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.514709:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.514711:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.514713:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.514715:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.514717:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.514718:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.514723:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.514726:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.514729:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.514730:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.514731:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.514733:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.514734:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.514735:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.514736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.514740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.514749:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:4.2:1619677474.514750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677474.514752:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.514753:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.514755:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677474.514757:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677474.514757:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.514759:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677474.514760:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.514761:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.514763:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.514766:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.514768:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.514769:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.514770:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.514771:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.514772:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.514775:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.514777:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.514780:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.514781:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.514781:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.514783:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.514783:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.514784:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.514785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.514789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.514797:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.514800:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.514801:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.514801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.514803:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.514806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.515039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.515047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.515050:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.515052:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.515053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:30.0:1619677474.515055:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.515056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.515056:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677474.515057:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.515058:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.515058:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.515060:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.515061:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.515062:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.515065:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.515066:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.515067:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.515068:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.515069:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.515069:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.515070:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677474.515071:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677474.515071:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677474.515072:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.515073:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.515074:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.515075:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.515076:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.515077:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.515077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677474.515078:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.515078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.515079:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.515079:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.515081:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.515081:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.515081:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.515082:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:30.0:1619677474.515082:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.515083:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:30.0:1619677474.515083:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.515084:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.515085:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.515085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.515086:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.515086:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.515087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.515090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.515096:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.515099:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.515100:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.515100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.515101:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.515103:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.515266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.515273:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.515275:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.515276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.515278:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.515281:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.515283:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677474.515284:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.515285:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.515286:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.515287:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.515289:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.515291:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.515293:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.515294:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677474.515298:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.515300:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.515300:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.515301:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.515301:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.515306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.515313:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.515315:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.515316:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.515317:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.515320:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.515321:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.515423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:20.2:1619677474.515430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677474.515431:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.515433:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:20.0:1619677474.515433:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677474.515434:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:20.0:1619677474.515436:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:26.0:1619677474.515437:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.515439:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.515439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.515439:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.515440:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:20.0:1619677474.515441:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.515442:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.515443:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.515443:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.515444:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.515445:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.515446:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:20.0:1619677474.515446:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.515447:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.515448:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677474.515452:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677474.515453:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.515455:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:20.0:1619677474.515456:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677474.515457:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.515458:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.515458:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:20.0:1619677474.515458:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.515459:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:26.0:1619677474.515460:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.515460:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:20.0:1619677474.515460:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677474.515461:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.515462:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:20.0:1619677474.515462:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.515462:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.515463:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.515464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.2:1619677474.515466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.2:1619677474.515468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677474.515473:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:21.0:1619677474.515475:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677474.515476:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.515477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.515477:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.515478:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.515478:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.515479:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.515480:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.515481:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677474.515482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.0:1619677474.515483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.515522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.515529:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.515531:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.515532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.515535:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.515537:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.515538:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.515539:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.515541:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.515542:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.515543:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.515546:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.515548:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.515551:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.515552:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.515552:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.515554:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.515554:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.515556:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.515557:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.515557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.515560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.515562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677474.515568:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677474.515568:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.515570:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.515570:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677474.515571:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677474.515571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:17.0:1619677474.515572:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.515573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.515574:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.515577:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.515980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677474.515986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.515989:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677474.515990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677474.515991:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.515994:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.515995:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677474.515996:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.515997:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.515998:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.515999:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677474.516002:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.516004:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:1.0:1619677474.516007:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.516008:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677474.516008:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677474.516010:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677474.516011:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.516011:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.516012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:25.2:1619677474.516016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677474.516024:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677474.516026:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677474.516027:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.516029:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.516031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677474.516032:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677474.516823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.516832:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.516834:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.516836:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.516838:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.516839:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.516839:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.516840:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.516842:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.516843:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.516843:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.516846:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.516848:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.516850:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.516851:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.516851:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.516853:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.516854:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.516854:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.516855:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.516859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.516866:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.516869:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.516870:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.516871:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.516873:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.516874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.517056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.517064:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.517066:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.517067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.517069:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.517072:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.517073:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.517074:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.517075:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.517076:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.517077:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.517080:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.517082:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.517085:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.517085:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.517086:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.517088:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.517088:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.517089:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.517089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.517094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.517102:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.517105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.517105:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.517106:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.517108:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.517111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.517299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.517307:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.517310:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.517312:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.517314:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.517314:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.517315:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.517317:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.517318:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.517319:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.517320:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.517323:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.517324:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.517327:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.517327:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.517328:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.517329:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.517330:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.517331:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.517331:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.517336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.517343:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.517345:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.517346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.517346:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.517348:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.517350:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.517365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677474.517373:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.517375:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677474.517376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677474.517378:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.517380:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.517381:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.517382:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.517383:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.517384:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.517385:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.517388:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.517389:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.517391:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.517392:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.517393:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.517394:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.517394:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.517395:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.517395:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.517399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.517402:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.517404:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677474.517405:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.517406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677474.517407:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.517409:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677474.517728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.517736:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.517739:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.517741:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.517743:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.517743:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.517745:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.517746:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.517747:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.517748:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.517749:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.517752:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.517753:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.517755:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.517756:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.517756:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.517758:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.517758:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.517760:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.517761:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.517762:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.517763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.517767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.517772:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.517773:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.517774:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.517775:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.517775:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.517776:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.517776:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.517777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.517778:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.517779:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.517783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.517789:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.517792:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.517793:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.517793:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.517794:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.517794:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.518178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.518184:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.518187:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.518187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.518190:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.518192:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.518194:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677474.518195:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.518196:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.518197:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.518199:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677474.518202:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.518204:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677474.518207:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.518208:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677474.518209:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677474.518210:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677474.518211:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.518212:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.518213:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.518216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.518223:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.518226:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677474.518226:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.518227:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.518229:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.518232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677474.518238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677474.518244:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.518247:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.518247:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677474.518248:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.518249:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.518249:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677474.518773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.518780:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.518782:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.518783:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.518785:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.518787:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.518788:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.518789:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.518791:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.518792:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.518793:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.518796:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.518798:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.518800:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.518801:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.518802:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.518804:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.518804:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.518805:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.518805:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.518810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.2:1619677474.518811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.518814:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.518816:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.518819:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.518819:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:21.0:1619677474.518820:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.518821:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.518821:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:20.0:1619677474.518822:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.518823:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677474.518823:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.518823:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.518825:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.518825:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:20.0:1619677474.518826:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.518827:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:2.0:1619677474.518829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:20.0:1619677474.518831:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.518833:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.518835:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.518836:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.518836:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.518838:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.518839:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.518839:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.518840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.518845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.518854:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.518857:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.518858:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.518858:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.518860:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.518863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.518898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.518906:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.518909:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.518911:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.518913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.518914:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.518915:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.518916:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.518918:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.518919:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.518920:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.518923:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.518925:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.518927:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.518928:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.518929:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.518931:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.518931:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.518933:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.518934:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.518934:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.518936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.518940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.518943:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.518945:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:4.2:1619677474.518945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677474.518946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.518946:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.518948:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677474.518948:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.518950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.518951:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.518951:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.518952:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:10.0:1619677474.518952:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.518954:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:9.2:1619677474.518954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677474.518955:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.518957:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.518959:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.518960:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:10.0:1619677474.518961:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.518962:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.518963:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:10.0:1619677474.518963:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.518964:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.518964:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:10.0:1619677474.518964:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.518964:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.518965:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:10.0:1619677474.518965:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.518968:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.518970:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.518973:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.518973:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.518974:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.518976:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.518976:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.518977:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.518977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.518993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.519000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677474.519001:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.519004:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.519005:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.519005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.519007:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:22.0:1619677474.519007:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.519009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677474.519010:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:25.2:1619677474.519011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.519011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.519012:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.519015:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.519016:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.519017:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.519018:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:26.0:1619677474.519019:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.519020:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.519020:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.519021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.519021:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.519022:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.519023:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000400:00000200:26.0:1619677474.519024:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.519024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677474.519026:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.519028:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.519028:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.519029:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.519031:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.519031:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.519033:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.519034:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.519034:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.519036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.519040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.519043:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.2:1619677474.519045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677474.519045:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.519046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.519046:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.519047:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677474.519048:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.519049:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.519050:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.519050:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.519051:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677474.519053:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.519055:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.519057:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.519059:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.519059:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.519061:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.519062:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.519063:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.519064:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.519065:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.519068:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.519070:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.519073:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.519074:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.519075:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.519077:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.519077:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.519078:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.519079:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.519083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.519091:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.519094:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.519095:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.519097:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.519097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.519100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.519109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.519116:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.519119:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.519121:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.519123:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.519124:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.519125:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.519127:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.519128:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.519129:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.519130:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.519133:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.519134:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.519137:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.519137:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.519138:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.519139:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.519140:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.519142:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677474.519143:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677474.519144:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677474.519145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.519149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:24.0:1619677474.519151:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.519153:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.519154:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.519155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.519156:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.519157:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:24.0:1619677474.519158:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.519160:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677474.519161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.519162:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:5.2:1619677474.519343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.519349:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.519352:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.519353:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.519355:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.519358:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.519359:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.519360:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.519362:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.519363:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.519364:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.519367:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.519368:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.519371:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.519372:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.519372:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.519374:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.519374:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.519375:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.519375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.519380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.519388:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.519391:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.519392:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.519392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.519394:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.519396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.519909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.519917:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.519920:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.519920:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.519923:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.519925:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.519927:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.519928:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.519929:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.519930:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.519931:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.519935:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.519936:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.519939:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.519940:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.519941:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.519943:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.519943:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.519944:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.519944:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.519949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.519956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.519959:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.519960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.519960:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.519963:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.519965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.520160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677474.520167:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.520170:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677474.520170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677474.520173:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.520175:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.520176:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677474.520178:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.520179:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.520180:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.520181:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677474.520185:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.520186:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677474.520189:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.520190:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:17.0:1619677474.520191:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677474.520193:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677474.520193:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.520194:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677474.520194:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:25.2:1619677474.520199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.520207:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.520210:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.520211:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.520213:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.520214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.520216:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.520479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677474.520486:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.520489:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677474.520490:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677474.520492:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.520495:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.520496:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.520498:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.520499:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.520500:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.520501:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.520504:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.520505:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.520508:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.520509:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.520509:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.520511:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.520511:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.520512:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677474.520513:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677474.520514:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677474.520517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:4.2:1619677474.520520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:8.0:1619677474.520522:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.520524:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.520525:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.520525:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.520527:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:8.0:1619677474.520527:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.520529:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:8.0:1619677474.520529:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:26.0:1619677474.520530:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.520531:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.520679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677474.520682:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.520684:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677474.520685:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.520685:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (23)--
00000800:00000200:8.0:1619677474.520685:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.520686:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677474.520774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.520782:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.520785:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.520787:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.520789:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.520790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.520790:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.520791:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.520793:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.520793:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.520794:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.520797:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.520799:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.520801:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.520802:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.520802:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.520804:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.520804:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.520805:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.520805:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.520812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.520818:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.520820:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.520821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.520821:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.520823:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.520825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.520931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.520938:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.520941:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.520942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.520944:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.520946:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.520948:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.520949:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.520951:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.520952:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.520953:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.520957:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.520959:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.520962:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.520963:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.520963:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.520965:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.520966:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.520967:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.520967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.520971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.520978:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.520981:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.520982:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.520984:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.520985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.520987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.520990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.520997:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.521000:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.521001:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.521001:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.521002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.521003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.521335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.521338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.521341:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.521343:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.521344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.521346:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.521346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:20.0:1619677474.521347:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.521349:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.521350:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.521351:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:20.0:1619677474.521352:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.521353:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.521353:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.521357:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.521358:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.521360:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.521361:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677474.521362:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.521362:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.521362:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:21.0:1619677474.521364:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677474.521364:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677474.521365:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.521366:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.521367:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.521368:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.521369:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.521370:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.521370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:21.0:1619677474.521372:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677474.521372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.521373:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.521373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.521374:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.521375:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.521375:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.521377:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677474.521377:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.521378:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.521378:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.521379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.521380:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.521380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.521381:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.521382:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.521383:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.521384:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.521387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.521389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677474.521393:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677474.521393:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.521395:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:24.0:1619677474.521396:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677474.521396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:24.0:1619677474.521397:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.521397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677474.521397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.521399:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.521402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.521444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.521452:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.521455:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.521457:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.521459:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.521459:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.521461:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.521462:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.521463:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.521464:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.521465:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.521469:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.521470:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.521472:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.521473:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.521473:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.521475:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.521476:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.521477:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.521477:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.521481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.521488:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.521491:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.521491:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.521493:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.521495:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.521496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.521631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677474.521639:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.521642:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677474.521643:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.521645:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.521646:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677474.521646:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.521647:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.521649:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.521649:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.521650:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.521653:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.521655:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.521657:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.521658:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.521659:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.521661:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.521661:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.521662:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.521662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677474.521667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.521674:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.521677:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.521678:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.521679:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.521680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.521683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.521693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.521701:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.521703:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.521704:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.521705:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.521706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.521706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.522008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:4.2:1619677474.522014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677474.522016:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.522019:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.522020:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.522022:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.522022:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677474.522024:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.522024:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.522025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677474.522025:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677474.522026:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.522027:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.522028:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.522029:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.522030:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:8.0:1619677474.522030:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.522031:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.522033:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.522033:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.522034:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.522035:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.522036:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:8.0:1619677474.522036:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.522037:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.522038:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.522039:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:8.0:1619677474.522039:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.522040:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.522041:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.522041:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.0:1619677474.522041:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.522042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.0:1619677474.522044:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.522045:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.522045:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:9.2:1619677474.522046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.0:1619677474.522048:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.522048:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.522049:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.522051:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.522051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677474.522052:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:25.2:1619677474.522053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.522054:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.522055:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.522056:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.522057:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.522057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677474.522058:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677474.522062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.522064:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.522065:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.522066:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677474.522066:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.522066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.522067:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.522068:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.522068:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.522070:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.522075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.522083:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.522085:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.522086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.522086:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.522087:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.522088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.522382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.522390:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.522393:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.522396:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.522398:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.522399:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.522400:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.522401:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.522403:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.522404:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.522405:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.522408:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.522409:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.522412:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.522412:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.522413:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.522415:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.522415:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.522416:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.522417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.522421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.522428:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.522431:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.522432:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.522432:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.522434:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.522436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.522439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.522445:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.522447:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.522448:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.522449:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.522449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.522450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.522509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.522515:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.522518:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.522519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.522521:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.522524:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.522526:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677474.522527:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.522528:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.522529:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.522531:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677474.522533:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.522535:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.522538:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.522539:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677474.522540:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.522542:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.522542:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.522543:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.522543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677474.522547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.522554:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.522557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.522557:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.522558:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.522560:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.522563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.522856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677474.522863:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.522866:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.522866:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677474.522868:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.522870:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.522871:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.522873:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.522874:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.522875:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.522876:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.522879:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.522881:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.522883:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.522884:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.522885:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.522886:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.522887:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.522887:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.522888:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.522892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.522895:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.522897:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677474.522898:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.522898:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677474.522900:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.522901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677474.523343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.523351:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.523353:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.523354:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.523356:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.523358:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.523359:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677474.523360:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.523362:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.523363:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.523363:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677474.523366:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.523367:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.523368:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.523369:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.523369:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.523370:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.523370:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.523371:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.523372:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.523376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.523383:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.523385:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.523385:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.523386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.523387:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.523387:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.523388:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677474.523389:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.523390:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.523391:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.523393:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.523394:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.523396:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.523397:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.523398:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.523399:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.523399:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:4.2:1619677474.523400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677474.523402:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.523404:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.523408:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.523409:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.523409:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.523410:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.523410:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.523412:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.523412:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:10.0:1619677474.523412:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.523413:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.523413:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677474.523414:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.523414:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.523415:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.523417:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677474.523418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677474.523418:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.523418:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.523419:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.523422:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.523423:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.523425:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.523425:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677474.523425:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.523426:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.523427:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.523428:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.523428:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677474.523428:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.523429:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.523429:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677474.523430:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.523432:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.523433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.523434:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677474.523440:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.523443:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.523444:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.523444:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.523446:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.523448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.523688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677474.523693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:26.0:1619677474.523695:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.523698:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.523698:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:24.0:1619677474.523700:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:26.0:1619677474.523701:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677474.523702:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:26.0:1619677474.523703:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.523703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:26.0:1619677474.523704:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:24.0:1619677474.523705:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.523706:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.523707:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.523708:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.523708:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.523709:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:24.0:1619677474.523710:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.523711:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96504137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.523712:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.523713:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:24.0:1619677474.523713:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.523714:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.523715:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.523718:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.523718:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:24.0:1619677474.523718:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.523719:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:24.0:1619677474.523719:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.523721:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.523721:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.523722:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.523722:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:24.0:1619677474.523722:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.523723:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.523723:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.523725:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.523726:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.523727:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677474.523728:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677474.523728:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:9.2:1619677474.523728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:25.2:1619677474.523732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.523734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:2.0:1619677474.523737:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:22.0:1619677474.523740:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677474.523740:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.523741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:8.0:1619677474.523741:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000400:00000200:2.0:1619677474.523741:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.523743:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:2.0:1619677474.523743:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:22.0:1619677474.523744:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.523744:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677474.523744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:6.0:1619677474.523744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.523745:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:22.0:1619677474.523746:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.523747:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.523748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677474.523749:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677474.523756:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.523759:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.523759:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.523759:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.523761:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.523761:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.523986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.523994:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.523997:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.523999:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.524002:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.524002:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.524003:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.524005:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.524006:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.524007:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.524008:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.524011:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.524013:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.524015:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.524016:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.524016:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.524018:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.524018:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.524019:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.524020:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.524023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.524030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.524032:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.524033:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.524033:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.524035:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.524038:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.524045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.524051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.524053:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.524054:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.524054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.524055:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.524056:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.524112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.524119:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.524122:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.524123:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.524125:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.524127:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.524129:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.524130:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.524132:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.524133:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.524134:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.524137:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.524139:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.524141:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.524142:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.524143:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.524144:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.524145:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.524145:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.524146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.524151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.524158:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.524160:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.524161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.524162:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.524164:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.524166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.524270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.524277:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.524280:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.524283:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.524284:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.524285:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.524287:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.524288:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.524289:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.524290:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.524291:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.524300:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.524301:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.524304:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.524304:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.524305:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.524307:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.524307:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.524308:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.524309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.524313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.524320:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.524323:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.524324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.524324:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.524326:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.524328:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.524502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.524510:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.524513:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.524514:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.524515:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.524518:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.524519:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.524520:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.524521:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.524522:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.524523:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.524526:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.524527:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.524529:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.524530:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.524531:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.524532:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.524532:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.524533:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.524533:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.524537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.524543:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.2:1619677474.524544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677474.524546:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.524547:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.524547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.524548:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.524550:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.524551:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.524554:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.524556:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.524558:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.524559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.524560:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.524561:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.524562:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.524563:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.524564:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.524567:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.524568:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.524570:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.524571:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.524571:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.524573:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.524573:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.524575:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.524576:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.524577:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.524578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.524582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.524584:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.524586:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.524587:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.524588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.524589:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.524590:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677474.524591:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.524593:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.524594:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.524595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.524949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.524957:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.524960:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.524962:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.524964:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.524964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.524966:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.524967:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.524968:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.524969:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.524970:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.524973:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677474.524973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:31.0:1619677474.524974:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.524976:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.524977:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.524977:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.524979:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.524979:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.524980:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.524980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677474.524980:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.524983:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:25.2:1619677474.524984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677474.524984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.524986:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.524989:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.524990:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.524991:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:2.0:1619677474.524992:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.524993:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.524994:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:2.0:1619677474.524994:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.524995:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.524995:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.524997:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.524998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.524999:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:2.0:1619677474.524999:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.525001:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.525004:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.525005:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677474.525005:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.525007:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677474.525008:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.525008:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.525009:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677474.525013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.525021:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.525023:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.525024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.525025:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.525027:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.525029:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.525086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.525093:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.525096:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.525097:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.525099:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.525101:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.525103:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.525104:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.525105:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.525106:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.525107:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.525110:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.525112:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.525115:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.525115:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.525116:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.525118:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.525118:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.525119:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.525121:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.525121:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677474.525124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.525127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:12.0:1619677474.525131:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.525133:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.525134:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.525134:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:12.0:1619677474.525135:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.525135:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.525136:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.525137:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677474.525138:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.525139:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.525146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677474.525152:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.525155:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.525155:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.525156:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.525157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.525157:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.526215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.526223:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.526225:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.526227:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.526229:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.526230:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.526231:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.526232:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.526233:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.526234:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.526235:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.526238:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.526239:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.526242:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.526242:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.526243:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.526244:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.526245:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.526246:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.526248:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.526249:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.526251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.526255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.526258:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.526261:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.526262:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.526262:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.526263:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.526263:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.526265:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677474.526265:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.526266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.526267:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.526613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.526618:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.526621:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.526623:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.526624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.526626:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.526627:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.526629:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.526630:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.526631:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.526632:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.526635:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.526637:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.526639:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.526640:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.526641:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.526642:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.526643:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.526644:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.526644:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.526649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.526657:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.526659:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.526660:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.526662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.526662:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.526664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.526701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677474.526707:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.526710:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677474.526711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677474.526713:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.526715:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.526717:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677474.526718:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.526719:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.526720:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.526721:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677474.526725:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.526726:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:6.0:1619677474.526729:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.526730:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:6.0:1619677474.526731:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677474.526733:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677474.526734:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.526734:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.526735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677474.526740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.526748:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.526751:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.526752:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.526752:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.526754:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.526757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677474.526786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.526791:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.526794:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.526796:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.526796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.526798:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.526800:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.526801:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.526802:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.526803:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.526804:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.526807:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.526809:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.526811:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.526812:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.526813:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.526814:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.526815:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.526816:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.526817:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.526818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.526820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.526823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.526830:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.526831:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.526831:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.526832:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.526832:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.526833:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.526834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.526835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.526836:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.526837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.526839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.526846:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.526849:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.526850:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.526850:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.526852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.526853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.527289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.527296:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.527299:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.527300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.527302:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.527305:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.527306:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.527308:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.527309:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.527310:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.527311:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.527315:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.527316:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.527319:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.527320:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.527321:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.527322:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.527323:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.527324:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.527324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.527329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.527337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.527339:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.527340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.527340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.527343:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.527345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.528168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.528176:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.528179:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.528179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.528181:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.528184:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.528186:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.528187:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.528189:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.528190:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.528191:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.528195:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.528197:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.528200:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.528201:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.528201:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.528203:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.528204:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.528205:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.528206:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.528207:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.528209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.528212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.528217:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.528219:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.528220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.528220:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677474.528221:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.528222:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.528223:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.528224:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.528225:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.528225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.528228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.528234:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.528237:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.528238:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.528238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.528239:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.528240:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.528280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.528288:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.528291:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.528292:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:4.2:1619677474.528297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:24.0:1619677474.528299:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.528303:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.528304:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677474.528306:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:24.0:1619677474.528306:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.528308:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.528309:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:24.0:1619677474.528309:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.528310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:24.0:1619677474.528311:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.528313:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.528316:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.528316:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677474.528318:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:24.0:1619677474.528318:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.528319:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.528321:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.528321:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.528322:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.528322:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:26.0:1619677474.528323:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:24.0:1619677474.528323:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.528325:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.528325:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.528326:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000400:00000200:26.0:1619677474.528327:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.528327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:26.0:1619677474.528329:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:25.2:1619677474.528331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.528332:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.528333:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.528333:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.528335:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.528336:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.528337:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.528337:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:6.0:1619677474.528340:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:9.2:1619677474.528342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677474.528343:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.528345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.528345:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.528347:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:25.2:1619677474.528349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.528350:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:6.0:1619677474.528352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.528354:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.528354:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.528356:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:6.0:1619677474.528356:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677474.528356:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.528357:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.528358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.528358:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000400:00000200:2.0:1619677474.528358:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.528359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677474.528361:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.528375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.528385:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.528389:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.528393:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.528395:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.528396:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.528398:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.528400:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.528402:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.528404:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.528405:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.528410:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.528413:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.528416:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.528418:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.528419:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.528421:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.528422:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.528423:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.528424:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.528427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.528433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.528436:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.528437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.528437:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.528439:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.528441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.528610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.528616:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.528619:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.528619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.528621:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.528624:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.528626:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.528627:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.528628:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.528629:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.528631:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.528634:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.528636:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.528639:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.528640:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.528641:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.528642:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.528643:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.528644:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.528644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.528649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.528658:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.528661:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.528662:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.528665:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.528668:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.528668:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.528707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.528716:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.528720:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.528724:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677474.528724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677474.528725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.528727:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.528729:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.528731:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.528731:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677474.528733:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.528735:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677474.528735:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677474.528736:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677474.528736:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.528739:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.528742:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.528743:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.528743:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.528745:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.528747:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677474.528747:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.528748:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.528749:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.528751:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.528752:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:17.0:1619677474.528752:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.528753:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.528755:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:17.0:1619677474.528755:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.528756:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677474.528756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677474.528758:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677474.528758:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.528758:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677474.528759:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677474.528759:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.528760:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:17.0:1619677474.528761:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.528761:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677474.528762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677474.528762:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.528763:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677474.528764:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:30.0:1619677474.528766:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677474.528767:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677474.528767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677474.528768:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:31.0:1619677474.528769:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.528769:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.528771:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.528772:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677474.528773:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.528774:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96504116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.528774:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:31.0:1619677474.528776:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.528777:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:31.0:1619677474.528778:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.528778:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.528778:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:31.0:1619677474.528780:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677474.528781:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.528784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.528784:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.528785:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:31.0:1619677474.528786:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.528790:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677474.528794:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.528795:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677474.528796:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.528799:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677474.528800:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.528801:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.528802:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.528805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.528813:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.528816:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.528817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.528817:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.528819:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.528823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.529173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.529181:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.529184:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.529185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.529188:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.529191:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.529192:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.529194:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.529196:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.529197:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.529198:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.529201:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.529203:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.529207:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.529208:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.529208:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.529210:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.529211:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.529212:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.529213:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.529214:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.529216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.529219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.529224:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.529227:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.529227:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.529228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.529228:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.529229:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.529230:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.529230:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.529232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.529233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.529285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.529292:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.529301:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677474.529303:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.529303:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.529307:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.529308:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.529310:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.529311:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.529312:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.529313:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.529317:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.529318:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.529320:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.529321:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.529322:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.529323:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.529323:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.529324:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.529325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.529329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.529336:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.529339:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.529339:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.529341:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.529343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.529344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.529345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.529353:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.529355:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.529356:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.529356:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.529357:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.529358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.529700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.529708:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.529711:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.529712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.529713:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.529716:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.529718:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.529719:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.529720:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.529721:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.529722:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.529725:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.529727:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.529730:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.529731:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.529731:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.529733:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.529733:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.529734:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.529735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.529739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.529748:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.529751:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.529752:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.529752:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.529754:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.529758:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.529759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.529766:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.529769:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.529770:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.529770:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.529771:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.529771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.529836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.529844:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.529847:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.529849:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.529851:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.529851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.529852:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.529853:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.529855:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.529855:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.529856:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.529860:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.529861:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.529863:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.529864:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.529865:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.529866:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.529866:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.529868:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.529869:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.529870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.529871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.529872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:21.2:1619677474.529876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.529878:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:30.0:1619677474.529880:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:22.0:1619677474.529881:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677474.529881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:30.0:1619677474.529882:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.529883:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.529883:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.529884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.529884:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.529885:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.529886:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.529886:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:22.0:1619677474.529886:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.529887:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.529888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677474.529888:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677474.529889:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.529890:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.529891:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.529892:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677474.529896:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.529897:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677474.529900:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.529901:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:22.0:1619677474.529901:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677474.529903:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677474.529904:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.529905:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677474.529906:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677474.529907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677474.529909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.529912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:18.0:1619677474.529917:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:6.0:1619677474.529917:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677474.529919:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:18.0:1619677474.529920:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:6.0:1619677474.529920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:18.0:1619677474.529921:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.529921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:2.0:1619677474.529922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.529924:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.529926:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.530004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.530012:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.530015:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.530017:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.530018:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.530019:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.530019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000400:00000200:31.0:1619677474.530021:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.530022:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.530023:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.530024:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.530025:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.530026:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.530028:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677474.530030:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.530030:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.530032:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.530033:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.530034:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.530035:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:30.0:1619677474.530035:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:31.0:1619677474.530036:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:30.0:1619677474.530036:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.530037:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:30.0:1619677474.530037:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.530038:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.530039:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.530040:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.530041:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.530041:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:30.0:1619677474.530042:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.530043:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677474.530043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.530044:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.530047:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.530048:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.530048:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.530049:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:30.0:1619677474.530050:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.530051:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.530052:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.530052:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:8.0:1619677474.530052:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.530053:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.530053:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.530055:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.530056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.530057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.530062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:2.0:1619677474.530064:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.530066:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.530067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.530068:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:2.0:1619677474.530068:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.530070:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:2.0:1619677474.530070:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.530071:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.530072:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.530073:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:2.0:1619677474.530073:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:24.0:1619677474.530074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.530079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.530086:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.530089:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.530089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.530090:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.530091:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.530091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.530803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.530808:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.530811:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.530812:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.530814:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.530816:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.530818:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677474.530819:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.530821:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.530821:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.530822:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.530826:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.530827:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.530830:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.530831:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.530832:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.530834:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.530834:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.530835:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.530835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677474.530840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.530848:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.530851:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.530852:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.530854:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.530855:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.530857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.530898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.530906:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677474.530907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:30.0:1619677474.530909:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.530911:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.530912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.530913:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.530913:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:30.0:1619677474.530915:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.530916:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.530916:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:30.0:1619677474.530917:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677474.530917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:30.0:1619677474.530918:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.530920:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677474.530920:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.530923:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.530923:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.530925:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:2.0:1619677474.530925:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.530926:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.530927:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.530928:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677474.530928:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.530929:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.530930:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:2.0:1619677474.530930:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.530932:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.530932:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.530933:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677474.530933:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.530934:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677474.530935:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:9.2:1619677474.530937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677474.530937:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.530938:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.530938:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.530940:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.530940:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.530942:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.530943:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.530943:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677474.530945:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:9.2:1619677474.530946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.530947:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.530948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.2:1619677474.530948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000400:00000200:8.0:1619677474.530948:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.530950:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.530952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677474.530954:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.530955:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.530956:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.530956:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:18.0:1619677474.530957:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.530957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.530958:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.530959:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.530959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677474.530960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.531734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.531741:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.531744:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.531745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.531747:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.531750:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.531751:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.531753:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.531754:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.531755:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.531756:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.531760:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.531761:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.531764:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.531765:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.531765:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.531767:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.531767:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.531768:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.531769:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677474.531774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.531782:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.531785:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.531786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.531786:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.531788:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.531791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.531967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.531975:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.531978:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.531980:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.531982:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.531983:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.531984:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.531985:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.531987:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.531988:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.531989:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.531992:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.531994:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.531996:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.531997:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.531997:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.531999:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.531999:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.532000:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.532001:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.532005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.532011:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.532014:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.532015:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.532015:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.532017:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.532019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.532205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.532211:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.532214:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.532215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.532217:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.532219:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.532221:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.532222:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.532224:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.532225:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.532226:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.532229:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.532231:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.532234:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.532235:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.532235:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.532237:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.532238:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.532238:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.532239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677474.532243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.532251:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.532253:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.532254:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.532254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.532256:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.532259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.532293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.532301:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.532304:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.532305:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.532307:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.532310:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.532311:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.532313:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.532314:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.532315:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.532316:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.532319:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.532321:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.532324:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.532324:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.532325:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.532327:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.532327:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.532328:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.532329:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.532330:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677474.532333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.532336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677474.532341:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677474.532343:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:13.0:1619677474.532343:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.532344:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.532345:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:13.0:1619677474.532345:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.532346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.532347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677474.532348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.532349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.532354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.532361:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.532364:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.532365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.532367:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.532369:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.532371:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.532372:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.532373:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.532374:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.532375:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.532379:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.532380:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677474.532381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677474.532383:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.532384:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.532384:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.532386:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.532386:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.532387:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.532387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677474.532388:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677474.532391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.532391:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.532392:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.532393:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.532396:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.532397:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.532398:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.532399:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677474.532400:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.532400:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.532401:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.532402:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.532402:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.532403:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.532405:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.532406:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.532408:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:25.2:1619677474.532409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.532410:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.532411:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.532412:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.532413:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677474.532413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.532414:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.532414:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677474.532414:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.532415:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.532416:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.532417:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.532417:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.532418:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:9.2:1619677474.532419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.532419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.532425:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.532428:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.532429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.532429:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.532431:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.532433:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.532438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.532444:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.532446:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.532447:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.532447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.532448:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.532449:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.532990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.532994:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.532996:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.532998:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.532999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.533000:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.533002:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.533003:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.533005:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.533006:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.533007:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.533010:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.533012:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.533014:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.533015:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.533016:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.533018:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.533018:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.533019:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.533019:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.533024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.533032:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.533034:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.533035:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.533037:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.533037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.0:1619677474.533038:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.533039:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677474.533044:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.533047:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.533048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.533049:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.533052:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.533054:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.533055:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.533056:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.533057:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.533058:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.533062:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.533063:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.533066:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.533067:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.533068:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.533070:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.533070:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.533071:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.533071:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.533076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:20.2:1619677474.533077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677474.533083:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:2.0:1619677474.533084:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677474.533086:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.533086:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.533087:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.533087:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.533088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677474.533090:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:2.0:1619677474.533090:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677474.533093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:2.0:1619677474.533093:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.533095:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.533096:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.533098:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.533098:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.533100:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.533103:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.533105:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.533107:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.533108:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.533109:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.533110:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.533111:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.533112:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.533113:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.533117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.533125:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.533127:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.533128:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.533128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.2:1619677474.533129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:17.0:1619677474.533130:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.533133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677474.533137:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.533141:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677474.533143:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.533145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677474.533145:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.533147:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.533148:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.533149:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.533150:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.533151:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.533155:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.533156:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.533159:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.533160:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.533161:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.533163:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.533163:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.533164:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.533165:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677474.533169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.533177:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.533180:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.533180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.533181:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.533183:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.533186:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.533374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.533381:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.533383:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.533384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.533386:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.533388:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.533390:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677474.533391:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.533392:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.533393:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.533394:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677474.533398:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.533399:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677474.533403:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.533403:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677474.533404:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677474.533406:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677474.533407:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.533408:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677474.533409:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677474.533409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677474.533413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.2:1619677474.533415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677474.533420:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:18.0:1619677474.533421:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:26.0:1619677474.533423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:18.0:1619677474.533423:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:10.0:1619677474.533423:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:18.0:1619677474.533424:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:10.0:1619677474.533424:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:5.2:1619677474.533424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677474.533425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:10.0:1619677474.533426:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.533429:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:1.0:1619677474.533430:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.533433:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.533434:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.533435:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.533438:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.533439:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.533440:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.533442:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.533443:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.533444:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.533447:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.533449:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.533452:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.533452:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.533453:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.533454:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.533455:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.533456:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.533457:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.533458:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.533460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.533463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677474.533468:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.533469:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.533470:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.533471:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.533471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677474.533472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.533472:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.533473:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.533474:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.533476:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.533494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.533503:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.533505:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.533508:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.533510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.533511:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.533512:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.533513:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.533515:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.533516:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.533517:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.533520:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.533522:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.533524:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.533525:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.533525:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.533527:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.533528:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.533529:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.533529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.533533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.533539:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.533542:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.533543:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.533544:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.533546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.533547:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.533651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.533659:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.533662:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.533664:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.533665:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.533666:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.533667:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.533668:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.533669:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.533670:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.533670:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.533673:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.533674:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.533677:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.533677:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.533678:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.533679:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.533680:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.533681:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.533683:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.533684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.533685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.533689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.533692:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.533695:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.533696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.533696:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.533697:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677474.533698:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.533700:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677474.533700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.533701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.533701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.534032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.534039:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.534041:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.534042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.534044:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.534046:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.534048:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.534049:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.534050:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.534051:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.534052:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.534054:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.534056:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.534059:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.534059:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.534060:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.534062:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.534062:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.534063:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.534063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.534068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.534075:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.534078:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.534079:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.534079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.534081:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.534084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.534107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.534114:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.534118:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.534119:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.534122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.534122:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.534123:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.534124:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.534126:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.534127:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.534128:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.534130:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.534132:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.534134:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.534134:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.534135:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.534136:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.534137:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.534138:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.534138:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.534142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.534149:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.534151:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.534152:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.534153:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.534155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.534156:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.534159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.534166:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.534169:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.534170:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.534171:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.534171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.534172:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.535085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.535092:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.535095:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.535096:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.535097:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.535100:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.535102:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.535103:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.535104:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.535105:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.535106:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.535109:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.535111:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.535113:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.535114:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.535115:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.535116:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.535117:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.535118:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.535119:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.535120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677474.535122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677474.535126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677474.535131:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677474.535133:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677474.535133:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.535134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.535135:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.535136:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.535136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.535137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677474.535137:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.535140:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.535367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.535373:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.535376:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.535378:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.535378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.535379:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.535381:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.535382:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.535383:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.535385:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.535386:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.535390:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.535391:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.535394:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.535395:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.535396:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.535398:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.535398:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.535399:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.535399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.535404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.535413:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.535415:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.535416:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.535418:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.535420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.535421:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.535512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.535520:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.535522:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.535525:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.535526:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.535527:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.535528:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.535529:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.535531:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.535532:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.535533:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.535536:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.535537:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.535540:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.535540:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.535541:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.535543:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.535543:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.535544:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.535545:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.535548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.535555:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.535558:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.535558:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.535559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.535560:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.535563:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.535776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677474.535783:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.535786:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677474.535787:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677474.535789:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.535792:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.535793:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.535794:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.535795:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.535796:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.535797:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.535800:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.535801:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.535804:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.535804:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.535805:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.535806:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.535807:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.535807:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.535808:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.535813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.535817:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.535819:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:8.0:1619677474.535820:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.535821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.535822:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.535824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:20.2:1619677474.536022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.536025:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.536027:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.536030:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.536030:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.536033:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.536034:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.536035:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.536037:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.536038:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.536039:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.536042:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.536044:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.536047:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.536047:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.536048:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.536050:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.536050:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.536051:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.536051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.536056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.536064:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.536067:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.536068:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.536068:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.536070:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.536072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.536076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.536083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.536085:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.536086:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.536087:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.536088:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.536088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.536430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.536438:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.536441:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.536443:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.536445:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.536446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.536447:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.536448:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.536450:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.536451:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.536452:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.536455:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.536457:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.536460:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.536461:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.536462:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.536463:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.536464:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.536466:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.536467:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.536468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.536470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.536474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.536482:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.536482:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.536485:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.536485:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677474.536486:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677474.536486:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.536487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677474.536488:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.536488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.536491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.536510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.536518:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.536521:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.536524:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.536525:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.536527:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.536528:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.536529:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.536531:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.536531:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.536532:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.536536:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.536537:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.536539:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.536540:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.536541:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.536542:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.536543:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.536544:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.536544:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.536549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.536555:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.536557:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.536558:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.536559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.536560:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.536562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.536566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.536572:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.536575:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.536576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.536576:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.536577:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.536578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.536763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.536767:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:4.2:1619677474.536767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677474.536770:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.536773:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.536773:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.536775:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:20.0:1619677474.536775:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.536777:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:26.0:1619677474.536778:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:20.0:1619677474.536778:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.536779:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677474.536780:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.536781:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.536781:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.536782:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677474.536784:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.536785:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677474.536786:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.536787:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.536788:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.536788:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:26.0:1619677474.536789:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.536790:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677474.536791:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.536792:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.536793:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:26.0:1619677474.536794:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.536794:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.536795:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:26.0:1619677474.536796:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677474.536796:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.536796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:26.0:1619677474.536799:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.536799:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.536800:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:9.2:1619677474.536800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677474.536802:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.536802:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.536803:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.536803:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:24.0:1619677474.536806:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:25.2:1619677474.536808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677474.536809:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.536810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.536810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.536812:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.536815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:12.0:1619677474.536815:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.536818:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.536819:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.536821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.536823:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.536825:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.537622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.537630:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.537632:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.537633:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.537635:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.537637:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.537639:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.537640:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.537641:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.537642:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.537643:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.537646:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.537648:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.537651:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.537652:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.537652:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.537653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677474.537654:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.537654:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.537655:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.537656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677474.537661:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677474.537661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.537664:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.537666:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.537668:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.537669:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677474.537669:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677474.537670:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.537671:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.537672:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.537672:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:31.0:1619677474.537673:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.537673:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.537673:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.537674:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:6.0:1619677474.537675:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.537678:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.537678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.537680:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.537683:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.537684:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.537684:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.537686:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.537687:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.537688:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.537688:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.537692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.537701:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.537703:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.537704:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.537706:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.537707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.537709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.537779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677474.537787:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677474.537790:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677474.537791:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677474.537793:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.537796:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.537797:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677474.537798:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.537800:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.537801:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.537802:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677474.537805:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.537807:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.537810:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.537811:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677474.537811:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677474.537813:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677474.537814:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677474.537815:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (22)++
00000800:00000200:10.0:1619677474.537816:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:10.0:1619677474.537816:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:25.2:1619677474.537819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677474.537824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (21)++
00000800:00000200:20.0:1619677474.537828:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.537831:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:26.0:1619677474.537832:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000400:00000200:20.0:1619677474.537832:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.537833:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.537834:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000800:00000200:21.0:1619677474.537834:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677474.537835:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677474.537836:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:20.0:1619677474.537836:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.537895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.537902:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.537905:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.537906:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.537908:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.537910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677474.537911:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.537912:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.537913:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.537915:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.537916:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.537917:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677474.537917:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677474.537920:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.537920:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.537921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677474.537922:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677474.537923:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.537924:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.537925:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.537926:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677474.537926:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.537928:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.537928:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000400:00000200:10.0:1619677474.537928:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:18.0:1619677474.537929:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000400:00000200:10.0:1619677474.537929:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677474.537930:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:10.0:1619677474.537930:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.537931:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.537932:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677474.537934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677474.537935:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.537936:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.537939:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.537940:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.537941:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677474.537941:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:10.0:1619677474.537943:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.537943:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677474.537943:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:10.0:1619677474.537944:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.537944:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:2.0:1619677474.537944:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.537945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.537946:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.537948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677474.537948:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.537949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.537956:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677474.537957:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:18.0:1619677474.537958:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.537959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677474.537960:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:18.0:1619677474.537960:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677474.537960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.537961:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.537963:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:18.0:1619677474.537963:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.537964:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.537965:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.537966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.537967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000400:00000200:18.0:1619677474.537967:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.537968:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.537969:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.537972:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.537974:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677474.537974:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.537977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:18.0:1619677474.537977:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.537977:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677474.537978:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677474.537978:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:18.0:1619677474.537979:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677474.537979:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.537980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677474.537981:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.537981:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.537982:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.537983:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.537984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.537987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677474.537989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.537993:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:2.0:1619677474.537995:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677474.537996:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:2.0:1619677474.537997:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:30.0:1619677474.537998:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.537998:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677474.537998:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677474.537999:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.538000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.538003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:5.2:1619677474.538159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.538165:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.538168:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.538169:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.538171:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.538174:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.538175:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.538176:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.538178:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.538179:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.538179:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.538182:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.538184:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.538186:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.538187:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.538188:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.538189:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.538190:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.538191:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.538191:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.538195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.538203:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.538205:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.538206:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.538206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.538208:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.538210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.538211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.538218:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.538221:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.538224:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.538225:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.538226:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.538228:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.538229:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.538231:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.538232:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.538233:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.538236:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.538238:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.538240:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.538241:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.538242:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.538243:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.538244:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.538245:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.538246:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.538249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.538256:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.538258:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.538259:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.538259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.538261:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.538263:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.538367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.538375:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.538378:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.538380:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.538382:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.538382:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.538383:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.538384:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.538386:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.538386:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.538387:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.538391:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.538392:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.538394:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.538395:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.538395:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.538397:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.538397:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.538398:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.538399:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.538403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.538410:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.538412:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.538413:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.538414:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.538416:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.538417:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.538737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.538743:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.538745:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.538747:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.538748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.538751:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.538752:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.538753:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.538755:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.538756:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.538757:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.538760:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.538762:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.538765:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.538765:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.538766:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.538767:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.538768:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.538769:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.538770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.538774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.538782:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.538784:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.538785:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.538786:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.538787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.538789:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.538908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.538916:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.538919:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.538919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.538921:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.538924:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.538926:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.538927:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.538928:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.538929:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.538930:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.538933:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.538935:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.538938:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.538939:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.538939:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.538941:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.538941:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.538942:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.538942:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.538947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.538955:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.538958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.538958:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.538959:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.538961:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.538964:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.539105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.539112:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.539115:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.539117:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.539119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.539120:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.539122:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677474.539123:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.539124:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.539125:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.539126:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677474.539129:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.539131:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677474.539133:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.539134:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677474.539135:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677474.539136:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677474.539137:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.539137:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.539138:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.539142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.539150:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.539153:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.539154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.539154:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.539156:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.539160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.539161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.539169:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.539172:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.539172:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.539172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.539173:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.539174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.539186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.539193:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.539196:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.539198:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.539200:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.539200:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.539201:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.539202:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.539204:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.539204:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.539205:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.539208:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.539210:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.539212:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.539213:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.539213:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.539215:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.539215:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.539216:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.539217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.539220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.539227:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.539229:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.539230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.539230:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.539232:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.539234:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.539571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677474.539578:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677474.539580:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677474.539581:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677474.539583:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.539586:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.539587:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677474.539588:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.539589:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.539590:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.539591:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677474.539594:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.539596:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.539599:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.539600:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677474.539600:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677474.539602:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677474.539602:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677474.539603:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677474.539604:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:25.2:1619677474.539609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.539617:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.539620:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.539621:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.539622:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.539623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.539625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.539854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.539862:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.539865:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.539868:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.539870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.539871:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.539872:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.539873:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.539875:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.539876:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.539876:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.539880:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.539881:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.539883:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.539884:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.539884:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.539886:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.539887:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.539888:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.539889:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.539890:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.539892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.539896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.539899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.539901:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.539902:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.539903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.539904:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.539904:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.539905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.539906:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677474.539906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.539907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.539908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:26.0:1619677474.539912:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.539915:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.539916:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.539917:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.539921:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.539922:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.539923:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.539924:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.539925:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.539926:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.539929:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.539930:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.539933:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.539934:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.539934:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.539936:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.539936:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.539938:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.539939:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.539939:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.539942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.539945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.539950:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.539952:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.539953:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.539954:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.539954:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677474.539954:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.539955:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677474.539956:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.539956:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.539959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.539962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.539969:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.539971:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.539972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.539972:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.539974:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.539974:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.540378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.540384:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.540387:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.540388:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.540390:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.540392:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.540394:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.540395:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.540397:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.540397:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.540399:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.540402:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.540403:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.540405:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.540406:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.540407:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.540408:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.540409:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.540410:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.540410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.540415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.540422:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.540425:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.540425:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.540426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.540427:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.540428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.540458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.540466:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.540468:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.540469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.540471:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.540474:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.540475:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.540477:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.540479:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.540480:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.540481:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.540484:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.540486:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.540489:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.540490:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.540490:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.540492:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.540493:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.540493:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.540494:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.540498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.540507:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.540509:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.540511:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.540511:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.540513:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.540516:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.540668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.540675:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.540678:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.540678:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.540680:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.540683:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.540684:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.540685:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.540686:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.540687:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.540688:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.540692:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.540693:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.540695:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.540696:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.540697:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.540698:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.540699:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.540699:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.540700:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.540704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.540711:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.540714:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.540714:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.540716:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.540718:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.540719:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.540722:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.540722:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.540723:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.540724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.540725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.541242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.541249:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.541251:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.541253:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.541255:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.541256:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.541257:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677474.541258:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.541260:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.541261:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.541261:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677474.541265:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.541266:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677474.541269:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.541270:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677474.541270:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677474.541272:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677474.541273:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.541274:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677474.541276:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677474.541276:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677474.541278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.541282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:2.0:1619677474.541286:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.541289:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.541290:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677474.541290:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:2.0:1619677474.541290:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.541292:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677474.541293:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:2.0:1619677474.541299:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.541302:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:2.0:1619677474.541303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.541406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.541410:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.541413:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.541415:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.541416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.541417:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.541419:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.541420:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.541421:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.541422:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.541423:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.541426:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.541428:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.541431:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.541432:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.541432:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.541434:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.541435:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.541436:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.541436:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.541441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.541448:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.541450:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.541451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.541452:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.541454:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.541456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.541459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.541465:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.541467:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.541468:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.541468:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.541469:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.541470:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.541798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.541803:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.541805:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.541807:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.541808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.541810:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.541811:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.541813:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.541814:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.541815:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.541816:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.541819:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.541821:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.541823:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.541824:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.541824:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.541826:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.541827:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.541828:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.541829:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.541830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.541832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.541835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677474.541836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677474.541839:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.541841:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.541841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677474.541841:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677474.541842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677474.541844:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677474.541844:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.541845:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.541845:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677474.541847:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:17.0:1619677474.541847:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.541849:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.541850:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677474.541851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.541852:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.541853:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.541854:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.541857:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.541858:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.541859:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.541862:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.541864:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.541867:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.541867:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.541868:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.541870:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.541871:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.541872:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.541872:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.541875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.541882:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.541884:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.541885:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.541885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.541887:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.541889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.542062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.542069:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.542072:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.542074:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.542076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.542076:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.542078:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.542079:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.542080:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.542081:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.542082:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.542085:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.542086:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.542090:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.542090:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.542091:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.542093:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.542094:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.542095:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.542097:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.542097:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.542099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.542103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.542107:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.542110:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.542110:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.542111:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677474.542112:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.542113:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.542113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.542114:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.542115:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.542115:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.542270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.542276:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.542279:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.542279:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677474.542282:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.542283:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.542285:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.542286:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.542288:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.542288:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.542290:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.542299:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.542301:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.542304:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.542305:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.542305:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.542307:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.542308:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.542308:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.542309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.542314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.542322:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.542324:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.542325:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.542327:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.542327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.542329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.542434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.542441:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.542444:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.542445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.542447:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.542450:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.542451:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.542453:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.542454:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.542455:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.542456:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.542459:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.542461:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.542464:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.542465:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.542465:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.542467:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.542468:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.542468:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.542469:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.542474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.542483:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.542485:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.542486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.542487:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.542489:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.542491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.542717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.542725:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.542728:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.542728:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677474.542730:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.542733:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.542735:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.542736:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.542737:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.542738:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.542739:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.542742:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.542743:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.542746:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.542747:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.542747:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.542749:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.542749:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.542750:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.542750:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.542755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.542762:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.542765:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.542766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.542766:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.542768:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.542771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.542835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677474.542835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.542839:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.542841:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:17.0:1619677474.542842:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:20.0:1619677474.542843:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677474.542844:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.0:1619677474.542845:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.542845:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.542845:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:20.0:1619677474.542847:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677474.542848:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.542849:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.542850:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.542850:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.542851:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.542852:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.542852:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.542853:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.542855:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.542856:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.542857:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677474.542857:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677474.542860:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.542861:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.542863:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.542863:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.542864:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.542864:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677474.542866:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.542866:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:17.0:1619677474.542866:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.542867:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:17.0:1619677474.542867:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:20.0:1619677474.542868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:17.0:1619677474.542868:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.542869:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.542870:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.542870:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.542871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677474.542872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:25.2:1619677474.542876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677474.542878:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.542881:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.542882:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.542882:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677474.542883:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:24.0:1619677474.542884:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.542886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677474.542886:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.542887:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.542887:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.542889:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.542891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.543068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.543076:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.543079:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.543082:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.543084:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.543084:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.543086:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.543087:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.543088:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.543089:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.543090:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.543094:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.543096:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.543098:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.543099:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.543099:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.543101:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.543101:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.543102:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.543103:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.543106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.543113:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.543116:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.543117:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.543118:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.543120:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.543121:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.543569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.543577:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.543580:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.543582:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.543584:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.543584:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.543586:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.543587:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.543588:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.543589:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.543590:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.543593:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.543595:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.543597:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.543598:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.543598:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.543600:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.543601:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.543601:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.543602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.543607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.543614:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.543616:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.543617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.543617:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.543619:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.543621:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.543790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.543798:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.543801:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.543803:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.543805:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.543805:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.543806:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.543807:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.543809:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.543810:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.543810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:30.0:1619677474.543811:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.543814:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.543815:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.543817:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.543818:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.543818:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.543818:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677474.543820:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.543820:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.543821:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677474.543821:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677474.543822:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677474.543822:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.543823:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677474.543825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:26.0:1619677474.543827:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.543828:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.543829:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.543830:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.543831:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.543832:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.543832:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.543834:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:26.0:1619677474.543835:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.543835:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.543835:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.543837:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.543837:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.543839:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.543839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.543840:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.543841:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.543842:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.543843:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.543843:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.543844:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.543849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.543857:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.543860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.543860:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.543861:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.543863:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.543865:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.544428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.544434:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.544437:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.544439:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.544440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.544442:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.544443:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.544444:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.544445:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.544446:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.544447:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.544450:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.544452:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.544454:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.544455:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.544455:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.544457:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.544458:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.544458:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.544459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.544464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.544472:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.544474:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.544475:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.544477:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.544478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.544479:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.544483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.544491:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.544494:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.544495:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.544495:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.544496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.544497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.544791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.544800:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.544803:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.544805:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.544807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.544808:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.544809:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.544810:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.544811:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.544813:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.544813:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.544817:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.544819:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.544821:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.544822:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.544823:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.544824:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.544825:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.544826:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.544826:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.544831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.544837:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.544840:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.544841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.544841:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.544842:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.544844:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.544954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.544960:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.544963:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.544964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.544966:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.544968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:1.0:1619677474.544969:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.544970:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677474.544972:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.544973:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.544974:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.544975:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.544976:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:1.0:1619677474.544978:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.544979:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.544980:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:1.0:1619677474.544980:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677474.544982:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677474.544983:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.544984:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.544984:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677474.544985:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:10.0:1619677474.544986:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.544987:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.544987:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677474.544987:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:10.0:1619677474.544988:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.544988:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:10.0:1619677474.544989:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677474.544989:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:10.0:1619677474.544990:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677474.544990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677474.544992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:10.0:1619677474.544993:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.544995:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677474.544996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:10.0:1619677474.544998:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.544999:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.545000:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.545000:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.545001:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.545002:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.545002:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:10.0:1619677474.545002:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.545002:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:4.2:1619677474.545002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:10.0:1619677474.545003:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.545004:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.545004:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:10.0:1619677474.545004:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:17.0:1619677474.545005:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.545006:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:10.0:1619677474.545006:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:25.2:1619677474.545007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.0:1619677474.545007:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677474.545008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.0:1619677474.545010:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.545012:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.545012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.545014:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.545014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:13.0:1619677474.545015:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:21.0:1619677474.545016:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.545017:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96504043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.545018:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677474.545018:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.545018:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.545020:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.545020:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:21.0:1619677474.545021:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677474.545021:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.545023:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.545024:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:13.0:1619677474.545024:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677474.545024:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.545025:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:21.0:1619677474.545026:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.545026:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.545026:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.0:1619677474.545028:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.545031:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.545032:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.545032:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.545034:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.545035:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.545036:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.545037:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.545038:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677474.545040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.545043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677474.545047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677474.545047:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.545049:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:30.0:1619677474.545050:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677474.545050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677474.545051:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.545051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677474.545052:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.545054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.545055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.545055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677474.545061:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.545064:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677474.545065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677474.545067:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.545070:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.545072:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677474.545073:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.545074:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.545075:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.545076:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677474.545080:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.545081:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677474.545084:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.545084:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677474.545085:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.545087:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677474.545087:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.545088:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677474.545088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677474.545093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.545101:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.545104:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.545105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.545105:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.545107:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.545109:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.545201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e7a00] (20)++
00000800:00000200:31.0:1619677474.545209:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:31.0:1619677474.545212:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.1.19@o2ib3
00000400:00000200:31.0:1619677474.545214:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.19@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.545216:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000400:00000200:31.0:1619677474.545217:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.545218:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:31.0:1619677474.545220:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.545221:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.545222:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.545223:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677474.545227:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.545228:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.545232:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.545232:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:31.0:1619677474.545233:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677474.545235:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677474.545236:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:31.0:1619677474.545236:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (22)--
00000800:00000200:31.0:1619677474.545237:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)--
00000800:00000200:25.2:1619677474.545241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.545248:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.545250:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.545251:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.545253:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.545254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.545256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.545292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e7a00] (20)++
00000800:00000200:30.0:1619677474.545301:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:30.0:1619677474.545304:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.19@o2ib3
00000400:00000200:30.0:1619677474.545306:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.19@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.545308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000400:00000200:30.0:1619677474.545309:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.545311:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.545312:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.545313:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.545314:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.545314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e7a00] (21)++
00000400:00000200:30.0:1619677474.545315:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.545318:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.545319:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.545321:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.545322:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)++
00000800:00000200:30.0:1619677474.545322:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.545323:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.545324:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.19@o2ib3
00000800:00000200:30.0:1619677474.545324:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.545325:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (23)++
00000400:00000200:31.0:1619677474.545326:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.19@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.545326:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (24)--
00000800:00000200:30.0:1619677474.545326:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (23)--
00000400:00000200:31.0:1619677474.545328:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.545328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000400:00000200:31.0:1619677474.545329:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:9.2:1619677474.545330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:31.0:1619677474.545331:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.545332:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.545333:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.545333:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.545336:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.545337:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:31.0:1619677474.545338:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.545339:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.545340:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.545340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.545341:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.545342:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.545342:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:24.0:1619677474.545342:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.545344:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.545344:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:24.0:1619677474.545344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.545345:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (22)--
00000800:00000200:31.0:1619677474.545346:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)--
00000800:00000200:25.2:1619677474.545350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:9.2:1619677474.545354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677474.545358:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.545360:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.545361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.545361:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677474.545362:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.545364:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.545364:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677474.545364:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.545365:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.545366:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.545367:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677474.545368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.545443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677474.545448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e7a00] (20)++
00000800:00000200:24.0:1619677474.545451:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.545453:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.545454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677474.545456:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)++
00000400:00000200:24.0:1619677474.545456:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.545459:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.19@o2ib3
00000400:00000200:24.0:1619677474.545459:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.545460:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677474.545461:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.545461:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.545463:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000400:00000200:24.0:1619677474.545463:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.545464:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.545464:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.545464:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677474.545465:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.545467:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.545468:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.545468:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.545469:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.545470:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.545470:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677474.545473:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.545475:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.545477:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.545477:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.545477:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.545478:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.545478:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.545479:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677474.545480:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.545480:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:24.0:1619677474.545480:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.545481:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677474.545482:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e7a00] (22)++
00000800:00000200:24.0:1619677474.545482:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:30.0:1619677474.545484:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (23)--
00000800:00000200:30.0:1619677474.545484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000800:00000200:24.0:1619677474.545484:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.545484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677474.545486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.2:1619677474.545486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677474.545490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:4.2:1619677474.545494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:17.0:1619677474.545495:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677474.545496:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.545498:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677474.545498:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:8.0:1619677474.545498:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.545499:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677474.545499:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.545499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.545500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.545500:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.545501:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.545502:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677474.545503:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)++
00000400:00000200:17.0:1619677474.545503:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.545503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:8.0:1619677474.545503:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.545506:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e7a00] (23)--
00000800:00000200:17.0:1619677474.545506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677474.545507:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000800:00000200:31.0:1619677474.545508:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)--
00000800:00000200:8.2:1619677474.545509:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.545510:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:4.2:1619677474.545511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e7a00] (20)++
00000800:00000200:24.0:1619677474.545512:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.545512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.545513:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:24.0:1619677474.545513:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.545513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.545514:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677474.545516:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677474.545519:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:1.0:1619677474.545519:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.545520:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.545520:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.545521:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:31.0:1619677474.545522:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.19@o2ib3
00000800:00000200:1.0:1619677474.545522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:31.0:1619677474.545525:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.19@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.545527:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.545527:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000400:00000200:31.0:1619677474.545528:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677474.545530:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.545531:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.545532:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.545533:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677474.545536:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.545538:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677474.545540:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.545541:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:31.0:1619677474.545541:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:31.0:1619677474.545543:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677474.545544:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:31.0:1619677474.545545:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (22)--
00000800:00000200:31.0:1619677474.545545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)--
00000800:00000200:21.2:1619677474.545546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:9.2:1619677474.545549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:30.0:1619677474.545554:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:26.0:1619677474.545555:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:30.0:1619677474.545556:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:26.0:1619677474.545558:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:30.0:1619677474.545559:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.545559:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.545559:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677474.545560:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.545561:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.545561:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.545562:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.545563:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.545564:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.545565:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.545566:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677474.545568:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:30.0:1619677474.545569:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.545571:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.545573:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.545574:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.545574:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.545576:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.545576:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.545577:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.545578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.545581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.545588:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.545590:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.545591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.545591:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.545593:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.545595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.545615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.545622:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.545625:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.545626:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.545627:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.545629:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.545631:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.545632:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.545633:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.545634:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.545635:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.545638:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.545639:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:4.2:1619677474.545640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e7a00] (20)++
00000800:00000200:10.0:1619677474.545641:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.545642:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.545642:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.545644:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.545644:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.545645:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.545645:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677474.545647:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:9.2:1619677474.545649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.545650:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.19@o2ib3
00000400:00000200:31.0:1619677474.545652:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.19@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.545654:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.545654:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000400:00000200:31.0:1619677474.545655:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677474.545655:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:31.0:1619677474.545656:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.545657:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.545657:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:31.0:1619677474.545658:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.545658:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.545659:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.545659:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.545660:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.545662:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.545663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.545664:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.545667:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.545667:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.545668:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.545670:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.545670:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:31.0:1619677474.545671:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (22)--
00000800:00000200:31.0:1619677474.545671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)--
00000800:00000200:25.2:1619677474.545675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.545683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.545686:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:4.2:1619677474.545686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e7a00] (20)++
00000400:00000200:22.0:1619677474.545687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.545687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.545689:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.545692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.545693:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:25.2:1619677474.545695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.545696:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.19@o2ib3
00000400:00000200:30.0:1619677474.545698:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.19@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.545700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000400:00000200:30.0:1619677474.545701:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.545702:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677474.545703:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.545704:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.545705:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.545706:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.545706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.545706:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677474.545707:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.545707:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677474.545708:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677474.545708:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677474.545711:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.545713:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.545716:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.545717:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.545717:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.545719:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.545720:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:30.0:1619677474.545721:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (22)--
00000800:00000200:30.0:1619677474.545721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)--
00000800:00000200:9.2:1619677474.545725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.545725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e7a00] (20)++
00000800:00000200:31.0:1619677474.545733:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:18.0:1619677474.545735:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:31.0:1619677474.545736:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.19@o2ib3
00000800:00000200:18.0:1619677474.545737:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:31.0:1619677474.545738:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.19@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.545739:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.545739:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:31.0:1619677474.545740:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.545740:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000400:00000200:18.0:1619677474.545741:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.545742:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677474.545743:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.545744:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.545745:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.545746:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677474.545749:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.545751:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.545752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.545754:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.545755:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.545755:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.545757:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.545757:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (21)++
00000800:00000200:31.0:1619677474.545759:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e7a00] (22)++
00000800:00000200:31.0:1619677474.545760:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e7a00] (23)--
00000800:00000200:31.0:1619677474.545761:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000800:00000200:25.2:1619677474.545762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.545765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:30.0:1619677474.545769:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.545772:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:26.0:1619677474.545772:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:4.2:1619677474.545772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e7a00] (21)++
00000400:00000200:30.0:1619677474.545773:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.545775:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.545775:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677474.545775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:31.0:1619677474.545777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.545778:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)++
00000800:00000200:30.0:1619677474.545778:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:26.0:1619677474.545778:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.545781:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e7a00] (23)--
00000800:00000200:31.0:1619677474.545781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (22)--
00000400:00000200:26.0:1619677474.545781:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.545782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e7a00] (21)--
00000400:00000200:26.0:1619677474.545782:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.545783:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.545785:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.545785:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.545786:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:25.2:1619677474.545788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:26.0:1619677474.545790:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.545791:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.545794:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677474.545795:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.545795:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.545796:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.545797:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.545798:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.545798:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:26.0:1619677474.545798:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.545798:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:31.0:1619677474.545799:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677474.545799:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:31.0:1619677474.545800:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:26.0:1619677474.545800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677474.545804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.545813:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.545815:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.545816:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.545818:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.545818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.545821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.545837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.545845:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.545848:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.545849:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.545849:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.545850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.545851:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.545861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.545867:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.545870:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.545871:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.545872:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.545875:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.545877:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.545878:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.545880:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.545880:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.545882:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.545885:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.545887:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.545889:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.545890:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.545891:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.545892:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.545893:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.545894:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.545894:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.545898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.545907:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.545910:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.545911:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.545911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.545913:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.545915:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.546481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.546490:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.546493:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.546495:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.546497:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.546497:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.546499:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.546500:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.546501:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.546502:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.546503:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.546507:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.546509:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.546512:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.546513:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.546513:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.546515:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.546516:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.546517:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.546519:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.546519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.546522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.546526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.546531:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.546532:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.546533:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.546534:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.546534:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.546535:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.546536:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.546536:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.546537:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.546538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.546699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.546705:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.546708:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.546709:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.546711:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.546713:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.546714:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677474.546715:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.546717:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.546718:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.546719:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677474.546723:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.546725:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677474.546728:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.546729:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677474.546729:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677474.546731:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677474.546731:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.546732:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.546733:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677474.546737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.546739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:31.0:1619677474.546744:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.546747:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:10.0:1619677474.546747:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:31.0:1619677474.546748:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.546750:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.546750:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677474.546750:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.546750:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:31.0:1619677474.546753:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:10.0:1619677474.546753:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.546755:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.546757:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.546758:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.546759:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.546760:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.546761:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.546765:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.546766:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.546769:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.546769:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.546770:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.546772:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.546772:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.546773:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.546773:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.546777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.546785:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.546787:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.546788:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.546790:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.546792:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.546793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.547047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.547054:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.547056:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.547057:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.547059:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.547061:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.547063:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.547064:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.547066:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.547067:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.547067:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.547071:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.547072:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.547075:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.547076:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.547076:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.547078:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.547079:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.547080:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.547081:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.547081:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.547084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.547087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.547092:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.547094:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.547094:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.547095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.547096:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677474.547096:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.547097:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.547098:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.547098:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.547100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.547106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.547113:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.547116:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.547116:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.547117:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.547118:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.547118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.547174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.547181:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.547184:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.547187:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.547188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.547189:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.547190:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.547192:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.547193:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.547194:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.547195:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.547198:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.547199:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.547202:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.547202:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.547203:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.547204:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.547205:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.547207:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.547208:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.547209:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.547210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.547215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.2:1619677474.547216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677474.547218:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677474.547219:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677474.547220:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.547221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.547221:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:8.0:1619677474.547221:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.547223:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677474.547224:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.547225:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:21.0:1619677474.547225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.547225:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.547226:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.547227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:20.0:1619677474.547227:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.547227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:20.0:1619677474.547229:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.547230:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.547232:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.547233:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.547234:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.547237:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.547239:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.547242:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.547242:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.547243:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.547245:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.547245:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.547246:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.547247:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.547251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.547260:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.547262:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.547263:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.547263:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.547266:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.547268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.547432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.547440:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.547443:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.547445:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.547448:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.547448:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.547449:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.547451:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.547452:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.547453:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.547454:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.547457:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.547459:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.547461:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.547462:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.547463:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.547464:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.547465:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.547466:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.547466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.547471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.547478:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.547480:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.547481:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.547482:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.547484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.547485:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.547494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.547500:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.547503:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.547504:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.547504:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.547505:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.547506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677474.547582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.547588:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.547591:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.547591:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.547593:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.547596:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.547598:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.547599:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.547600:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.547601:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.547602:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.547606:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.547607:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.547610:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.547611:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.547611:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.547613:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.547613:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.547614:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.547615:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.547619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.547627:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.547629:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.547630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.547630:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.547632:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.547634:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.548265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.548272:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.548275:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.548276:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.548277:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.548281:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.548283:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.548284:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96504020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.548285:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.548286:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.548287:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.548291:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.548293:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.548300:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.548301:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.548301:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.548303:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.548303:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.548304:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.548305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.548309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.548317:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.548320:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.548321:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.548321:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.548323:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.548326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.548459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.548466:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.548469:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.548470:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.548472:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.548474:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.548475:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.548477:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.548478:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.548479:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.548480:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.548483:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.548485:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.548487:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.548488:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.548488:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.548490:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.548491:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.548492:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.548492:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.548496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.548503:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.548505:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.548506:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.548507:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.548508:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.548509:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.548516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.548523:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.548525:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.548526:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.548526:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.548526:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.548527:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.548568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.548576:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.548579:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.548581:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.548583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.548583:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.548584:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.548586:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.548587:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.548588:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.548589:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.548592:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.548593:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.548597:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.548598:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.548599:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.548600:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.548601:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.548602:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.548602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.548607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.548610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.0:1619677474.548616:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677474.548618:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677474.548618:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.548619:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.548621:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:21.0:1619677474.548621:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.548621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.548622:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.548624:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.548624:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:26.0:1619677474.548628:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.548629:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.548630:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.548632:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.548633:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.548634:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.548637:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.548638:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.548641:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.548642:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.548643:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.548644:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.548645:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.548645:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.548646:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.548650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.548658:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.548660:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.548661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.548662:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.548664:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.548667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.548797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.548804:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.548807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.548807:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.548809:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.548812:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.548813:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.548814:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.548815:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.548816:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.548817:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.548820:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.548821:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.548823:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.548824:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.548825:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.548826:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.548827:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.548827:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.548829:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.548829:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677474.548832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.548835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677474.548839:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677474.548841:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:13.0:1619677474.548842:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.548842:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.548843:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.548844:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677474.548844:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677474.548845:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677474.548846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.548847:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.549383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677474.549386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.549392:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.549394:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677474.549394:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:31.0:1619677474.549396:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.549396:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.549398:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.549399:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.549400:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.549401:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.549403:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.549403:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.549404:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.549407:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.549409:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.549409:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.549411:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.549411:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:30.0:1619677474.549411:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677474.549412:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677474.549412:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.549413:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677474.549413:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.549414:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.549415:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.549416:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.549417:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.549417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.549418:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677474.549418:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.549419:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:25.2:1619677474.549420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.549421:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.549422:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.549422:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.549422:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.549423:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.549423:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.549423:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.549424:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:31.0:1619677474.549425:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.549425:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.549426:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.549426:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.549427:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677474.549428:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:9.2:1619677474.549430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.549437:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.549440:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.549441:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.549441:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.549442:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.549445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.549695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.549699:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.549702:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.549704:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.549705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.549706:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.549708:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.549709:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.549710:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.549711:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.549712:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.549715:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.549717:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.549720:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.549720:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.549721:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.549723:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.549723:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.549724:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.549725:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.549726:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.549728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.549731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.2:1619677474.549733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677474.549735:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677474.549736:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677474.549737:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677474.549737:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.549737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:24.0:1619677474.549738:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.549738:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677474.549738:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.549739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:21.0:1619677474.549740:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:24.0:1619677474.549741:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.549741:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.549742:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.549744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:21.0:1619677474.549746:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.549748:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.549749:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677474.549749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:21.0:1619677474.549750:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.549751:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.549752:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.549755:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.549755:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.549757:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677474.549758:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.549758:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.549759:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.549759:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677474.549760:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677474.549760:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.549761:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677474.549761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677474.549762:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.549763:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.549764:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.549764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.549768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.549776:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.549778:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.549779:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.549781:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.549782:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.549783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.549975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.549983:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.549986:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.549987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.549989:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.549992:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.549993:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.549994:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.549996:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.549997:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.549998:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.550001:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.550002:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.550005:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.550005:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.550006:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.550007:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.550008:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.550009:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.550010:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.550010:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.550013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.550016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.550019:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.550021:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.550022:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.550022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.550023:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677474.550024:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.550025:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.550026:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.550026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.550027:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.550080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.550087:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.550090:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.550091:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.550093:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.550095:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.550097:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.550098:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.550099:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.550100:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.550101:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.550105:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.550106:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.550109:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.550110:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.550110:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.550112:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.550112:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.550113:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.550113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.550118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.550126:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.550128:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.550129:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.550131:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.550132:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.550133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.550234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.550241:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.550244:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.550246:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.550248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.550249:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.550251:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.550252:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.550253:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.550254:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.550255:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.550259:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.550260:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.550263:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.550264:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.550264:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.550266:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.550266:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.550267:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.550268:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.550271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.550278:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.550280:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.550281:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.550281:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.550283:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.550285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.550966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.550972:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.550975:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.550976:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.550978:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.550981:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.550982:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.550984:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.550985:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.550986:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.550987:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.550990:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.550992:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.550995:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.550996:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.550996:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.550998:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.550999:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.551000:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.551000:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.551005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.551013:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.551016:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.551017:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.551018:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.551020:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.551022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.551082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.551089:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.551092:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.551094:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.551096:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.551097:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.551098:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.551099:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.551101:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.551101:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.551103:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.551106:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.551108:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.551110:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.551111:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.551111:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.551113:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.551114:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.551115:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.551115:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.551119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.551126:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.551128:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.551129:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.551129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.551131:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.551133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.551906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.551913:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.551916:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.551916:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.551918:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.551920:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.551922:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.551923:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.551925:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.551925:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.551927:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.551930:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.551932:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.551935:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.551935:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.551936:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.551938:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.551939:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.551939:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.551940:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.551945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.551954:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.551956:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.551957:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.551959:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.551959:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.551962:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.551970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.551977:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.551980:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.551981:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.551982:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.551985:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.551987:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.551988:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.551990:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.551991:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.551991:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.551995:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.551996:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.551998:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.551999:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.552000:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.552001:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.552002:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.552002:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.552003:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.552008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.552017:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.552019:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.552020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.552021:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.552023:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.552025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.552118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.552126:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.552129:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.552131:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.552133:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.552134:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.552134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.552136:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.552137:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.552138:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.552139:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.552141:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.552143:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.552145:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.552146:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.552146:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.552147:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.552148:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.552149:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.552151:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.552151:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.552153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.552157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.552162:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.552163:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.552164:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.552165:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.552166:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677474.552166:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.552167:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.552167:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.552168:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.552169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677474.552811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.552817:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.552820:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.552821:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.552823:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.552826:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.552827:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.552829:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.552830:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.552831:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.552833:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.552836:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.552838:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.552840:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.552841:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.552842:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.552844:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.552844:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.552845:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.552846:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.552846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.552850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.552851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.552856:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.552858:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.552858:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.552859:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.552860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.552861:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.552862:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.552863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.552864:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.552867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.553065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.553074:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.553077:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.553080:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.553081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.553082:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.553083:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.553085:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96504002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677474.553085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:31.0:1619677474.553086:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.553087:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.553088:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.553091:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.553093:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677474.553093:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677474.553095:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.553096:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677474.553096:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:31.0:1619677474.553097:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677474.553097:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:31.0:1619677474.553098:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.553099:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:26.0:1619677474.553099:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677474.553100:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.553101:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.553102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:26.0:1619677474.553102:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.553103:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.553104:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96504001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.553105:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.553106:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.553107:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:9.2:1619677474.553107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.553108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:26.0:1619677474.553110:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.553111:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.553113:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.553114:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:8.0:1619677474.553114:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:26.0:1619677474.553115:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:30.0:1619677474.553116:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:26.0:1619677474.553116:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.553117:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.553117:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.0:1619677474.553117:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677474.553118:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:26.0:1619677474.553118:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:24.0:1619677474.553118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.553118:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.553119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.553120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:8.0:1619677474.553120:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.553122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677474.553122:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677474.553131:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.553134:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.553135:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.553137:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.553139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.553140:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.553141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677474.553148:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:5.2:1619677474.553150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:12.0:1619677474.553151:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.553151:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.553152:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.553153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.553153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677474.553156:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.553159:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677474.553159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677474.553162:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.553164:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.553166:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677474.553167:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96504000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.553168:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.553170:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.553171:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677474.553174:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.553176:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677474.553178:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.553179:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:17.0:1619677474.553179:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677474.553182:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677474.553182:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.553183:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677474.553183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677474.553188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.553196:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.553199:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.553200:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.553200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.553202:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.553205:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.553337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.553343:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.553346:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.553348:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.553348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.553350:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.553352:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.553354:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.553355:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.553356:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.553357:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.553360:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.553362:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.553365:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.553366:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.553367:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.553369:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.553369:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.553370:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.553371:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.553375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.553382:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.553385:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.553386:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.553386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.553388:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:9.2:1619677474.553390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.553392:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.553394:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.553396:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.553397:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.553397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.553398:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.553398:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.553427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.553434:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.553436:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.553437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.553439:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.553441:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.553443:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.553445:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.553446:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.553447:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.553448:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.553452:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.553453:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.553456:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.553457:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.553457:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.553459:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.553460:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.553460:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.553461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.553465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.553473:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.553476:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.553477:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.553477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.553480:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.553482:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.553513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.553520:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.553523:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.553524:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.553526:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.553528:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.553530:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.553531:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.553532:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.553533:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.553534:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.553537:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.553539:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.553541:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.553542:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.553543:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.553544:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.553545:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.553545:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.553546:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.553550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.553557:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.553560:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.553561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.553561:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.553563:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.553566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.553568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.553575:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.553578:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.553579:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.553579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.553580:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.553581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.553788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.553796:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.553799:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.553801:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.553804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.553804:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.553805:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.553806:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.553808:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.553809:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.553809:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.553813:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.553814:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.553816:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.553817:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.553817:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.553819:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.553820:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.553821:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.553821:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.553825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.553831:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.553834:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.553834:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.553836:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.553838:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.553839:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.554213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.554217:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.554219:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677474.554222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.554222:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.554224:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.554226:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.554227:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.554228:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.554229:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.554230:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.554233:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.554235:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.554238:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.554239:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.554240:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.554242:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.554242:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.554243:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.554244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.554248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.554255:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.554258:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.554259:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.554261:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.554262:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.554263:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.554541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.554549:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.554552:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.554553:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.554555:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.554558:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.554559:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.554560:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.554561:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.554562:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.554563:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.554566:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.554568:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.554570:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.554571:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.554572:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.554573:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.554574:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.554574:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.554575:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.554580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.554586:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.554589:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.554590:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.554590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.554591:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.554594:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.554714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677474.554722:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.554724:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677474.554725:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677474.554726:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.554728:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.554729:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.554730:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.554732:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.554732:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.554733:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.554736:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.554737:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.554740:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.554740:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.554741:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.554742:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.554743:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.554743:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.554744:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.554748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.554752:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.554754:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.554755:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.554755:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.554757:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.554759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677474.554765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.554773:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.554776:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.554778:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.554780:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.554781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.554782:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.554783:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.554784:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.554785:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.554786:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.554789:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.554790:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.554792:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.554793:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.554794:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.554795:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.554796:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.554797:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.554797:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.554801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.554807:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.554809:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.554810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.554811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.554812:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.554814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.555199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.555206:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.555209:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.555211:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.555213:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.555213:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.555215:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.555216:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.555217:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.555218:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.555219:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.555222:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.555223:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.555226:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.555227:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.555228:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.555229:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.555230:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.555231:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.555231:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.555231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677474.555236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.555239:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.555241:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.555243:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.555244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.555245:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.555246:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.555246:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.555247:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.555248:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.555248:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.555250:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.555251:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.555252:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.555255:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.555257:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.555258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.555259:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.555259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.555260:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.555260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677474.555261:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.555262:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.555263:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.555264:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.555266:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.555266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.555268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.555272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.555279:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.555279:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.555279:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677474.555280:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.555280:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.555281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.555282:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.555282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.555283:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677474.555284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.555710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06ffaac00] (20)++
00000800:00000200:26.0:1619677474.555718:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06ffaac00] (21)++
00000800:00000200:26.0:1619677474.555721:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.11@o2ib3
00000800:00000200:10.0:1619677474.555721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaac00] (22)--
00000400:00000200:26.0:1619677474.555724:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.11@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.555726:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.555728:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.555729:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.555730:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.555731:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.555732:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.555735:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.555737:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.555739:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.555740:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.555741:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.555742:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.555743:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06ffaac00] (21)++
00000800:00000200:26.0:1619677474.555744:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06ffaac00] (22)--
00000800:00000200:26.0:1619677474.555744:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06ffaac00] (21)--
00000800:00000200:25.2:1619677474.555749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.555756:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:20.2:1619677474.555759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:12.0:1619677474.555759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.555760:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.555761:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.555763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.555765:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677474.555765:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.555768:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.555769:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.555771:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.555774:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.555775:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.555776:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.555778:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.555779:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.555780:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.555783:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.555785:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.555788:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.555789:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.555789:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.555791:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.555792:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.555793:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.555793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.555797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.555803:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.555806:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.555807:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.555809:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.555810:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.555812:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.556696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.556703:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.556706:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.556707:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.556709:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.556711:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.556713:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.556714:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.556715:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.556717:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.556718:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.556720:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.556722:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.556725:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.556726:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.556726:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.556728:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.556728:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.556729:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.556730:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.556736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.556745:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.556748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.556748:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.556749:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.556751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:6.0:1619677474.556751:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.556753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.556756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:30.0:1619677474.556760:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.556763:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677474.556763:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:30.0:1619677474.556765:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.556766:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.556767:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.556767:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.556767:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:30.0:1619677474.556768:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.556768:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000400:00000200:30.0:1619677474.556769:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:22.0:1619677474.556769:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:30.0:1619677474.556771:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.556772:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.556773:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.556774:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.556777:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.556779:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.556781:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.556782:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.556782:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.556783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.556784:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.556785:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.556786:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.556786:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677474.556788:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.556790:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:9.2:1619677474.556790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.556791:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.556791:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.556793:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.556794:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.556795:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.556796:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.556797:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.556797:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:30.0:1619677474.556798:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677474.556799:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.556800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.556800:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.556801:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.556802:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677474.556802:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.556804:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.556805:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.556805:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.556806:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.556807:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.556808:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.556809:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.556809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.556813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.556820:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.556822:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.556823:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.556824:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.556826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.556827:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.556831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.556838:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.556840:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.556841:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.556842:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.556842:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.556843:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.556966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.556973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.556976:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.556976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.556979:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.556982:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.556983:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.556984:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.556986:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.556987:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.556987:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.556990:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.556992:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.556994:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.556994:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.556995:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.556997:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.556997:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.556998:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.556999:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.557003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.557010:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.557012:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.557013:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.557014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.557015:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.557016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.557066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce0bf6400] (20)++
00000800:00000200:1.0:1619677474.557073:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:17.0:1619677474.557076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)--
00000800:00000200:1.0:1619677474.557076:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000400:00000200:1.0:1619677474.557079:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.557082:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.557083:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.557085:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.557086:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.557087:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.557088:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.557091:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.557093:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.557095:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.557096:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.557097:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.557099:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.557099:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:1.0:1619677474.557100:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (22)--
00000800:00000200:1.0:1619677474.557100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)--
00000800:00000200:9.2:1619677474.557104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.557111:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.557113:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.2:1619677474.557114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce0bf6400] (20)++
00000800:00000200:8.0:1619677474.557114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.557115:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.557117:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.557119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677474.557119:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:17.0:1619677474.557122:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:1.0:1619677474.557123:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)--
00000400:00000200:17.0:1619677474.557125:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.557127:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.557128:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677474.557130:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.557131:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.557132:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.557133:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677474.557136:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.557138:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677474.557140:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.557141:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677474.557142:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677474.557144:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677474.557144:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:17.0:1619677474.557145:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (22)--
00000800:00000200:17.0:1619677474.557145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)--
00000800:00000200:21.2:1619677474.557148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce0bf6400] (20)++
00000800:00000200:25.2:1619677474.557150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677474.557153:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:4.2:1619677474.557155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:1.0:1619677474.557156:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:17.0:1619677474.557157:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)++
00000800:00000200:12.0:1619677474.557157:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:1.0:1619677474.557158:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.557159:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000400:00000200:17.0:1619677474.557160:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.557160:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:1.0:1619677474.557160:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.557161:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.557161:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:26.0:1619677474.557163:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:12.0:1619677474.557163:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.557163:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677474.557164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677474.557165:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:1.0:1619677474.557165:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.557166:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677474.557166:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.557168:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677474.557170:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:12.0:1619677474.557170:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:17.0:1619677474.557172:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.557173:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.557173:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677474.557173:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.557175:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.557176:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677474.557176:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:26.0:1619677474.557177:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.557177:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.557178:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.557179:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.557179:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677474.557180:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.557181:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677474.557182:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.557183:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.557184:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.557186:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.557186:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (23)++
00000400:00000200:26.0:1619677474.557187:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677474.557187:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.557188:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce0bf6400] (24)++
00000800:00000200:26.0:1619677474.557190:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677474.557190:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677474.557190:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (25)--
00000800:00000200:1.0:1619677474.557191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:8.2:1619677474.557192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.557192:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:1.0:1619677474.557193:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:26.0:1619677474.557194:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.557194:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.557195:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677474.557196:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677474.557196:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:1.0:1619677474.557196:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.557197:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:1.0:1619677474.557198:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.557199:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677474.557199:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:1.0:1619677474.557199:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677474.557200:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677474.557200:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:26.0:1619677474.557201:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:1.0:1619677474.557201:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96503978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.557202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677474.557202:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:1.0:1619677474.557202:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.557203:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.557203:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.557205:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.557205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:1.0:1619677474.557205:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.557206:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.557206:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (24)++
00000800:00000200:17.0:1619677474.557206:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (25)--
00000800:00000200:17.0:1619677474.557207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:20.0:1619677474.557208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:17.0:1619677474.557208:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:9.2:1619677474.557208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:1.0:1619677474.557209:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.557210:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:10.0:1619677474.557211:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:17.0:1619677474.557212:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.557212:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677474.557214:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:8.2:1619677474.557214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:26.0:1619677474.557215:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:17.0:1619677474.557215:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.557215:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.557215:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.557216:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677474.557216:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:17.0:1619677474.557217:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677474.557217:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677474.557218:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.557219:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:17.0:1619677474.557219:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96503977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.557219:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.557220:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.557220:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:17.0:1619677474.557221:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.557222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:17.0:1619677474.557222:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.557223:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000400:00000200:17.0:1619677474.557223:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.0:1619677474.557224:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.557225:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:9.2:1619677474.557225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677474.557226:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.557227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:17.0:1619677474.557227:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.557228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677474.557228:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677474.557229:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.557229:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (24)++
00000800:00000200:9.2:1619677474.557230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677474.557230:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (25)--
00000800:00000200:18.0:1619677474.557231:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:10.0:1619677474.557231:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:1.0:1619677474.557231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:17.0:1619677474.557232:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.557232:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.557232:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:17.0:1619677474.557233:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.557233:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:18.0:1619677474.557234:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.557234:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.557234:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677474.557234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)--
00000800:00000200:17.0:1619677474.557235:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.557235:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:2.0:1619677474.557235:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:1.0:1619677474.557235:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)++
00000800:00000200:17.0:1619677474.557236:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:17.0:1619677474.557236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)--
00000400:00000200:18.0:1619677474.557237:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.557237:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)++
00000800:00000200:2.0:1619677474.557237:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677474.557237:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:2.0:1619677474.557238:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.557238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677474.557239:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:2.0:1619677474.557239:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.557240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:1.0:1619677474.557240:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.557241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.2:1619677474.557241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:17.0:1619677474.557242:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.557242:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.557244:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.557245:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.557246:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.557247:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.557248:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.557249:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.557251:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677474.557252:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96503975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.557252:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.557254:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.557254:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.557255:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.557256:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677474.557259:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677474.557260:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.557260:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.557260:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.557261:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677474.557262:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.557263:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:17.0:1619677474.557264:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.557265:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:17.0:1619677474.557265:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677474.557265:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce0bf6400] (24)++
00000800:00000200:1.0:1619677474.557266:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (25)--
00000800:00000200:17.0:1619677474.557267:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677474.557267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:17.0:1619677474.557268:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:17.0:1619677474.557268:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (25)--
00000800:00000200:1.0:1619677474.557268:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:25.2:1619677474.557269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.557269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:1.0:1619677474.557269:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:17.0:1619677474.557271:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.557272:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.557272:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.557272:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.557274:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.557274:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.557275:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:6.0:1619677474.557276:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:1.0:1619677474.557276:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96503974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.557277:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.557278:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.557279:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:1.0:1619677474.557279:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677474.557280:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.557280:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:6.0:1619677474.557280:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.557281:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:17.0:1619677474.557282:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000400:00000200:6.0:1619677474.557283:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.557283:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.557285:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.557285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677474.557285:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:17.0:1619677474.557287:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.557288:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677474.557289:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677474.557289:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677474.557289:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (22)++
00000400:00000200:17.0:1619677474.557291:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96503973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.557291:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (23)--
00000400:00000200:17.0:1619677474.557292:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.557300:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.557300:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:8.2:1619677474.557300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (22)++
00000800:00000200:1.0:1619677474.557301:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (24)++
00000800:00000200:1.0:1619677474.557302:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (25)--
00000800:00000200:1.0:1619677474.557303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (24)--
00000400:00000200:17.0:1619677474.557304:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.557304:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677474.557305:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:1.0:1619677474.557305:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:17.0:1619677474.557306:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677474.557306:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)--
00000800:00000200:1.0:1619677474.557307:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)++
00000800:00000200:17.0:1619677474.557308:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.557308:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:17.0:1619677474.557309:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:21.0:1619677474.557310:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (23)++
00000800:00000200:17.0:1619677474.557310:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:1.0:1619677474.557311:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.557312:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (24)--
00000800:00000200:17.0:1619677474.557312:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:1.0:1619677474.557312:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.557313:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.557313:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:17.0:1619677474.557313:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:17.0:1619677474.557314:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)--
00000400:00000200:1.0:1619677474.557314:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.557315:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.557315:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)++
00000800:00000200:20.0:1619677474.557316:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:17.0:1619677474.557316:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000400:00000200:1.0:1619677474.557316:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96503972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.557318:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677474.557319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:1.0:1619677474.557319:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677474.557320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (22)++
00000400:00000200:1.0:1619677474.557320:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.557323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000400:00000200:17.0:1619677474.557323:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.557324:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:1.0:1619677474.557324:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.557326:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.557326:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677474.557326:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.557327:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.557328:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:21.0:1619677474.557328:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (25)--
00000800:00000200:20.0:1619677474.557328:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (23)++
00000400:00000200:17.0:1619677474.557328:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:21.0:1619677474.557329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (24)--
00000800:00000200:20.0:1619677474.557329:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (25)--
00000400:00000200:17.0:1619677474.557329:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96503971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.557329:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:21.0:1619677474.557330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.557330:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.557330:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677474.557331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.557331:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.557332:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.557333:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.557333:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.557334:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.557335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677474.557336:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.557337:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.557337:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:17.0:1619677474.557338:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.557339:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.557340:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677474.557340:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:2.0:1619677474.557341:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677474.557342:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.557342:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce0bf6400] (24)++
00000800:00000200:17.0:1619677474.557343:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:1.0:1619677474.557343:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (25)--
00000800:00000200:17.0:1619677474.557344:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677474.557344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:25.2:1619677474.557345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.557345:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:17.0:1619677474.557346:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677474.557346:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:17.0:1619677474.557347:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (24)++
00000800:00000200:17.0:1619677474.557347:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (25)--
00000800:00000200:17.0:1619677474.557348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:17.0:1619677474.557348:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:17.0:1619677474.557349:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce0bf6400] (24)--
00000400:00000200:1.0:1619677474.557349:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.557350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)--
00000800:00000200:17.0:1619677474.557350:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)++
00000400:00000200:1.0:1619677474.557351:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.557352:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:9.2:1619677474.557352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:1.0:1619677474.557352:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677474.557353:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96503970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.557354:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677474.557355:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.557355:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.557356:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677474.557357:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.557358:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.557358:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.557358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677474.557360:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:17.0:1619677474.557360:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.557361:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.557361:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:1.0:1619677474.557361:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.557363:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:17.0:1619677474.557363:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96503969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.557364:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.557364:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677474.557364:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:17.0:1619677474.557365:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.557366:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.557367:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.557368:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677474.557369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677474.557371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677474.557371:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677474.557372:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.557372:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)++
00000800:00000200:1.0:1619677474.557373:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (23)++
00000800:00000200:17.0:1619677474.557374:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.557375:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (24)--
00000800:00000200:1.0:1619677474.557375:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (24)--
00000800:00000200:1.0:1619677474.557375:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (23)++
00000800:00000200:18.0:1619677474.557376:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:17.0:1619677474.557376:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.557376:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:17.0:1619677474.557377:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (23)++
00000800:00000200:1.0:1619677474.557377:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)--
00000800:00000200:17.0:1619677474.557378:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (24)++
00000800:00000200:1.0:1619677474.557378:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)++
00000800:00000200:2.0:1619677474.557379:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (25)++
00000800:00000200:18.0:1619677474.557380:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:17.0:1619677474.557380:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (25)--
00000800:00000200:2.0:1619677474.557380:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (26)--
00000800:00000200:1.0:1619677474.557380:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:18.0:1619677474.557381:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (26)++
00000800:00000200:17.0:1619677474.557381:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (23)++
00000400:00000200:2.0:1619677474.557381:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.557382:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (25)--
00000800:00000200:17.0:1619677474.557382:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (24)--
00000800:00000200:18.0:1619677474.557383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (24)--
00000800:00000200:17.0:1619677474.557383:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (23)--
00000400:00000200:1.0:1619677474.557383:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.557384:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.557385:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (23)++
00000800:00000200:17.0:1619677474.557385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)--
00000800:00000200:18.0:1619677474.557386:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (24)--
00000400:00000200:18.0:1619677474.557386:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.557386:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.557387:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.557389:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.557389:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:1.0:1619677474.557389:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677474.557391:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.557392:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.557393:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.557394:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.557395:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:1.0:1619677474.557397:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.557399:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677474.557401:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.557402:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677474.557403:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677474.557404:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:1.0:1619677474.557405:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:1.0:1619677474.557406:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce0bf6400] (22)++
00000800:00000200:1.0:1619677474.557407:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (23)--
00000800:00000200:1.0:1619677474.557407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)--
00000800:00000200:25.2:1619677474.557411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677474.557412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:17.0:1619677474.557418:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)++
00000800:00000200:30.0:1619677474.557419:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:17.0:1619677474.557420:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce0bf6400] (23)--
00000800:00000200:17.0:1619677474.557420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)--
00000800:00000200:17.0:1619677474.557421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)--
00000800:00000200:30.0:1619677474.557422:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.557423:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.557425:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.557427:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.557428:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.557428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce0bf6400] (20)++
00000800:00000200:25.2:1619677474.557430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677474.557433:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:1.0:1619677474.557436:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:31.0:1619677474.557437:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677474.557437:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)--
00000400:00000200:1.0:1619677474.557439:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.557440:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.557441:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.557441:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:1.0:1619677474.557441:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.557442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:1.0:1619677474.557443:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677474.557444:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:1.0:1619677474.557444:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.557445:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.557446:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.557447:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.557450:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.557452:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.557454:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.557455:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.557455:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.557457:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.557458:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:1.0:1619677474.557459:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (22)--
00000800:00000200:1.0:1619677474.557459:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)--
00000800:00000200:9.2:1619677474.557463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.557470:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.557473:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.557474:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.557475:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.557477:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.557479:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.557484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce0bf6400] (20)++
00000800:00000200:17.0:1619677474.557489:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:17.0:1619677474.557492:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:1.0:1619677474.557493:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)--
00000400:00000200:17.0:1619677474.557495:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.557497:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.557499:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.557500:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.557501:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.557502:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.557503:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.557505:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.557507:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.557509:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.557510:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.557510:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.557512:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.557512:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:17.0:1619677474.557513:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (22)--
00000800:00000200:17.0:1619677474.557513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)--
00000800:00000200:8.2:1619677474.557518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.557526:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.557528:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.557529:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.557530:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.557531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.557533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677474.557544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.557550:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677474.557551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce0bf6400] (20)++
00000800:00000200:1.0:1619677474.557553:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.557555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.557555:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.557557:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:17.0:1619677474.557558:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000400:00000200:1.0:1619677474.557558:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.557559:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.557561:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.557562:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.557563:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.557564:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.557566:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.557567:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.557568:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.557569:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.557570:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.557571:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.557572:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.557573:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.557573:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.557576:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.557576:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.557577:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.557577:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.557578:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.557579:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.557580:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.557580:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.557581:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.557581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677474.557582:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.557582:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.557582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)--
00000800:00000200:17.0:1619677474.557584:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:25.2:1619677474.557585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.557585:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:17.0:1619677474.557585:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (22)--
00000800:00000200:17.0:1619677474.557586:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)--
00000800:00000200:25.2:1619677474.557590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.557595:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.557597:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.557598:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.557599:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.557600:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.557600:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.557601:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.557602:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.557603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.557604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.557605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.557605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.557620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce0bf6400] (20)++
00000800:00000200:1.0:1619677474.557626:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:1.0:1619677474.557629:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.57@o2ib3
00000800:00000200:17.0:1619677474.557630:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)--
00000400:00000200:1.0:1619677474.557632:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.57@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.557634:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.557635:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677474.557637:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.557638:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.557639:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.557639:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677474.557642:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.557643:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.557644:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.557645:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.557645:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.557646:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.557647:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:1.0:1619677474.557648:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce0bf6400] (22)++
00000800:00000200:1.0:1619677474.557649:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce0bf6400] (23)--
00000800:00000200:1.0:1619677474.557649:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)--
00000800:00000200:8.2:1619677474.557652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.2:1619677474.557655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce0bf6400] (21)++
00000800:00000200:1.0:1619677474.557661:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)++
00000800:00000200:17.0:1619677474.557662:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.557662:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce0bf6400] (23)--
00000800:00000200:17.0:1619677474.557663:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.557663:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.557663:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (22)--
00000800:00000200:1.0:1619677474.557664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce0bf6400] (21)--
00000400:00000200:17.0:1619677474.557665:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.557665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.557666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677474.558032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.558035:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.558037:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.558040:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.558040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:21.0:1619677474.558041:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.558043:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.558045:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.558047:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.558048:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000400:00000200:20.0:1619677474.558048:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.558049:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.558051:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:20.0:1619677474.558051:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677474.558052:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677474.558055:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.558055:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.558057:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677474.558058:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.558060:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.0:1619677474.558060:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.558060:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:24.0:1619677474.558061:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96503961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.558061:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.558062:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:24.0:1619677474.558063:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.558063:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677474.558064:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.558064:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677474.558065:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.558065:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:24.0:1619677474.558069:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677474.558069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677474.558072:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.558075:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.558075:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:24.0:1619677474.558076:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:1.0:1619677474.558077:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (24)++
00000800:00000200:24.0:1619677474.558079:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (25)--
00000800:00000200:24.0:1619677474.558079:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.558080:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:1.0:1619677474.558080:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (24)--
00000800:00000200:24.0:1619677474.558081:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:17.0:1619677474.558082:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.558082:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.558084:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.558085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.558088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.558090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.558091:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.558092:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.558093:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.558094:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677474.558094:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.558095:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:17.0:1619677474.558096:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (24)--
00000800:00000200:17.0:1619677474.558097:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.558097:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.558098:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.558378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677474.558383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677474.558386:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.558390:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.558391:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.558391:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677474.558392:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.558394:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677474.558395:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.558395:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677474.558397:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.558397:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.558399:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.558400:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.558400:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.558401:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.558401:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.558403:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.558403:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.558405:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.558406:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.558407:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.558407:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.558409:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.558411:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.558412:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.558413:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.558413:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.558413:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.558415:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.558416:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.558416:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.558416:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.558416:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.558417:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.558417:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.558418:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.558419:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.558420:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.558420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.558422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.558431:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677474.558434:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677474.558434:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:6.0:1619677474.558435:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:22.0:1619677474.558436:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.558436:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.558438:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.558438:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.558441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.558442:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.558443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.558449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.558456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.558459:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.558460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.558460:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.558461:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.558461:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.558805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.558813:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.558816:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.558818:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.558820:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.558821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.558821:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.558823:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.558824:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.558825:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.558826:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.558829:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.558831:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.558833:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.558834:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.558834:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.558836:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.558836:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.558838:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.558839:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.558840:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.558841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.558846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.558850:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.558851:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.558852:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.558854:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.558854:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.558855:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.558855:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.558856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.558857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.558858:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.558987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.558993:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.558995:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.558997:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.558998:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.558999:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.559001:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:9.2:1619677474.559001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:21.0:1619677474.559002:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.559004:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.559005:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.559006:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.559007:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.559009:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.559009:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.559010:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.559010:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.559011:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.559011:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.559012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677474.559014:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.559015:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.559015:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.559017:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.559018:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.559018:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.559019:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.559023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.559030:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.559032:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.559033:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.559035:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.559036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.559038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.559174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.559182:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.559184:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.559185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.559187:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.559190:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.559191:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.559193:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.559194:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.559195:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.559196:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.559199:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.559200:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.559203:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.559204:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.559205:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.559206:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.559207:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.559208:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.559209:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.559210:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.559212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.559215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.559220:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.559222:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.559223:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.559224:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677474.559224:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.559224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.559225:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.559226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677474.559226:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.559229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.559836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677474.559844:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.559846:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677474.559847:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677474.559849:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.559852:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.559853:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.559854:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.559856:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.559857:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.559858:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.559861:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.559862:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.559864:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.559865:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.559865:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.559867:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.559867:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.559868:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677474.559870:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677474.559870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677474.559873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:4.2:1619677474.559876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:8.0:1619677474.559877:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.559880:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.559881:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.559881:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.559882:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.559883:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677474.559885:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:8.0:1619677474.559885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:26.0:1619677474.559886:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.559887:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.559895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677474.559898:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.559900:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677474.559901:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.559901:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677474.559902:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.559902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677474.560252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.560260:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.560263:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.560265:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.560267:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.560268:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.560268:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.560269:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.560270:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.560271:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.560272:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.560275:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.560277:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.560279:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.560280:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.560280:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.560282:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.560282:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.560283:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.560284:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.560288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.560295:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.560299:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.560301:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.560302:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.560304:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.560305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.560692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.560700:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.560703:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.560704:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.560706:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.560707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.560708:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.560709:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.560710:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.560711:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.560712:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.560715:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.560716:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.560718:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.560719:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.560719:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.560720:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.560721:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.560722:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.560722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.560727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.560733:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.560736:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.560737:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.560738:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.560740:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.560741:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.561161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.561165:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.561168:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.561170:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.561171:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.561173:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.561175:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.561176:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.561177:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.561178:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.561179:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.561182:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.561184:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.561186:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.561187:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.561187:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.561189:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.561190:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.561190:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.561191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.561195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.561203:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.561205:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.561206:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.561206:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.561208:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.561210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.561244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.561252:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.561255:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.561256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.561257:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.561260:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.561261:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.561262:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.561264:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.561265:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.561266:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.561269:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.561271:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.561273:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.561274:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.561275:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.561276:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.561277:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.561277:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.561278:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.561282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.561288:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.561291:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.561292:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.561292:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.561299:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.561300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.561304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.561311:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.561313:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.561314:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.561314:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.561315:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.561316:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.561495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.561503:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.561505:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.561506:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.561508:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.561510:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.561511:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.561513:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.561514:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.561515:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.561516:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.561519:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.561520:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.561523:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.561524:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.561524:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.561526:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.561526:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.561527:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.561528:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.561532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.561540:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.561543:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.561544:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.561544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.561546:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.561549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.561698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.561706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.561709:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.561711:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.561712:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.561713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.561714:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.561715:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.561717:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.561718:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.561718:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.561721:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.561723:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.561727:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.561727:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.561728:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.561730:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.561730:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.561731:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.561732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.561736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.561744:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.561746:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.561747:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.561749:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.561750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.561752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.561835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.561842:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.561844:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.561845:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.561847:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.561850:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.561852:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.561853:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.561854:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.561855:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.561856:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.561859:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:21.2:1619677474.561861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:10.0:1619677474.561861:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.561863:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.561864:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.561865:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.561866:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.561867:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.561867:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.561868:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:1.0:1619677474.561868:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.561871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.561871:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:9.2:1619677474.561873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:1.0:1619677474.561873:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.561875:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.561877:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.561878:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.561879:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.561880:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:1.0:1619677474.561880:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.561881:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677474.561883:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.561884:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:1.0:1619677474.561884:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.561885:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.561887:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.561887:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677474.561889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677474.561890:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.561890:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.561891:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.561893:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.561893:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.561894:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.561895:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.561899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.561905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.561911:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.561912:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.561912:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.561913:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.561913:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.561916:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.561916:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.561918:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.561919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:21.0:1619677474.561920:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.561920:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.561921:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.561922:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.561923:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.561925:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.561926:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.561929:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.561931:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.561933:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.561934:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.561935:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.561936:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.561937:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.561938:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.561938:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.561942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.561949:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.561951:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.561952:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.561954:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.561955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.561956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.562312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.562320:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.562323:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.562325:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677474.562325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677474.562327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.562328:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.562330:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.562331:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.562332:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.562332:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677474.562333:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.562334:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.562335:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.562335:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677474.562338:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.562339:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.562341:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.562342:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.562343:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677474.562344:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.562345:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.562345:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:17.0:1619677474.562345:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.562346:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.562347:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:17.0:1619677474.562347:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.562348:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:17.0:1619677474.562348:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.562349:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.562351:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000400:00000200:17.0:1619677474.562351:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.562352:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:17.0:1619677474.562353:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677474.562353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.562356:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.562357:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.562357:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.2:1619677474.562358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677474.562359:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.562359:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.562360:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.562360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677474.562361:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.562363:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.562364:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.562365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677474.562365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.562366:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.562367:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.562368:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.562369:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677474.562369:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.562370:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.562373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.562376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.562376:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.562377:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.562379:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.562381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.562387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.562393:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.562396:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.562397:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.562397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.562398:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.562398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.562896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.562903:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.562906:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.562907:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.562909:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.562911:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.562913:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.562914:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.562915:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.562916:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.562917:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.562920:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.562922:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.562924:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.562925:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.562925:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.562927:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.562928:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.562929:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.562930:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.562931:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.562933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.562936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677474.562941:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677474.562942:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.562944:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677474.562944:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.562945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.562945:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677474.562945:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.562947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677474.562947:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.562949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.562954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.562962:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.562965:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.562966:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.562966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.562967:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.562967:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.562981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677474.562989:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.562992:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.562993:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.562995:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.562998:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.562999:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677474.563000:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.563001:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.563003:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.563004:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677474.563007:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.563009:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677474.563011:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.563012:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.563013:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677474.563014:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.563014:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.563015:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.563016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.563019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.563023:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.563026:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.563027:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.563027:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.563028:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.563030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677474.563203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.563210:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.563213:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.563214:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.563215:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.563218:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.563219:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.563220:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.563222:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.563222:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.563223:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.563226:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.563228:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.563230:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.563231:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.563232:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.563233:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.563234:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.563235:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.563236:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.563237:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677474.563239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677474.563242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677474.563248:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677474.563250:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677474.563250:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.563251:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.563252:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677474.563252:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.563253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677474.563253:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.563254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677474.563257:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.563317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.563324:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.563327:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.563328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.563330:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.563332:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.563334:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.563335:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.563337:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.563338:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.563339:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.563342:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.563343:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.563346:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.563347:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.563347:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.563349:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.563350:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.563350:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.563351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.563356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.563364:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.563367:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.563368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.563368:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.563371:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.563373:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.563623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.563631:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.563634:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.563636:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.563638:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.563639:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.563640:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.563642:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.563643:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.563644:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.563645:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.563648:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.563650:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.563651:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.563652:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.563652:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.563654:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.563654:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.563655:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.563656:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.563660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.563666:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.563669:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.563669:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.563671:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.563673:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.563674:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.563950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.563957:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.563960:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.563962:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.563963:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.563964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.563965:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.563966:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.563967:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.563968:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.563968:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.563971:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.563973:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.563975:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.563976:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.563976:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.563978:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.563978:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.563979:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.563980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.563984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.563991:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.563993:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.563994:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.563994:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.563996:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.563998:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.564308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.564316:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.564318:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.564319:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.564321:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.564324:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.564325:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.564326:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.564328:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.564328:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.564329:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.564332:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.564334:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.564336:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.564337:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.564338:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.564339:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.564340:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.564340:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.564341:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.564346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.564354:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.564357:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.564358:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.564358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.564360:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.564363:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.564366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.564373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.564375:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.564376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.564376:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.564377:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.564378:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.564749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:4.2:1619677474.564752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677474.564756:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.564759:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:8.0:1619677474.564760:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677474.564760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.564762:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.564763:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.564763:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:18.0:1619677474.564764:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.564765:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.564765:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.564766:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.564768:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.564769:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.564769:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.564770:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.564771:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.564772:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96503935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.564774:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.564774:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.564775:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.564776:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.564777:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.564779:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.564780:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:8.0:1619677474.564780:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.564781:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.564782:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.564782:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.564783:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.564783:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.564784:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.564785:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677474.564785:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.564785:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.564786:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:9.2:1619677474.564788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.564788:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.564788:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.564789:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.564790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.2:1619677474.564791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677474.564794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.564797:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.564798:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.564799:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.564799:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.564800:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.564800:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.564801:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.564802:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:17.0:1619677474.564802:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:2.0:1619677474.564802:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.564803:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677474.564805:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.564806:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.564807:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.564809:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.564811:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.565538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.565546:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.565549:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.565551:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.565553:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.565553:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.565555:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.565556:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.565557:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.565558:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.565559:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.565563:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.565565:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.565566:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.565567:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.565568:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.565569:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.565570:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.565571:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.565572:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.565573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.565575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.565579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.565584:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.565585:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.565586:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.565587:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.565587:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.565588:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.565589:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.565590:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.565591:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.565592:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.565592:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.565593:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.565593:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.565594:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.565594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.565697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677474.565702:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.565704:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677474.565707:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.565707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677474.565709:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.565710:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677474.565712:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.565713:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.565714:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.565715:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677474.565717:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.565719:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.565721:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.565722:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.565723:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.565724:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.565725:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.565726:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677474.565726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.565731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.565739:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.565741:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.565742:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.565743:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.565744:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.565745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.565854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.565862:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.565864:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.565866:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.565868:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.565868:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.565870:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.565871:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.565872:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.565873:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.565874:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.565877:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.565879:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.565881:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.565882:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.565882:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.565884:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.565885:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.565886:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.565886:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.565890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.565896:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.565899:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.565900:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.565900:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.565901:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.565904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.565911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.565918:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.565920:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.565921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.565921:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.565922:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.565923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.565943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677474.565950:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.565953:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.565953:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677474.565956:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.565958:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.565960:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.565961:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.565962:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.565963:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.565964:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.565967:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.565968:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.565970:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.565971:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.565972:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.565973:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.565973:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.565974:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.565974:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.565978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.565981:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.565983:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677474.565984:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.565985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677474.565986:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.565988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677474.566299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.566307:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.566310:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.566312:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.566314:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.566315:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.566316:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.566317:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.566318:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.566319:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.566320:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.566323:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.566325:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.566328:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.566329:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.566330:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.566332:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.566332:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.566333:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.566333:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.566338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.566342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.0:1619677474.566346:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.566349:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677474.566350:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:21.0:1619677474.566350:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.566352:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:21.0:1619677474.566352:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.566352:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.566353:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.566355:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.566355:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.566356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000400:00000200:26.0:1619677474.566358:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.566360:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.566360:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.566362:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.566363:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.566364:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677474.566364:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.566366:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.566367:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.566367:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.566368:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.566368:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.566369:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.0:1619677474.566369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677474.566372:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.566373:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.566373:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.566375:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.566375:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.566377:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.566378:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.566379:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.566382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.566384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.566390:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.566391:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.566393:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677474.566393:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677474.566393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.566394:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.566394:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.566395:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677474.566396:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.566399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.566811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.566815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.566817:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.566819:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.566820:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.566821:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.566823:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.566824:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.566825:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.566826:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.566827:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.566831:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.566832:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.566835:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.566836:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.566837:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.566839:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.566839:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.566840:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.566840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.566845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.566851:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.566854:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.566855:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.566855:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.566857:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.566860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.566970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.566975:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.566978:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:5.2:1619677474.566979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677474.566980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.566981:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677474.566984:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.566986:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:1.0:1619677474.566986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:21.0:1619677474.566987:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.566988:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:21.0:1619677474.566989:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.566989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:21.0:1619677474.566990:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.566991:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.566992:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.566994:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677474.566994:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.566995:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:21.0:1619677474.566996:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.566997:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.566999:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677474.566999:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.567000:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.567000:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:1.0:1619677474.567000:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.567001:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677474.567002:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.567002:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.567003:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.567003:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677474.567005:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.567007:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.567009:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.567010:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.567011:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.567012:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.567013:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.567014:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.567014:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.567019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:8.2:1619677474.567019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677474.567026:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:20.0:1619677474.567027:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.567029:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:6.0:1619677474.567029:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.567030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:20.0:1619677474.567030:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.567030:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.567032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.567032:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000400:00000200:6.0:1619677474.567033:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.567034:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677474.567035:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677474.567039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.567047:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.567050:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:4.2:1619677474.567050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.0:1619677474.567051:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.567051:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.567052:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.567053:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677474.567058:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.567061:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.567062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.567064:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.567066:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.567068:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.567069:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.567071:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.567071:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.567073:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.567076:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.567078:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.567080:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.567082:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.567082:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.567084:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.567084:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.567085:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.567087:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.567087:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.567090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.567093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.2:1619677474.567094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677474.567096:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.567098:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:20.0:1619677474.567098:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677474.567099:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.567100:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677474.567100:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.567100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.567102:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.567102:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:21.0:1619677474.567103:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.567103:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.567103:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.567105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:20.0:1619677474.567105:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.567105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677474.567106:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.567107:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.567109:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.567110:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.567111:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.567114:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.567116:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.567118:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.567119:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.567119:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.567121:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.567121:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.567122:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.567123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677474.567133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:8.2:1619677474.567137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:31.0:1619677474.567140:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.567143:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000800:00000200:21.0:1619677474.567144:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:31.0:1619677474.567146:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.567147:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.567147:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.567148:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677474.567149:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.567149:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.567150:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677474.567150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.567151:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:31.0:1619677474.567152:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.567153:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.567154:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.567155:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677474.567159:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.567160:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.567163:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.567164:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677474.567165:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677474.567167:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677474.567167:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.567169:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:31.0:1619677474.567171:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677474.567171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:9.2:1619677474.567173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.567177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:18.0:1619677474.567181:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.567184:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:30.0:1619677474.567185:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000800:00000200:2.0:1619677474.567185:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.567186:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.567188:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000400:00000200:18.0:1619677474.567188:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.567189:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.567190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:18.0:1619677474.567192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.567197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.567205:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.567208:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.567208:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.567209:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.567210:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.567210:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.567485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.567493:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.567496:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.567499:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.567501:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.567502:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.567503:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.567505:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.567506:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.567507:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.567508:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.567511:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.567512:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.567515:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.567515:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.567516:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.567517:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.567518:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.2:1619677474.567518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.567519:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.567521:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677474.567523:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.567524:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:9.2:1619677474.567524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:30.0:1619677474.567526:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.567527:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.567527:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.567528:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.567530:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.567531:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.567531:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.567531:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:30.0:1619677474.567532:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677474.567533:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.567534:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.567534:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.567536:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.567537:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.567538:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677474.567539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.567540:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.567541:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.567541:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.567543:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.567543:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.567544:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.567544:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.567548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.567555:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.567557:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.567558:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.567559:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.567561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.567562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.568123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677474.568131:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.568133:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.568134:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.568136:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.568138:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.568139:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677474.568141:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.568142:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.568143:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.568144:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677474.568146:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.568148:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677474.568150:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.568151:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.568151:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677474.568152:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.568153:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.568154:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.568154:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.568158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.568161:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.568163:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677474.568164:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.568165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677474.568166:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.568168:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677474.568238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.568245:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.568248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.568248:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.568250:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.568253:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.568254:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.568255:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.568256:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.568257:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.568258:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.568261:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.568263:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.568265:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.568266:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.568267:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.568268:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.568269:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.568270:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.568270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.568274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.568283:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.568285:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.568286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.568287:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.568289:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.568292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.568319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.568326:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:4.2:1619677474.568328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.568329:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.568331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.568332:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:10.0:1619677474.568332:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.568333:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.568335:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.568337:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.568337:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677474.568338:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.568340:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.568341:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.568342:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677474.568344:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.568346:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.568346:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.568347:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.568348:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677474.568349:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.568350:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.568351:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.568354:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.568354:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.568355:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677474.568356:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.568356:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677474.568358:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677474.568359:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.568359:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.568360:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.568360:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.568360:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.568361:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677474.568362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.568363:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.568363:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.568365:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:9.2:1619677474.568365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677474.568366:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.568366:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677474.568368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677474.568372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677474.568375:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.568377:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.568377:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:18.0:1619677474.568378:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.568378:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:10.0:1619677474.568379:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:2.0:1619677474.568379:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.568380:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.568381:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:2.0:1619677474.568381:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.568382:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.568383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677474.568385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.568385:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.568386:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677474.568386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677474.568524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.568530:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.568533:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.568533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.568536:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.568537:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.568539:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.568540:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.568541:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.568542:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.568543:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.568547:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.568548:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.568551:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.568552:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.568553:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.568555:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.568555:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.568556:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.568556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.568561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.568568:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.568570:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.568571:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.568573:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.568575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.568576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.568738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.568745:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.568748:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.568750:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.568752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.568752:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.568753:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.568755:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.568756:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.568757:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.568758:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.568761:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.568763:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.568765:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.568766:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.568767:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.568768:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.568769:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.568770:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.568770:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.568774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.568780:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.568783:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.568783:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.568785:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.568786:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.568787:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.568832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.568839:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.568842:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.568843:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.568845:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.568847:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.568849:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.568850:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.568851:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.568852:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.568853:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.568856:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.568857:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.568860:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.568861:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.568861:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.568863:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.568863:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.568864:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.568865:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.568869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.568877:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.568880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.568880:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.568881:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.568883:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.568886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.569455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.569464:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.569466:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.569469:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.569470:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.569471:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.569472:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.569473:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.569474:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.569475:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.569476:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.569479:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.569480:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.569483:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.569483:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.569484:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.569485:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.569486:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.569488:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.569489:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.569490:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.569492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.569496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.569499:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.569502:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.569503:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.569503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.569504:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.569505:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.569507:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.569507:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677474.569507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.569509:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.569859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.569867:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.569869:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.569870:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.569872:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.569875:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.569876:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.569877:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.569878:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.569879:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.569880:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.569883:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.569884:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.569886:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.569887:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.569888:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.569889:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.569889:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.569890:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.569891:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.569894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.569901:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.569903:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.569904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.569904:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.569906:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.569908:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.570131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.570138:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.570141:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.570143:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.570146:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.570146:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.570147:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.570148:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.570149:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.570150:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.570151:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.570154:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.570156:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.570158:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.570158:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.570159:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.570160:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.570161:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.570162:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.570163:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.570163:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.570166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.570169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.570174:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.570175:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.570176:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.570177:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.570177:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.570178:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.570178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.570179:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.570180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.570181:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.570421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.570425:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.570428:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.570430:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.570431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.570432:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.570433:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.570435:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.570436:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.570437:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.570438:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.570440:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.570442:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.570445:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.570446:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.570446:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.570448:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.570449:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.570450:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.570451:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.570452:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.570455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.570462:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.570464:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.570465:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.570465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.570468:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.570470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.570470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:9.2:1619677474.570471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.570475:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.570477:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:8.0:1619677474.570477:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.570478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.570479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677474.570480:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.570481:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.570481:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.570482:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.570483:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.570669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.570677:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.570679:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.570680:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.570682:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.570685:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.570686:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.570688:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.570689:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.570690:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.570691:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.570694:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.570696:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.570699:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.570699:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.570700:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.570702:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.570702:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.570703:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.570703:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.570708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.570716:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.570719:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.570719:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.570720:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.570722:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.570725:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.571050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677474.571051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677474.571058:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.571058:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677474.571061:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677474.571061:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.571062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677474.571064:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.571065:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.571066:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.571067:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.571068:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:24.0:1619677474.571068:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.571069:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677474.571070:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.571071:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.571071:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.571072:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.571072:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.571073:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677474.571073:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.571074:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677474.571078:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.571079:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.571080:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677474.571081:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677474.571082:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.571082:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.571083:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.571084:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677474.571084:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.571085:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.571085:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.571085:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677474.571086:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.571086:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677474.571087:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.571088:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.571088:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.571089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.571090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.2:1619677474.571093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677474.571097:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.571099:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.571100:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.571100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:31.0:1619677474.571101:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.571102:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:31.0:1619677474.571104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.571105:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:17.0:1619677474.571105:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:31.0:1619677474.571106:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:17.0:1619677474.571106:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.571106:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.571108:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.571109:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:25.2:1619677474.571109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000400:00000200:31.0:1619677474.571110:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.571112:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.571112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677474.571113:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.571114:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.571115:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.571115:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.571116:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.571117:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.571117:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000400:00000200:31.0:1619677474.571118:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.571118:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.571118:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677474.571119:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.571119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:31.0:1619677474.571122:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.571123:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.571126:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.571126:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.571127:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.571129:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.571129:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.571130:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.571131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.571135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.571142:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.571144:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.571145:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.571145:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.571147:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.571149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.571302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.571307:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.571310:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.571311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.571313:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.571316:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.571317:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.571318:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.571319:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.571321:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.571321:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.571325:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.571327:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.571329:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.571330:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.571331:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.571332:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.571333:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.571334:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.571335:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.571335:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.571339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.571341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.571347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677474.571349:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.571349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677474.571350:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.571351:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677474.571352:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.571353:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.571355:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.571356:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.571357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677474.571738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.571744:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.571747:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.571748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.571749:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.571752:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.571753:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.571755:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.571756:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.571757:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.571758:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.571761:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.571762:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.571765:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.571766:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.571766:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.571768:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.571768:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.571769:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.571770:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.571771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.2:1619677474.571772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:25.2:1619677474.571775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.571776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677474.571778:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.571780:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.571782:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:17.0:1619677474.571782:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.571782:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:2.0:1619677474.571783:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.571784:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.571785:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.571785:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.571786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.571786:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.571786:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.571787:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.571789:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:2.0:1619677474.571789:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.571790:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.571791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:2.0:1619677474.571792:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677474.571793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000400:00000200:2.0:1619677474.571793:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.571794:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.571797:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.571799:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677474.571800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677474.571801:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.571801:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:22.0:1619677474.571802:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677474.571802:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677474.571803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.571804:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.571804:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677474.571804:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677474.571805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677474.571805:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.571806:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.571806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.571810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.571818:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.571820:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.571821:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.571821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.571823:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.571825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.571988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.571992:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.571994:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.571997:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.571998:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.571999:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.572000:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.572002:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.572003:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.572004:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.572005:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.572008:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.572010:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.572012:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.572013:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.572014:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.572015:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.572016:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.572017:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.572018:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.572021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.572029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.572032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.572032:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.572033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.572035:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.572037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.572166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.572174:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.572178:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.572180:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.572182:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.572183:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.572185:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677474.572186:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.572188:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.572189:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.572190:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677474.572193:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.572194:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677474.572197:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.572198:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:13.0:1619677474.572199:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:13.0:1619677474.572200:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677474.572201:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.572202:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.572202:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677474.572206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.572214:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.572217:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.572218:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.572218:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.572220:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.572224:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.572284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.572292:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.572299:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.572300:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.572302:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.572305:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.572306:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.572308:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.572309:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.572310:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.572311:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.572313:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.572315:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.572317:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.572317:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.572318:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.572319:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.572320:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.572321:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.572321:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.572326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.572333:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.572335:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.572336:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.572336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.572338:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.572340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.572463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.572471:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.572474:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.572476:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.572478:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.572479:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.572479:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.572480:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.572481:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.572482:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.572483:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.572486:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.572488:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.572490:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.572490:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.572491:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.572492:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.572493:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.572494:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.572494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.572498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.572505:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.572507:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.572508:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.572509:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.572511:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.572512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.572796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677474.572804:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.572807:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677474.572808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677474.572809:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.572812:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.572814:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677474.572815:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.572816:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.572817:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.572818:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677474.572821:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.572823:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677474.572826:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.572826:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:8.0:1619677474.572827:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677474.572828:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.0:1619677474.572829:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.572830:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.572830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:25.2:1619677474.572834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.572841:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.572844:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.572844:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.572846:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.572848:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.572849:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.572963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.572971:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.572973:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.572974:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.572976:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.572979:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.572980:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.572981:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.572982:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.572983:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.572984:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.572987:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.572988:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.572991:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.572991:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.572992:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.572993:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.572994:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.572994:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.572995:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.572999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.573006:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.573008:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.573009:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.573010:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.573012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.573014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.573017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677474.573023:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.573026:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.573027:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.573028:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.573028:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.573029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.573047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (20)++
00000800:00000200:31.0:1619677474.573055:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:31.0:1619677474.573058:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000400:00000200:31.0:1619677474.573060:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.573062:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.573063:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677474.573063:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000400:00000200:31.0:1619677474.573065:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.573066:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.573067:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.573068:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.573071:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677474.573071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.573073:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.573075:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.573075:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.573076:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.573077:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:4.2:1619677474.573077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:31.0:1619677474.573078:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (22)++
00000800:00000200:31.0:1619677474.573079:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (23)--
00000800:00000200:30.0:1619677474.573079:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.573080:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:30.0:1619677474.573080:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677474.573081:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:31.0:1619677474.573083:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000400:00000200:30.0:1619677474.573083:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677474.573083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:31.0:1619677474.573085:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.573085:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.573086:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.573087:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.573089:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.573089:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.573089:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:30.0:1619677474.573090:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.2:1619677474.573090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:31.0:1619677474.573092:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.573092:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:31.0:1619677474.573093:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677474.573093:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.573093:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.573093:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:31.0:1619677474.573095:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.573095:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:24.0:1619677474.573095:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.573096:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:31.0:1619677474.573097:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.573097:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.573099:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677474.573099:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.573100:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:24.0:1619677474.573101:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.573102:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.573102:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:31.0:1619677474.573103:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.573103:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.573104:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.573105:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.573105:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.573106:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000400:00000200:2.0:1619677474.573106:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677474.573107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.573108:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.573108:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677474.573108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:2.0:1619677474.573108:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.573109:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677474.573109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:9.2:1619677474.573110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:2.0:1619677474.573110:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677474.573111:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677474.573111:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (21)++
00000400:00000200:2.0:1619677474.573111:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96503893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.573112:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:31.0:1619677474.573112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)--
00000400:00000200:2.0:1619677474.573113:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.573114:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.573115:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:9.2:1619677474.573116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:8.0:1619677474.573116:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:2.0:1619677474.573118:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.573119:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.573120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.573120:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.573121:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677474.573122:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.573123:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677474.573125:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.573125:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677474.573126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (20)++
00000800:00000200:2.0:1619677474.573126:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.573126:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.573127:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.573128:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.573129:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:18.0:1619677474.573130:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.573130:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.573131:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.573131:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:2.0:1619677474.573133:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677474.573134:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:25.2:1619677474.573134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.573134:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:30.0:1619677474.573137:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000400:00000200:30.0:1619677474.573140:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.573141:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000400:00000200:30.0:1619677474.573142:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.573142:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:9.2:1619677474.573142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000400:00000200:30.0:1619677474.573144:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677474.573144:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:30.0:1619677474.573145:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.573145:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.573147:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.573147:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:30.0:1619677474.573148:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677474.573148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:30.0:1619677474.573149:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.573149:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:13.0:1619677474.573150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677474.573152:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677474.573153:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.573153:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.573153:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.573154:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:30.0:1619677474.573155:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.573155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:30.0:1619677474.573158:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.573159:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.573160:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.573161:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.573162:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:4.2:1619677474.573163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (22)++
00000800:00000200:30.0:1619677474.573164:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3b0e00] (22)++
00000800:00000200:30.0:1619677474.573166:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (24)--
00000800:00000200:30.0:1619677474.573167:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (23)--
00000800:00000200:30.0:1619677474.573168:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)++
00000800:00000200:25.2:1619677474.573168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.573170:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000800:00000200:31.0:1619677474.573172:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (23)++
00000400:00000200:30.0:1619677474.573172:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.573173:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3b0e00] (24)--
00000400:00000200:30.0:1619677474.573173:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.573174:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (23)--
00000800:00000200:31.0:1619677474.573175:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000400:00000200:30.0:1619677474.573175:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:22.0:1619677474.573175:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677474.573176:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.573177:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.573178:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.573178:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677474.573178:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.573179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.573180:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.573182:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.573183:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677474.573184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677474.573185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.573186:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677474.573188:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.573189:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:30.0:1619677474.573189:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677474.573190:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677474.573191:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:30.0:1619677474.573191:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:30.0:1619677474.573192:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:30.0:1619677474.573192:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)--
00000800:00000200:21.0:1619677474.573192:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.573194:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.573195:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677474.573196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:21.0:1619677474.573197:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.573198:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.573199:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677474.573200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000400:00000200:21.0:1619677474.573201:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.573202:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.573203:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:12.0:1619677474.573203:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.573205:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:21.0:1619677474.573206:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677474.573206:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.573207:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.573208:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:12.0:1619677474.573208:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.573209:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677474.573210:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.573210:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.573211:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.573211:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:12.0:1619677474.573211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:6.0:1619677474.573211:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.573212:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677474.573213:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.573214:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.573214:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677474.573214:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.573215:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.573217:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.573217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.573220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:25.2:1619677474.573223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.2:1619677474.573223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:13.0:1619677474.573229:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677474.573230:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.573231:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.573231:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677474.573232:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.573232:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.573233:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677474.573233:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677474.573234:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.573234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:13.0:1619677474.573234:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:21.0:1619677474.573235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677474.573235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677474.573236:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677474.573237:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.573238:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:4.2:1619677474.573258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (20)++
00000800:00000200:31.0:1619677474.573265:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:31.0:1619677474.573268:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000400:00000200:31.0:1619677474.573270:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.573272:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.573272:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000400:00000200:31.0:1619677474.573274:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:31.0:1619677474.573275:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.573277:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.573278:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.573279:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677474.573282:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.573284:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677474.573286:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.573287:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:31.0:1619677474.573287:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:31.0:1619677474.573289:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:31.0:1619677474.573289:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:31.0:1619677474.573290:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:31.0:1619677474.573291:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)--
00000800:00000200:9.2:1619677474.573296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.573303:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.573305:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.573306:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.573306:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.573308:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.573310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677474.573312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (20)++
00000800:00000200:30.0:1619677474.573320:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:30.0:1619677474.573323:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000400:00000200:30.0:1619677474.573325:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.573327:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.573328:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000400:00000200:30.0:1619677474.573329:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677474.573330:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.573331:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.573332:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.573333:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677474.573336:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.573337:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677474.573340:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.573341:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:30.0:1619677474.573341:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677474.573343:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:30.0:1619677474.573344:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:30.0:1619677474.573345:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:30.0:1619677474.573345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)--
00000800:00000200:8.2:1619677474.573350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.573357:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:4.2:1619677474.573359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (20)++
00000800:00000200:20.0:1619677474.573361:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.573363:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.573365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.573365:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.573368:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:20.0:1619677474.573368:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.573369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:31.0:1619677474.573371:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000400:00000200:31.0:1619677474.573373:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.573375:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000400:00000200:31.0:1619677474.573376:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.573377:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:21.0:1619677474.573377:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:31.0:1619677474.573379:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.573380:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677474.573381:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.573381:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000400:00000200:31.0:1619677474.573382:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.573382:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677474.573382:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.573382:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.573384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:31.0:1619677474.573386:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.573388:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.573389:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.573390:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.573391:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677474.573391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:31.0:1619677474.573392:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.573393:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (22)++
00000800:00000200:31.0:1619677474.573394:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3b0e00] (23)++
00000800:00000200:31.0:1619677474.573395:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (24)--
00000800:00000200:31.0:1619677474.573396:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (23)--
00000800:00000200:31.0:1619677474.573397:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)++
00000800:00000200:31.0:1619677474.573399:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000800:00000200:25.2:1619677474.573399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677474.573400:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.573400:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (23)++
00000800:00000200:30.0:1619677474.573401:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3b0e00] (24)--
00000400:00000200:31.0:1619677474.573402:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.573402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (23)--
00000400:00000200:31.0:1619677474.573403:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.573404:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.573404:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:31.0:1619677474.573405:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.573405:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.573406:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.573406:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.573406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:31.0:1619677474.573407:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.573408:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.573410:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.573410:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.573411:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.573411:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:30.0:1619677474.573412:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.573414:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.573414:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.573414:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.573415:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.573415:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677474.573416:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.573417:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (21)++
00000400:00000200:30.0:1619677474.573417:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.573418:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:31.0:1619677474.573418:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)--
00000400:00000200:30.0:1619677474.573419:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.573420:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.573420:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.573421:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.573422:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677474.573422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:30.0:1619677474.573423:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.573424:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.573427:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.573428:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:30.0:1619677474.573429:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.573430:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:30.0:1619677474.573431:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677474.573431:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.573432:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.573432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.573433:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:24.0:1619677474.573433:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.573434:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:30.0:1619677474.573435:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.573436:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.573436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677474.573436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:9.2:1619677474.573440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.573446:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.573448:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.573449:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:8.0:1619677474.573449:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.573449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (20)++
00000800:00000200:24.0:1619677474.573450:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677474.573450:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.573451:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)++
00000800:00000200:24.0:1619677474.573452:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (24)--
00000800:00000200:8.0:1619677474.573452:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (24)--
00000800:00000200:24.0:1619677474.573453:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.573454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.573456:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:31.0:1619677474.573459:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000400:00000200:31.0:1619677474.573461:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.573463:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.573463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000400:00000200:31.0:1619677474.573464:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677474.573465:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.573467:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.573467:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.573468:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677474.573471:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.573473:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677474.573475:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.573475:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:31.0:1619677474.573476:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:31.0:1619677474.573478:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677474.573478:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:31.0:1619677474.573479:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:31.0:1619677474.573480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)--
00000800:00000200:25.2:1619677474.573483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.573491:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.573493:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.573494:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.573495:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.573497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.573498:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.573501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (20)++
00000800:00000200:30.0:1619677474.573508:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:30.0:1619677474.573511:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000400:00000200:30.0:1619677474.573513:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.573515:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000400:00000200:30.0:1619677474.573515:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.573517:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677474.573518:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.573519:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.573520:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.573521:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677474.573524:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.573526:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677474.573529:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.573530:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:30.0:1619677474.573530:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677474.573532:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:30.0:1619677474.573532:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:30.0:1619677474.573533:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:30.0:1619677474.573534:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)--
00000800:00000200:8.2:1619677474.573538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.573547:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:4.2:1619677474.573548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (20)++
00000800:00000200:17.0:1619677474.573550:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.573550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.573551:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.573553:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.573555:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:17.0:1619677474.573556:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677474.573558:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000400:00000200:31.0:1619677474.573560:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677474.573561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:30.0:1619677474.573562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000400:00000200:31.0:1619677474.573563:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.573565:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:31.0:1619677474.573566:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.573567:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.573568:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.573569:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.573569:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000400:00000200:31.0:1619677474.573572:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.573572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.573572:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.573573:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.573574:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:31.0:1619677474.573575:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.573575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.573577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677474.573578:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.573579:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:31.0:1619677474.573579:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677474.573581:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:31.0:1619677474.573582:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:31.0:1619677474.573583:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3b0e00] (22)++
00000800:00000200:24.0:1619677474.573584:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677474.573585:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (23)--
00000800:00000200:31.0:1619677474.573585:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:24.0:1619677474.573586:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.2:1619677474.573587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677474.573588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (21)++
00000400:00000200:24.0:1619677474.573589:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.573592:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.573593:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677474.573593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.573595:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96503881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.573596:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)++
00000400:00000200:24.0:1619677474.573596:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.573596:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:24.0:1619677474.573597:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.573598:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.573598:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:30.0:1619677474.573599:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000400:00000200:17.0:1619677474.573599:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.573599:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:30.0:1619677474.573601:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.573602:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.573603:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (23)++
00000800:00000200:31.0:1619677474.573603:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3b0e00] (24)--
00000400:00000200:24.0:1619677474.573603:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.573604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (23)--
00000800:00000200:31.0:1619677474.573605:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000400:00000200:30.0:1619677474.573605:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.573605:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.573606:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:30.0:1619677474.573607:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.573608:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.573608:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677474.573609:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.573609:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:30.0:1619677474.573610:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.573610:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677474.573611:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677474.573611:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.573612:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.573613:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:30.0:1619677474.573614:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.573615:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.573615:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677474.573616:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677474.573617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.573619:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.573620:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.573620:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.573621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677474.573622:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.573622:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:30.0:1619677474.573623:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:30.0:1619677474.573624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)--
00000800:00000200:22.0:1619677474.573625:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.573628:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.2:1619677474.573628:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677474.573629:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.573629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.573632:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.573633:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.573634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.573635:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.573636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677474.573636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677474.573637:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.573639:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.573640:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.573640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3b0e00] (20)++
00000400:00000200:21.0:1619677474.573642:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.573644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.573646:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:31.0:1619677474.573648:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:31.0:1619677474.573651:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.3@o2ib3
00000400:00000200:31.0:1619677474.573653:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.3@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.573655:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (22)--
00000400:00000200:31.0:1619677474.573656:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.573657:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677474.573659:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.573660:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.573661:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.573662:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677474.573666:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.573667:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.573669:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.573670:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.573670:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.573672:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.573673:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (21)++
00000800:00000200:31.0:1619677474.573673:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3b0e00] (22)--
00000800:00000200:31.0:1619677474.573674:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3b0e00] (21)--
00000800:00000200:25.2:1619677474.573678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.573684:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.573687:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.573687:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.573689:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.573691:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.573692:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.573731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.573739:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.573741:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.573742:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.573744:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.573746:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.573747:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.573749:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.573750:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.573751:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.573751:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.573755:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.573756:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.573759:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.573760:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.573760:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.573762:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.573762:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.573763:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.573765:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.573765:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.573768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.573771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.573775:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.573778:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.573778:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.573779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.573779:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.573780:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.573781:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.573781:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.573782:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.573784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.574691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.574699:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.574702:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.574704:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.574706:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.574706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.574707:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.574709:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.574710:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.574711:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.574712:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.574715:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.574716:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.574718:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.574719:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.574720:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.574721:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.574722:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.574723:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.574723:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.574728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.574735:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.574737:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.574738:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.574738:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.574740:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.574743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.574745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.574751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.574754:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.574754:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.574755:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.574755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.574756:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.574948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.574952:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.574955:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.574957:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.574958:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.574959:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.574960:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.574961:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.574963:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.574964:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.574965:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.574968:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.574970:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.574972:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.574973:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.574973:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.574975:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.574976:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.574977:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.574978:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.574979:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677474.574981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.574985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.574990:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.574991:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.574991:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.574992:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:20.0:1619677474.574992:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.574993:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.574993:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.574994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.574996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677474.574996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:21.2:1619677474.575018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.575025:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.575028:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.575030:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.575032:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.575033:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.575034:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.575036:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.575037:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.575038:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.575039:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.575042:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.575044:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.575047:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.575047:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.575048:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.575049:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.575050:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.575052:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.575053:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.575053:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.575055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.575059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.575061:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.575064:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.575065:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.575065:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.575067:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.575067:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.575069:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.575069:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.575070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.575071:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.575340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677474.575347:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.575350:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.575351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.575353:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.575356:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.575357:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677474.575358:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.575360:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.575361:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.575362:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677474.575365:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.575366:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677474.575368:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.575369:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.575370:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677474.575371:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.575372:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.575372:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677474.575374:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677474.575374:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677474.575377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.575380:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:4.2:1619677474.575380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:24.0:1619677474.575382:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677474.575383:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.575383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677474.575385:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.575387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:10.0:1619677474.575387:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677474.575389:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677474.575390:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.575391:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.575401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:8.0:1619677474.575405:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.575407:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677474.575408:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:8.0:1619677474.575408:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.575409:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (23)--
00000800:00000200:8.0:1619677474.575409:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677474.575496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.575504:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.575506:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.575507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.575508:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.575511:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.575512:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.575513:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.575514:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.575515:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.575516:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.575519:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.575520:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.575522:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.575522:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.575523:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.575525:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.575525:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.575526:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.575526:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.575531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.575537:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:4.2:1619677474.575537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677474.575539:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.575540:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.575540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.575542:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.575544:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.575544:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.575547:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.575548:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.575549:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.575552:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.575553:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.575554:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677474.575554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:10.0:1619677474.575556:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.575556:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.575557:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677474.575560:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:10.0:1619677474.575560:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.575561:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.0:1619677474.575562:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.575564:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677474.575564:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.575565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:10.0:1619677474.575565:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:21.0:1619677474.575566:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.575566:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:21.0:1619677474.575567:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677474.575567:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.575568:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.575568:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:21.0:1619677474.575569:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.575569:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:21.0:1619677474.575570:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.575571:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.575572:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677474.575573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:21.0:1619677474.575575:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.575577:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.575580:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.575581:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:21.0:1619677474.575581:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.575581:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.575583:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.575583:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677474.575584:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:21.0:1619677474.575584:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000400:00000200:22.0:1619677474.575585:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.575585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.575585:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.575587:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.575589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.575592:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.575593:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.575595:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.575596:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.575596:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.575597:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.575600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.575833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.575841:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.575844:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.575846:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.575848:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.575848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.575850:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.575851:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.575852:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.575853:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.575854:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.575857:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.575859:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.575862:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.575863:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.575864:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.575865:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.575866:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.575867:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.575867:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.575871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.575880:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.575882:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.575883:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.575885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.575885:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.575888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.576654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.576662:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.576665:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.576667:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.576669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.576670:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.576672:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.576673:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.576674:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.576675:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.576676:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.576679:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.576681:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.576683:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.576684:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.576684:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.576686:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.576686:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.576688:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.576690:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.576690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.576692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.576696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.576699:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.576702:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.576703:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.576703:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.576705:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677474.576705:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.576707:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677474.576707:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.576708:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.576709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:5.2:1619677474.576947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.576953:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.576956:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.576957:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.576959:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.576962:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.576963:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.576964:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.576966:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.576967:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.576968:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.576971:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.576973:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.576975:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.576976:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.576977:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.576979:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.576979:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.576980:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.576980:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.576985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.576992:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.576995:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.576996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.576996:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.576998:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.577000:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.577031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.577039:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.577041:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.577044:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.577045:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.577046:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.577048:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.577049:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.577050:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.577051:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.577052:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.577055:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.577057:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.577060:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.577060:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.577061:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.577063:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.577064:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.577064:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.577065:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.577068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.577075:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.577077:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.577078:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.577078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.577080:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.577083:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.577089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.577095:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.577098:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.577098:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.577098:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.577099:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.577100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.577631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.577639:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.2:1619677474.577639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677474.577641:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.577642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.577643:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.577645:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:24.0:1619677474.577646:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.577647:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:24.0:1619677474.577648:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.577649:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.577650:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.577650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:24.0:1619677474.577651:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.577652:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.577652:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.577653:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.577655:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.577656:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.577657:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677474.577658:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.577659:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.577660:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.577660:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.577663:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.577664:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:21.0:1619677474.577664:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.577665:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.577666:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.577666:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.577667:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.577668:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.577668:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677474.577669:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.577670:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.577670:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.577672:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:25.2:1619677474.577673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.577673:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.577673:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.577674:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.577678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.577682:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.577684:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.577685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.577686:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.577687:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:22.0:1619677474.577688:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.577689:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.577690:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.577691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677474.577691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:25.2:1619677474.577692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000400:00000200:17.0:1619677474.577693:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.577695:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677474.577700:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.577703:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.577704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.577704:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.577705:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.2:1619677474.577706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677474.577706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.577715:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.577718:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.577720:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.577722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.577723:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.577724:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.577725:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.577727:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.577728:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.577729:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.577733:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.577734:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.577737:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.577738:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.577738:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.577740:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.577741:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.577742:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.577742:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.577746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.577753:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.577755:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.577756:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.577756:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.577758:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.577760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.578060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.578067:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.578070:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.578071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.578073:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.578076:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.578077:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.578078:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.578080:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.578080:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.578081:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.578084:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.578086:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.578088:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.578088:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.578089:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.578090:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.578091:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.578091:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.578092:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.578096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.578102:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.578104:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.578105:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.578105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.578107:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.578109:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.578397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.578405:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.578407:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.578409:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.578411:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.578412:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.578413:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.578414:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.578415:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.578416:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.578417:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.578420:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.578421:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.578423:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.578424:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.578425:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.578426:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.578427:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.578428:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.578428:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.578444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.578451:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.578453:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.578454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.578454:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.578456:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.578458:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.578624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.578631:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.578634:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.578635:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.578636:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.578638:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.578640:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.578641:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.578642:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677474.578643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677474.578643:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.578644:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.578647:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.578648:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.578650:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.578651:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.578652:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677474.578653:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677474.578653:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.578654:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:30.0:1619677474.578655:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.578655:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.578655:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.578656:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677474.578657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.578657:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.578658:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.578659:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.578660:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677474.578660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:30.0:1619677474.578661:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.578662:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.578665:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.578667:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.578669:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.578669:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:30.0:1619677474.578670:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.578670:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.578672:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.578672:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677474.578672:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.578673:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.578673:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677474.578674:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.578675:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:18.0:1619677474.578675:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.578676:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.578677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677474.578679:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.578681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.578683:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.578686:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.578687:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.578687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.578689:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.578689:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.578691:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677474.578691:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.578692:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.578693:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.578756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677474.578763:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.578766:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.578767:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.578768:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.578771:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.578772:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677474.578773:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.578775:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.578776:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.578777:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677474.578780:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.578781:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677474.578784:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.578784:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.578785:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677474.578786:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.578787:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.578787:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.578788:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.578792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.578796:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.578798:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.578799:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.578799:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.578800:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.578802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677474.578961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.578968:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.578971:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.578972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.578973:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.578976:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.578977:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.578978:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.578979:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.578980:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.578981:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.578983:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.578985:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.578987:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.578987:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.578988:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.578989:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.578990:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.578991:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.578991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.578995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.579001:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.579003:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.579004:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.579005:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.579006:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.579008:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.579016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.579022:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.579024:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.579025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.579025:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.579026:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.579026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.579114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.579117:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.579120:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.579122:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.579123:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.579124:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.579126:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.579127:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.579128:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.579129:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.579130:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.579134:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.579136:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.579138:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.579139:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.579140:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.579142:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.579142:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.579143:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.579144:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.579147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.579154:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.579156:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.579157:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.579157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.579159:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.579162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.579195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.579201:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.579203:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.579205:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.579206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.579208:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.579210:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.579211:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.579213:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.579213:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.579215:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.579218:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.579219:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.579222:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.579222:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.579223:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.579225:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.579225:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.579226:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.579227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.579231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.579239:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.579241:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.579242:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.579243:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.579244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.579245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.580205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.580213:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.580215:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.580217:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.580220:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.580220:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.580221:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.580222:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.580223:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.580224:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.580226:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.580229:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.580231:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.580233:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.580234:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.580234:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.580236:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.580236:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.580237:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.580238:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.580242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.580249:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.580252:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.580252:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.580254:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.580256:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.580257:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.580340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.580347:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.580350:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.580351:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.580353:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.580355:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.580357:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.580358:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.580359:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.580360:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.580361:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.580364:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.580365:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.580368:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.580369:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.580369:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.580371:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.580371:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.580372:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.580373:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.580378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.580386:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.580389:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.580390:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.580390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.580392:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.2:1619677474.580392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:22.0:1619677474.580395:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677474.580398:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.580401:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.580402:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.580404:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677474.580406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:2.0:1619677474.580407:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.580409:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.580410:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677474.580412:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:2.0:1619677474.580412:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677474.580413:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677474.580414:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.580415:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.580418:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677474.580421:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:2.0:1619677474.580421:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.580423:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.580423:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.580424:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:18.0:1619677474.580425:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:2.0:1619677474.580425:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:18.0:1619677474.580426:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.580426:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.580427:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.580428:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.580429:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.580430:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.580431:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.580432:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:18.0:1619677474.580433:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:9.2:1619677474.580433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.580434:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.580434:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.580436:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.580436:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.580437:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677474.580437:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.580438:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.580439:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:2.0:1619677474.580439:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.580440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.580441:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.580441:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.580441:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677474.580442:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.580443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.580448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.580456:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.580459:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.580460:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.580461:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.580462:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.580465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.580719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.580727:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.580730:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.580732:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.580733:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.580734:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.580736:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.580737:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.580738:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.580739:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.580740:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.580743:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.580745:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.580748:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.580748:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.580749:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.580751:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.580751:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.580752:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.580753:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.580756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.580763:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.580765:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.580766:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.580766:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.580768:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.580770:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.580772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.580778:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.580780:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.580781:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.580781:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.580782:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.580782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.581115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.581122:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.581124:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.581125:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.581127:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.581130:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.581131:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.581132:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.581133:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.581134:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.581135:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.581138:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.581139:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.581141:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.581142:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.581143:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.581144:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.581145:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.581146:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.581147:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.581147:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.581150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.581153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.581156:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.581158:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.581159:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.581160:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677474.581160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.581161:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.581162:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.581163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.581163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.581164:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677474.581283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.581291:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.581299:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:31.0:1619677474.581300:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.581301:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.581304:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.581305:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.581306:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.581307:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.581308:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.581309:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.581312:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.581313:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.581316:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.581316:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.581317:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.581318:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.581319:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.581320:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.581320:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.581325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.581332:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.581334:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.581335:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.581335:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.581337:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.581339:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677474.581693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.581699:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.581702:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.581702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.581705:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.581707:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.581708:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.581710:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.581711:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.581712:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.581713:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.581716:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.581717:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.581720:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.581720:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.581721:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.581723:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.581723:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.581724:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.581724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.581729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.581737:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.581740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.581740:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.581741:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.581743:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.581745:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.581752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.581759:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.581761:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.581762:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.581762:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.581763:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.581763:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.581807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.581815:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.581818:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.581820:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.581822:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.581823:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.581825:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.581826:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.581827:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.581828:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.581829:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.581833:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.581834:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.581838:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.581838:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.581839:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.581841:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.581842:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.581843:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.581843:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.581847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.581856:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.581858:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.581859:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.581861:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.581862:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.581864:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.582238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.582246:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.582249:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.582249:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.582251:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.582254:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.582255:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.582257:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.582258:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.582259:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.582260:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.582263:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.582265:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.582268:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.582269:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.582269:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.582271:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.582271:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.582272:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.582274:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.582274:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677474.582277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.582280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677474.582286:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677474.582288:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:20.0:1619677474.582289:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.582290:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.582290:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:20.0:1619677474.582291:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.582291:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.582292:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.0:1619677474.582293:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677474.582299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.582302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.582303:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.582305:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.582306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.582306:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.582307:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.582308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.582384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.582392:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.582394:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.582397:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.582399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.582399:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.582401:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.582402:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.582403:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.582404:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.582405:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.582408:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.582409:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.582412:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.582412:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.582413:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.582414:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.582415:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.582416:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.582417:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.582421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.582427:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.582430:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.582431:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.582431:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.582433:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.582435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.582480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:4.2:1619677474.582486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677474.582487:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.582490:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.582491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:24.0:1619677474.582493:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677474.582493:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.582495:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.582496:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.582497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677474.582497:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.582499:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.582500:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.582501:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.582502:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.582503:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.582503:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677474.582505:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.582507:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96503843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.582507:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.582509:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.582510:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:24.0:1619677474.582511:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.582512:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.582513:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.582514:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.582514:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677474.582515:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.582516:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.582516:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:24.0:1619677474.582517:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.582517:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.582518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:24.0:1619677474.582521:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.582522:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.2:1619677474.582522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677474.582523:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.582525:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.582525:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.582526:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.582527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.582531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.582531:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.582533:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.582534:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.582536:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.582536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.582538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677474.582539:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.582542:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.582543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.582543:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.582545:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.582547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.582649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.582656:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.582659:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.582662:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.582663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.582665:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.582666:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.582667:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.582668:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.582670:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.582670:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.582674:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.582676:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.582678:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.582679:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.582679:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.582681:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.582681:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.582682:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.582682:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.582686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.582693:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.582695:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.582696:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.582698:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.582699:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.582700:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.582703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.582709:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677474.582711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:31.0:1619677474.582712:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.582713:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.582713:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.582714:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677474.582714:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:31.0:1619677474.582715:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.0:1619677474.582717:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.582719:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.582720:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.582721:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.582723:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.582724:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.582725:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.582726:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.582728:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.582731:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.582733:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.582735:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.582736:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.582737:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.582738:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.582739:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.582740:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.582741:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.582742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677474.582744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.582747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677474.582751:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.582752:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677474.582753:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.582754:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.582754:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.582755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677474.582755:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.582756:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.582756:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677474.582759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.582872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.582880:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.582883:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.582884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.582886:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.582888:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.582890:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.582891:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.582892:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.582893:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.582894:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.582898:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.582900:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.582902:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.582903:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.582904:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.582906:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.582906:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.582907:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.582907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.582912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.582919:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.582922:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.582924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.582924:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.582926:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.582929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.583032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.583037:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.583040:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.583042:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.583042:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.583044:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.583046:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.583047:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.583049:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.583050:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.583051:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.583055:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.583057:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.583060:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.583060:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.583061:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.583063:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.583064:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.583064:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.583065:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.583069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.583076:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.583079:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.583079:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.583081:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.583083:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.583084:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.583089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.583095:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.583098:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.583099:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.583099:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.583100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.583102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.583386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.583394:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.583397:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.583399:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.583400:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.583401:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:31.0:1619677474.583402:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.583402:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.583404:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.583405:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.583405:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.583408:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.583410:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.583412:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.583413:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.583414:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.583416:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.583416:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.583418:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.583419:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.583420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.583422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.583426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.583429:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.583431:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.583432:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.583432:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.583434:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.583434:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.583436:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.583436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.583437:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.583438:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.583488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.583496:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.583499:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.583500:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.583502:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.583505:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.583506:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.583507:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.583509:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677474.583510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677474.583510:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.583510:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.583513:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677474.583513:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.583515:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677474.583515:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.583518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.583518:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.583518:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.583519:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.583520:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677474.583521:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.583521:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:20.0:1619677474.583522:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677474.583522:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.583522:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677474.583523:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.583523:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677474.583525:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.583526:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677474.583527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677474.583527:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.583530:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.583532:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.583534:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.583535:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.583536:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677474.583536:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.583537:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.583538:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.583538:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677474.583539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.583539:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677474.583539:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.583540:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.583542:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.583543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677474.583545:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677474.583552:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.583554:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.583555:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.583556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.583558:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.583560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.583564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.583571:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.583573:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.583574:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.583575:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.583575:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.583576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.584043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.584051:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.584054:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.584056:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.584057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.584058:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.584060:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.584061:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.584062:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.584063:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.584064:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.584068:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.584070:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.584072:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.584073:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.584073:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.584075:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.584075:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.584076:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.584077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.584080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.584087:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.584090:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.584090:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.584092:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.584093:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.584094:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.584266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.584274:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.584277:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.584278:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.584279:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.584283:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.584284:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.584285:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.584286:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.584287:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.584288:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.584291:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.584292:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.584299:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.584300:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.584300:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.584302:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.584302:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.584303:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.584304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.584308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.584315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.584317:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.584318:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.584319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.584320:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.584322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.584406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.584414:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.584417:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.584419:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.584420:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.584421:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677474.584422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.584423:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.584424:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.584425:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.584425:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.584428:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.584430:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.584432:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.584433:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.584434:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.584435:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.584436:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.584437:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.584439:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.584439:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.584441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.584445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.584447:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.584449:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.584450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.584450:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.584452:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.584453:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.0:1619677474.584454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.584455:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.584456:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.584457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677474.584603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.584609:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.584611:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.584613:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.584614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.584616:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.584617:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.584619:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.584620:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.584621:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.584622:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.584625:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.584627:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.584628:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.584629:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.584630:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.584631:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.584632:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.584633:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.584634:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.584635:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.584637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.584640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.584647:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.584648:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.584648:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.584648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.584649:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677474.584650:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.584650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.584651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.584652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.584653:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.584657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:4.2:1619677474.584664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677474.584665:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.584669:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.584670:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.584671:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.584672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:10.0:1619677474.584672:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677474.584673:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677474.584675:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.584676:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.584678:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.584680:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.584682:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.584684:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.584685:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.584686:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.584687:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.584690:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.584691:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.584694:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.584695:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.584695:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.584697:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.584698:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.584699:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.584700:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.584700:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.584703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.584706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.584711:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.584713:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.584714:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.584715:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.584715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.584716:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.584717:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677474.584717:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.584717:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.584720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.584766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.584774:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.584776:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.584779:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.584780:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.584781:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.584782:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.584784:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.584785:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.584786:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.584787:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.584790:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.584792:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.584794:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.584794:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.584795:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.584796:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.584797:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.584798:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.584798:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.584802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.584809:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.584811:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.584812:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.584813:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.584815:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.584816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.585060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677474.585067:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.585070:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.585071:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.585072:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.585075:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.585076:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677474.585078:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.585079:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.585080:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.585081:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677474.585083:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.585085:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677474.585087:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.585088:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.585088:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677474.585090:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.585090:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.585091:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.585091:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.585095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.585098:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.585100:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677474.585101:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.585102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677474.585103:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.585105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:20.2:1619677474.586181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.586188:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.586191:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.586191:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.586194:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.586196:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.586197:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.586199:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.586200:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.586201:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.586202:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.586206:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.586208:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.586211:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.586211:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.586212:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.586214:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.586214:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.586216:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.586217:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.586217:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.586220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677474.586223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.586228:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:2.0:1619677474.586229:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677474.586231:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:2.0:1619677474.586231:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:30.0:1619677474.586232:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.586232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.586233:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677474.586234:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.586236:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.586237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.586330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.586337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.586340:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.586342:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.586344:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.586345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.586346:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.586347:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.586348:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.586349:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.586350:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.586353:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.586355:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.586357:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.586358:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.586359:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.586360:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.586361:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.586362:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.586363:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.586366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:9.2:1619677474.586367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677474.586374:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.586374:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:10.0:1619677474.586376:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.586377:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.586377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.586377:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:10.0:1619677474.586378:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.586378:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.586380:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.586381:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.586382:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.0:1619677474.586382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:10.0:1619677474.586383:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.586385:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.586385:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.586386:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:9.2:1619677474.586386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:4.2:1619677474.586388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:10.0:1619677474.586389:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.586391:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.586392:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677474.586393:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.586393:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.586394:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.586394:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:10.0:1619677474.586394:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.586395:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:10.0:1619677474.586395:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.586395:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.586396:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.586396:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677474.586396:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677474.586397:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:10.0:1619677474.586397:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.586397:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:21.0:1619677474.586399:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677474.586400:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.586400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.586401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:21.0:1619677474.586402:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.586403:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.586405:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.586406:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.586407:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.586407:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.586409:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.586410:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.586411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:21.0:1619677474.586411:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677474.586412:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.586413:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.586414:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677474.586415:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.586416:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.586417:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.586419:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.586419:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.586420:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.586421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.586421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:25.2:1619677474.586426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.586427:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.586430:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.586431:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.586431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.586432:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.586432:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677474.586434:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.586436:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.586438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.586438:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.586440:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.586442:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.586445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.586454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.586458:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.586459:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.586460:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.586461:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.586463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.586860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.586868:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.586871:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.586872:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.586874:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.586876:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.586878:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.586879:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.586881:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.586882:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.586883:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.586886:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.586888:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.586891:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.586892:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.586892:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.586894:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677474.586895:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.586896:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.586897:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.586898:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677474.586900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677474.586904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677474.586910:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:10.0:1619677474.586912:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.586913:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.586914:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677474.586914:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:10.0:1619677474.586915:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:18.0:1619677474.586916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.586916:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.586919:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.586923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.586936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:5.2:1619677474.586939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677474.586943:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.586946:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.586947:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.586948:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.586948:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:26.0:1619677474.586952:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.586952:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:26.0:1619677474.586953:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:1.0:1619677474.586953:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:26.0:1619677474.586955:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.586957:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.586957:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.586958:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.586959:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:17.0:1619677474.586960:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.586963:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.586963:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677474.586965:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:17.0:1619677474.586965:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.586967:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.586968:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.586968:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000400:00000200:17.0:1619677474.586968:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.586969:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.586970:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.586970:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:17.0:1619677474.586970:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.586971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000400:00000200:17.0:1619677474.586971:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.2:1619677474.586975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000400:00000200:17.0:1619677474.586977:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.586979:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:17.0:1619677474.586980:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677474.586981:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.586982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.586982:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.586983:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677474.586984:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.586985:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.586986:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677474.586986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:17.0:1619677474.586988:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.586989:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.586990:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.586991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.586994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.587004:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.587007:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.587009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.587009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.587013:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.587015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.587283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.587292:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.587299:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677474.587301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.587301:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.587304:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.587305:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.587307:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.587308:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.587309:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.587310:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.587314:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.587316:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.587318:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.587319:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.587320:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.587321:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.587322:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.587323:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.587323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.587328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.587335:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.587337:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.587338:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.587340:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.587341:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.587342:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.587371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.587379:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.587382:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.587383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.587384:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.587388:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.587389:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.587391:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.587392:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.587393:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.587394:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.587397:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.587398:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.587402:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.587402:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.587403:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.587405:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.587405:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.587406:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.587407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.587412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.587422:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.587427:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.587429:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.587429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.587433:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.587437:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.587957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.587964:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.587966:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.587967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.587969:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.587972:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.587973:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.587975:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.587976:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.587977:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.587978:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.587982:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.587983:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.587985:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.587986:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.587987:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.587988:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.587989:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.587990:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.587990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.587994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.588001:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.588004:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.588004:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.588005:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.588006:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.588007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.588015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.588023:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.588025:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.588026:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.588026:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.588026:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.588027:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.588068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.588076:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.588079:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.588081:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.588083:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.588083:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.588085:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.588086:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.588087:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.588088:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.588089:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.588093:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.588094:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.588097:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.588098:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.588098:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.588100:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.588100:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.588102:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.588103:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.588104:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.588105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.588110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.588114:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.588115:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.588116:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.588118:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.588118:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.588118:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.588119:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.588119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.588120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.588121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.588125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.588132:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.588135:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.588135:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.588136:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.588137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.588138:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.588371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.588378:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.588381:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.588382:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.588383:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.588385:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.588387:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.588388:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.588389:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.588390:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.588391:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.588394:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.588396:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.588398:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.588399:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.588400:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.588401:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.588402:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.588403:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.588404:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.588405:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.588409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.588411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.588417:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.588418:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.588419:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.588420:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677474.588420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.588421:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.588421:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.588422:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677474.588423:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.588425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.588739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.588748:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.588751:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.588752:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.588754:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.588755:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.588755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.588756:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.588758:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.588759:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.588760:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.588763:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.588765:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.588766:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.588767:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.588768:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.588769:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.588770:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.588771:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.588771:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.588775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.588782:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.588784:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.588784:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.588786:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.588788:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.588789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.589130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.589133:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.589135:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.589137:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.589138:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.589139:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.589141:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.589142:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.589144:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.589145:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.589146:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.589149:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.589151:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.589153:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.589154:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.589155:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.589156:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.589157:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.589158:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.589158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.589163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.589172:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.589175:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.589176:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.589176:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.589178:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.589180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.589338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.589345:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.589348:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.589350:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.589352:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.589353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.589353:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677474.589355:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.589356:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.589357:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.589358:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677474.589361:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.589363:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.589364:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.589365:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.589366:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.589367:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.589368:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.589369:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677474.589371:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677474.589371:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:25.2:1619677474.589373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.589377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.589382:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.589384:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.589384:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.589386:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.589386:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677474.589386:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677474.589387:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.589388:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.589389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677474.589390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:21.2:1619677474.589475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.589483:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.589485:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.589487:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.589489:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.589489:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.589490:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.589491:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.589492:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.589493:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.589494:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.589497:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.589498:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.589500:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.589500:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.589501:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.589502:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.589502:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.589503:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.589504:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.589508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.589514:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.589517:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.589517:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.589519:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.589520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.589521:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.589522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.589530:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.589532:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.589533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.589535:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.589538:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.589539:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.589540:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.589541:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.589542:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.589543:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.589546:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.589547:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.589550:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.589551:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.589552:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.589553:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.589554:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:4.2:1619677474.589554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.589555:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.589557:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677474.589558:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677474.589559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.589560:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.589561:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.589562:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.589564:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.589565:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.589566:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.589567:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:10.0:1619677474.589567:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.589568:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.589569:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.589570:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.589571:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.589571:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.589571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.589573:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.589574:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.589576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677474.589576:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.589577:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.589577:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.589579:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.589579:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.589580:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.589581:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.589585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.589592:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.589594:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.589595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.589595:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.589598:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.589599:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.590094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.590102:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.590105:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.590106:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.590108:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.590108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.590109:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.590111:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.590112:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.590113:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.590114:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.590117:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.590118:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.590121:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.590121:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.590122:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.590124:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.590124:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.590125:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.590126:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.590129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.590137:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.590139:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.590140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.590140:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.590142:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.590144:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.590343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.590348:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.590351:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.590353:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.590354:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.590356:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.590357:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.590358:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.590360:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.590361:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.590362:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.590364:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.590366:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.590368:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.590369:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.590370:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.590371:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.590372:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.590372:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.590373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.590378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.590386:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.590388:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.590389:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.590390:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.590391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.590393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.590476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.590481:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.590483:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.590485:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.590486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.590487:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.590489:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.590490:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.590491:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.590492:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.590493:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.590495:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.590497:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.590499:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.590500:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.590500:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.590502:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.590502:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.590503:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.590503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.590508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.590517:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.590520:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.590521:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.590521:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.590524:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.590526:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.590716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.590724:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.590727:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.590729:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.590731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.590732:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.590733:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.590734:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.590736:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.590737:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.590738:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.590741:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.590743:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.590746:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.590747:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.590747:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.590749:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.590750:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.590751:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.590751:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.590755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.590763:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.590765:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.590766:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.590768:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.590769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.590771:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.591099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.591106:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.591109:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.591110:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.591111:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.591113:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.591115:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.591116:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.591118:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.591119:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.591120:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.591123:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.591125:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.591128:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.591129:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.591130:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.591131:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.591132:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.591133:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.591133:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.591138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.591146:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.591149:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.591149:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.591151:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.591153:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.591154:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677474.591256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.591262:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.591265:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.591265:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.591267:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.591270:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.591272:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677474.591273:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.591274:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.591275:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.591276:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677474.591279:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.591280:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677474.591284:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.591284:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677474.591285:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677474.591286:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677474.591287:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.591288:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.591288:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677474.591293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.591301:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.591304:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.591304:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.591305:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.591307:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.591310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.591426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.591434:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.591436:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.591437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.591439:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.591441:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.591442:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.591443:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.591445:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.591446:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.591447:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.591450:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.591452:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.591455:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.591456:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.591456:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.591458:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.591459:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.591459:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.591459:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.591465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.591473:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.591476:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.591476:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677474.591477:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.591479:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.591481:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.591484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677474.591491:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.591494:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.591495:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.591495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.591496:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.591497:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.591678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.591684:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.591687:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.591687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.591690:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.591693:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.591694:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677474.591696:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.591697:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.591699:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.591700:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677474.591703:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.591705:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.591708:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.591709:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677474.591710:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.591712:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.591712:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.591714:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677474.591715:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677474.591715:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677474.591718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.591720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:24.0:1619677474.591725:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677474.591725:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677474.591727:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:24.0:1619677474.591728:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677474.591728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:24.0:1619677474.591729:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.591729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677474.591729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:24.0:1619677474.591731:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.591735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.591775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.591781:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.591784:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.591785:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.591787:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.591789:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.591791:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.591792:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.591793:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.591794:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.591795:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.591799:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.591800:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.591803:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.591804:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.591804:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.591806:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.591806:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.591808:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.591809:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.591809:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.591812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.591814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.591820:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.591820:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.591822:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.591822:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.591823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677474.591824:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.591824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.591824:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.591826:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.591829:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.591835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.591842:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.591845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.591845:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.591846:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.591847:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.591847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.591935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.591943:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.591946:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.591949:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.591951:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.591952:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.591953:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.591954:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.591956:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.591957:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.591958:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.591961:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.591963:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.591966:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.591966:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.591967:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.591969:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.591969:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.591970:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.591971:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.591974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.591981:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.591983:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.591984:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.591984:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.591986:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.591988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.592126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.592133:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.592136:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.592137:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.592138:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.592141:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.592142:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.592143:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.592145:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.592146:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.592146:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.592149:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.592151:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.592153:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.592154:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.592154:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.592156:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.592156:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.592157:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.592157:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.592161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.592165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:24.0:1619677474.592167:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.592169:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.592170:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.592171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.592172:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.592172:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:24.0:1619677474.592174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.592174:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677474.592175:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677474.592176:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.592179:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.592180:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.592181:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.592182:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.592183:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.592184:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.592186:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.592188:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.592190:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.592190:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.592191:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.592192:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.592193:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.592194:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677474.592195:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677474.592195:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677474.592198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.592201:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:4.2:1619677474.592201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:8.0:1619677474.592204:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:8.0:1619677474.592204:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.592205:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.592206:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.592208:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677474.592208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:26.0:1619677474.592210:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677474.592211:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.592212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.592225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677474.592227:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.592230:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677474.592230:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.592231:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (23)--
00000800:00000200:8.0:1619677474.592231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677474.592232:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677474.592247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.592254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.592256:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.592257:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.592258:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.592261:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.592263:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.592264:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.592265:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.592266:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.592267:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.592270:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.592271:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.592274:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.592275:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.592275:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.592277:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.592278:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.592279:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.592280:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.592281:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.592283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.592286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.592290:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.592293:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.592294:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.592300:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677474.592301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.592302:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677474.592302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.592303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.592304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.592306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.592307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.592315:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.592317:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.592318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.592318:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.592319:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.592320:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.592513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.592522:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.592525:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.592527:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.592529:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.592530:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.592530:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.592532:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.592533:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.592534:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.592535:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.592538:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.592540:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.592542:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.592542:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.592543:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.592544:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.592545:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.592546:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.592547:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.592548:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.592550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.592554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.592558:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.592559:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.592560:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.592562:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.592562:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.592563:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.592563:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.592564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.592565:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.592566:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.592673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.592680:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.592683:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.592683:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677474.592685:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.592687:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.592689:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.592690:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.592692:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.592693:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.592694:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.592697:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.592699:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.592701:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.592702:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.592703:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.592705:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.592705:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.592706:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.592706:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.592711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.592720:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.592722:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.592723:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.592725:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.592726:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.592727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.592837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.592842:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.592845:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.592845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.592847:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.592850:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.592851:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677474.592853:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.592854:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.592855:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.592856:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677474.592860:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.592862:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.592864:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.592865:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677474.592866:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.592868:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.592868:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.592869:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.592870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677474.592874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.592881:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.592883:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.592884:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.592885:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.592887:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.592889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.593288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.593295:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.593301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.593303:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.593305:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.593307:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.593309:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.593310:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.593311:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.593312:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.593313:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.593317:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.593318:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.593321:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.593322:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.593323:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.593324:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.593325:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.593325:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.593326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.593331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.593339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.593342:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.593343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.593343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.593345:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.593348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.593475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.593483:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.593486:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.593488:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.593490:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.593491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.593491:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.593493:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.593494:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.593495:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.593496:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.593499:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.593500:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.593502:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.593503:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.593504:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.593505:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.593505:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.593506:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.593507:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.593511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.593517:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.593520:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.593520:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.593522:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.593524:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.593524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.593529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.593536:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.593538:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.593539:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.593539:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.593540:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.593541:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.594080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.594087:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.594089:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.594091:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.594093:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.594093:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.594094:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.594095:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.594096:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.594097:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.594098:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.594101:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.594102:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.594104:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.594105:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.594106:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.594107:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.594108:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.594109:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.594111:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.594111:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.594113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.594117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.594120:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.594122:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.594123:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.594124:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.594125:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.594125:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.594127:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677474.594127:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.594128:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.594129:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.594435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.594442:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.594445:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.594446:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.594447:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.594450:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.594452:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.594453:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.594454:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.594455:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.594456:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.594459:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.594460:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.594463:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.594464:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.594464:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.594466:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.594467:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.594467:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.594468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.594473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.594479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:22.0:1619677474.594481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.594484:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.594485:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.594485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.594488:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.594488:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:22.0:1619677474.594491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:12.0:1619677474.594492:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.594496:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.594497:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.594500:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.594502:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677474.594504:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.594506:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.594509:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.594510:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677474.594516:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.594518:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677474.594522:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.594523:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677474.594524:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677474.594526:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677474.594527:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.594528:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.594529:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.594532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.594538:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.594541:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.594541:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.594542:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.594544:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.594546:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677474.594567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677474.594574:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.594576:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677474.594577:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.594577:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.594578:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.594578:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677474.594972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.594975:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.594978:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.594980:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.594982:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.594982:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.594983:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.594984:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.594986:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.594987:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.594988:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.594991:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.594993:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.594995:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.594996:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.594997:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.594998:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.594999:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.594999:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.595000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.595005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.595006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:17.0:1619677474.595014:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:2.0:1619677474.595014:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677474.595017:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.595018:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.595018:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:1.0:1619677474.595018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:18.0:1619677474.595020:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:17.0:1619677474.595021:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.595023:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:2.0:1619677474.595023:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.595027:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.595030:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.595032:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.595035:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.595036:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.595038:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.595043:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.595046:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.595049:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.595050:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.595051:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.595054:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.595055:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.595057:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.595058:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.595059:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.595060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.595064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:21.2:1619677474.595072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677474.595072:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.595073:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.595074:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.595075:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.595075:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.595076:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.595078:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.595078:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.595079:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677474.595080:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.595081:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.595083:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.595086:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.595088:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.595088:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.595090:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.595091:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.595093:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.595094:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.595095:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.595098:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.595099:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.595103:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.595104:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.595104:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.595106:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.595107:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.595108:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.595108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.595112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.595120:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.595123:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.595124:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.595125:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.595126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.595128:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.595155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.595162:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.595165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.595165:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.595168:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.595170:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.595172:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.595173:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.595174:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.595175:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.595176:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.595179:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.595181:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.595184:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.595184:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.595185:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.595187:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.595187:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.595188:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.595188:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.595193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.595202:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.595206:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677474.595207:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.595208:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.595211:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.595215:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.595256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.595263:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.595266:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.595267:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.595268:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.595271:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.595273:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.595274:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.595275:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.595276:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.595277:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.595279:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.595281:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.595283:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.595283:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.595284:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.595285:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.595286:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.595287:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.595288:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.595289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.595291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.595295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.595298:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.595300:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.595301:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.595301:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.595302:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677474.595303:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.595305:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:8.0:1619677474.595305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.595306:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677474.595306:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.595312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.595318:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.595321:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.595321:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.595322:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.595322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.595323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.595418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.595422:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.595424:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677474.595427:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.595427:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.595429:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.595430:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.595431:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.595432:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.595434:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.595435:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.595438:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.595439:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.595442:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.595443:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.595443:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.595445:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.595445:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.595446:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.595447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.595451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.595459:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.595461:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.595462:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.595464:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.595464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.595466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.595747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.595754:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.595757:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.595760:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.595762:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.595763:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.595764:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.595766:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.595767:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.595768:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.595769:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.595772:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.595774:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.595776:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.595777:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.595778:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.595779:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.595780:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.595781:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.595782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.595785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.595791:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.595794:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.595795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.595795:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.595797:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.595799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.595805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.595811:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.595813:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.595814:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.595814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.595815:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.595816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.595851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677474.595858:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.595861:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677474.595862:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677474.595863:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.595865:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.595867:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.595868:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.595869:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.595870:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.595871:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.595874:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.595875:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.595877:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.595878:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.595879:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.595880:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.595880:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.595881:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.595881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.595886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.595889:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.595892:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.595893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.595893:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.595894:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.595896:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677474.596327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677474.596335:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.596338:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677474.596340:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.596342:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.596343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677474.596343:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.596344:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.596346:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.596346:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.596347:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.596350:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.596352:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.596354:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.596355:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.596355:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.596357:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.596357:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.596358:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.596359:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677474.596363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.596370:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.596372:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.596373:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.596374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.596375:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.596377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.596768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.596774:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.596777:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.596778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.596780:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.596782:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.596784:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.596785:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.596786:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.596787:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.596788:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.596792:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.596794:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.596796:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.596797:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.596797:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.596799:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.596799:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.596800:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.596801:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.596805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.596813:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.596815:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.596816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.596816:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.596818:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.596821:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.596884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.596887:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.596890:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.596892:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.596893:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.596894:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.596895:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.596897:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.596898:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.596899:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.596900:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.596903:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.596905:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.596907:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.596908:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.596909:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.596910:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.596911:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.596912:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.596913:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.596914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.596917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.596919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.596924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677474.596925:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677474.596926:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.596927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677474.596927:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677474.596928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677474.596928:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.596928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.596930:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.596932:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.596939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.596947:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.596949:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.596950:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.596950:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.596951:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.596952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.597195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.597203:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.597206:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.597208:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.597209:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.597211:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.597212:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.597213:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.597215:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.597216:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.597217:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.597220:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.597222:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.597224:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.597225:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.597226:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.597227:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.597228:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.597229:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.597229:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.597233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.597239:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.597241:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.597242:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.597244:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.597245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:31.0:1619677474.597246:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.597247:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.597250:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.597253:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.597254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.597256:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.597259:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.597260:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677474.597262:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.597263:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.597264:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.597265:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.597268:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.597270:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.597273:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.597273:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.597274:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.597276:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.597277:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.597278:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.597278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677474.597282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.597290:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.597293:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.597298:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.597300:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.597302:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.597305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.597350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.597358:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.597361:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.597361:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.597364:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.597366:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.597367:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.597369:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.597370:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.597371:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.597372:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.597375:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.597377:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.597379:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.597380:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.597380:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.597382:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.597383:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.597384:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.597384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.597389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677474.597393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:17.0:1619677474.597397:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677474.597400:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:17.0:1619677474.597400:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.597401:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.597401:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677474.597403:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:17.0:1619677474.597404:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:10.0:1619677474.597404:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.597406:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.597406:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:26.0:1619677474.597409:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.597410:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677474.597411:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.597412:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.597413:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.597414:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677474.597418:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.597420:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677474.597422:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.597423:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.597423:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677474.597425:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.597425:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.597426:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.597426:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.597430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.597434:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.597436:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.597437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.597437:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.597439:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.597441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677474.597632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.597639:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.597642:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.597643:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.597644:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.597647:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.597649:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.597650:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.597651:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.597652:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.597653:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.597656:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.597657:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.597660:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.597661:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.597661:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.597663:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.597664:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.597665:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.597665:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.597670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.597678:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.597681:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.597682:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.597682:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.597684:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.597687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.597844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.597852:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.597855:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.597857:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.597859:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.597859:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.597860:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.597861:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.597862:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.597863:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.597864:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.597867:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.597869:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.597871:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.597872:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.597873:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.597874:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.597875:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.597876:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.597876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.597881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.597887:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.597890:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.597890:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.597891:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.597893:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.597895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.597991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.597998:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.598000:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.598001:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.598003:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.598005:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.598006:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.598008:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.598009:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.598010:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.598010:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.598014:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.598015:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.598017:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.598018:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.598018:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.598020:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.598020:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.598021:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.598021:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.598025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.598031:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.598034:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.598035:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.598035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.598036:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.598038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.598139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.598146:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.598149:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.598151:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.598153:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.598153:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.598154:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.598155:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.598156:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.598157:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.598158:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.598161:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.598163:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.598165:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.598165:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.598166:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.598168:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.598168:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.598170:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.598171:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.598172:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.598173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.598177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.598182:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.598183:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.598184:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.598186:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.598186:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.598187:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.598187:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.598188:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.598189:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.598190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.598511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.598519:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.598522:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.598524:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677474.598526:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.598526:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.598528:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:13.0:1619677474.598529:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.598530:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.598531:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.598532:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:13.0:1619677474.598535:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.598537:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:13.0:1619677474.598539:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.598540:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:13.0:1619677474.598540:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:13.0:1619677474.598542:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:13.0:1619677474.598542:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.598543:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.598544:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.598548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.598556:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.598558:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.598559:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.598559:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.598561:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.598564:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.598769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.598776:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.598779:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.598780:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.598781:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.598783:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.598784:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.598785:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.598787:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.598788:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.598788:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.598791:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.598793:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.598795:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.598796:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.598796:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.598798:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.598798:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.598799:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.598799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.598803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.598810:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.598812:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.598813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.598813:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.598815:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.598817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.599315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.599323:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.599325:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.599326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.599327:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.599330:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.599331:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.599332:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.599334:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.599335:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.599336:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.599339:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.599340:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.599343:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.599343:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.599344:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.599345:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.599346:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.599347:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.599347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.599352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.599360:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.599363:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.599364:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.599364:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.599366:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.599369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.599413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.599420:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.599422:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.599423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.599425:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.599428:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.599430:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677474.599431:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.599433:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.599434:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.599434:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677474.599437:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.599438:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677474.599441:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.599442:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677474.599442:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:10.0:1619677474.599444:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677474.599444:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.599445:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.599446:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.599450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677474.599457:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.599459:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.599460:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.599462:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.599464:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.599465:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677474.599871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.599879:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.599882:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.599884:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.599886:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.599887:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.599888:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.599889:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.599890:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.599891:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.599892:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.599895:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.599897:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.599899:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.599900:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.599901:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.599902:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.599903:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.599904:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.599904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.599908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.599915:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.599917:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.599918:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.599918:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.599920:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.599922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.600399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677474.600406:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677474.600409:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677474.600410:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677474.600411:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.600414:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.600415:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677474.600416:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.600418:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.600419:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.600419:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677474.600422:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.600424:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677474.600427:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.600427:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677474.600428:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.600430:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.600430:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677474.600431:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677474.600431:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:8.2:1619677474.600436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.600445:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.600448:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.600449:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.600449:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.600452:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.600455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.600802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.600809:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.600812:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.600813:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.600815:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.600818:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.600819:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.600821:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.600822:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.600823:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.600824:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.600827:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.600828:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.600831:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.600831:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.600832:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.600834:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.600834:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.600835:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.600835:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.600841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.600848:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.600851:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.600852:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.600852:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.600855:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.600858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.601131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.601138:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.601141:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.601142:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.601144:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.601147:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.601148:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.601149:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.601151:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.601152:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.601152:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.601156:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.601157:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.601159:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.601160:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.601160:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.601162:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.601162:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.601163:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.601163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.601167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.601174:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.601176:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.601177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.601177:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.601179:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.601181:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.601187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.601193:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.601196:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.601197:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.601197:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.601198:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.601198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.601441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.601449:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.601452:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.601454:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.601456:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.601457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.601458:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:30.0:1619677474.601459:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.601460:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.601461:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.601462:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:30.0:1619677474.601465:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.601466:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677474.601469:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.601469:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:30.0:1619677474.601470:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:30.0:1619677474.601471:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:30.0:1619677474.601472:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.601473:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.601475:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.601476:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.601478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.2:1619677474.601482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:10.0:1619677474.601485:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.601487:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.601488:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.601488:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.601490:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:10.0:1619677474.601490:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.601492:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:10.0:1619677474.601492:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:31.0:1619677474.601493:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.601494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.601804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677474.601811:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.601813:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.601814:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.601815:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.601818:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.601819:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677474.601820:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.601821:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.601822:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.601823:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677474.601825:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.601827:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677474.601829:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.601830:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.601830:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677474.601832:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.601832:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.601833:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.601833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.601838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.601842:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.601844:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.601845:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.601845:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.601847:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.601849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677474.601888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.601896:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.601899:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.601901:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.601903:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.601903:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.601904:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.601905:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.601907:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.601908:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.601909:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.601912:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.601913:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.601915:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.601916:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.601916:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.601918:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.601918:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.601919:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.601920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.601924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.601930:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.601933:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.601933:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.601935:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.601937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.601938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.601947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.601954:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.601956:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.601957:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.601958:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.601958:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.601959:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.602101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677474.602107:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.602109:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.602112:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.602112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.602114:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.602116:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.602117:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.602118:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.602119:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.602120:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.602124:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.602126:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.602128:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.602129:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.602130:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.602131:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.602132:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.602133:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677474.602133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:4.2:1619677474.602137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:25.2:1619677474.602138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.602144:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677474.602146:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.602147:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.602148:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:6.0:1619677474.602148:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.602149:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.602150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677474.602150:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.602151:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.602152:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.602154:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:6.0:1619677474.602154:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677474.602155:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.602156:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.602157:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.602158:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:5.2:1619677474.602160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677474.602162:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.602163:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677474.602165:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.602166:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677474.602166:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677474.602166:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.602167:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.602168:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.602168:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:17.0:1619677474.602169:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677474.602169:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:1.0:1619677474.602170:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.602172:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677474.602173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:17.0:1619677474.602174:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.602175:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.602177:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.602178:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.602179:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:17.0:1619677474.602179:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.602180:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.602181:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.602182:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.602183:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677474.602184:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.602184:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.602186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:17.0:1619677474.602186:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.602188:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.602189:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.602190:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.602192:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.602192:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.602193:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.602194:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.602198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.602205:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.602208:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.602209:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.602209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.602211:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.602213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.602221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.602228:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.602230:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.602231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.602232:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.602233:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.602233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.602503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.602510:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.602513:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.602514:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.602516:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.602519:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.602520:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.602522:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.602523:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.602524:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.602525:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.602529:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.602530:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.602533:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.602534:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.602535:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.602537:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.602537:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.602538:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.602539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.602543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.602551:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.602554:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.602555:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.602555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.602557:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.602560:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.602565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.602571:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.602574:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.602575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.2:1619677474.602575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:6.0:1619677474.602575:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.602576:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.602577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677474.602581:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.602584:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.602585:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.602587:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.602589:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.602591:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.602592:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.602594:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.602595:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.602596:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.602599:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.602601:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.602604:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.602605:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.602605:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.602607:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.602608:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.602608:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.602609:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.602613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.602620:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.602622:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.602623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.602624:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.602626:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.602628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.602640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.602648:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.602651:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.602653:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.602655:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.602656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.602656:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.602657:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.602659:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.602660:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.602661:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.602664:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.602666:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.602669:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.602669:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.602670:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.602672:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.602672:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.602673:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.602674:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.602677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.602683:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.602685:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.602686:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.602686:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.602688:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.602690:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.602696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.602701:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.602704:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.602704:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.602705:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.602706:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.602706:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.602812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.602820:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.602823:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.602825:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.602826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.602827:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.602828:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.602829:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.602831:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.602832:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.602833:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.602836:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.602838:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.602840:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.602841:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.602841:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.602843:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.602843:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.602844:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.602845:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.602848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.602855:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.602857:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.602858:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.602859:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.602861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.602862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.603040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.603047:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.603050:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.603051:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.603052:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.603054:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.603056:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.603057:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.603058:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.603059:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.603060:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.603063:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.603065:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.603067:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.603068:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.603068:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.603070:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.603071:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.603072:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.603073:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.603074:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.603076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.603079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.603084:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.603086:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.603087:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.603088:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677474.603088:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.603088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.603089:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677474.603090:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.603090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.603093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.604026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.604034:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.604037:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.604038:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.604039:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.604042:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.604044:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.604045:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.604046:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.604047:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.604048:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.604050:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.604052:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.604054:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.604055:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.604056:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.604057:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.604058:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.604058:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.604059:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.604064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.604072:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.604075:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.604076:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.604076:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.604078:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.604081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.604098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.604104:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.604107:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.604108:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.604110:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.604113:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.604114:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677474.604115:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.604117:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.604118:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.604119:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.604122:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.604123:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.604126:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.604127:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677474.604128:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.604129:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677474.604130:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.604131:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.604131:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.604135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.604142:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.604145:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.604146:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.604146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.0:1619677474.604146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.604148:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.604151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.604155:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.604157:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.604159:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.604161:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.604162:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.604163:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.604164:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.604165:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.604166:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.604167:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.604171:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.604172:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.604175:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.604176:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.604176:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.604178:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.604178:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.604179:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.604180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.604183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.604190:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.604192:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.604193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.604193:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.604195:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.604196:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.604342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.604350:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.604352:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.604353:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.604355:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.604358:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.604359:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.604360:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.604362:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.604363:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.604363:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.604367:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.604368:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.604371:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.604372:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.604372:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.604374:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.604375:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.604375:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.604377:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.604377:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.604381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.604382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.604389:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.604390:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.604392:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.604392:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677474.604392:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677474.604393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.604393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.604394:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.604396:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.2:1619677474.604396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:22.0:1619677474.604398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677474.604403:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.604406:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.604406:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.604409:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.604411:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.604412:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.604413:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.604415:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.604416:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.604417:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.604420:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.604422:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.604424:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.604424:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.604425:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.604427:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.604427:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.604428:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.604429:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.604433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.604440:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.604442:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.604443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.604443:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.604445:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.604446:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.604501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.604507:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.604510:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.604510:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.604512:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.604513:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.604515:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.604516:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.604517:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.604518:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.604519:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.604523:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.604524:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.604527:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.604528:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.604529:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.604530:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.604531:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.604531:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.604532:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.604537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.604544:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.604547:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.604548:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.604550:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.604552:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.604553:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.604560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677474.604566:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.604569:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.604570:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.604571:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.604571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.604573:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.605333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.605340:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.605342:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.605344:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.605345:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.605348:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.605350:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.605351:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.605352:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.605353:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.605354:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.605358:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.605359:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.605362:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.605363:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.605364:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.605366:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.605366:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.605367:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.605367:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.605372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.605380:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.605383:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.605384:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.605384:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.605386:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.605390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.605397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.605404:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.605406:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.605407:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.605408:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.605408:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.605409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.605610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.605618:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.605621:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.605623:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.605626:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.605626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.605627:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.605629:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.605630:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.605631:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.605632:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.605635:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.605637:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.605639:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.605640:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.605640:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.605642:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.605642:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.605643:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.605644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.605648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.605655:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.605657:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.605658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.605658:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.605660:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.605663:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.605738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.605743:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.605746:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.605747:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.605749:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.605751:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.605752:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.605753:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.605755:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.605756:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.605757:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.605760:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.605762:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.605765:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.605766:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.605766:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.605768:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.605768:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.605769:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.605769:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.605774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.605782:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.605785:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.605785:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.605787:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.605788:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.605790:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.605795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.605802:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.605805:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.605806:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.605807:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.605807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.605809:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.605853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.605858:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.605860:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.605863:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.605863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.605865:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.605866:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.605868:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.605869:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.605870:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.605871:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.605874:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.605876:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.605879:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.605880:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.605881:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.605883:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.605883:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.605884:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.605886:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.605886:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677474.605889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.605891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.605895:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677474.605896:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677474.605897:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.605898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677474.605899:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677474.605899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677474.605900:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.605901:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.605903:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.605904:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.605918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.605926:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.605928:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.605930:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.605931:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.605934:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.605936:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.605937:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.605938:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.605939:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.605940:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.605943:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.605945:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.605948:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.605949:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.605950:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.605951:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.605952:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.605952:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.605954:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.605954:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677474.605957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.605960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677474.605964:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.605966:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:26.0:1619677474.605967:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:13.0:1619677474.605967:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.605968:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.605969:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677474.605969:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.605970:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677474.605971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.605972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.606905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.606914:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.606917:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.606918:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.606919:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.606922:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.606923:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.606924:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.606926:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.606927:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.606927:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.606930:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.606932:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.606934:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.606935:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.606935:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:5.2:1619677474.606935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677474.606937:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.606937:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.606938:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.606938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677474.606942:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677474.606943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677474.606944:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.606945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.606947:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.606950:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677474.606950:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.606951:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677474.606952:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:17.0:1619677474.606952:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.606953:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.606954:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.606954:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:17.0:1619677474.606955:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.606957:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.606957:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677474.606959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:17.0:1619677474.606960:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.606963:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.606965:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.606966:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.606967:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.606968:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.606969:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.606969:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.606970:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.606975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.606983:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.606986:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.606987:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.606987:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.606989:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.606991:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.607110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.607118:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.607121:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.607123:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.607125:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.607125:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.607127:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.607128:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.607129:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.607130:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.607132:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.607135:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.607137:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.607140:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.607141:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.607141:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.607143:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.607143:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.607144:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.607145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.607148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.607155:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.607157:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.607158:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.607159:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.607161:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.607162:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.607580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c77bd8600] (20)++
00000800:00000200:8.0:1619677474.607588:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)++
00000800:00000200:8.0:1619677474.607591:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.28@o2ib3
00000800:00000200:24.0:1619677474.607592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)--
00000400:00000200:8.0:1619677474.607594:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.28@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.607597:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.607598:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:8.0:1619677474.607599:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.607601:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.607602:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.607603:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:8.0:1619677474.607606:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.28@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.607607:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677474.607610:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.607611:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:8.0:1619677474.607611:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:8.0:1619677474.607613:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:8.0:1619677474.607614:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c77bd8600] (21)++
00000800:00000200:8.0:1619677474.607615:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c77bd8600] (22)--
00000800:00000200:8.0:1619677474.607615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)--
00000800:00000200:9.2:1619677474.607620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.607628:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:4.2:1619677474.607630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c77bd8600] (20)++
00000800:00000200:2.0:1619677474.607631:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.607632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.607632:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.607634:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.607637:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)++
00000800:00000200:2.0:1619677474.607638:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677474.607640:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.28@o2ib3
00000800:00000200:8.0:1619677474.607641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)--
00000400:00000200:24.0:1619677474.607643:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.28@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.607646:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.607647:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677474.607649:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.607651:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.607652:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.607653:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:24.0:1619677474.607656:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.28@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.607658:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677474.607660:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.607660:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:24.0:1619677474.607661:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:24.0:1619677474.607663:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:24.0:1619677474.607663:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c77bd8600] (21)++
00000800:00000200:24.0:1619677474.607664:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c77bd8600] (22)--
00000800:00000200:4.2:1619677474.607664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c77bd8600] (22)++
00000800:00000200:25.2:1619677474.607668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:24.0:1619677474.607668:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)--
00000800:00000200:24.0:1619677474.607669:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)++
00000800:00000200:24.0:1619677474.607671:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.28@o2ib3
00000800:00000200:8.0:1619677474.607672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)--
00000400:00000200:24.0:1619677474.607673:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.28@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.607675:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.607676:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:13.0:1619677474.607676:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:24.0:1619677474.607677:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96503735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677474.607678:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:24.0:1619677474.607679:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.607679:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.607679:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.607680:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:13.0:1619677474.607681:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.607683:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:24.0:1619677474.607684:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.28@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.607684:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:24.0:1619677474.607686:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677474.607689:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.607690:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:24.0:1619677474.607690:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677474.607692:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677474.607693:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c77bd8600] (21)++
00000800:00000200:24.0:1619677474.607694:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c77bd8600] (22)++
00000800:00000200:24.0:1619677474.607696:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c77bd8600] (23)--
00000800:00000200:24.0:1619677474.607696:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)--
00000800:00000200:8.2:1619677474.607698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.607702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c77bd8600] (21)++
00000800:00000200:20.0:1619677474.607708:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.607710:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:8.0:1619677474.607710:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)++
00000400:00000200:20.0:1619677474.607711:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.607712:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c77bd8600] (23)--
00000800:00000200:8.0:1619677474.607712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)--
00000400:00000200:20.0:1619677474.607713:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.607713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)--
00000800:00000200:21.0:1619677474.607715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.607717:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.607718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c77bd8600] (20)++
00000800:00000200:24.0:1619677474.607725:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)++
00000800:00000200:24.0:1619677474.607727:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.28@o2ib3
00000800:00000200:8.0:1619677474.607728:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)--
00000400:00000200:24.0:1619677474.607730:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.28@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.607730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:24.0:1619677474.607733:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.607734:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.607735:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.607737:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.607738:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:24.0:1619677474.607738:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.607738:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677474.607740:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.607743:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.607743:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.28@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.607745:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.607745:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.607745:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677474.607746:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.607747:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.607748:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.607748:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000400:00000200:30.0:1619677474.607749:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.607749:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:30.0:1619677474.607750:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.607750:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.607750:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.607751:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c77bd8600] (21)++
00000800:00000200:24.0:1619677474.607752:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c77bd8600] (22)--
00000800:00000200:24.0:1619677474.607752:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)--
00000400:00000200:30.0:1619677474.607753:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.607755:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.2:1619677474.607756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677474.607757:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.607758:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.607759:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.607760:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.607761:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.607762:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.607764:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.607764:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:1.0:1619677474.607765:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:9.2:1619677474.607766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.607768:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.607769:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.607769:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.607770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:4.2:1619677474.607771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c77bd8600] (20)++
00000400:00000200:1.0:1619677474.607771:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.607774:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677474.607777:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:31.0:1619677474.607778:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.0:1619677474.607778:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.607779:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.607780:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)++
00000800:00000200:31.0:1619677474.607781:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677474.607781:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.28@o2ib3
00000400:00000200:8.0:1619677474.607781:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.607782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.607783:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:24.0:1619677474.607784:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.28@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.607784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.607785:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.607786:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677474.607787:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.607787:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.607788:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.607788:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.607789:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:8.0:1619677474.607789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)--
00000400:00000200:24.0:1619677474.607790:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.607790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:24.0:1619677474.607792:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.607793:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.607794:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677474.607797:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.28@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.607798:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677474.607801:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.607802:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677474.607803:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677474.607804:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677474.607805:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c77bd8600] (21)++
00000800:00000200:24.0:1619677474.607806:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c77bd8600] (22)--
00000800:00000200:24.0:1619677474.607806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)--
00000800:00000200:8.2:1619677474.607810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.607818:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.607821:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.607822:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.607823:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.607824:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.607826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.607840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.607847:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.607850:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.607852:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.607852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c77bd8600] (20)++
00000800:00000200:30.0:1619677474.607854:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.607855:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.607856:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.607857:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.607859:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.607859:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)++
00000400:00000200:31.0:1619677474.607860:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.607861:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677474.607862:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)--
00000800:00000200:8.0:1619677474.607862:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.28@o2ib3
00000400:00000200:31.0:1619677474.607865:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.607865:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.28@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.607867:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677474.607867:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.607869:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677474.607869:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677474.607870:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:8.0:1619677474.607870:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96503730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.607871:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:8.0:1619677474.607871:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.607872:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:8.0:1619677474.607872:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.607873:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677474.607873:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.607874:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.607875:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.607875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677474.607876:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.28@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677474.607878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677474.607878:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677474.607881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.607881:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.607881:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.607882:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.607884:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.607884:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c77bd8600] (21)++
00000800:00000200:8.0:1619677474.607885:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c77bd8600] (22)--
00000800:00000200:8.0:1619677474.607885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)--
00000800:00000200:30.0:1619677474.607886:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.607887:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.607888:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.607889:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.607889:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.607889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.607890:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.607890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.607891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.607892:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.607893:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.607897:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.607900:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.607901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.607902:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.607904:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.607907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.607920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c77bd8600] (20)++
00000800:00000200:24.0:1619677474.607928:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)++
00000800:00000200:24.0:1619677474.607930:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.28@o2ib3
00000800:00000200:8.0:1619677474.607931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)--
00000400:00000200:24.0:1619677474.607933:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.28@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.607936:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.607937:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.607939:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.607940:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.607941:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.607942:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.607945:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.28@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.607946:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.607948:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.607949:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.607950:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.607950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c77bd8600] (21)++
00000800:00000200:24.0:1619677474.607951:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.607953:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c77bd8600] (22)++
00000800:00000200:24.0:1619677474.607954:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c77bd8600] (23)++
00000800:00000200:24.0:1619677474.607956:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c77bd8600] (24)--
00000800:00000200:25.2:1619677474.607957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.607957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (23)--
00000800:00000200:24.0:1619677474.607958:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)++
00000800:00000200:24.0:1619677474.607959:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.28@o2ib3
00000800:00000200:8.0:1619677474.607960:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c77bd8600] (23)++
00000400:00000200:24.0:1619677474.607961:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.28@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.607961:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c77bd8600] (24)--
00000800:00000200:8.0:1619677474.607962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (23)--
00000800:00000200:8.0:1619677474.607963:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)--
00000400:00000200:24.0:1619677474.607964:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.607964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:24.0:1619677474.607965:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677474.607966:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96503728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.607967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.607967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:24.0:1619677474.607968:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.607968:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.607968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.607970:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677474.607970:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.607973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:24.0:1619677474.607974:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.28@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677474.607975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.607976:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677474.607978:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.607979:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677474.607979:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677474.607981:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677474.607981:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c77bd8600] (21)++
00000800:00000200:24.0:1619677474.607982:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c77bd8600] (22)--
00000800:00000200:24.0:1619677474.607982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)--
00000800:00000200:22.0:1619677474.607983:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.607985:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677474.607986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677474.607986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.607987:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.607988:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.607988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677474.607993:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:4.2:1619677474.607994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c77bd8600] (20)++
00000800:00000200:26.0:1619677474.607996:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.607997:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.607997:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677474.607999:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.608001:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)++
00000800:00000200:26.0:1619677474.608002:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.0:1619677474.608004:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.28@o2ib3
00000800:00000200:24.0:1619677474.608005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (22)--
00000400:00000200:8.0:1619677474.608006:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.28@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.608009:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.608010:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677474.608011:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.608013:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.608014:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.608014:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677474.608017:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.28@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.608017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677474.608019:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.608021:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.608021:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677474.608022:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677474.608023:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677474.608024:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.608024:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c77bd8600] (21)++
00000800:00000200:8.0:1619677474.608025:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c77bd8600] (22)--
00000800:00000200:8.0:1619677474.608025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c77bd8600] (21)--
00000800:00000200:10.0:1619677474.608026:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.608027:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.608029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:10.0:1619677474.608029:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.608031:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.608032:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.608033:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.608034:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.608035:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:10.0:1619677474.608035:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.608036:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.608038:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.608039:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.608039:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.608040:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.608041:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.608042:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.608043:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:10.0:1619677474.608043:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.608044:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.608044:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.608046:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.608046:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.608047:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.608047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.608051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:9.2:1619677474.608052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.608058:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677474.608059:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:30.0:1619677474.608061:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.608061:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:30.0:1619677474.608062:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:24.0:1619677474.608062:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.608062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.608063:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.608063:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.608064:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:24.0:1619677474.608064:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.608066:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.608173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.608181:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.608183:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.608184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.608186:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.608188:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.608190:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.608191:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.608192:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.608193:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.608194:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.608197:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.608198:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.608201:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.608202:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.608202:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.608204:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.608204:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.608205:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.608206:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.608207:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.608210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.608212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.608217:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.608219:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.608220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.608220:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.608221:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.608222:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.608222:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677474.608223:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.608224:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.608226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.608689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.608697:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.608699:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.608701:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.608702:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.608705:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.608707:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.608708:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.608709:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.608710:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.608711:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.608714:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.608716:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.608718:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.608719:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.608720:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.608722:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.608722:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.608723:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.608723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.608728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.608736:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.608739:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.608740:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.608740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.608742:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.608745:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.609083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.609090:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.609093:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.609094:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.609096:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.609098:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.609100:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.609101:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.609103:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.609104:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.609105:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.609108:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.609110:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.609113:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.609114:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.609115:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.609116:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.609117:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.609118:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.609119:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.609120:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.609123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677474.609125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677474.609130:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677474.609131:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677474.609133:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:2.0:1619677474.609133:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:13.0:1619677474.609134:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.609134:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.609135:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677474.609136:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.609138:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677474.609138:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.2:1619677474.609391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.609399:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.609402:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.609404:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.609406:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.609407:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.609408:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.609409:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.609410:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.609411:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.609412:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.609416:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.609418:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.609420:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.609421:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.609422:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.609423:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.609424:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.609425:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.609425:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.609430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.609437:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.609439:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.609440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.609440:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.609442:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.609445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.609687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.609695:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.609697:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.609700:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.609701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.609702:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.609703:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.609704:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.609706:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.609707:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.609708:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.609711:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.609713:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.609715:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.609716:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.609716:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.609718:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.609718:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.609719:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.609720:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.609734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.609740:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.609742:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.609743:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.609744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.609745:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.609748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.609818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.609826:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.609829:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.609829:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.609831:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.609834:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.609835:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.609836:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.609838:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.609839:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.609839:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.609842:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.609844:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.609847:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.609848:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.609848:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.609850:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.609850:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.609851:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.609851:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.609856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.609864:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.609867:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.609868:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.609868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.609870:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.609873:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.609880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677474.609887:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.609890:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.609890:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677474.609891:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.609892:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.609892:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.609905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677474.609912:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.609915:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677474.609916:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677474.609918:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.609921:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.609922:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.609924:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.609925:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.609926:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.609927:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.609929:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.609931:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.609934:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.609935:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.609935:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.609937:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.609938:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.609938:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677474.609939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677474.609943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.609951:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.609954:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.609954:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.609955:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.609957:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.609960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.610169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.610176:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.610179:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.610180:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.610182:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.610185:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.610186:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.610187:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.610189:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.610190:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.610190:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.610194:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.610195:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.610197:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.610198:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.610198:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.610200:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.610200:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.610201:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.610202:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.610203:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.610208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.610215:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:9.2:1619677474.610216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.610217:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.610218:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.610219:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677474.610222:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.610225:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.610226:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.610226:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.610228:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.610230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.610239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.610245:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.610248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.610248:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.610249:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.610249:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.610250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.611230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677474.611236:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.611239:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677474.611240:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677474.611242:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677474.611245:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.611246:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.611248:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.611249:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.611250:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.611251:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.611253:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677474.611255:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.611257:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.611258:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.611259:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.611260:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.611261:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.611261:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677474.611262:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:25.2:1619677474.611267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.611271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:6.0:1619677474.611276:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.611278:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.611279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:12.0:1619677474.611279:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:6.0:1619677474.611279:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.611281:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.611282:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.611284:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.611284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:13.0:1619677474.611286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.611286:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.611288:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677474.611289:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.611291:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.611292:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.611298:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677474.611302:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.611304:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677474.611306:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.611307:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677474.611307:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677474.611309:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677474.611309:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.611311:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677474.611312:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677474.611313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:25.2:1619677474.611315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677474.611317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:4.2:1619677474.611319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:30.0:1619677474.611323:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:1.0:1619677474.611324:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:30.0:1619677474.611325:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.611326:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.611326:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.611327:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:13.0:1619677474.611327:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:30.0:1619677474.611329:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.611329:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:31.0:1619677474.611330:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677474.611330:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:1.0:1619677474.611330:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.611331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:30.0:1619677474.611332:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:1.0:1619677474.611332:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.611334:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677474.611335:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.611337:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.611338:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.611339:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677474.611343:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.611344:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677474.611347:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.611348:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677474.611349:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677474.611351:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677474.611351:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.611352:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.611353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.2:1619677474.611357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:9.2:1619677474.611357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.611364:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:2.0:1619677474.611364:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677474.611367:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:2.0:1619677474.611367:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:26.0:1619677474.611368:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.611368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677474.611368:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:2.0:1619677474.611369:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.611371:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:2.0:1619677474.611373:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.611374:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:2.0:1619677474.611374:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.611375:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.611377:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.611378:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677474.611379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:2.0:1619677474.611379:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.611382:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.611383:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677474.611385:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677474.611385:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.611386:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.611387:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677474.611388:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.611388:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:2.0:1619677474.611388:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677474.611389:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677474.611389:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677474.611390:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677474.611390:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:10.0:1619677474.611391:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:2.0:1619677474.611391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.611395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.611402:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.611404:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.611404:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.611405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.611407:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.611408:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.611414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.611421:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.611424:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.611425:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.611425:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.611426:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.611426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.611432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.611439:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.611442:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.611444:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.611447:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.611448:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677474.611448:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.611449:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.611451:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.611452:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.611453:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.611457:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.611458:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.611461:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.611461:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.611462:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.611464:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.611465:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.611466:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.611466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.611470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.611476:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.611479:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.611480:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.611480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.611482:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.611484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.611555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.611564:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.611568:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677474.611571:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.611571:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.611574:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.611577:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.611578:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.611581:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.611582:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.611584:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.611589:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.611591:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.611595:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.611596:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.611597:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.611599:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.611600:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.611601:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.611602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.611604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.611611:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.611613:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.611614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.611614:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.611616:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.611618:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.611820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.611827:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.611829:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.611830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.611832:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.611835:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.611837:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.611838:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.611839:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.611840:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.611841:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.611844:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.611846:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.611849:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.611849:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.611850:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.611852:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.611852:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.611853:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.611853:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.611858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.611868:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.611872:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.611874:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.611874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.611878:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.611882:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677474.611962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.611969:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.611972:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.611972:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.611974:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.611976:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.611977:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.611979:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.611980:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.611981:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.611982:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.611986:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.611988:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.611991:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.611992:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.611993:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.611995:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.611996:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.611997:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.611997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677474.612001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.612008:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.612011:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.612012:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.612013:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.612016:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.612017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677474.612215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.612223:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.612228:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.612229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.612232:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.612236:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.612238:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.612240:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.612242:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.612244:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.612246:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.612251:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.612254:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.612257:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.612259:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.612260:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.612262:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.612263:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.612265:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.612266:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.612267:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.612268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.612272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.612276:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.612278:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.612279:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.612280:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.612280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.612281:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.612283:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677474.612284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.612284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677474.612284:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.612286:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.612289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677474.612293:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677474.612298:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.612301:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677474.612302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.612302:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677474.612303:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677474.612303:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000400:00000200:31.0:1619677474.612304:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.612304:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.612305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:31.0:1619677474.612306:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.612307:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.612309:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.612310:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.612311:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.612312:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.612316:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.612318:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.612321:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.612321:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.612322:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.612324:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.612324:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.612325:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.612326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.612330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.612337:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.612340:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.612341:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.612341:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.612343:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.612345:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.612735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.612743:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.612746:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.612747:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.612748:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.612751:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.612752:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.612753:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.612755:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.612756:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.612756:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.612759:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.612761:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.612764:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.612765:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.612765:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.612767:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.612767:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.612768:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.612769:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.612773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.612780:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.612783:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.612784:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.612784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.612787:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.612789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.612886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.612894:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.612897:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.612900:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.612901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.612902:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.612903:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.612904:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.612906:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.612907:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.612907:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.612911:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.612913:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.612914:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.612915:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.612915:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.612917:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.612917:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.612919:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.612921:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.612921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.612923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.612927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.612931:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.612932:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.612933:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.612934:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.612935:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.612935:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.612936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.612936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.612937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.612938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.612946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.612952:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.612955:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.612956:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.612957:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.612957:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.612958:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.613031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.613037:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.613040:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.613041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.613042:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.613045:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.613047:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.613048:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.613049:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.613050:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.613051:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.613054:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.613055:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.613058:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.613059:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.613060:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.613061:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.613062:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.613062:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.613063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.613067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.613075:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.613077:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.613078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.613078:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.613080:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.613083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.613091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.613098:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.613100:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.613101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.613102:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.613102:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.613103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.613368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.613376:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.613379:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.613381:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.613384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.613384:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.613385:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.613386:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.613387:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.613388:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.613389:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.613392:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.613394:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.613396:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.613397:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.613397:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.613399:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.613400:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.613400:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.613401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.613405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.613413:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.613415:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.613416:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.613416:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.613418:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.613420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.613505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.613513:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.613515:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.613516:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.613518:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.613521:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.613522:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.613523:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.613525:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.613526:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.613527:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.613529:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.613531:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.613534:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.613535:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.613535:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.613537:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.613537:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.613538:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.613538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.613542:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.613543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.613543:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.613545:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.613546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.613547:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.613549:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.613551:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677474.613551:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677474.613552:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.613553:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.613553:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.613554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677474.613554:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.613554:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677474.613554:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.613557:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.613559:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.613561:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.613563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677474.613565:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.613565:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.613566:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.613567:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.613568:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.613569:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.613569:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.613573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.613581:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.613583:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.613584:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.613584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.613586:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.613588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.613624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.613627:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.613630:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.613632:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.613633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.613635:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.613637:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.613638:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.613639:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.613640:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.613641:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.613644:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.613646:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.613648:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.613649:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.613650:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.613651:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.613652:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.613653:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.613653:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.613658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.613666:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.613668:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.613669:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.613669:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.613671:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.613674:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.614338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.614344:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.614347:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.614348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.614350:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.614353:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.614354:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.614356:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.614357:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.614358:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.614359:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.614362:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.614364:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.614366:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.614367:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.614367:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.614369:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.614370:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.614370:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.614371:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.614376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.614385:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.614387:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.614388:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.614388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.614390:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.614392:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.615008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677474.615013:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.615016:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.615017:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.615018:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.615020:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.615022:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.615023:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.615024:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.615025:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.615026:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.615029:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.615030:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.615033:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.615034:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.615035:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.615036:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.615037:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.615038:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677474.615039:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677474.615039:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:9.2:1619677474.615043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.615044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:24.0:1619677474.615049:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:17.0:1619677474.615050:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:24.0:1619677474.615052:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:17.0:1619677474.615052:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:17.0:1619677474.615053:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:8.0:1619677474.615053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677474.615053:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000400:00000200:24.0:1619677474.615054:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.615056:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.615059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.615068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.615075:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.615077:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.615078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.615078:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.615079:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.615080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.615161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.615168:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.615170:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.615171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.615173:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.615175:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.615177:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.615178:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.615179:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.615180:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.615181:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.615185:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.615187:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.615189:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.615190:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.615191:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.615192:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.615193:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.615193:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.615194:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677474.615199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.615206:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.615209:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.615210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.615210:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.615212:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.615215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.615250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677474.615258:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.615261:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677474.615263:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.615265:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.615265:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677474.615267:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677474.615268:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.615270:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.615270:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.615271:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677474.615274:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677474.615275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:31.0:1619677474.615276:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677474.615278:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.615279:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:31.0:1619677474.615279:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677474.615281:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677474.615281:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:31.0:1619677474.615282:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.615283:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:17.0:1619677474.615284:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:31.0:1619677474.615285:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:1.0:1619677474.615285:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:31.0:1619677474.615286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:25.2:1619677474.615287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:17.0:1619677474.615287:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.615290:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677474.615291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000400:00000200:17.0:1619677474.615291:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.615292:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677474.615294:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:17.0:1619677474.615301:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677474.615301:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:30.0:1619677474.615302:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000400:00000200:17.0:1619677474.615302:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677474.615302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677474.615302:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.615303:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677474.615304:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000400:00000200:12.0:1619677474.615304:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.615305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.615306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000400:00000200:17.0:1619677474.615307:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.615307:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:17.0:1619677474.615309:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.615312:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.615313:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.615313:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.615315:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.615316:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.615317:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677474.615318:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.615319:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.615322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.615324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.615330:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.615332:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.615332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677474.615333:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677474.615333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677474.615335:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.615336:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.615338:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.615338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.615340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.615513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677474.615519:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.615522:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677474.615523:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677474.615524:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.615526:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.615528:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677474.615529:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.615531:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.615532:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.615533:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677474.615536:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.615537:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677474.615539:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.615540:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677474.615540:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677474.615542:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677474.615542:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.615543:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677474.615544:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677474.615544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:9.2:1619677474.615549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.2:1619677474.615550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.615555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677474.615556:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:18.0:1619677474.615557:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677474.615557:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.615558:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:18.0:1619677474.615559:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.615560:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.615560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.615563:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.615565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.615749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.615757:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.615759:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.615762:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.615763:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.615765:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.615766:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.615767:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.615769:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.615770:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.615771:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.615774:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.615775:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.615779:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.615780:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.615780:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.615782:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.615783:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.615784:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.615785:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.615786:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.615788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.615792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.615796:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.615799:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.615800:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.615800:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.615800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.615802:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677474.615802:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.615803:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.615804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.615805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.616179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.616187:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.616190:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.616191:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.616193:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.616196:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.616197:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.616198:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.616199:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.616200:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.616201:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.616204:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.616206:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.616208:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.616209:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.616209:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.616211:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.616211:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.616212:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.616213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.616217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.616223:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.616225:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.616226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.616226:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.616228:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.616230:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.616304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.616311:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.616314:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.616316:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.616319:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.616319:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.616320:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.616321:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.616322:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.616323:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.616324:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.616327:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.616329:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.616330:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.616331:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.616331:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.616333:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.616334:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.616334:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.616335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.616339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.616347:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.616349:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.616350:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.616351:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.616354:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.616354:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677474.616462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.616469:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.616472:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.616473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.616474:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.616476:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.616478:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.616479:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.616480:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.616481:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.616482:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.616485:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.616486:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.616489:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.616490:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.616490:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677474.616492:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.616493:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.616493:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.616494:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.616498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.616507:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.616510:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.616511:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.616511:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.616513:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.616516:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.616893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.616901:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.616903:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.616905:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.616907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.616907:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.616908:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.616909:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.616910:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.616911:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.616912:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.616915:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.616917:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.616919:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.616919:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.616920:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.616921:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.616921:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.616922:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.616923:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.616927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.616933:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.616936:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.616937:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.616938:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.616940:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.616941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677474.616942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.616948:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.616951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.616951:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677474.616953:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.616956:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.616957:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.616958:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.616960:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.616961:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.616962:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.616965:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.616967:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.616970:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.616970:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.616971:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.616973:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.616973:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.616974:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.616974:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.616979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.616986:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.616989:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.616990:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.616990:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.616992:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.616994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.617420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.617424:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.617427:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677474.617430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.617430:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.617432:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.617434:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.617435:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.617436:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.617437:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.617438:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.617442:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.617444:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.617447:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.617447:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.617448:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.617450:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.617450:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.617451:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.617452:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.617456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677474.617463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.617466:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.617468:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.617469:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.617471:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.617471:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.617473:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677474.617474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.617475:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:30.0:1619677474.617475:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.617476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.617477:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.617478:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.617479:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.617481:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.617481:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.617482:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.617485:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.617487:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.617489:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.617490:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.617491:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.617492:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.617493:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.617494:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.617494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.617498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.617504:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.617507:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.617507:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.617509:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.617510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.617511:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.617594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.617602:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.617604:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.617605:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.617607:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.617607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:26.0:1619677474.617610:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.617611:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.617612:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.617614:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.617614:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.617615:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:26.0:1619677474.617615:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.617617:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:26.0:1619677474.617619:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.617620:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.617621:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:26.0:1619677474.617621:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.617622:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.617623:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677474.617624:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.617625:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.617625:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:31.0:1619677474.617626:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.617626:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677474.617627:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.617627:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677474.617628:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.617628:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.617629:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.617630:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:31.0:1619677474.617631:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.617631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677474.617632:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677474.617633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.617636:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677474.617636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677474.617637:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.617637:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:31.0:1619677474.617639:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:31.0:1619677474.617639:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.617640:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.617641:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.617642:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:10.0:1619677474.617643:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.617644:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:10.0:1619677474.617645:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677474.617645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000400:00000200:22.0:1619677474.617646:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.617646:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677474.617646:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:10.0:1619677474.617647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677474.617647:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.617648:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.617649:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.617652:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.617652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.617653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.617663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.617670:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.617673:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.617674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.617674:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.617675:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.617676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.618045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.618052:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.618055:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.618057:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.618059:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.618060:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.618061:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.618063:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.618064:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.618065:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.618066:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.618069:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.618070:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.618072:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.618073:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.618074:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.618075:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.618076:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.618077:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.618079:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.618079:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.618080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.618085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.618089:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.618090:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.618091:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.618093:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.618093:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.618093:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.618094:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.618094:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.618095:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.618096:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.618099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.618105:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.618108:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.618109:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.618109:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.618110:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.618110:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.618254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.618262:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.618264:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.618266:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.618267:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.618270:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.618271:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.618273:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.618274:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.618275:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.618276:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.618279:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.618281:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.618283:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.618284:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.618285:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.618286:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.618287:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.618287:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.618288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.618292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.618300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.618303:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.618304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.618304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.618306:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.618309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.618309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.618316:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.618319:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.618320:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.618321:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.618324:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.618326:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.618327:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.618328:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.618329:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.618330:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.618333:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.618334:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.618337:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.618338:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.618339:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.618340:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.618341:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.618341:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.618342:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.618347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.618354:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.618357:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.618358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.618358:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.618360:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.618363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.618852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.618859:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.618862:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.618863:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.618864:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.618867:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.618869:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.618870:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.618871:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.618872:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.618873:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.618877:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.618878:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.618880:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.618880:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.618881:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.618882:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.618883:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.618883:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.618884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.618888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.618894:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.618897:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.618898:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.618898:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.618900:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.618902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.618909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.618915:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.618918:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.618919:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.618919:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.618919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.618920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.619182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677474.619188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.619190:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.619193:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.619193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.619196:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.619197:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677474.619199:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.619200:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.619201:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.619202:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677474.619205:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.619207:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677474.619210:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.619211:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677474.619212:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677474.619214:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677474.619214:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.619215:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677474.619216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:9.2:1619677474.619220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.619226:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.619229:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.619230:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.619230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.619232:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.619235:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677474.619239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:9.2:1619677474.619245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677474.619246:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.619248:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.619249:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.619251:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.619251:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.619253:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.619254:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:18.0:1619677474.619254:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.619254:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:18.0:1619677474.619255:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:10.0:1619677474.619255:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:18.0:1619677474.619256:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.619256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:18.0:1619677474.619257:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.619258:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.619259:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.619262:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.619264:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.619265:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.619266:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.619267:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.619268:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.619269:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.619270:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.619271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.619274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.619281:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.619284:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.619284:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.619285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.619286:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.619287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.619414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.619417:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.619419:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.619422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.619422:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.619424:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.619426:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.619427:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.619428:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.619429:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.619430:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.619434:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.619436:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.619438:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.619439:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.619440:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.619441:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.619442:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.619442:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.619443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.619448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.619457:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.619459:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.619460:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.619460:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.619462:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.619465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.619496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.619504:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.619507:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.619507:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.619509:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.619512:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.619513:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677474.619515:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.619517:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.619518:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.619519:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677474.619523:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.619525:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677474.619528:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.619529:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677474.619530:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677474.619532:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677474.619532:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.619533:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.619534:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677474.619538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.619545:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.619547:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.619548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.619549:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.619551:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.619554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.619676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.619684:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.619687:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.619689:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.619691:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.619692:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.619693:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.619694:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.619695:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.619696:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.619697:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.619701:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.619703:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.619706:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.619707:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.619707:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.619709:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.619710:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.619711:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.619711:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.619715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.619724:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.619726:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.619727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.619728:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.619730:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.619732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.620725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.620732:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.620735:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.620736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.620737:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.620741:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.620742:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677474.620744:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.620745:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.620746:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.620747:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677474.620749:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.620751:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.620752:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.620753:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.620753:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.620755:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.620755:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.620757:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677474.620758:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677474.620759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:9.2:1619677474.620761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.620765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.620770:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.620771:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.620772:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.620773:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.620774:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677474.620774:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.620775:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677474.620775:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.620776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677474.620776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.621221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.621227:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.621230:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.621231:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.621233:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.621236:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.621238:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.621239:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.621240:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.621241:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.621242:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.621246:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.621248:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.621251:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.621252:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.621253:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.621255:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.621255:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.621256:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.621256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:25.2:1619677474.621261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.621269:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.621271:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.621272:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.621274:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.621276:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.621277:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.621627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.621635:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.621638:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.621640:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.621642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.621643:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.621643:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.621644:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677474.621645:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677474.621645:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.621646:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.621647:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.621648:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.621649:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.621652:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.621654:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.621655:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.621656:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.621657:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.621657:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.621657:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677474.621659:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.621659:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677474.621660:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.621661:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.621662:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.621663:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.621664:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.621664:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.621665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677474.621666:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.621666:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.621667:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677474.621667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.621671:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.621671:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.621672:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.621673:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.621674:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.621674:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.621675:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.621675:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677474.621675:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.621675:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.621677:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677474.621677:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.621678:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.621689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.621697:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.621700:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.621701:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.621701:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.621703:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.621706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.621711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.621718:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:5.2:1619677474.621720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677474.621721:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.621722:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.621722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.621724:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.621724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677474.621726:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.621729:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.621730:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.621732:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.621734:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.621736:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.621737:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.621738:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.621739:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.621740:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.621743:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.621745:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.621747:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.621748:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.621749:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.621750:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.621750:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.621751:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.621752:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.621756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.621763:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.621766:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.621767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.621767:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.621769:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.621771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.621776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.621783:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.621785:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.621786:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.621787:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.621787:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.621788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.621951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677474.621956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:26.0:1619677474.621959:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.621962:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.621962:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.621963:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:26.0:1619677474.621964:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.621965:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:26.0:1619677474.621966:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.621966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:26.0:1619677474.621968:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677474.621968:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.621969:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.621970:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.621970:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.621971:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.621971:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.621972:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677474.621973:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.621974:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.621975:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.621975:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.621980:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.621980:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.621982:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.621982:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.621985:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.621985:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.621986:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.621986:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.621987:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.621987:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.621988:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.2:1619677474.621988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b5000] (20)++
00000800:00000200:26.0:1619677474.621989:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.621989:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.621989:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:26.0:1619677474.621990:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.621990:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:26.0:1619677474.621991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.2:1619677474.621991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677474.621991:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:17.0:1619677474.621994:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b5000] (21)++
00000800:00000200:25.2:1619677474.621995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:9.2:1619677474.621995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677474.621997:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.2@o2ib3
00000800:00000200:18.0:1619677474.621998:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677474.621998:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b5000] (22)--
00000400:00000200:17.0:1619677474.622000:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.4.2@o2ib3 : GET - for me
00000800:00000200:18.0:1619677474.622001:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:24.0:1619677474.622002:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:2.0:1619677474.622002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677474.622003:0:19930:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.4.2@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:6.0:1619677474.622003:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.622004:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:18.0:1619677474.622004:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.622005:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.622005:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.622006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:18.0:1619677474.622006:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.622006:0:19930:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.4.2@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:6.0:1619677474.622006:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:24.0:1619677474.622007:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.622007:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b5000] (21)++
00000400:00000200:6.0:1619677474.622007:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.622008:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:17.0:1619677474.622008:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b5000] (22)--
00000800:00000200:24.0:1619677474.622009:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:18.0:1619677474.622010:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.622011:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.4.2@o2ib3 local destination
00000400:00000200:6.0:1619677474.622011:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:18.0:1619677474.622012:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.622013:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.622014:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677474.622015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:17.0:1619677474.622017:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.4.2@o2ib3(10.51.4.2@o2ib3:10.51.4.2@o2ib3) : REPLY try# 0
00000400:00000200:18.0:1619677474.622019:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677474.622020:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.4.2@o2ib3
00000800:00000200:18.0:1619677474.622021:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677474.622022:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918ce37f3b00] -> 10.51.4.2@o2ib3 (2) version: 12
00000800:00000200:17.0:1619677474.622023:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918ce07b5000] (21)++
00000800:00000200:18.0:1619677474.622024:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.622024:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b5000] (22)++
00000800:00000200:18.0:1619677474.622025:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:17.0:1619677474.622025:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918ce07b5000] (23)--
00000800:00000200:18.0:1619677474.622026:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677474.622026:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b5000] (22)--
00000800:00000200:18.0:1619677474.622028:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.622029:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.622030:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.622030:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.2:1619677474.622031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b5000] (21)++
00000800:00000200:25.2:1619677474.622034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677474.622037:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b5000] (22)++
00000800:00000200:1.0:1619677474.622039:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b5000] (23)--
00000400:00000200:1.0:1619677474.622039:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.622040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b5000] (22)--
00000800:00000200:31.0:1619677474.622041:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:1.0:1619677474.622042:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.4.2@o2ib3: REPLY: OK
00000800:00000200:31.0:1619677474.622044:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:1.0:1619677474.622044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b5000] (21)--
00000400:00000200:31.0:1619677474.622045:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.622047:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.622048:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.622050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.622050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:25.2:1619677474.622054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677474.622057:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.622060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.622060:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:30.0:1619677474.622061:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:10.0:1619677474.622063:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.622064:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.622064:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.622065:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:10.0:1619677474.622065:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.622066:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.622066:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:10.0:1619677474.622067:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.622068:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.622069:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.622070:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.622071:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.622075:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.622077:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.622079:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.622080:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.622080:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.622082:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.622082:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.622083:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677474.622084:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677474.622085:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677474.622087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.622091:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:4.2:1619677474.622091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:8.0:1619677474.622094:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.622095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.622095:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.622096:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.622098:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:8.0:1619677474.622098:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:26.0:1619677474.622100:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677474.622101:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.622102:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.622107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677474.622110:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.622112:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677474.622113:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.622114:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.622114:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:8.0:1619677474.622114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:4.2:1619677474.622127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f390200] (20)++
00000800:00000200:10.0:1619677474.622135:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f390200] (21)++
00000800:00000200:10.0:1619677474.622138:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.2@o2ib3
00000800:00000200:26.0:1619677474.622139:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f390200] (22)--
00000400:00000200:10.0:1619677474.622140:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.622143:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.622144:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.622145:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.622146:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.622147:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.622148:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.622151:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.622152:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.622154:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.622155:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.622155:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.622157:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.622157:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f390200] (21)++
00000800:00000200:10.0:1619677474.622158:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f390200] (22)--
00000800:00000200:10.0:1619677474.622159:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f390200] (21)--
00000800:00000200:24.2:1619677474.622162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.622166:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.622168:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.622169:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.622169:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.622170:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.622172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677474.622197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.622204:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.622206:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.622209:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.622211:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.622211:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.622212:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.622213:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.622215:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.622216:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.622217:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.622220:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.622221:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.622223:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.622224:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.622224:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.622226:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.622226:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.622227:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.622228:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.622232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.622239:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.622241:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.622242:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.622243:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.622245:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.622246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.622563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.622569:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.622572:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.622573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.622575:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.622578:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.622580:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677474.622581:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.622582:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.622583:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.622584:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677474.622587:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.622589:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.622592:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.622592:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677474.622593:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.622595:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677474.622595:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.622597:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677474.622598:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.622598:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:25.2:1619677474.622601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.622604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:22.0:1619677474.622609:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677474.622610:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:22.0:1619677474.622612:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:18.0:1619677474.622612:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000400:00000200:22.0:1619677474.622613:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.622613:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:6.0:1619677474.622613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677474.622614:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:22.0:1619677474.622615:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.622618:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.622847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.622855:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.622858:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.622858:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.622861:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.622863:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.622864:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.622865:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677474.622865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677474.622867:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.622868:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.622869:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.622872:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677474.622872:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.622873:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.622875:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677474.622876:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.622877:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.622877:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.622877:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:31.0:1619677474.622879:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.622879:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.622879:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677474.622880:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.622880:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677474.622880:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.622881:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677474.622882:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.622883:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.622884:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.622884:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:9.2:1619677474.622885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:31.0:1619677474.622887:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.622889:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.622890:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.622891:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:24.0:1619677474.622891:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:31.0:1619677474.622892:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.622893:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.622893:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.622894:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.622894:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:31.0:1619677474.622895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:24.0:1619677474.622895:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.622895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.622897:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.622899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.622899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.622905:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.622908:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.622909:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.622910:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.622912:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.622913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.623134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.623139:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.623142:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.623145:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.623145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.623148:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.623149:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.623150:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677474.623150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:21.0:1619677474.623152:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.623153:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.623154:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.623157:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.623157:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677474.623159:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.623159:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.623160:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.0:1619677474.623161:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.623162:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.623163:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.623163:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.623164:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.623165:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:10.0:1619677474.623165:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.623166:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.623166:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:10.0:1619677474.623166:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.623168:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.623169:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.623170:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677474.623171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.623171:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.623174:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.623174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677474.623176:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.623178:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.623178:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.623179:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.623180:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.623180:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.623181:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.623181:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:8.0:1619677474.623181:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677474.623181:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.623182:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.623183:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.623184:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:8.0:1619677474.623184:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:6.0:1619677474.623184:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.623185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677474.623185:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677474.623187:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.623187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.623188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:8.0:1619677474.623190:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.623190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677474.623191:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.623193:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96503659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.623194:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.623195:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:8.0:1619677474.623195:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.623196:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.623197:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.623197:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.623198:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.623198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.623199:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:8.0:1619677474.623199:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.623200:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677474.623200:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.623201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.0:1619677474.623201:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.623203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677474.623203:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.623204:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.623205:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.623206:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.623207:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.623208:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.623209:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.623210:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.623212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.623215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.623219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.623222:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.623222:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.623223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.623223:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.623224:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.623225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677474.623225:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.623226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.623228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.623233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.623240:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.623242:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.623243:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.623243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.623244:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.623245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.623747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.623755:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.623758:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.623760:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.623762:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.623762:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.623763:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.623765:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.623766:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.623767:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.623768:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.623771:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.623773:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.623776:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.623777:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.623777:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.623779:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.623780:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.623781:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.623783:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.623783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.623786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.623790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.623794:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.623796:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.623797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.623798:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.623798:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.623800:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677474.623800:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.623801:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.623802:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.623803:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.623839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677474.623844:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.623847:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677474.623847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677474.623849:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.623852:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.623854:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677474.623855:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.623857:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.623858:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.623859:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677474.623862:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.623863:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677474.623866:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.623866:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677474.623867:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677474.623869:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677474.623870:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.623870:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677474.623871:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677474.623875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.623883:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.623886:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.623886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.623887:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.623889:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.623892:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.624266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.624273:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.624276:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.624279:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.624280:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.624282:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.624283:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.624285:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.624286:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.624287:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.624288:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.624292:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.624297:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.624300:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.624301:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.624301:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.624303:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.624304:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.624305:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.624307:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.624307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.624309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.624313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.624316:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.624319:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.624320:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.624320:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.624321:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.624322:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.624324:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.624324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.624325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.624325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.624330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.624335:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.624338:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.624339:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.624339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.624340:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.624340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.624531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.624540:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.624544:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.624548:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.624550:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.624551:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.624553:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:4.2:1619677474.624555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:13.0:1619677474.624556:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.624558:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.624560:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.624562:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677474.624563:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.624566:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.624566:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677474.624567:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.624569:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.624570:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:26.0:1619677474.624572:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.624573:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:13.0:1619677474.624574:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.624575:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.624576:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677474.624576:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:26.0:1619677474.624577:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677474.624577:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:26.0:1619677474.624578:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:13.0:1619677474.624579:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677474.624580:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000400:00000200:26.0:1619677474.624581:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.624581:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.624582:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:26.0:1619677474.624583:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:9.2:1619677474.624585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.624586:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.624586:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677474.624587:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (23)++
00000800:00000200:26.0:1619677474.624589:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (24)--
00000800:00000200:26.0:1619677474.624589:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.624590:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.624590:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:2.0:1619677474.624593:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (23)++
00000800:00000200:2.0:1619677474.624595:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (24)--
00000800:00000200:18.0:1619677474.624596:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (24)++
00000800:00000200:18.0:1619677474.624597:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (24)--
00000400:00000200:2.0:1619677474.624597:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.624598:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.624599:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677474.624600:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.624603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.624604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.624604:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.624608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677474.624615:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.624618:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677474.624618:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.624619:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.624620:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.624620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.624620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.624627:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.624630:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.624631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.624632:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.624635:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.624637:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.624638:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.624639:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.624640:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.624641:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.624644:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.624646:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.624648:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.624648:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.624649:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.624650:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.624651:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.624652:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.624653:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.624653:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.624655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.624659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.624662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.624664:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.624665:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.624666:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677474.624666:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.624667:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.624668:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.624669:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.624669:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.624670:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.625496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.625503:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.625506:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.625508:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.625511:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.625511:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.625512:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.625513:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.625515:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.625515:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.625517:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.625520:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.625521:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.625524:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.625525:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.625525:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.625527:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.625528:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.625529:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.625529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.625535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.625543:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.625546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.625546:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.625547:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.625549:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.625551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.625764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.625768:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.625771:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.625773:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.625774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.625775:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.625777:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.625778:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.625779:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.625780:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.625781:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.625784:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.625786:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.625788:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.625789:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.625789:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.625791:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.625791:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.625792:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.625793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.625808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.625816:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.625819:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.625819:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.625821:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.625822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.625823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.626062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.626069:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.626071:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.626073:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.626074:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.626077:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.626078:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.626080:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.626081:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.626082:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.626083:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.626087:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.626088:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.626090:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.626091:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.626092:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.626093:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.626094:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.626095:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.626096:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.626096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.626099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.626102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.626108:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.626109:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.626109:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677474.626110:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.626110:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.626110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.626111:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.626112:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.626112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677474.626113:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.626499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.626507:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.626510:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.626512:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.626514:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.626515:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.626517:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.626518:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.626519:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.626520:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.626521:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.626525:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.626527:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.626529:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.626530:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.626530:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.626532:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.626532:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.626534:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.626534:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.626538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.626545:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.2:1619677474.626546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:30.0:1619677474.626548:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.626548:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.626550:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.626552:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.626553:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:17.0:1619677474.626553:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:25.2:1619677474.626556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:17.0:1619677474.626556:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.626556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.626559:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.626561:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.626562:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:17.0:1619677474.626562:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.626563:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.626565:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.626565:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000400:00000200:17.0:1619677474.626565:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.626566:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000400:00000200:17.0:1619677474.626566:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.626567:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.626567:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:17.0:1619677474.626567:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.626570:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.626572:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.626574:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.626575:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.626576:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.626577:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.626578:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.626578:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.626579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.626595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.626603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.626606:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.626606:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.626608:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.626610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.626611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.626614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.626622:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.626624:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.626625:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.626626:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.626627:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.626627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.626893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.626900:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.626902:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.626904:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.626905:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.626907:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.626908:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.626909:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.626911:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.626912:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.626913:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.626916:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.626918:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.626921:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.626921:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.626922:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.626924:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.626924:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.626925:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.626926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.626930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.626938:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.626940:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.626941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.626941:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.626943:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.626946:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.626951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.626958:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.626961:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.626962:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.626962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.626963:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.626964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677474.627001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.627006:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.627008:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.627010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.627011:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.627013:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.627015:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.627016:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.627017:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.627018:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.627019:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.627022:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.627024:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.627026:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.627027:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.627028:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.627029:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.627030:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.627030:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.627031:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.627035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.627043:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.627046:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.627047:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.627047:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.627049:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.627051:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.627311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.627319:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.627322:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.627324:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.627326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.627326:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.627327:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.627328:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.627330:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.627331:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.627332:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.627335:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.627336:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.627338:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.627339:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.627340:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.627341:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.627342:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.627343:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.627343:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.627347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.627354:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.627357:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.627358:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.627359:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.627361:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.627362:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.627412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.627420:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.627423:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.627424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.627425:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.627428:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.627429:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.627430:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.627432:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.627432:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.627433:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.627436:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.627438:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.627441:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.627441:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.627442:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.627444:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.627444:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.627445:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.627445:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.627450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.627457:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.627460:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.627461:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.627461:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.627463:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.627466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.627582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.627589:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.627592:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.627593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.627594:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.627597:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.627599:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.627600:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.627601:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.627602:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.627603:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.627606:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.627607:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.627610:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.627610:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.627611:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.627612:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.627613:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.627613:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.627614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.627618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.627626:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.627628:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.627629:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.627629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.627631:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.627634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.628160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.628168:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.628171:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.628173:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.628174:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.628175:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.628177:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.628178:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.628179:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.628180:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.628181:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.628184:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.628185:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.628189:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.628189:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.628190:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.628191:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.628192:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.628193:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.628193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.628198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.628200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677474.628205:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.628207:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.628208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.628208:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.628209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.628210:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.628211:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.628211:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.628213:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.628214:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.628216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000400:00000200:10.0:1619677474.628216:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.628217:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.628218:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.628220:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.628221:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.628222:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.628223:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.628225:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.628226:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.628226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.628227:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.628227:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.628228:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.628228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677474.628229:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.628229:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.628230:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.628232:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.628232:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.628233:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.628233:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.628237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.628244:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.628247:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.628248:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.628248:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.628249:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.628252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.628512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.628517:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.628520:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.628522:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.628523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.628524:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.628526:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.628527:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.628528:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.628529:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.628530:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.628534:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.628535:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.628538:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.628539:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.628539:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.628541:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.628541:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.628542:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.628543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.628547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.628556:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.628558:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.628559:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.628560:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.628561:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.628563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.628596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.628604:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.628606:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.628607:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.628609:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.628612:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.628613:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.628615:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.628616:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.628617:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.628618:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.628621:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.628622:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.628625:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.628626:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.628626:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.628628:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.628629:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.628629:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.628630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.628634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.628641:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.628644:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.628645:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.628645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.628647:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.628650:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.629139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.629147:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.629150:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.629151:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.629152:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.629155:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.629156:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677474.629157:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.629158:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.629159:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.629160:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677474.629163:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.629164:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.629166:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.629166:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.629167:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.629168:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.629169:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.629169:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.629170:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.629174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.629180:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.629182:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.629182:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.629183:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.629184:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.629185:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.629799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677474.629806:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.629809:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677474.629811:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.629814:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.629814:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677474.629815:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.629816:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.629818:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.629818:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.629819:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.629822:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.629824:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.629826:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.629827:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.629827:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.629829:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.629830:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.629830:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.629831:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.629835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.629843:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.629845:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.629846:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.629846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.629848:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.629850:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.630335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677474.630339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.630344:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.630346:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.630348:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:26.0:1619677474.630349:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.630349:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677474.630351:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.630351:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.630352:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.630354:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.630356:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.630356:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.630357:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.630360:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.630361:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.630361:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677474.630363:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.630363:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.630364:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.630365:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.630366:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.630368:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.630369:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:10.0:1619677474.630369:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.630370:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.630371:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.630372:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.630372:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.630373:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.630374:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.630374:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.630375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677474.630375:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.630375:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.630376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.630377:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.630378:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:9.2:1619677474.630378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.630379:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.630380:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.630381:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677474.630383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677474.630384:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:4.2:1619677474.630386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:8.0:1619677474.630387:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.630388:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.630389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:12.0:1619677474.630390:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:8.0:1619677474.630390:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.630392:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677474.630393:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.630393:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.630394:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:12.0:1619677474.630395:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.630396:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.630396:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:13.0:1619677474.630397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.630398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677474.630398:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.630401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677474.630407:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.630410:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.630411:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.630411:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.630412:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.630413:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677474.630618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:20.2:1619677474.630621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.630625:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677474.630627:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:20.0:1619677474.630628:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.630631:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.630631:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:21.0:1619677474.630632:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.630633:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.630633:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:20.0:1619677474.630634:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.630636:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.630638:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.630638:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.630639:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.630640:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677474.630643:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.630644:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.630645:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:18.0:1619677474.630646:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677474.630647:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.630648:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.630649:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:18.0:1619677474.630649:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.630651:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.630651:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.630652:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:18.0:1619677474.630652:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.630653:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677474.630654:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.630655:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:8.2:1619677474.630657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:18.0:1619677474.630661:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.630664:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.0:1619677474.630666:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.630668:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:18.0:1619677474.630668:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677474.630669:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.630670:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:21.0:1619677474.630671:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.630671:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:18.0:1619677474.630671:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.630673:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677474.630674:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677474.630674:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.630676:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677474.630677:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677474.630678:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:25.2:1619677474.630679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677474.630683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:13.0:1619677474.630686:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.630689:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.630690:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.630691:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000400:00000200:13.0:1619677474.630692:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.630693:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.630694:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677474.630694:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677474.630695:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.630697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:21.2:1619677474.631158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.631165:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.631168:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.631170:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.631173:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.631173:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.631174:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.631176:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.631177:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.631178:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.631179:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.631182:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.631184:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.631186:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.631187:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.631188:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.631189:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.631190:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.631191:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.631191:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.631195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.631201:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.631204:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.631205:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.631206:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.631208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.631209:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.631609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f390200] (20)++
00000800:00000200:10.0:1619677474.631617:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f390200] (21)++
00000800:00000200:10.0:1619677474.631620:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.2@o2ib3
00000800:00000200:26.0:1619677474.631621:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f390200] (22)--
00000400:00000200:10.0:1619677474.631622:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.631625:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.631626:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677474.631627:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.631628:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.631629:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.631630:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677474.631633:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.631634:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677474.631637:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.631637:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677474.631638:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677474.631639:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677474.631640:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f390200] (21)++
00000800:00000200:10.0:1619677474.631640:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f390200] (22)--
00000800:00000200:10.0:1619677474.631641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f390200] (21)--
00000800:00000200:24.2:1619677474.631645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.631649:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.631652:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677474.631653:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.631654:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677474.631655:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.631657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:5.2:1619677474.631792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.631798:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.631801:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.631802:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.631804:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.631807:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.631808:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.631810:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.631811:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.631812:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.631813:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.631816:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.631818:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.631820:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.631821:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.631822:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.631823:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.631824:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.631825:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.631826:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.631827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.631830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.631832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.631837:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.631838:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.631839:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.631839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.631840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.631841:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.631842:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.631843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.631844:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.631847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.631850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.631857:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.631859:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.631860:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.631860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.631861:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.631862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.631871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.631878:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.631881:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.631882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.631884:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.631887:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.631888:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.631889:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.631891:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.631892:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.631893:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.631896:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.631898:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.631901:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.631901:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.631902:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.631904:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.631905:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.631906:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.631907:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.631907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.631910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.631912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.2:1619677474.631913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677474.631918:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677474.631919:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:2.0:1619677474.631919:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677474.631920:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.631921:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.631922:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:2.0:1619677474.631922:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.631922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:12.0:1619677474.631923:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:10.0:1619677474.631923:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:2.0:1619677474.631924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:26.0:1619677474.631925:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.631925:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.631926:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:26.0:1619677474.631927:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.631929:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677474.631930:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.631932:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.631933:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.631934:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677474.631937:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.631938:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677474.631938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677474.631941:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.631942:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677474.631943:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.631944:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.631945:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677474.631946:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677474.631946:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.631947:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.631948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:31.0:1619677474.631949:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.631951:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677474.631951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:31.0:1619677474.631953:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.631954:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.631954:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677474.631954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000400:00000200:31.0:1619677474.631955:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.631956:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.631957:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.631958:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677474.631959:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:10.0:1619677474.631961:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:31.0:1619677474.631962:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.631962:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.631963:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.631963:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:1.0:1619677474.631963:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677474.631964:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677474.631964:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677474.631964:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:17.0:1619677474.631965:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.631966:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.631967:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.631967:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.631969:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:17.0:1619677474.631969:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677474.631970:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677474.631970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677474.631971:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.631973:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.631973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.631975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:1.0:1619677474.631978:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677474.631979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677474.631980:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.631981:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.631982:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.631982:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677474.631983:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:1.0:1619677474.631983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677474.631985:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.631986:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.631988:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.631988:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677474.631988:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.631989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.631990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.631991:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.631992:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.632323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.632331:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.632333:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.632334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.632336:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.632339:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.632340:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.632342:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.632343:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.632344:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.632345:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.632348:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.632349:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.632352:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.632353:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.632353:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.632355:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.632356:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.632357:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.632358:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.632358:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.632362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.632364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.632370:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.632372:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.632373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.632373:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.632374:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677474.632374:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.632375:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.632376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677474.632376:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.632379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.632500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.632507:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.632510:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.632511:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.632513:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.632515:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.632517:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.632518:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.632519:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.632520:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.632521:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.632524:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.632525:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.632528:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.632529:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.632529:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.632531:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.632531:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.632532:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.632532:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.632537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.632545:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.632548:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.632549:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.632549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.632552:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.632554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.632800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677474.632803:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677474.632806:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677474.632808:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.632809:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677474.632810:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.632811:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.632813:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.632814:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.632815:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.632816:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.632819:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.632821:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.632823:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.632824:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.632824:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.632826:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.632827:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677474.632827:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677474.632828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.632833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.632841:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.632843:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.632844:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.632845:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.632846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.632847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.633108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.633114:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.633117:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.633118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.633121:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.633124:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.633125:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.633126:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.633128:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.633129:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.633130:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.633133:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.633134:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.633136:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.633137:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.633137:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.633139:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.633139:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.633140:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.633141:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.633145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.633152:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.633154:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.633154:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.633155:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.633156:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.633157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.633434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4000] (20)++
00000800:00000200:6.0:1619677474.633441:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)++
00000800:00000200:22.0:1619677474.633444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000800:00000200:6.0:1619677474.633444:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.44@o2ib3
00000400:00000200:6.0:1619677474.633447:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.44@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.633449:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.633451:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:6.0:1619677474.633453:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.633454:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.633455:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.633456:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:6.0:1619677474.633460:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.44@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.633461:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:6.0:1619677474.633464:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.633465:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:6.0:1619677474.633466:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:6.0:1619677474.633468:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:6.0:1619677474.633469:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d4000] (21)++
00000800:00000200:6.0:1619677474.633470:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d4000] (22)--
00000800:00000200:6.0:1619677474.633471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)--
00000800:00000200:4.2:1619677474.633475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.2:1619677474.633479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:26.0:1619677474.633486:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677474.633486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4000] (20)++
00000800:00000200:26.0:1619677474.633488:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.633488:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)++
00000800:00000200:10.0:1619677474.633489:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2fa00] (23)--
00000400:00000200:26.0:1619677474.633491:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.633491:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.633492:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)++
00000400:00000200:10.0:1619677474.633493:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.51@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.633494:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.633494:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.44@o2ib3
00000800:00000200:6.0:1619677474.633495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000400:00000200:26.0:1619677474.633496:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677474.633497:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.44@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.633498:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.633499:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.633500:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.633500:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.633501:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.633502:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677474.633503:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.633503:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:10.0:1619677474.633504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.633504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (21)--
00000400:00000200:22.0:1619677474.633505:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.633506:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.633507:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.633511:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.44@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.633513:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.633516:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.633518:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.633518:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.633519:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.633519:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.633521:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.633521:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.633521:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.633521:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d4000] (21)++
00000800:00000200:26.0:1619677474.633522:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.633523:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d4000] (22)++
00000800:00000200:26.0:1619677474.633524:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.633524:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.633524:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d4000] (23)--
00000800:00000200:21.2:1619677474.633524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4000] (23)++
00000800:00000200:26.0:1619677474.633525:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.633526:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.633527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (23)--
00000800:00000200:25.2:1619677474.633528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.633530:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)++
00000800:00000200:22.0:1619677474.633532:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.44@o2ib3
00000800:00000200:6.0:1619677474.633532:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:6.0:1619677474.633533:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677474.633534:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.633535:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.44@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.633536:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.633537:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.633538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.633539:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (23)++
00000400:00000200:22.0:1619677474.633540:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:6.0:1619677474.633540:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d4000] (24)--
00000400:00000200:22.0:1619677474.633541:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.633541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (23)--
00000800:00000200:6.0:1619677474.633542:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:22.0:1619677474.633543:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.633543:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.633544:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.633545:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.633545:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.633546:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:22.0:1619677474.633550:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.44@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.633550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677474.633552:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677474.633552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.633553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000800:00000200:6.0:1619677474.633554:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.633555:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.633555:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.633556:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:6.0:1619677474.633556:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.633557:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677474.633557:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.633557:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.633558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.633559:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:6.0:1619677474.633559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.633560:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d4000] (21)++
00000800:00000200:22.0:1619677474.633560:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d4000] (22)--
00000800:00000200:8.0:1619677474.633560:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.633561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677474.633561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)--
00000400:00000200:8.0:1619677474.633562:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677474.633565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:8.0:1619677474.633565:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.633567:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.633568:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96503618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.633570:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.633571:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:8.0:1619677474.633571:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.633572:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.633574:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.633575:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.633575:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.633575:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.633577:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.633577:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.633580:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.0:1619677474.633580:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.633581:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.633582:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.633583:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.633584:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.633584:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.633585:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.633591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.633598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4000] (20)++
00000800:00000200:6.0:1619677474.633601:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.633603:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.633604:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)++
00000400:00000200:6.0:1619677474.633604:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.633606:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.44@o2ib3
00000400:00000200:6.0:1619677474.633606:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.633608:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.44@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.633609:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.633610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:22.0:1619677474.633611:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.633611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000400:00000200:22.0:1619677474.633614:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677474.633615:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.633616:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.633617:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.633618:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677474.633621:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.44@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.633623:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677474.633625:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.633626:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:22.0:1619677474.633627:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677474.633629:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677474.633629:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d4000] (21)++
00000800:00000200:22.0:1619677474.633630:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d4000] (22)--
00000800:00000200:22.0:1619677474.633630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)--
00000800:00000200:9.2:1619677474.633635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.633644:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.633646:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.633647:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.633647:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.633649:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.633652:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.633658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.2:1619677474.633662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4000] (20)++
00000800:00000200:2.0:1619677474.633666:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677474.633668:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)++
00000800:00000200:2.0:1619677474.633668:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.633669:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.633670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677474.633670:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.44@o2ib3
00000800:00000200:2.0:1619677474.633670:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.633671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677474.633672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000400:00000200:6.0:1619677474.633673:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.44@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.633675:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.633677:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677474.633678:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.633679:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.633680:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.633681:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677474.633684:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.44@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.633686:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677474.633688:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.633689:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677474.633690:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677474.633692:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677474.633692:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d4000] (21)++
00000800:00000200:6.0:1619677474.633693:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d4000] (22)--
00000800:00000200:6.0:1619677474.633693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)--
00000800:00000200:25.2:1619677474.633697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.633705:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.633708:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.633709:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.633710:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.633712:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.633713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.633764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.633771:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.633774:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.633774:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.633776:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.633779:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.633781:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.633782:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.633783:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.633784:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.633785:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.633788:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.633789:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.633792:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.633793:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.633794:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.633796:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.633796:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.633797:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.633797:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.2:1619677474.633799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4000] (20)++
00000800:00000200:25.2:1619677474.633801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.633806:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)++
00000800:00000200:22.0:1619677474.633809:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.44@o2ib3
00000800:00000200:6.0:1619677474.633810:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:22.0:1619677474.633812:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.44@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.633812:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.633813:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.633814:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.633815:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.633816:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677474.633817:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.633818:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.633819:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.633820:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:6.0:1619677474.633823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.633824:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.44@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.633824:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000800:00000200:6.0:1619677474.633825:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.633826:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.633829:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.633830:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677474.633831:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.633833:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.633833:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d4000] (22)++
00000800:00000200:21.2:1619677474.633833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4000] (21)++
00000800:00000200:22.0:1619677474.633835:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d4000] (23)++
00000800:00000200:22.0:1619677474.633836:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d4000] (24)--
00000800:00000200:22.0:1619677474.633837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (23)--
00000800:00000200:22.0:1619677474.633838:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)++
00000800:00000200:9.2:1619677474.633838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.633839:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.44@o2ib3
00000800:00000200:6.0:1619677474.633839:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (23)--
00000400:00000200:22.0:1619677474.633841:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.44@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.633841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4000] (22)++
00000400:00000200:22.0:1619677474.633842:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.633844:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677474.633845:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:22.0:1619677474.633845:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.633847:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:22.0:1619677474.633847:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.633847:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (23)++
00000400:00000200:22.0:1619677474.633848:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.633848:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677474.633848:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d4000] (24)--
00000400:00000200:24.0:1619677474.633849:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.633849:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677474.633849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (23)--
00000800:00000200:6.0:1619677474.633850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000400:00000200:24.0:1619677474.633852:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.633854:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.44@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.633856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.633857:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677474.633859:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.633860:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677474.633861:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.2:1619677474.633862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4000] (21)++
00000800:00000200:22.0:1619677474.633863:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677474.633863:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d4000] (22)++
00000800:00000200:22.0:1619677474.633864:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d4000] (23)--
00000800:00000200:22.0:1619677474.633865:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000800:00000200:9.2:1619677474.633865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677474.633866:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)++
00000800:00000200:22.0:1619677474.633868:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.44@o2ib3
00000800:00000200:6.0:1619677474.633868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000800:00000200:25.2:1619677474.633869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:22.0:1619677474.633870:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.44@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.633871:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677474.633872:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.633874:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.0:1619677474.633874:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.633875:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:22.0:1619677474.633875:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.633875:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.633876:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.633876:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677474.633877:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677474.633877:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:22.0:1619677474.633878:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.633878:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677474.633879:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:22.0:1619677474.633881:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.44@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677474.633881:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.633883:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.633884:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677474.633884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677474.633886:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.633886:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677474.633887:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:22.0:1619677474.633887:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677474.633889:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:22.0:1619677474.633889:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d4000] (21)++
00000800:00000200:22.0:1619677474.633890:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d4000] (22)--
00000800:00000200:22.0:1619677474.633891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)--
00000800:00000200:8.2:1619677474.633895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.2:1619677474.633896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4000] (20)++
00000800:00000200:6.0:1619677474.633901:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)++
00000800:00000200:17.0:1619677474.633904:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677474.633904:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.44@o2ib3
00000800:00000200:22.0:1619677474.633905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000800:00000200:17.0:1619677474.633907:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:6.0:1619677474.633907:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.44@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.633908:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.633908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:6.0:1619677474.633909:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.633910:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.633911:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677474.633912:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.633913:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.633914:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.633915:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677474.633918:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.44@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.633919:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677474.633919:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677474.633922:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.633922:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:6.0:1619677474.633923:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677474.633924:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677474.633925:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d4000] (21)++
00000800:00000200:6.0:1619677474.633926:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d4000] (22)--
00000800:00000200:6.0:1619677474.633926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)--
00000800:00000200:8.2:1619677474.633930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.633938:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.633941:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.633941:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.633942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.633943:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.633945:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.633948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4000] (20)++
00000800:00000200:22.0:1619677474.633953:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)++
00000800:00000200:22.0:1619677474.633956:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.44@o2ib3
00000800:00000200:6.0:1619677474.633956:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000400:00000200:22.0:1619677474.633958:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.44@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.633960:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.633961:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677474.633963:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.633964:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.633965:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.633966:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677474.633968:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.44@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.633970:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677474.633972:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.633972:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:22.0:1619677474.633973:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677474.633974:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677474.633975:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d4000] (21)++
00000800:00000200:22.0:1619677474.633976:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d4000] (22)++
00000800:00000200:22.0:1619677474.633977:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d4000] (23)--
00000800:00000200:22.0:1619677474.633977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000800:00000200:9.2:1619677474.633980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.633982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d4000] (21)++
00000800:00000200:6.0:1619677474.633987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)++
00000800:00000200:18.0:1619677474.633989:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:6.0:1619677474.633989:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d4000] (23)--
00000800:00000200:6.0:1619677474.633989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (22)--
00000800:00000200:6.0:1619677474.633990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d4000] (21)--
00000800:00000200:18.0:1619677474.633991:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.633992:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.633992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.633994:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.633997:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.634120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:4.2:1619677474.634123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.634125:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.634127:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.634129:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:20.0:1619677474.634129:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.634131:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.634131:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.634133:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.634133:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.634134:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.634136:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.634137:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.634138:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.634141:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.634141:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:21.0:1619677474.634142:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.634143:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.634143:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.634145:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.634146:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.634147:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677474.634151:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:21.0:1619677474.634154:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.634156:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677474.634158:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677474.634158:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.634159:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.634159:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.634160:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.634160:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.634161:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:20.0:1619677474.634161:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.634162:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.634162:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.634162:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:30.0:1619677474.634163:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.634163:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.634163:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.634164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.634164:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (23)++
00000800:00000200:31.0:1619677474.634165:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677474.634165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (24)--
00000400:00000200:30.0:1619677474.634166:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.634166:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677474.634166:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (24)--
00000400:00000200:30.0:1619677474.634167:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677474.634167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:30.0:1619677474.634169:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677474.634169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:30.0:1619677474.634170:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.634171:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.634172:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:4.2:1619677474.634173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000400:00000200:30.0:1619677474.634176:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.634176:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:30.0:1619677474.634177:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.634177:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.634178:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.634178:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.634178:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:9.2:1619677474.634178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.634179:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.634180:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.634180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:30.0:1619677474.634181:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.634181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:30.0:1619677474.634182:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:30.0:1619677474.634182:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (23)++
00000800:00000200:21.0:1619677474.634182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677474.634185:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (24)--
00000800:00000200:30.0:1619677474.634185:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.634185:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)++
00000800:00000200:30.0:1619677474.634186:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.634187:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677474.634187:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000400:00000200:24.0:1619677474.634189:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.634189:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.634190:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.634193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:9.2:1619677474.634200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.634206:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.634208:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.634209:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:8.0:1619677474.634209:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.634211:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677474.634211:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.634212:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677474.634213:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677474.634213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.634214:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677474.634214:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:4.2:1619677474.634264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677474.634271:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.634273:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677474.634274:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677474.634276:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.634278:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.634280:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677474.634281:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.634282:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.634283:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.634284:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677474.634287:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.634289:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.634291:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.634292:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677474.634292:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677474.634299:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677474.634299:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:25.2:1619677474.634300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677474.634300:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677474.634300:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:31.0:1619677474.634308:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.634310:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.634311:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.634312:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.634315:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.634315:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.634661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677474.634668:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.634671:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677474.634673:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.634674:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.634675:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677474.634676:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677474.634677:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.634678:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.634679:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.634680:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677474.634683:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.634684:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677474.634688:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.634689:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677474.634689:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.634691:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677474.634691:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.634692:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677474.634693:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677474.634697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.634705:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.634708:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.634709:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.634709:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.634711:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.634715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.634817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.634824:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.634827:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.634830:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.634832:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.634833:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.634834:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.634835:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.634837:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.634838:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.634839:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.634842:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.634843:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.634846:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.634846:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.634847:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.634848:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.634849:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.634850:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.634850:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.634854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.634861:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.634863:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.634864:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.634865:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.634867:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.634868:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.634871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.634878:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.634880:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.634881:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.634882:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.634882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.634883:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.635924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.635932:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.635935:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.635937:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.635938:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.635939:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.635940:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.635940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.635941:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.635942:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.635943:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.635946:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.635947:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.635949:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.635949:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.635950:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.635951:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.635951:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.635952:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.635953:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.635957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.635964:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.635966:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.635967:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.635969:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.635971:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.635971:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677474.636135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.636143:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.636146:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.636146:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.636148:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.636151:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.636152:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.636153:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.636155:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.636155:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.636156:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.636159:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.636161:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.636163:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.636163:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.636164:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.636165:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.636166:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.636166:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.636167:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.636171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.636178:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.636180:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.636181:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.636182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.636183:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.636185:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.636864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.636871:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.636874:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.636876:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.636877:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.636879:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.636879:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.636880:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.636881:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.636882:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.636883:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.636886:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.636887:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.636889:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.636890:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.636891:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.636892:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.636893:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.636894:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.636894:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.636898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.636906:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.636908:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.636909:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.636909:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.636910:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.636913:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.637008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.637015:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.637017:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.637018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.637020:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.637022:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.637024:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.637025:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.637026:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.637027:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.637029:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.637031:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.637033:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.637036:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.637036:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.637037:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.637039:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.637039:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.637040:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.637040:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.637045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.637053:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.637056:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.637057:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.637057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.637059:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.637061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.637267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.637270:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.637273:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.637275:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.637276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.637277:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.637279:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.637280:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.637282:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.637283:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.637284:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.637287:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.637289:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.637292:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677474.637292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677474.637293:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.637299:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677474.637300:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.0:1619677474.637301:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.637301:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.637302:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:26.0:1619677474.637303:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:20.0:1619677474.637303:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.637304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677474.637304:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.637306:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677474.637307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:26.0:1619677474.637309:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.637310:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.2:1619677474.637310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677474.637312:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.637313:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.637314:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.637314:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:26.0:1619677474.637315:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677474.637315:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.637315:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.637315:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.637316:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.637317:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.637317:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.637319:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.637319:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.637320:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.637321:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677474.637322:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.637324:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.637325:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.637326:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.2:1619677474.637327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:26.0:1619677474.637328:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.637328:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.637329:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.637330:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.637334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677474.637335:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.637337:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.637338:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.637339:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.637339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.637341:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677474.637342:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.637345:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.637346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.637346:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.637348:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.637351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.637736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.637743:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.637746:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.637747:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.637749:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.637752:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.637753:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.637754:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.637755:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.637756:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.637757:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.637760:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.637762:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.637764:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.637765:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.637766:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.637767:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.637768:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.637769:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.637769:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.637773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.637781:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.637783:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.637784:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.637784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.637787:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.637790:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.637795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.637802:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.637805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.637805:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.637806:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.637807:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.637807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.638252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.638256:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.638258:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.638260:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.638261:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.638262:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.638263:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.638264:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.638266:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.638267:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.638268:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.638271:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.638273:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.638275:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.638276:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.638277:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.638278:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.638279:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.638279:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.638280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.638284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.638292:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.638297:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.638299:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.638300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.638302:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.638304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.638479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.638487:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.638490:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.638491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.638492:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.638495:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.638497:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.638498:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.638499:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.638500:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.638501:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.638505:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.638507:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.638510:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.638511:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.638511:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.638513:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.638514:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.638514:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.638515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.638519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.638527:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.638530:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.638531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.638531:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.638533:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.638536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.638742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.638751:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.638754:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.638756:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.638758:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.638758:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.638759:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.638760:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.638762:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.638763:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.638764:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.638767:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.638768:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.638770:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.638771:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.638772:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.638773:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.638773:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.638775:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.638776:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.638777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.638779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.638783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.638785:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.638788:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.638789:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.638789:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.638790:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.638791:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.638793:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.638793:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.638794:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.638794:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.638932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782c00] (20)++
00000800:00000200:18.0:1619677474.638939:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782c00] (21)++
00000800:00000200:18.0:1619677474.638942:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.18@o2ib3
00000800:00000200:2.0:1619677474.638943:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782c00] (22)--
00000400:00000200:18.0:1619677474.638945:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.18@o2ib3 : GET - for me
00000400:00000200:18.0:1619677474.638948:0:19933:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.18@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:18.0:1619677474.638951:0:19933:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.18@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:18.0:1619677474.638952:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919065782c00] (21)++
00000800:00000200:18.0:1619677474.638954:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782c00] (22)++
00000800:00000200:4.2:1619677474.638954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677474.638955:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919065782c00] (23)--
00000400:00000200:18.0:1619677474.638957:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.18@o2ib3 local destination
00000800:00000200:26.0:1619677474.638961:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.2:1619677474.638961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782c00] (22)++
00000400:00000200:18.0:1619677474.638961:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.18@o2ib3(10.51.5.18@o2ib3:10.51.5.18@o2ib3) : REPLY try# 0
00000800:00000200:18.0:1619677474.638963:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.18@o2ib3
00000800:00000200:26.0:1619677474.638964:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:18.0:1619677474.638965:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c610fe00] -> 10.51.5.18@o2ib3 (2) version: 12
00000800:00000200:10.0:1619677474.638965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.638967:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.638967:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919065782c00] (23)++
00000800:00000200:18.0:1619677474.638967:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919065782c00] (24)++
00000800:00000200:2.0:1619677474.638967:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782c00] (23)++
00000800:00000200:18.0:1619677474.638968:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919065782c00] (26)--
00000400:00000200:26.0:1619677474.638969:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.638969:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782c00] (25)--
00000800:00000200:2.0:1619677474.638969:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782c00] (26)--
00000800:00000200:18.0:1619677474.638970:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782c00] (22)--
00000800:00000200:2.0:1619677474.638970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782c00] (23)--
00000400:00000200:26.0:1619677474.638971:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.638972:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.638973:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.638974:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677474.638974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919065782c00] (21)++
00000400:00000200:26.0:1619677474.638975:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.638978:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.638979:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919065782c00] (22)++
00000800:00000200:26.0:1619677474.638980:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.638981:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919065782c00] (23)--
00000800:00000200:26.0:1619677474.638982:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.638982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782c00] (22)--
00000400:00000200:2.0:1619677474.638982:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.638983:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.638984:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:2.0:1619677474.638984:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.18@o2ib3: REPLY: OK
00000800:00000200:26.0:1619677474.638985:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.638986:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:2.0:1619677474.638986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919065782c00] (21)--
00000800:00000200:26.0:1619677474.638987:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.638989:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.638989:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.638992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.638995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.639000:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.639002:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677474.639002:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677474.639003:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.639003:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.639004:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.639005:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677474.639005:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677474.639006:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.639008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.639910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.639919:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.639921:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.639923:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.639926:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.639926:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.639927:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.639928:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.639929:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.639930:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.639931:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.639934:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.639936:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.639937:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.639938:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.639938:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.639940:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.639940:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.639941:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.639942:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.639946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.639953:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.639956:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.639956:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.639958:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.639960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.639961:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.640013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.640019:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.640022:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.640023:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.640025:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.640026:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.640028:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.640029:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.640031:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.640032:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.640033:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.640037:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.640038:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.640042:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.640042:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.640043:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.640045:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.640046:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.640047:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.640047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.640061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.640068:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.640071:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.640072:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.640074:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.640075:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.640077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.640393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.640399:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.640402:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.640403:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.640406:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.640408:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.640409:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.640411:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.640412:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.640413:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.640414:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.640417:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.640419:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.640421:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.640422:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.640423:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.640424:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.640425:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.640425:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.640426:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.640431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.640440:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.640442:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.640443:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.640445:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.640446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.640447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.640656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.640664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.640667:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.640669:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.640672:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.640672:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.640673:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.640675:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.640676:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.640677:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.640678:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.640681:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.640683:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.640685:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.640686:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.640686:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.640688:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.640689:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.640689:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.640690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.640694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.640701:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.640704:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.640705:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.640705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.640707:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.640709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.640714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.640720:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.640722:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.640723:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.640723:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.640724:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.640725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.641764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:5.2:1619677474.641764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.641771:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677474.641773:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677474.641773:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.641774:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677474.641776:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677474.641777:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.641779:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.641779:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.641781:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.641781:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677474.641781:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677474.641782:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677474.641782:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.641784:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.641784:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.641785:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.641786:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.641786:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677474.641789:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.641790:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677474.641791:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.641794:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677474.641795:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.641797:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677474.641797:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.641798:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677474.641799:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.641799:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.641800:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:17.0:1619677474.641800:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:31.0:1619677474.641801:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677474.641801:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.641801:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677474.641802:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:17.0:1619677474.641802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677474.641803:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.641804:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.641805:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.641807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:25.2:1619677474.641811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677474.641816:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:30.0:1619677474.641818:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:6.0:1619677474.641818:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.641819:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.641819:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.641821:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.641821:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.641821:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.641823:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.641823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.641825:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.641826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.641827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.641834:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.641837:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.641838:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.641838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.641839:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.641840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.641874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.641878:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:4.2:1619677474.641878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677474.641882:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.641884:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.641884:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.641886:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.641887:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677474.641888:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.641889:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.641890:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.641891:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677474.641891:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.641892:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.641894:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.641894:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.641897:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677474.641897:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.641899:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.641899:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.641900:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.641901:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.641902:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.641903:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.641903:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.641903:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.641904:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677474.641905:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.641905:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.641906:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.641906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:10.0:1619677474.641908:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.641909:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:25.2:1619677474.641911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.641912:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.641913:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.641913:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.641915:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.641915:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.641916:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.641916:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.641918:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:9.2:1619677474.641920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.641920:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.641921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.641921:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.641923:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.641925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677474.641927:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.641930:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:25.2:1619677474.641931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000400:00000200:24.0:1619677474.641931:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.641931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.641933:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.641935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.641938:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.641940:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.641941:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.641941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.641942:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.641943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.642132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.642140:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.642142:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.642143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.642145:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.642147:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.642149:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.642150:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.642152:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.642153:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.642153:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.642157:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.642158:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.642161:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.642162:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.642162:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.642164:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.642165:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.642166:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.642167:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.642168:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.642170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.642174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.642177:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.642180:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.642181:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.642181:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:6.0:1619677474.642181:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.642183:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.642183:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.642183:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.642184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.642186:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.642234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.642239:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.642242:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.642244:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.642245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.642246:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.642247:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.642249:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.642250:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.642251:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.642252:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.642255:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.642257:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.642258:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.642259:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.642260:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.642261:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.642262:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.642263:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.642263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.642268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.642276:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.642278:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.642279:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.642280:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.642281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.642282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.642474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677474.642481:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.642484:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.642485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.642487:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.642490:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.642491:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677474.642493:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.642494:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.642495:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.642496:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677474.642499:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.642501:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677474.642503:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.642504:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.642505:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677474.642506:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.642506:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.642507:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.642508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.642512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.642516:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.642518:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.642519:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.642519:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.642521:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.642523:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677474.642750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677474.642755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677474.642759:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.642761:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677474.642761:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:4.2:1619677474.642762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000400:00000200:31.0:1619677474.642763:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.642764:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:31.0:1619677474.642765:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.642765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:30.0:1619677474.642766:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.642767:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677474.642767:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.642768:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.642769:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.642769:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000400:00000200:2.0:1619677474.642769:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.642770:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.642771:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677474.642772:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677474.642772:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677474.642773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:2.0:1619677474.642773:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.642774:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.642775:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.642776:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.642777:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677474.642778:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.642779:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.642780:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.642781:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:24.0:1619677474.642781:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677474.642783:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.642783:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:24.0:1619677474.642783:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96503579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.642783:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.642784:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.642785:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:24.0:1619677474.642785:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.642785:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.642786:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677474.642786:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.642786:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:24.0:1619677474.642787:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.642787:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677474.642788:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:2.0:1619677474.642788:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677474.642789:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:2.0:1619677474.642789:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677474.642790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677474.642790:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.642790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:2.0:1619677474.642790:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.642791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677474.642792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.642792:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.642795:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.642796:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.2:1619677474.642796:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677474.642796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.642797:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.642799:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.642799:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:21.0:1619677474.642800:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677474.642801:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:24.0:1619677474.642801:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:30.0:1619677474.642802:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:24.0:1619677474.642802:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000400:00000200:30.0:1619677474.642803:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.642803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:21.0:1619677474.642803:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677474.642804:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677474.642804:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:30.0:1619677474.642805:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.642805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.642806:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.642806:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677474.642806:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.642806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677474.642807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677474.642807:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.642807:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.642808:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677474.642808:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.642808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000800:00000200:2.0:1619677474.642808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.642809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.642810:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.642810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677474.642810:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:31.0:1619677474.642811:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677474.642811:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96503578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.642811:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.642812:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.642812:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.642813:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677474.642813:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677474.642813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:21.0:1619677474.642814:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.642815:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:21.0:1619677474.642815:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.642816:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000400:00000200:22.0:1619677474.642817:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.642817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:21.0:1619677474.642818:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.642818:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677474.642819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000400:00000200:22.0:1619677474.642819:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.642819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:21.0:1619677474.642820:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677474.642822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.642823:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.642824:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.642825:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.642827:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.642827:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.642828:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.642830:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.642830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677474.642832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.642835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677474.642839:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677474.642839:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.642841:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:30.0:1619677474.642842:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677474.642842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677474.642843:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.642843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677474.642845:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.642846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.642847:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.642852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.642859:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.642862:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.642863:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.642863:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.642864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.642865:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.642880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.642888:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.642890:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.642891:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.642893:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.642895:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.642897:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.642898:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.642899:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.642900:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.642901:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.642905:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.642906:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.642909:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.642910:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.642911:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.642912:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.642913:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.642914:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.642914:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.642918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.642926:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.642929:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.642929:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.642930:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.642932:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.642935:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.643877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.643885:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.643888:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.643889:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.643891:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.643891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:8.0:1619677474.643893:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.643895:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.643896:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.643897:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.643898:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:8.0:1619677474.643898:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.643899:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.643901:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.643901:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:8.0:1619677474.643903:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.643904:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.643904:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.643907:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.643907:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.643908:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677474.643908:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.643908:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677474.643909:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.643910:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.643910:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.643910:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677474.643911:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.643912:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:8.0:1619677474.643912:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.643913:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.643913:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677474.643915:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.643916:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:25.2:1619677474.643917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.643918:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.643919:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.643919:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:4.2:1619677474.643919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677474.643920:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.643921:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.643921:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.643922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677474.643925:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.643926:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.2:1619677474.643926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677474.643927:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.643928:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677474.643928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.643929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677474.643929:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.643930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.643931:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.643932:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:22.0:1619677474.643933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677474.643934:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.643935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.643935:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.643937:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.643938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677474.644114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.644121:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.644123:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.644124:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.644126:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.644128:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.644130:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.644131:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.644132:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.644133:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.644134:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.644137:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.644139:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.644142:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.644142:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.644143:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.644145:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.644145:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.644146:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.644147:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677474.644152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.644159:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.644162:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.644163:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.644164:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.644165:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.644168:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677474.644292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.644297:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.644299:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.644301:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.644302:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.644304:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.644305:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.644307:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.644308:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.644309:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.644310:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.644313:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.644315:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.644318:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.644318:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.644319:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.644321:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.644322:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.644322:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.644323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.644328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.644336:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.644338:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.644339:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.644340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.644342:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.644344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.644642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.644650:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.644653:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.644655:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.644657:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.644658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.644658:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.644660:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.644661:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.644662:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.644663:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.644667:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.644669:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.644671:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.644672:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.644673:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.644674:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.644675:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.644676:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.644676:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.644680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.644687:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.644690:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.644690:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.644691:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.644693:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.644695:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.644730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.644738:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.644740:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.644741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.644743:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.644746:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.644747:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.644748:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.644750:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.644751:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.644752:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.644756:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.644757:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.644760:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.644761:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.644762:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.644764:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.644764:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.644765:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.644766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.644770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.644776:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.644779:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.644780:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.644782:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.644783:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.644784:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.645011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.645019:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.645022:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.645023:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.645024:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.645026:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.645028:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.645029:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.645030:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.645031:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.645032:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.645035:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.645036:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.645039:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.645039:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.645040:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.645042:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.645042:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.645043:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.645043:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.645047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.645053:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.645056:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.645056:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.645057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.645058:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.645060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.645203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.645210:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.645213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.645213:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677474.645215:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.645217:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.645218:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677474.645219:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.645220:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.645221:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.645222:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677474.645224:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.645225:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.645227:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.645227:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.645227:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.645229:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.645229:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.645230:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.645230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.645234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.645240:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.645242:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.645243:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.645244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.645245:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.645246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.645331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.645336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.645339:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.645341:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.645342:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.645343:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.645345:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.645346:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.645348:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.645349:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.645350:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.645352:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.645354:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.645357:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.645358:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.645358:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.645360:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.645361:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.645362:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.645362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.645366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.645374:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.645377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.645377:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.645378:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.645380:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.645383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.645415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.645423:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.645426:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.645428:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.645430:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.645431:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.645431:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.645432:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.645434:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.645434:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.645435:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.645439:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.645440:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.645443:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.645444:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.645445:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.645446:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.645447:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.645448:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.645448:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.645452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.645459:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.645461:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.645461:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.645463:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.645465:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.645466:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.645860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.645867:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.645870:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.645871:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.645873:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.645874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.645875:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677474.645876:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.645877:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.645878:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.645879:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677474.645883:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.645884:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677474.645887:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.645888:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677474.645889:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677474.645891:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677474.645891:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.645893:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.645894:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.645895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.2:1619677474.645897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677474.645902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:1.0:1619677474.645905:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.645908:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.645909:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.645909:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.645910:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:1.0:1619677474.645911:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.645913:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.645914:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:1.0:1619677474.645914:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677474.645915:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.646645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.646652:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.646655:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.646656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.646658:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.646660:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.646661:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677474.646662:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.646663:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.646664:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.646665:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:21.2:1619677474.646667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677474.646667:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.646669:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.646670:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.646671:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.646671:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.646672:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.646673:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.646674:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:30.0:1619677474.646675:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.646675:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.646676:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:30.0:1619677474.646678:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:9.2:1619677474.646679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:30.0:1619677474.646680:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.646682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677474.646683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.646683:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.646684:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.646685:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.646686:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.646687:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.646687:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:30.0:1619677474.646688:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.646688:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.646689:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.646691:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.646691:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.646691:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.646692:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.646692:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.646693:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:30.0:1619677474.646694:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.646694:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677474.646694:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:30.0:1619677474.646696:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.646697:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.646698:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:9.2:1619677474.646698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:30.0:1619677474.646699:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.646700:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.646700:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.646701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677474.646704:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:9.2:1619677474.646705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.646707:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.646708:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.646708:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.646709:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.646709:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677474.646711:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.646714:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.646715:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.646715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.646717:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.646719:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.646726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.646732:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.646735:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.646736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.646736:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.646737:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.646738:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.646883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.646890:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.646893:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.646894:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.646895:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.646897:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.646898:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.646899:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.646901:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.646902:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.646902:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.646905:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.646907:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.646910:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.646910:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.646911:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.646913:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.646913:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.646914:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.646914:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.646919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.646926:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.646929:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.646929:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.646931:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.646934:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.646935:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.646938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677474.646944:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.646946:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.646947:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.646948:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.646948:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.646949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.646970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.646977:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.646979:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.646980:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.646981:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.646984:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.646985:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.646986:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.646988:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.646989:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.646989:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.646993:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.646994:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.646997:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.646998:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.646998:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.647000:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.647000:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.647001:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.647002:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.647006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.647014:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.647017:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.647018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.647018:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.647021:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.647024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.647042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.647048:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.647050:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.647051:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.647053:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.647055:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.647057:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.647058:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.647059:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.647060:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.647061:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.647064:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.647065:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.647068:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.647069:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.647069:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.647071:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.647071:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.647072:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.647073:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.647077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.647084:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.647087:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.647088:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.647088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.647090:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.647092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.647400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.647408:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.647411:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.647413:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.647416:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.647416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.647417:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.647418:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.647420:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.647421:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.647422:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.647425:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.647427:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.647429:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.647430:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.647430:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.647432:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.647433:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.647434:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.647436:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.647436:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.647438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.647443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.647448:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.647449:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.647450:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.647451:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.647452:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.647452:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.647452:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.647454:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.647455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.647455:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.647475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.647481:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:4.2:1619677474.647482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677474.647484:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.647486:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.647487:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.647490:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677474.647490:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.647492:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:26.0:1619677474.647493:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.647493:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677474.647493:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.647495:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.647496:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.647497:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.647498:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677474.647501:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.647503:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:26.0:1619677474.647504:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.647505:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:2.0:1619677474.647506:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.647507:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.647507:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:26.0:1619677474.647508:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.647508:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:26.0:1619677474.647510:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.647510:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677474.647510:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:26.0:1619677474.647511:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:2.0:1619677474.647511:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.647512:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:26.0:1619677474.647514:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677474.647515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.647516:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.647519:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.647520:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.647520:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.647522:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.647523:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.647523:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.647524:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.647524:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.647525:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:10.0:1619677474.647525:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.647526:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.647527:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.647528:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.647528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.647531:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677474.647532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.647534:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.647537:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.647537:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.647538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.647539:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677474.647539:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.647541:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677474.647541:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.647542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.647543:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.647650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.647657:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.647659:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.647660:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.647662:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.647665:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.647666:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.647667:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.647669:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.647670:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.647671:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.647674:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.647676:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.647678:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.647679:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.647680:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.647681:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.647681:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.647682:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.647682:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.647687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.647695:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.647698:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.647698:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.647700:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.647701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.647703:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.647894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677474.647901:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.647904:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677474.647906:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.647907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677474.647908:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.647910:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677474.647911:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.647912:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.647914:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.647915:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677474.647918:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.647919:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677474.647922:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.647923:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:30.0:1619677474.647923:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:30.0:1619677474.647925:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:30.0:1619677474.647925:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.647926:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.647927:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677474.647930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.647937:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.647939:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.647940:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.647940:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.647942:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.647945:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677474.648173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.648178:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.648181:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.648183:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.648184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.648186:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.648187:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.648188:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.648190:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.648191:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.648192:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.648194:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.648196:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.648198:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.648199:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.648200:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.648201:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.648202:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.648203:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.648204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677474.648206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.2:1619677474.648207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.648215:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677474.648215:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677474.648217:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.648218:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.648218:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.648219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677474.648220:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.648221:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.648222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:8.0:1619677474.648222:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.648224:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.648226:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.648228:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677474.648231:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:8.0:1619677474.648234:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.648235:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.648236:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.0:1619677474.648239:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:8.0:1619677474.648239:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.648241:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.648242:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677474.648242:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.0:1619677474.648243:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.648243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.648244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.648244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.0:1619677474.648245:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.648246:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.648246:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.648248:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.648248:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.648249:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.648250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677474.648251:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.648254:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.648255:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.648256:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.648258:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.648259:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.648261:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.648262:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.648263:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.648263:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.2:1619677474.648264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:26.0:1619677474.648266:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.648268:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.648270:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.648270:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.648271:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.648272:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677474.648272:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677474.648273:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.648273:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.648274:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677474.648275:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.648276:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.648276:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:9.2:1619677474.648278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677474.648279:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.648281:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677474.648284:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.648287:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.648288:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677474.648288:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000400:00000200:24.0:1619677474.648290:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.648292:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.648293:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677474.648297:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.648300:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.648301:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.648301:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.648302:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.648303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.649235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.649242:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.649245:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.649246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.649248:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.649250:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.649251:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.649253:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.649254:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.649255:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.649256:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.649259:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.649261:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.649263:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.649264:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.649265:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.649266:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.649267:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.649268:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.649268:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.649273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.649282:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.649285:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.649286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.649286:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.649288:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.649291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.649294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.649302:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.649305:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.649306:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.649306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.649307:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.649307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.649496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.649504:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.649507:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.649508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.649509:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.649511:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.649513:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677474.649514:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.649516:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.649517:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.649518:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.649521:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.649522:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.649524:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.649525:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677474.649525:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.649527:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.649527:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.649528:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.649528:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.649533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.649541:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.649543:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.649544:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.649545:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.649547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.649548:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.649570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.649577:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.649580:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.649582:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.649584:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.649584:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.649585:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.649586:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.649587:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.649588:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.649589:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.649592:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.649593:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.649595:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.649595:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.649596:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.649597:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.649598:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.649599:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.649599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.649603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.649609:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.649612:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.649613:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.649614:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.649616:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.649616:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:25.2:1619677474.649620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.649627:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.649629:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.649630:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.649631:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.649631:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.649632:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.649842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.649849:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.649853:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.649854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.649857:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.649859:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.649860:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.649862:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.649863:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.649864:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.649865:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.649868:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.649870:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.649873:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.649874:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.649875:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.649876:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.649877:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.649878:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.649879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.649883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.649890:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.649893:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.649894:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.649895:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.649898:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.649899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677474.650041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.650045:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.650047:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.650049:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.650050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.650051:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.650053:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.650054:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.650055:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.650056:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.650057:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.650060:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.650062:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.650064:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.650065:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.650066:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.650067:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.650068:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.650069:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.650070:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.650071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.650074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.650076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.650081:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677474.650082:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677474.650083:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.650084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.650085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.650085:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.650086:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.650086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.650089:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.650091:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.650344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.650352:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.650355:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.650356:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.650358:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.650360:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.650361:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.650363:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.650364:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.650365:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.650366:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.650369:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.650371:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.650373:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.650373:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.650374:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.650376:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.650376:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.650377:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.650377:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.650382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.650389:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.650391:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.650392:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.650393:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.650394:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.650396:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.650764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.650772:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.650775:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.650777:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.650779:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.650780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.650780:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.650781:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.650782:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.650783:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.650784:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.650787:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.650788:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.650791:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.650791:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.650792:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.650793:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.650794:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.650795:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.650795:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.650799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.650805:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.650807:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.650808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.650808:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.650810:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.650812:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.650816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.650822:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.650824:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.650825:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.650826:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.650826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.650827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.651818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.651826:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.651828:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.651829:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.651831:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.651833:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.651835:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.651836:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.651837:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.651838:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.651839:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.651842:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.651843:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.651846:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.651847:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.651848:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.651849:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.651850:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.651850:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.651851:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.651856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.651865:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.651867:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.651868:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:20.2:1619677474.651869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000400:00000200:6.0:1619677474.651869:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.651871:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.651873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677474.651875:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:25.2:1619677474.651876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:21.0:1619677474.651877:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.651879:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.651880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.651882:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.651883:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:21.0:1619677474.651883:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:21.0:1619677474.651885:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.651886:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:21.0:1619677474.651886:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.651887:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000400:00000200:21.0:1619677474.651887:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.651887:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.651888:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000400:00000200:21.0:1619677474.651888:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677474.651889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:21.0:1619677474.651892:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.651894:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.0:1619677474.651897:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.651898:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677474.651899:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677474.651901:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677474.651901:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.651902:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677474.651903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677474.651906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677474.651914:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677474.651917:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677474.651918:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.651920:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.651922:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677474.651923:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:25.2:1619677474.651927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:12.0:1619677474.651934:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677474.651936:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677474.651937:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.651938:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677474.651939:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.651940:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677474.652038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.652047:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.652049:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.652052:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.652054:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.652054:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.652055:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.652056:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.652058:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.652059:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.652060:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.652063:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.652065:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.652067:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.652068:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.652069:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.652070:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.652071:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.652072:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.652072:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.652076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.652083:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.652085:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.652086:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.652087:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.652089:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.652090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.652322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.652328:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.652330:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.652331:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.652334:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.652336:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:5.2:1619677474.652338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677474.652338:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.652339:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.652340:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.652341:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.652342:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677474.652344:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677474.652345:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.652347:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677474.652347:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.652348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677474.652349:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.652350:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:1.0:1619677474.652350:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.652351:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.652352:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.652352:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.652353:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:1.0:1619677474.652353:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.652354:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:1.0:1619677474.652354:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.652355:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.652357:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.652358:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.652359:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677474.652360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:1.0:1619677474.652362:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.652363:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.652366:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.652366:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.652367:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:18.0:1619677474.652368:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:1.0:1619677474.652369:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.652369:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677474.652370:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:1.0:1619677474.652370:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:18.0:1619677474.652371:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.652371:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677474.652371:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.652372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677474.652373:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.652375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:18.0:1619677474.652375:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.652377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.652383:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.652383:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.652385:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.652386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.652386:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.652387:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.652387:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.652387:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.652389:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.652391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.652494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.652502:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.652505:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.652506:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.652508:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.652510:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.652512:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.652513:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.652515:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.652516:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.652517:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.652519:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.652521:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.652524:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.652525:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.652525:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.652527:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.652528:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.652528:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.652529:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.652533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.652541:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.652544:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.652544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.652545:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.652547:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.652550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.652875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.652878:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.652881:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.652883:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.652884:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.652885:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.652887:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.652888:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.652891:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.652892:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.652893:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.652897:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.652898:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.652901:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.652902:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.652902:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.652904:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.652905:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.652905:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.652906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.652910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.652917:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.652920:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.652921:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.652922:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.652924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.652925:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.653380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.653388:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.653391:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.653392:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.653393:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.653396:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.653398:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.653399:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.653401:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.653402:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.653403:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.653407:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.653408:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.653411:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.653412:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.653413:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.653414:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.653415:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.653415:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.653416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.653420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.653429:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.653432:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.653432:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.653433:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.653435:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.653438:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.653513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.653521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677474.653522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:31.0:1619677474.653524:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.653526:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.653528:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.653529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.653530:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677474.653530:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:31.0:1619677474.653531:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.653533:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.653533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677474.653533:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:31.0:1619677474.653534:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.653535:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.653536:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.653538:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.653539:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.653540:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.653541:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677474.653542:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.653542:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677474.653542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.653543:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.653544:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677474.653544:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.653545:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:10.0:1619677474.653545:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.653546:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677474.653546:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.653547:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.653549:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.653549:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677474.653551:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677474.653551:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.653552:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:25.2:1619677474.653552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.653553:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.653554:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.653555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677474.653557:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.653557:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.653558:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:10.0:1619677474.653558:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.653558:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:26.0:1619677474.653559:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.653560:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.653561:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:26.0:1619677474.653561:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.653561:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:30.0:1619677474.653562:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.653562:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.653563:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.653563:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.653563:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.653564:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.653564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677474.653564:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:30.0:1619677474.653565:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.653565:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677474.653566:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:31.0:1619677474.653567:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:26.0:1619677474.653567:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.653567:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677474.653567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.653568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677474.653569:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.653571:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.653572:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.653572:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.653574:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.653574:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.653575:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:21.0:1619677474.653575:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677474.653576:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.653577:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677474.653577:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.653578:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.653579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:21.0:1619677474.653580:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.653580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:4.2:1619677474.653582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677474.653583:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677474.653585:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.653588:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.653589:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.653589:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.653589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.653591:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.653591:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.653592:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.653593:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677474.653594:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677474.653596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.653604:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.653607:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.653608:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.653608:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.653609:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.653610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.653646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.653649:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.653651:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.653654:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.653654:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.653656:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.653658:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.653659:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.653660:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.653661:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.653662:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.653666:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.653667:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.653670:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.653670:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.653671:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.653673:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.653673:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.653674:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.653676:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.653676:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677474.653679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.653681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.653687:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677474.653687:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.653689:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:6.0:1619677474.653689:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677474.653690:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:6.0:1619677474.653690:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.653691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.653691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677474.653693:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.653695:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.653940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.653945:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.653948:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.653950:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.653950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.653952:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.653954:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.653955:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.653956:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.653958:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.653958:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.653962:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.653963:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.653965:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.653966:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.653967:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.653968:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.653969:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.653970:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.653970:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.653974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.653981:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.653983:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.653984:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.653986:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.653987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.653988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.654379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.654387:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.654390:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.654392:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.654394:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.654395:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.654396:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.654397:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.654399:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.654400:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.654401:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.654404:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.654405:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.654408:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.654408:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.654409:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.654411:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.654411:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.654412:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.654413:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.654417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.654425:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.654427:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.654428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.654428:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.654430:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.654432:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.654550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.654557:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.654560:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.654561:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.654562:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.654565:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.654566:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.654567:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.654569:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.654569:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.654570:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.654573:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.654575:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.654578:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.654578:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.654579:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.654581:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.654582:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.654583:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.654584:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.654585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677474.654587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677474.654590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677474.654595:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677474.654598:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677474.654598:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.654599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.654599:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.654600:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.654600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677474.654601:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.654602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677474.654604:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.655207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677474.655207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:30.0:1619677474.655215:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.655215:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:30.0:1619677474.655218:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677474.655218:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.655219:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:30.0:1619677474.655221:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.655222:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677474.655222:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677474.655224:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.655225:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.655225:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.655227:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.655227:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:30.0:1619677474.655228:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.655228:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.655230:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.655230:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.655231:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.655231:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.655233:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:30.0:1619677474.655235:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.655237:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.655237:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.655239:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677474.655240:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.655241:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.655241:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.655241:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.655242:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.655242:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:30.0:1619677474.655243:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.655243:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.655244:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677474.655244:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:30.0:1619677474.655245:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677474.655245:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.655245:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.655246:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677474.655249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:25.2:1619677474.655250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677474.655255:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.655258:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:13.0:1619677474.655258:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:24.0:1619677474.655259:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.655259:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.655260:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.655260:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.655261:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.655263:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:13.0:1619677474.655263:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.655265:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.655266:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.655916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.655924:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.655927:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.655928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.655929:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.655932:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.655933:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.655934:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.655935:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.655936:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.655937:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.655940:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.655941:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.655943:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.655944:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.655944:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.655946:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.655946:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.655947:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.655947:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.655952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.655959:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.655962:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.655962:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.655963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.655964:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.655966:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.655977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.655983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.655985:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.655986:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.655987:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.655987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677474.655987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.2:1619677474.656119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677474.656126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677474.656127:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.656130:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.656132:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.656133:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:31.0:1619677474.656134:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.656135:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.656135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.656136:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.656136:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.656136:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677474.656138:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.656139:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.656139:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.656140:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.656141:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.656143:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.656143:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.656144:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.656145:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.656146:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.656147:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.656147:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.656148:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.656148:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677474.656148:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.656150:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.656150:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.656151:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677474.656151:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.656152:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677474.656153:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:25.2:1619677474.656156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.656156:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.656156:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.656157:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.656158:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.656159:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.656160:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.656160:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677474.656163:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.656165:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:25.2:1619677474.656165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:30.0:1619677474.656166:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.656167:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.656169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.656170:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.656173:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.656175:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.656176:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.656177:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.656179:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.656182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.656232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677474.656237:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.656240:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677474.656241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677474.656243:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.656245:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.656247:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677474.656248:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.656250:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.656251:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.656252:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677474.656255:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.656257:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677474.656260:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.656260:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677474.656261:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677474.656263:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677474.656264:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.656264:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.656264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677474.656269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.656278:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.656280:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.656281:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.656281:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.656283:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.656285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.656292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.656300:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.656302:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.656303:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.656304:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.656305:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.656305:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.656878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.656886:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.656888:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.656891:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.656892:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.656893:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.656893:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.656895:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.656896:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.656897:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.656898:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.656901:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.656903:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.656905:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.656906:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.656907:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.656908:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.656908:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.656910:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.656912:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.656912:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.656914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.656918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.656922:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.656924:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.656925:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.656925:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677474.656926:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.656926:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.656927:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.656928:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.656929:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.656930:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.656932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.656938:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.656941:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.656942:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.656942:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.656943:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.656944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677474.657001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.657007:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.657010:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.657011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.657012:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.657015:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.657016:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.657018:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.657019:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.657020:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.657021:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.657024:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.657026:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.657029:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.657029:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.657030:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.657032:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.657032:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.657033:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.657033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.657038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.657045:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.657048:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.657049:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.657049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.657051:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.657053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.657335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.657341:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.657344:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.657346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.657347:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.657349:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.657351:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.657352:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.657354:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.657355:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.657356:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.657359:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.657361:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.657363:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.657364:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.657364:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.657366:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.657367:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.657368:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.657369:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.657370:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.657373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.657375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.657381:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677474.657381:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677474.657384:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.657384:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677474.657384:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.657385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.657385:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677474.657385:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.657387:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.657389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.657396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.657404:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.657407:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.657408:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.657410:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.657413:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.657414:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.657416:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.657417:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.657418:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.657419:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.657423:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.657425:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.657428:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.657428:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.657429:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.657431:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.657431:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.657432:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.657432:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.657438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.657446:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.657449:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.657450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.657450:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.657453:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.657456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.657656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.657662:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.657665:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.657667:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.657667:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.657669:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.657671:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.657672:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.657674:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.657675:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.657676:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.657679:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.657680:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.657683:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.657683:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.657684:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.657686:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.657686:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.657687:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.657687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.657692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.657700:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.657702:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.657703:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.657703:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.657705:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.657707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.658282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.658289:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.658292:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.658293:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.658302:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.658305:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.658306:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677474.658307:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.658308:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.658309:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.658310:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677474.658313:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.658314:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.658316:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.658317:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.658317:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.658319:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.658319:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.658320:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.658320:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.658325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.658332:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.658334:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.658334:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.658335:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.658336:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.658338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.658473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.658480:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.658483:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.658484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.658485:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.658488:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.658490:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.658491:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.658492:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.658493:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.658494:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.658497:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.658498:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.658501:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.658502:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.658502:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.658504:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.658504:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.658505:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.658505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.658510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.658518:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.658521:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.658522:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.658522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.658524:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.658527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.658532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.658539:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.658541:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.658542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.658542:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.658543:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.658544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.658568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.658575:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.658578:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.658579:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.658581:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.658582:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.658584:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.658585:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.658586:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.658587:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.658588:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.658591:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.658593:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.658596:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.658596:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.658597:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.658598:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.658599:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.658600:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.658600:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.658604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.658612:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.658615:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.658616:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.658616:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.658618:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.658621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.658804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.658813:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.658815:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.658818:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.658820:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.658820:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.658821:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.658822:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.658824:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.658825:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.658826:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.658829:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.658831:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.658833:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.658834:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.658834:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.658836:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.658836:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.658837:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.658838:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.658841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.658848:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.658850:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.658851:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.658851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.658853:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.658856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.659035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.659038:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.659041:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.659043:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.659044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.659046:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.659047:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.659048:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.659050:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.659051:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.659052:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.659055:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.659057:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.659059:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.659060:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.659060:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.659062:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.659063:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.659064:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.659064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.659068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.659076:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.659078:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.659079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.659079:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677474.659081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:6.0:1619677474.659081:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.659083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677474.659088:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.659091:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.659092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.659094:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.659096:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.659097:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.659099:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.659100:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.659101:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.659102:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.659105:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.659107:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.659109:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.659110:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.659111:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.659112:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.659113:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.659114:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.659115:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.659115:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.659119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.659122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.659127:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.659129:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.659129:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.659129:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.659130:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.659130:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.659131:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.659132:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677474.659132:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.659133:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.659140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.659148:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.659150:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.659151:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.659151:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.659151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.659152:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.659700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.659706:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.659708:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.659710:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.659711:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.659713:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.659714:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.659715:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.659717:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.659718:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.659718:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.659721:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.659723:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.659725:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.659726:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.659727:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.659728:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.659729:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.659729:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.659730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.659735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.659743:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.659745:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.659746:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.659748:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.659748:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.659750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.659915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677474.659923:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.659926:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677474.659926:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:8.0:1619677474.659928:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.659931:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.659932:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677474.659934:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.659935:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.659936:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.659937:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677474.659941:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.659942:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677474.659944:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.659945:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677474.659946:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677474.659948:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677474.659948:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.659949:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677474.659950:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677474.659951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.2:1619677474.659954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677474.659956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:26.0:1619677474.659961:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.659963:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:24.0:1619677474.659963:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000400:00000200:26.0:1619677474.659964:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.659964:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677474.659965:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000400:00000200:26.0:1619677474.659966:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.659966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677474.659967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:26.0:1619677474.659968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.660221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.660230:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.660232:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.660234:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.660236:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.660237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.660238:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.660239:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.660240:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.660241:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.660242:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.660245:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.660246:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.660248:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.660249:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.660250:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.660251:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.660252:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.660252:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.660253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.660257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.660263:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.660265:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.660266:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.660266:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.660268:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.660270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.660340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.660345:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.660347:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.660350:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.660350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.660352:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.660354:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.660355:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.660356:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.660357:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.660358:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.660361:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.660363:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.660365:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.660366:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.660366:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.660368:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.660369:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.660369:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.660370:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.660375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.660383:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.660385:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.660386:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.660386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.660388:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.660391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.660934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.660942:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.660944:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.660945:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.660947:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.660949:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.660951:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.660952:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.660953:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.660954:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.660955:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.660958:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.660960:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.660963:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.660964:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.660964:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.660966:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.660966:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.660968:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.660969:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.660969:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.660972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.660975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.660979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.660982:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.660982:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.660983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.660983:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.660984:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.660985:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.660985:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.660986:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.660988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.660992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.660999:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.661001:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.661002:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.661002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.661003:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.661004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.661225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.661232:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.661235:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.661236:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.661238:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.661240:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.661242:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.661243:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.661244:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.661246:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.661246:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.661249:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.661251:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.661253:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.661254:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.661254:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.661255:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.661256:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.661257:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.661258:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.661259:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.661261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.661264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.661268:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.661270:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.661271:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.661271:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.661272:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677474.661273:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.661274:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.661274:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.661275:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677474.661276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.661361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.661369:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.661372:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.661374:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.661376:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.661377:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.661378:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.661379:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.661380:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.661381:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.661382:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.661384:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.661386:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.661388:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.661389:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.661389:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.661391:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.661391:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.661392:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.661393:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.661397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.661404:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.661406:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.661407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.661407:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.661409:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.661411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.661414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.661420:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.661422:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.661423:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.661423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.661424:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.661425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.662196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.662205:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.662207:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.662209:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.662211:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.662211:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.662213:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.662214:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.662215:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.662216:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.662217:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.662220:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.662221:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.662224:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.662224:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.662225:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.662226:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.662227:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.662228:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.662230:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.662230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.662232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.662236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.662238:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.662240:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.662241:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.662241:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.662243:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.662244:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677474.662245:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.662246:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.662247:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.662248:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677474.662256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.662262:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.662265:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.662266:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.662268:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.662270:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.662271:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.662272:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.662274:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.662275:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.662276:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.662279:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.662281:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.662284:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.662284:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.662285:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.662287:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.662287:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.662288:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.662288:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.662293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.662302:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.662304:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.662305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.662305:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.662307:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.662309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.662432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.662436:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.662438:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.662440:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.662441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.662443:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.662444:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.662445:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.662447:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.662448:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.662449:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.662453:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.662454:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.662457:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.662458:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.662459:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.662461:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.662461:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.662462:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.662463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.662467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.662475:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.662478:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.662479:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.662480:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.662482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.662483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.662709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.662715:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.662718:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.662719:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.662721:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.662723:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.662725:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.662726:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.662728:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.662729:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.662730:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.662733:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.662734:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.662738:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.662739:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.662740:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.662742:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.662742:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.662743:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.662744:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.662748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.662756:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.662759:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.662760:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.662761:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.662764:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.662765:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.662835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.662843:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.662845:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.662846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.662848:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.662851:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.662853:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.662854:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.662855:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.662856:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.662857:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.662860:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.662862:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.662864:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.662865:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.662866:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.662868:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.662868:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.662869:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.662869:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.662874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.662882:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.662885:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.662885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.662886:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.662888:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.662891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.662892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:5.2:1619677474.662895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:6.0:1619677474.662899:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.662901:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:6.0:1619677474.662902:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.662903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.662903:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677474.662904:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:6.0:1619677474.662904:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.662904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677474.662905:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677474.662907:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677474.662910:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.662911:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677474.662912:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.662914:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.662915:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.662916:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677474.662919:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677474.662920:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677474.662923:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.662924:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677474.662925:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677474.662926:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677474.662927:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.662928:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677474.662928:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677474.662932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.662939:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.662941:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.662942:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.662943:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.662945:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677474.662948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.663066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.663074:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.663077:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.663078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.663079:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.663082:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.663083:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.663084:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.663085:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.663086:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.663087:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.663090:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.663092:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.663095:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.663095:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:4.2:1619677474.663095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (20)++
00000800:00000200:24.0:1619677474.663096:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.663098:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.663098:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.663099:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.663099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677474.663103:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (21)++
00000800:00000200:25.2:1619677474.663104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.663105:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000800:00000200:26.0:1619677474.663106:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000400:00000200:10.0:1619677474.663108:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.663110:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.663112:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677474.663112:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:10.0:1619677474.663113:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.663114:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:10.0:1619677474.663115:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.663115:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.663115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.663116:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.663116:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677474.663118:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.663120:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.663122:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677474.663123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677474.663124:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.663125:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:10.0:1619677474.663125:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677474.663127:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677474.663128:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (21)++
00000800:00000200:10.0:1619677474.663129:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f343800] (22)++
00000800:00000200:10.0:1619677474.663131:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (23)--
00000800:00000200:10.0:1619677474.663131:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000800:00000200:4.2:1619677474.663133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (21)++
00000800:00000200:26.0:1619677474.663139:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (22)++
00000800:00000200:26.0:1619677474.663142:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000800:00000200:10.0:1619677474.663142:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (23)++
00000800:00000200:10.0:1619677474.663143:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f343800] (24)--
00000400:00000200:26.0:1619677474.663144:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.663144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (23)--
00000800:00000200:9.2:1619677474.663144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677474.663145:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000400:00000200:26.0:1619677474.663146:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.663148:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:26.0:1619677474.663148:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.663150:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.663150:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.663150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:26.0:1619677474.663151:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.663152:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.663153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.663153:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.663154:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.663155:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:8.0:1619677474.663155:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.663155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (21)++
00000800:00000200:8.0:1619677474.663157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677474.663158:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.663159:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677474.663159:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677474.663162:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677474.663162:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (22)++
00000800:00000200:26.0:1619677474.663163:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (23)--
00000800:00000200:10.0:1619677474.663163:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (23)++
00000800:00000200:26.0:1619677474.663164:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (23)--
00000800:00000200:26.0:1619677474.663165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000800:00000200:10.0:1619677474.663165:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000400:00000200:10.0:1619677474.663167:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.663169:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.663170:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.663171:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.663172:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.663173:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.663174:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.663177:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.663178:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677474.663178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.663179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (21)++
00000800:00000200:10.0:1619677474.663180:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.663181:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.663182:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.663184:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:10.0:1619677474.663184:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.663185:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (22)++
00000800:00000200:24.0:1619677474.663186:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:10.0:1619677474.663186:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (23)--
00000800:00000200:10.0:1619677474.663186:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000400:00000200:24.0:1619677474.663187:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.663187:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (21)++
00000800:00000200:8.0:1619677474.663188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.663189:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.663190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.663190:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000800:00000200:26.0:1619677474.663191:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000800:00000200:24.0:1619677474.663191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:10.0:1619677474.663193:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.663195:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.663196:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677474.663197:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.663197:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677474.663199:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.663200:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.663200:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677474.663200:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.663201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.663201:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.663204:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.663205:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.663207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677474.663208:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:4.2:1619677474.663208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (21)++
00000800:00000200:10.0:1619677474.663212:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.663213:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677474.663213:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677474.663215:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.663216:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (22)++
00000800:00000200:10.0:1619677474.663216:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (22)++
00000800:00000200:26.0:1619677474.663217:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000800:00000200:10.0:1619677474.663217:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (24)--
00000800:00000200:10.0:1619677474.663218:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (23)--
00000800:00000200:10.0:1619677474.663219:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000400:00000200:26.0:1619677474.663220:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.663222:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677474.663223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:26.0:1619677474.663224:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677474.663225:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.663226:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.663227:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.663228:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677474.663231:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.663231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (21)++
00000800:00000200:1.0:1619677474.663231:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677474.663233:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677474.663234:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.663235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.663235:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.663237:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677474.663237:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.663238:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677474.663238:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677474.663240:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.663240:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (22)++
00000800:00000200:26.0:1619677474.663241:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (23)++
00000800:00000200:10.0:1619677474.663241:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000800:00000200:1.0:1619677474.663241:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677474.663243:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f343800] (24)++
00000800:00000200:26.0:1619677474.663244:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (25)--
00000400:00000200:10.0:1619677474.663244:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.663245:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (24)--
00000800:00000200:26.0:1619677474.663246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (23)--
00000800:00000200:25.2:1619677474.663246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677474.663246:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.663248:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677474.663249:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.663250:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677474.663250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (22)++
00000400:00000200:10.0:1619677474.663251:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.663253:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:10.0:1619677474.663253:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.663255:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.663256:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.663256:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.663257:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (23)++
00000400:00000200:31.0:1619677474.663258:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.663259:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f343800] (24)--
00000800:00000200:26.0:1619677474.663259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (23)--
00000800:00000200:10.0:1619677474.663259:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.663260:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677474.663260:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (22)++
00000800:00000200:31.0:1619677474.663261:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:26.0:1619677474.663262:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000800:00000200:10.0:1619677474.663262:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.663263:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677474.663264:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:26.0:1619677474.663265:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.663266:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.663266:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677474.663266:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (23)++
00000800:00000200:10.0:1619677474.663267:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (24)--
00000400:00000200:26.0:1619677474.663268:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677474.663268:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (23)--
00000800:00000200:10.0:1619677474.663269:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000400:00000200:26.0:1619677474.663270:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.663271:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.663272:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677474.663272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:26.0:1619677474.663273:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.663276:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.663277:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.663279:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.663280:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.663280:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:26.0:1619677474.663281:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.663281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (21)++
00000800:00000200:26.0:1619677474.663282:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.663282:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:26.0:1619677474.663283:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (22)++
00000800:00000200:26.0:1619677474.663283:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (23)--
00000800:00000200:2.0:1619677474.663283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.663284:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.663285:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000800:00000200:26.0:1619677474.663286:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (21)++
00000400:00000200:18.0:1619677474.663286:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.663288:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000800:00000200:25.2:1619677474.663288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.663289:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000400:00000200:26.0:1619677474.663290:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.663291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.663293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (21)++
00000800:00000200:12.0:1619677474.663294:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:26.0:1619677474.663300:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.663302:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:12.0:1619677474.663302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.663303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.663303:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (22)++
00000400:00000200:26.0:1619677474.663304:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677474.663304:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677474.663304:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000400:00000200:12.0:1619677474.663305:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.663306:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677474.663306:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.663307:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677474.663307:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)++
00000400:00000200:26.0:1619677474.663308:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:13.0:1619677474.663308:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (24)--
00000800:00000200:13.0:1619677474.663310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:26.0:1619677474.663311:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.663311:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.663313:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677474.663314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:10.0:1619677474.663314:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.663315:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.663316:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.663317:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:10.0:1619677474.663317:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.663318:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.663319:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677474.663320:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.663321:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.663322:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.663323:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677474.663324:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (23)++
00000800:00000200:9.2:1619677474.663324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.663325:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (24)--
00000800:00000200:26.0:1619677474.663326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (23)--
00000800:00000200:26.0:1619677474.663327:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (22)++
00000400:00000200:10.0:1619677474.663327:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.663328:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000800:00000200:10.0:1619677474.663328:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:26.0:1619677474.663330:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.663331:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.663332:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.663332:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677474.663333:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:26.0:1619677474.663334:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677474.663335:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.663335:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:26.0:1619677474.663336:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.663337:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.663338:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677474.663339:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (23)++
00000800:00000200:10.0:1619677474.663339:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f343800] (24)++
00000800:00000200:10.0:1619677474.663341:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (25)--
00000800:00000200:10.0:1619677474.663341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (24)--
00000400:00000200:26.0:1619677474.663342:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.663342:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:8.2:1619677474.663342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677474.663343:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.663344:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677474.663344:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.663346:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.663346:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.663347:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677474.663347:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.663348:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677474.663348:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (23)++
00000800:00000200:10.0:1619677474.663349:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f343800] (24)--
00000800:00000200:10.0:1619677474.663349:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (23)--
00000800:00000200:26.0:1619677474.663350:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677474.663350:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (22)++
00000800:00000200:17.0:1619677474.663350:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:10.0:1619677474.663350:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.663351:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (23)--
00000800:00000200:26.0:1619677474.663352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000800:00000200:10.0:1619677474.663352:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.663352:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.663353:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (21)--
00000800:00000200:17.0:1619677474.663353:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:10.0:1619677474.663353:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.663353:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.663354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:1.0:1619677474.663354:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:25.2:1619677474.663355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:17.0:1619677474.663355:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.663357:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.663360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677474.663362:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:8.2:1619677474.663362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:30.0:1619677474.663365:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.663366:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.663368:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.663370:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:1.0:1619677474.663370:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677474.663371:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:1.0:1619677474.663372:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.663373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.663374:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:25.2:1619677474.663375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677474.663375:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.663375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677474.663382:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.663385:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.663386:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.663386:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.663387:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.663388:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.663417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.663423:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.663426:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.663428:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.663429:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.663432:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.663433:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.663435:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.663436:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.663437:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.663438:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.663442:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.663443:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.663446:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.663447:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.663447:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.663449:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.663450:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.663450:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.663451:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.663452:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.663456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.663457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.663460:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.663462:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.663463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.663463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.663463:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.663465:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.663466:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.663467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.663468:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.663471:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.663644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (20)++
00000800:00000200:10.0:1619677474.663652:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (21)++
00000800:00000200:10.0:1619677474.663655:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000800:00000200:26.0:1619677474.663656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000400:00000200:10.0:1619677474.663658:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.663659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677474.663660:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.663661:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677474.663663:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.663664:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.663665:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.663666:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677474.663666:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677474.663669:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677474.663670:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.663671:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.663672:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677474.663673:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.663673:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.663674:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.663675:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.663675:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.663676:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677474.663676:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:30.0:1619677474.663677:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.663677:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.663678:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.663678:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677474.663679:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (21)++
00000800:00000200:10.0:1619677474.663679:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (22)--
00000800:00000200:10.0:1619677474.663680:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (21)--
00000400:00000200:30.0:1619677474.663681:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.663683:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.2:1619677474.663684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677474.663685:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.663686:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.663687:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.663688:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.663689:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.663690:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.663690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677474.663691:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:25.2:1619677474.663694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:1.0:1619677474.663694:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.663695:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.663695:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.663697:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.663700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677474.663701:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.663703:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.663704:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.663706:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.663707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.663708:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.663730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.663733:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.663736:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.663738:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.663739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.663741:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.663742:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.663744:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.663745:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.663746:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.663747:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.663750:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.663752:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.663754:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.663755:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.663756:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.663758:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.663758:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.663759:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.663760:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.663764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.663770:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.663773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.663773:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.663774:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.663776:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.663778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.663782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.663788:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.663790:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.663791:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.663791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.663793:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.663793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.663856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (20)++
00000800:00000200:26.0:1619677474.663864:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (21)++
00000800:00000200:26.0:1619677474.663866:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000800:00000200:10.0:1619677474.663867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000400:00000200:26.0:1619677474.663869:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.663872:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.663874:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:26.0:1619677474.663875:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.663876:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.663877:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.663878:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677474.663881:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.663883:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:26.0:1619677474.663886:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.663887:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:26.0:1619677474.663887:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677474.663889:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:26.0:1619677474.663889:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (21)++
00000800:00000200:26.0:1619677474.663890:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (22)--
00000800:00000200:4.2:1619677474.663891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (22)++
00000800:00000200:26.0:1619677474.663894:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000800:00000200:26.0:1619677474.663895:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (21)++
00000800:00000200:8.2:1619677474.663895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677474.663897:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.12@o2ib3
00000400:00000200:26.0:1619677474.663899:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.12@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.663899:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000400:00000200:26.0:1619677474.663901:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.663902:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677474.663903:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.663903:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:26.0:1619677474.663904:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.663905:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.663905:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:21.0:1619677474.663905:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.663906:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.663908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677474.663909:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.663910:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.663912:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.663913:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.663913:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677474.663913:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677474.663914:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.663915:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.663915:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f343800] (21)++
00000800:00000200:26.0:1619677474.663916:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f343800] (22)++
00000800:00000200:26.0:1619677474.663918:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f343800] (23)--
00000800:00000200:26.0:1619677474.663918:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000800:00000200:9.2:1619677474.663921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:4.2:1619677474.663924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f343800] (21)++
00000800:00000200:10.0:1619677474.663929:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.663930:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.663931:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.663933:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f343800] (22)++
00000800:00000200:26.0:1619677474.663934:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f343800] (23)--
00000400:00000200:10.0:1619677474.663934:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.663935:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (22)--
00000800:00000200:10.0:1619677474.663935:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.663936:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677474.663937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f343800] (21)--
00000800:00000200:4.2:1619677474.664003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.664010:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.664013:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.664014:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.664015:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.664017:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.664018:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.664019:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.664021:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.664022:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.664022:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.664025:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.664027:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.664029:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.664030:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.664030:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.664032:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.664032:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.664033:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.664034:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.664038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.664047:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.664050:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.664050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.664051:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.664053:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.664055:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.664249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677474.664254:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.664257:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.664257:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677474.664260:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.664262:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.664263:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677474.664264:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.664266:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.664267:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.664268:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677474.664271:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.664273:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677474.664275:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.664276:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677474.664277:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677474.664279:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677474.664279:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.664280:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677474.664281:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677474.664282:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677474.664285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.664287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.664292:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:2.0:1619677474.664293:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:6.0:1619677474.664298:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677474.664299:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677474.664299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:18.0:1619677474.664299:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.664303:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.664304:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.664305:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.664308:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.665372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.665375:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.665378:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.665380:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.665380:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.665382:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.665384:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.665385:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.665386:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.665387:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.665389:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.665392:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.665394:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.665396:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.665397:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.665398:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.665399:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.665400:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.665400:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.665401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.665406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.665413:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.665415:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.665416:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.665418:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.665418:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.665420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.665424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.665432:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.665434:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.665435:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.665436:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.665437:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.665438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.665708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.665716:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.665719:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.665720:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.665722:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.665724:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.665726:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.665727:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.665728:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.665730:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.665730:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.665734:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.665736:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.665739:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.665739:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.665740:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.665742:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.665742:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.665743:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.665743:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.665748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.665757:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.665759:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.665760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.665761:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.665763:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.665766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.665862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.665870:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.665873:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.665875:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.665877:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.665878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.665879:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.665880:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.665881:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.665882:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.665883:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.665886:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.665888:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.665890:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.665891:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.665891:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.665893:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.665893:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.665894:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.665895:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.665898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.665905:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.665908:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.665908:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.665910:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.665911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.665912:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.665988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.665993:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.665996:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.665997:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.665998:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.666000:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.666001:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.666002:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.666004:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.666005:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.666005:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:4.2:1619677474.666005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:21.0:1619677474.666008:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.666010:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677474.666012:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.0:1619677474.666013:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.666014:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677474.666015:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:21.0:1619677474.666015:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.666016:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677474.666016:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.0:1619677474.666017:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:26.0:1619677474.666018:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.666018:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.666019:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.666020:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:26.0:1619677474.666021:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.666022:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:9.2:1619677474.666022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:26.0:1619677474.666023:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.666024:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677474.666024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000400:00000200:26.0:1619677474.666025:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.666026:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.666028:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677474.666029:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:26.0:1619677474.666030:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.666031:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.666031:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.666031:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.666032:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677474.666032:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.666033:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.666033:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.666034:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.666035:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.666035:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:8.0:1619677474.666035:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.666037:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.666037:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.666038:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.0:1619677474.666038:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677474.666039:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.666043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.666051:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.666054:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.666054:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.666055:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.666057:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.666059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.666063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.666069:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.666072:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.666073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.666073:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.666074:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.666075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.666743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.666751:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.666754:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.666756:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.666758:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.666759:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.666759:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.666761:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.666762:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.666763:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.666764:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.666767:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.666769:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.666771:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.666771:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.666772:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.666773:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.666774:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.666775:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.666776:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.666780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.666787:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.666789:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.666790:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.666791:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.666793:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.666794:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.666796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.666803:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.666806:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.666807:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.666807:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.666808:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.666809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.666877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.666885:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.666887:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.666888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.666890:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.666892:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.666894:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.666895:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.666896:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.666897:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.666898:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.666901:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.666902:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.666904:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.666905:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.666905:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.666907:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.666907:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.666908:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.666908:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.666912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.666920:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.666922:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.666923:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.666923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.666924:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.666927:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.666929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.666937:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.666940:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.666940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.666942:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.666945:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.666947:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.666948:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.666949:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.666951:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.666952:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.666955:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.666957:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.666958:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.666959:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.666960:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.666962:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.666962:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.666963:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.666964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.666968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.666975:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.666977:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.666977:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.666978:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.666979:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.666980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.667464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.667471:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.667474:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.667475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.667477:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.667479:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.667480:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.667482:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.667483:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.667484:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.667485:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.667488:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.667489:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.667492:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.667493:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.667493:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.667495:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.667495:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.667496:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.667496:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.667501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.667507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.667511:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:13.0:1619677474.667511:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:20.0:1619677474.667513:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.667515:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:13.0:1619677474.667515:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.667516:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.667517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.667518:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.667519:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.667519:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:20.0:1619677474.667520:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.667521:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.667523:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677474.667523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:20.0:1619677474.667524:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.667525:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.667528:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.667530:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.667532:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.667533:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.667534:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.667536:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.667536:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.667537:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.667538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.667542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.667549:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.667552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.667552:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.667553:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.667555:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.667558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677474.667575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.667582:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.667585:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.667585:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.667588:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.667590:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.667591:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.667593:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.667594:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.667595:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.667596:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.667599:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.667601:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.667604:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.667605:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.667605:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.667607:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.667607:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.667608:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.667609:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.667613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.667621:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.667624:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.667625:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.667625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.667627:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.667629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.667697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.667704:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.667707:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.667707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.667709:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.667711:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.667713:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.667714:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.667716:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.667717:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.667718:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677474.667719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677474.667722:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.667723:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.667726:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.667727:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677474.667727:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.667728:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:30.0:1619677474.667730:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677474.667730:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.667731:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:30.0:1619677474.667732:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.667732:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.667732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677474.667734:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.667735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.667735:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.667736:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677474.667736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:30.0:1619677474.667737:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.667738:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.667739:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.667743:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.667743:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:30.0:1619677474.667744:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:12.0:1619677474.667745:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.667746:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.667747:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.667748:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.667748:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:12.0:1619677474.667748:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.667750:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.667750:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677474.667750:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:30.0:1619677474.667752:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:12.0:1619677474.667752:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677474.667753:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.667754:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.667755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.667759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.667761:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.667764:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.667765:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.667765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.667767:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.667767:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.667769:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677474.667769:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.667770:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.667771:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.667951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677474.667955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677474.667958:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.667961:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.667962:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.667962:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677474.667964:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.667965:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.667966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677474.667966:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.667967:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.667968:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.667968:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.667970:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.667970:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.667971:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677474.667971:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.667972:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.667973:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.667974:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.667975:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.667976:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.667980:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.667980:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.667982:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.667983:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.667985:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.667985:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.667985:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.667986:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.667986:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.667987:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.667988:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.667988:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.667989:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.667989:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.667989:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677474.667990:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:8.0:1619677474.667990:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.667990:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677474.667991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.667994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:25.2:1619677474.667995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.667996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.668000:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.668002:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.668003:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.668003:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.668004:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.668004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.668005:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.668005:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.668006:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677474.668006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.668007:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.668007:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:6.0:1619677474.668007:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.668008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677474.668009:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.668011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677474.668013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.668019:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.668022:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.668023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.668023:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.668024:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.668024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:25.2:1619677474.668044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.668051:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.668054:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677474.668056:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.116@o2ib7 : GET - for me
00000400:00000200:31.0:1619677474.668058:0:19945:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.116@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:30.0:1619677474.668058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:31.0:1619677474.668060:0:19945:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.116@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:31.0:1619677474.668062:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.668062:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:31.0:1619677474.668064:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677474.668068:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:31.0:1619677474.668069:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.668072:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.668072:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.668073:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.668074:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.668075:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:25.2:1619677474.668080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.668086:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.668089:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.668090:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.668091:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: REPLY: OK
00000800:00000200:31.0:1619677474.668093:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.668094:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.668693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.668700:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.668703:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.668705:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.668707:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.668708:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.668708:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.668709:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.668710:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.668712:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.668712:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.668716:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.668717:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.668719:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.668720:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.668720:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.668722:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.668722:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.668723:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.668724:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.668728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.668735:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.668738:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.668738:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.668740:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.668741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.668742:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.669537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.669544:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.669547:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.669548:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.669550:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.669552:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.669553:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.669554:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.669555:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.669556:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.669557:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.669560:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.669561:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.669563:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.669564:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.669564:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.669566:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.669566:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.669567:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.669567:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.669572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.669579:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.669581:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.669582:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.669583:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.669584:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.669586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.669682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.669688:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.669690:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.669692:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.669693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.669695:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.669696:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.669697:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.669699:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.669699:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.669700:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.669703:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.669705:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.669708:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.669708:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.669709:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.669710:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.669711:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.669712:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.669713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.669717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.669725:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.669728:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.669729:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.669729:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.669731:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.669733:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.669735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.669743:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.669745:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677474.669746:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.669747:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.669747:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.669748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.670096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677474.670099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677474.670102:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677474.670103:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.670105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677474.670106:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.670107:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677474.670108:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.670110:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.670111:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.670112:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677474.670114:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.670116:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.670118:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.670119:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.670120:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.670121:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.670122:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677474.670122:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677474.670123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.670127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.670135:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.670137:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.670138:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.670140:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.670141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.670142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.670577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.670585:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.670587:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.670588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.670590:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.670592:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.670594:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.670595:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.670597:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.670598:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.670599:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.670602:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.670604:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.670607:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.670607:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.670608:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.670610:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.670610:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.670611:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.670611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.670616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.670625:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.670627:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.670628:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.670629:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.670631:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.670633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.670660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.670667:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.670670:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.670671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.670671:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.670672:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.670672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.671148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.671152:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.671154:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677474.671157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.671157:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.671159:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.671160:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.671162:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.671163:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.671164:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.671165:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.671168:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.671170:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.2:1619677474.671172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677474.671172:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.671173:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.671173:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.671175:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.671176:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.671176:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.671177:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677474.671180:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677474.671182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677474.671183:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.671185:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.671186:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.671188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.671188:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.671189:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.671189:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:30.0:1619677474.671190:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.671191:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.671191:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:30.0:1619677474.671192:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:21.0:1619677474.671192:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.671194:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.671194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:30.0:1619677474.671196:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.671196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677474.671198:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.671201:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.671201:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.671202:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.671203:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.671204:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.671205:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.671206:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.671209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.671216:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.671219:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.671220:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.671221:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.671222:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.671223:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.671226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.671229:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.671232:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.671234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.671234:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.671236:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.671238:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.671239:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.671240:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.671241:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.671242:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.671245:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.671247:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.671249:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.671250:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.671250:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.671252:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.671252:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.671253:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.671254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.671257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.671265:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.671267:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.671268:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.671270:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.671270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.671272:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.672168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.672176:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.672179:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.672180:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.672182:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.672185:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.672186:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.672187:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.672188:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.672189:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.672190:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.672193:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.672195:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.672197:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.672198:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.672198:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.672200:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.672200:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.672201:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.672201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.672206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.672213:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.672215:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.672216:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.672217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.672218:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.672220:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.672390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.672395:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.672398:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.672399:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.672401:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.672404:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.672405:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.672406:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.672408:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.672409:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.672410:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.672413:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.672415:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.672417:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.672418:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.672419:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.672420:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.672421:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.672422:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.672423:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.672423:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.672427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.672429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.672435:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.672435:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.672437:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.672437:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677474.672438:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.672438:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.672439:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.672439:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.672441:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.672443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.672449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.672457:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.672460:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.672462:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.672464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.672464:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.672465:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.672467:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.672468:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.672469:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.672470:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.672473:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.672475:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.672477:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.672478:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.672479:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.672480:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.672481:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.672482:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.672482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.672486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.672493:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.672496:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.672496:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.672498:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.672499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.672500:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.673043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.673051:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.673053:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.673054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.673056:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.673058:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.673059:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.673060:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.673061:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.673062:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.673062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:26.0:1619677474.673063:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.673066:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.673066:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:26.0:1619677474.673068:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.673068:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.673070:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677474.673071:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.673071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:26.0:1619677474.673072:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.673072:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677474.673072:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.673074:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.673074:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677474.673074:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677474.673075:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677474.673075:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.673076:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:20.0:1619677474.673076:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.673077:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.673078:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677474.673080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677474.673082:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.673084:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.673086:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.673087:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.673088:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:6.0:1619677474.673088:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:20.0:1619677474.673090:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (25)--
00000800:00000200:20.0:1619677474.673090:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677474.673091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677474.673091:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677474.673091:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:20.0:1619677474.673092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677474.673092:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.673094:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.673095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.673098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.673100:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.673102:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.673102:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.673103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.673104:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.673106:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.673170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.673178:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.673181:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.673183:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.673185:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.673185:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.673186:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.673187:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.673188:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.673189:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.673190:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.673193:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.673195:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.673197:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.673198:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.673199:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.673200:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.673201:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.673202:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.673204:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.673204:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.673206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.673210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.673214:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.673216:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.673216:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.673218:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.673218:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.673219:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.673219:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.673220:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.673221:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.673222:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.673223:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.673223:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.673224:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.673224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.673225:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.673377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.673384:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.673387:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.673388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.673389:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.673392:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.673393:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.673395:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.673396:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.673397:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.673398:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.673400:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.673402:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.673405:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.673406:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.673407:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.673408:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.673409:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.673410:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.673411:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.673412:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.673415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.673418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.673422:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.673425:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.673425:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.673425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.673426:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.673427:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677474.673428:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.673428:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.673430:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.673431:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.673434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.673441:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.673444:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.673445:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.673445:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.673446:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.673447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.674136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.674144:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.674147:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.674149:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.674151:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.674151:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.674152:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.674154:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.674155:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.674156:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.674157:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.674160:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.674161:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.674163:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.674164:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.674164:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.674166:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.674167:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.674168:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.674168:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.674172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.674178:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.674181:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.674182:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.674182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.674184:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.674186:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.674426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.674432:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.674435:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.674436:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.674438:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.674441:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.674442:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.674444:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.674445:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.674446:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.674447:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.674450:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.674452:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.674455:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.674456:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.674456:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.674458:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.674459:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.674459:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.674460:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.674465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.674474:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.674476:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.674477:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.674478:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.674480:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.674482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.674726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.674733:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.674736:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.674737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.674739:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.674742:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.674743:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677474.674745:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.674746:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.674747:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.674748:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677474.674752:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.674753:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677474.674756:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.674757:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677474.674758:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677474.674759:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677474.674760:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.674761:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.674762:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.674763:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677474.674765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677474.674769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:17.0:1619677474.674773:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677474.674776:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:17.0:1619677474.674776:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.674777:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.674777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.674778:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.674779:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:17.0:1619677474.674779:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.674779:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677474.674783:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.675186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.675193:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.675196:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.675197:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.675199:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.675202:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.675203:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.675204:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.675205:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.675206:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.675207:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.675210:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.675211:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.675213:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.675214:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.675214:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.675216:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.675216:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.675217:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.675218:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.675219:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.675222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.675224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.675228:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:10.0:1619677474.675231:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.675231:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.675232:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.675232:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.675233:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.675234:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677474.675234:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.675235:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.675236:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.675240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.675246:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.675248:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.675249:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.675250:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.675250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.675251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.675312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.675319:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.675322:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.675322:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.675325:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.675327:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.675329:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.675330:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.675332:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.675333:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.675334:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.675337:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.675339:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.675340:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.675341:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.675342:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.675343:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.675344:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.675344:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.675345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.675350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.675357:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.675359:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.675360:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.675360:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.675362:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.675363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.675373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.675380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.675380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:18.0:1619677474.675383:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.675384:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.675384:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.675384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.675385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677474.675387:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.675390:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.675391:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.675393:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.675395:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.675396:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.675398:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.675399:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.675400:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.675401:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.675404:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.675405:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.675408:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.675409:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.675410:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.675412:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.675412:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.675413:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.675413:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.675418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.675426:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.675428:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.675429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.675430:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.675432:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.675434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.675622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.675628:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.675631:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.675633:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.675633:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.675635:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.675637:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.675638:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.675640:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.675641:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.675642:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.675645:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.675647:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.675649:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.675650:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.675650:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.675652:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.675653:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.675654:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.675655:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.675656:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.675659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.675661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677474.675663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677474.675664:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.675666:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.675667:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.675667:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677474.675667:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.675670:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:31.0:1619677474.675671:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677474.675671:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.675671:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.675673:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.675674:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677474.675675:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:31.0:1619677474.675677:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.675679:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.675679:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.675680:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.675681:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.675683:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.675684:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.675684:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.675688:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.675689:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.675691:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.675692:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.675693:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.675694:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.675695:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.675696:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.675696:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.675700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.675706:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.675709:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.675709:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.675711:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.675713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.675714:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.676096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.676100:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.676102:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.676105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.676105:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.676107:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.676109:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.676110:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.676112:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.676112:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.676113:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.676117:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.676118:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.676121:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.676122:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.676123:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.676125:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.676125:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.676126:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.676127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.676131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.676137:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.676140:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.676141:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.676141:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.676143:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.676146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.676576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.676584:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.676587:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.676589:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.676591:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.676592:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.676592:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.676593:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.676594:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.676595:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.676596:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.676600:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.676601:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.676603:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.676604:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.676604:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.676606:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.676606:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.676607:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.676608:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.676612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.676619:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.676622:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.676623:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.676624:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.676626:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.676627:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.677119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.677125:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.677127:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.677129:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.677130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.677132:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.677133:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.677135:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.677136:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.677137:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.677138:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.677141:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.677143:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.677144:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.677145:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.677146:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.677147:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.677148:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.677149:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.677150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.677154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.677162:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.677165:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.677165:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.677167:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.677168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.677170:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.677465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.677470:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.677473:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.677475:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.677476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.677477:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.677479:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.677480:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.677481:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.677482:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.677483:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.677486:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.677487:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.677490:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.677491:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.677491:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.677493:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.677493:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.677495:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.677496:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.677496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.677499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.677501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.677505:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677474.677506:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677474.677507:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.677508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677474.677509:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677474.677509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:24.0:1619677474.677510:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.677510:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.677513:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.677515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.677517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.677523:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.677526:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.677526:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.677527:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.677528:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.677528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677474.677560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677474.677564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677474.677567:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.677570:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.677571:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677474.677572:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677474.677573:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.677575:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677474.677575:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.677577:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677474.677578:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.677578:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.677580:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.677580:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677474.677580:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.677581:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.677582:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677474.677582:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677474.677584:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.677586:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.677586:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.677587:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.677588:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677474.677589:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:31.0:1619677474.677591:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.677592:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.677593:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.677593:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.677594:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677474.677595:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.677595:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.677596:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677474.677597:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.677597:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677474.677598:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.677599:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.677600:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.677601:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677474.677601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.677602:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.677604:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.677604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677474.677604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677474.677606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677474.677609:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:21.2:1619677474.677610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677474.677612:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.677612:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.677613:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.677613:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.677614:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677474.677615:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.677616:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.677616:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.677617:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677474.677618:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677474.677618:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.677618:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.677618:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:20.0:1619677474.677619:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.677620:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.677620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.677621:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677474.677621:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.677622:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677474.677622:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677474.677623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:10.0:1619677474.677623:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.677625:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.677626:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.677627:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.677630:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.677632:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.677634:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.677635:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.677635:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.677637:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.677637:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.677638:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.677639:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.677642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.677649:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.677651:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.677652:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.677652:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.677654:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.677656:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.677887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.677894:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.677897:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.677899:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.677901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.677902:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.677904:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.677905:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.677906:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.677907:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.677908:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.677911:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.677912:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.677916:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.677916:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.677917:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.677919:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.677919:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.677920:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.677920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.677925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.677932:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.677935:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.677936:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.677936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.677938:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.677942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.678348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.678356:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.678358:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.678359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.678361:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.678363:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.678364:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.678365:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.678367:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.678367:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.678368:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.678371:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.678372:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.678375:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.678375:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.678376:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.678377:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.678378:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.678378:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.678379:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.678383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.678391:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.678393:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.678394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.678394:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.678396:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.678398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.678669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.678676:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.678679:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.678680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.678681:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.678684:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.678685:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.678686:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.678687:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.678688:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.678689:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.678692:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.678693:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.678696:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.678697:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.678697:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.678699:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.678700:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.678700:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.678700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.678705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.678712:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.678715:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.678716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.678716:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.678718:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.678721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.679073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.679081:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.679084:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.679086:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.679088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.679089:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.679090:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.679091:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.679092:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.679093:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.679094:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.679097:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.679099:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.679101:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.679102:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.679102:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.679104:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.679104:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.679105:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.679106:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.679109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.679116:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.679119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.679119:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.679120:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.679122:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.679124:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.679652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.679660:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.679663:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.679665:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.679667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.679668:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.679669:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.679670:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.679671:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.679672:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.679673:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.679676:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.679678:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677474.679680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677474.679681:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.679682:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.679682:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.679684:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.679684:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.679685:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.679686:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:10.0:1619677474.679687:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677474.679690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.679690:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.679691:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.679693:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.679695:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.679697:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677474.679698:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677474.679698:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.679699:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.679700:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.679700:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.679701:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.679702:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.679702:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.679705:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.679705:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.679707:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.679708:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677474.679710:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.679711:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.679711:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.679713:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.679713:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.679715:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.679716:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.679717:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.679719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.679722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.679726:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.679729:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.679729:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.679730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.679730:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.679731:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.679732:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.679732:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.679733:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.679734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.679738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.679745:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.679748:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.679748:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.679749:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.679750:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.679750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.679800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.679807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.679810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.679810:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677474.679813:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.679815:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.679816:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677474.679818:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.679819:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.679820:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.679821:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677474.679825:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.679827:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677474.679830:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.679831:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677474.679831:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677474.679833:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677474.679833:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.679835:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677474.679836:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677474.679836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677474.679839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.2:1619677474.679842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:10.0:1619677474.679845:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:17.0:1619677474.679848:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:10.0:1619677474.679848:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.679849:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.679849:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.679850:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677474.679850:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.679851:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:10.0:1619677474.679852:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.679855:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.680170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.680176:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.680179:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.680179:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677474.680181:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.680184:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.680185:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.680187:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.680188:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.680189:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.680190:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.680193:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.680194:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.680196:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.680197:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.680198:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.680199:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.680200:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.680200:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.680202:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.680202:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.680206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.680207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.680213:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677474.680215:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.680215:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.680215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.680216:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677474.680218:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.680219:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.680220:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.680221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.680223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.680227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.680235:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.680238:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.680239:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.680240:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.680240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.680241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.680335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.680342:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.680345:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.680347:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.680349:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.680351:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.680352:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.680353:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.680355:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.680356:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.680357:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.680360:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.680362:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.680364:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.680364:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.680365:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.680366:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.680367:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.680368:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.680369:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.680373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.680379:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.680382:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.680383:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.680384:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.680386:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.680387:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.680416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.680424:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.680426:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.680427:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.680429:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.680431:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.680433:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.680434:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.680435:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.680436:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.680437:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.680440:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.680442:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.680444:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.680445:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.680445:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.680447:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.680447:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.680448:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.680449:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.680453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.680460:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.680462:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.680463:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.680463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.680464:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.680466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.680471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.680477:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.680480:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.680480:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.680480:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.680481:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.680482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.680853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.680859:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.680862:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.680863:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.680865:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.680867:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.680868:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.680870:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.680871:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.680872:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.680873:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.680877:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.680878:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.680881:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.680882:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.680883:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.680885:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.680886:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.680887:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.680888:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.680888:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.680891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677474.680893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677474.680898:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677474.680899:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677474.680900:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:18.0:1619677474.680901:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:31.0:1619677474.680902:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.680902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.680903:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677474.680904:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.680904:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.680907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.681249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.681256:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.681259:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.681261:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.681262:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.681263:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.681264:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.681265:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.681266:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.681267:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.681268:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.681270:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.681272:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.681274:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.681275:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.681276:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.681277:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.681278:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.681279:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677474.681279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.681282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.681283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.681284:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.681286:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:31.0:1619677474.681288:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:30.0:1619677474.681288:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.681289:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.681289:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.681290:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.681291:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.681291:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.681292:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.681301:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.681301:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.681302:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.681304:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.681306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.681306:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677474.681306:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.681307:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.681308:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.681308:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.681309:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.681309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.681310:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.681310:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.681311:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.681311:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.681313:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.681314:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.681315:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.681315:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.681320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.681329:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.681332:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.681333:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.681333:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.681335:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.681339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.681403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.681409:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:4.2:1619677474.681412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:1.0:1619677474.681412:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.681413:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.681414:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.681416:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677474.681417:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.681418:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:1.0:1619677474.681419:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677474.681421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677474.681421:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.681422:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.681422:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.681423:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.681424:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.681424:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.681426:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677474.681428:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.681429:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677474.681430:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.681431:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677474.681431:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.681432:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:1.0:1619677474.681434:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.681435:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:20.0:1619677474.681436:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.681436:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.681437:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.681438:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677474.681438:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.681438:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.681439:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.0:1619677474.681441:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.681441:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677474.681442:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:25.2:1619677474.681443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.681444:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677474.681445:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.681446:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.681447:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.681447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677474.681450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677474.681451:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:4.2:1619677474.681452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677474.681454:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.681455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.681455:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.681457:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.681458:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:18.0:1619677474.681458:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:6.0:1619677474.681459:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.681460:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:25.2:1619677474.681461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.681461:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677474.681461:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677474.681462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677474.681462:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.681462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.681464:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.681467:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677474.681468:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.681471:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.681472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.681473:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.681474:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.681474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.681649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.681657:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.681660:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.681662:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.681664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.681665:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.681666:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.681668:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.681669:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.681670:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.681671:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.681674:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.681676:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.681679:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.681680:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.681681:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.681683:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.681683:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.681685:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.681685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.681689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.681696:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.681699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.681699:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.681700:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.681702:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.681705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.682020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.682028:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.682030:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.682033:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.682034:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.682036:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.682038:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.682039:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.682040:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.682041:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.682042:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.682046:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.682047:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.682049:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.682050:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.682050:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.682051:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.682052:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.682053:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.682055:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.682055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.682057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.682061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.682065:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.682067:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.682067:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.682069:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.682069:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.682070:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.682070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.682071:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.682072:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.682072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.682111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.682116:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.682119:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.682120:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.682122:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.682124:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.682126:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.682127:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677474.682128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000400:00000200:2.0:1619677474.682128:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.682129:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.682131:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.682134:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000400:00000200:2.0:1619677474.682134:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.682136:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.682136:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.682138:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.682138:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.682139:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:18.0:1619677474.682140:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.682140:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:18.0:1619677474.682141:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:2.0:1619677474.682142:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.682143:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.682144:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.682145:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.682146:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.2:1619677474.682148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:18.0:1619677474.682149:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.682150:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.682151:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.682151:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.682152:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677474.682153:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.682154:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.682154:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (23)++
00000800:00000200:2.0:1619677474.682154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.682155:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (24)--
00000800:00000200:2.0:1619677474.682155:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (24)++
00000800:00000200:18.0:1619677474.682156:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.682156:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (25)--
00000400:00000200:2.0:1619677474.682156:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.682157:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.682157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:2.0:1619677474.682158:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.682159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.682160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:9.2:1619677474.682161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.682169:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.682171:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.682171:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.682172:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:18.0:1619677474.682173:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.682173:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.682174:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.682174:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (23)++
00000800:00000200:18.0:1619677474.682175:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.682175:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.682175:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.682312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.682318:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.682321:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.682323:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.682323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.682326:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.682327:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.682328:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.682330:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.682331:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.682331:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.682334:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.682336:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.682337:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.682338:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.682339:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.682341:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.682341:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.682342:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.682343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.682347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.682356:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.682358:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.682359:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.682361:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.682361:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.682363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677474.682823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.682829:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.682832:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.682833:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.682835:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.682837:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.682839:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.682840:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.682842:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.682843:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.682844:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.682847:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.682849:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.682851:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.682852:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.682853:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.682854:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.682855:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.682856:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.682856:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677474.682858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677474.682861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.682866:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.682868:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.682869:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677474.682870:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:10.0:1619677474.682871:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.682872:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:10.0:1619677474.682873:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.682873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.682874:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.682876:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677474.682877:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.682877:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.682879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:10.0:1619677474.682879:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.682880:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.682881:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.682884:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.682886:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.682888:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.682889:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.682890:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.682891:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.682892:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.682893:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.682894:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.682895:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.682897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.682900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.682905:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.682907:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.682908:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.682908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.682908:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.682910:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677474.682910:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.682911:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.682912:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.682913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.683590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.683597:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.683600:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.683601:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.683602:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.683605:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.683607:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.683608:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.683610:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.683611:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.683612:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677474.683612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:24.0:1619677474.683615:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.683616:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.683619:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.683619:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.683620:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.683620:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.683622:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:24.0:1619677474.683622:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.683622:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.683623:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677474.683624:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.683624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:30.0:1619677474.683626:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.683627:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.683627:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.683628:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677474.683629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677474.683630:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.683630:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.683631:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.683634:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.683636:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.683637:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.683639:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.683639:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.683640:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.683640:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.683641:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000400:00000200:22.0:1619677474.683641:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.683643:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.683643:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:22.0:1619677474.683643:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.683645:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.683646:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:22.0:1619677474.683646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.683647:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.683649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.683652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.683656:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.683658:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.683659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.683659:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.683660:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.683661:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.683662:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.683663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677474.683663:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.683665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.683666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:4.2:1619677474.683668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677474.683673:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.683676:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.683676:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.683676:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.683676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.683678:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.683678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677474.683679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.683679:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.683681:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.683684:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.683685:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.683686:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.683687:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.683689:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.683689:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.683692:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.683694:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.683697:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.683697:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.683698:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.683699:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.683700:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.683701:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.683701:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.683705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.683712:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.683715:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.683716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.683716:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.683718:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.683721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.683916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.683923:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.683926:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.683927:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.683929:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.683931:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.683933:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.683934:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.683936:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.683937:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.683937:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.683940:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.683942:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.683944:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.683945:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.683945:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.683947:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.683947:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.683948:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.683949:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.683950:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:4.2:1619677474.683955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.683962:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:25.2:1619677474.683963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.683964:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.683965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.683966:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677474.683971:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.683974:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.683975:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.683976:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.683978:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.683979:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.684197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.684204:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.684207:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.684209:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.684211:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.684211:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.684212:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.684214:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.684215:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.684216:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.684217:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.684220:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.684221:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.684223:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.684224:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.684225:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.684226:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.684227:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.684228:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.684228:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.684232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.684238:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.684241:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.684242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.684242:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.684244:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.684246:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.684497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.684504:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.684506:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.684507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.684509:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.684511:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.684513:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.684514:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.684515:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.684516:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.684517:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.684520:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.684521:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.684524:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.684525:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.684526:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.684527:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.684528:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.684528:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.684529:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.684534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.684542:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.684545:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.684546:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.684546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.684548:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.684551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.685551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.685559:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.685562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.685562:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.685565:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.685567:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.685569:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.685570:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.685571:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.685572:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.685573:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.685576:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.685578:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.685580:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.685581:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.685581:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.685583:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.685583:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.685584:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.685585:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.685589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.685596:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.685599:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.685600:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.685600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.685601:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.685604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.685640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677474.685644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.685649:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.685651:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677474.685652:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.685654:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.685654:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.685656:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.685656:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.685658:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.685659:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.685660:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.685661:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.685662:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.685663:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.685665:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.685665:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.685666:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.685667:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.685667:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.685668:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.685669:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.685672:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.685673:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.685673:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.685673:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.685674:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.685674:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.685675:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.685676:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.685677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.685678:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.685678:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.685678:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.685679:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.685679:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.685680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.685681:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.685682:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.685682:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.685683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.0:1619677474.685686:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:25.2:1619677474.685687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.685688:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.685689:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.685689:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.685691:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.685693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677474.685694:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.685697:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.685698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.685699:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.685701:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.685704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.685806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.685813:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.685816:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.685817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.685819:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.685821:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.685823:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.685824:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.685826:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.685827:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.685828:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.685831:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.685833:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.685836:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.685837:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.685838:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.685840:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.685841:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.685842:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.685842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.685846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.685854:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.685856:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.685857:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.685859:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.685860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.685861:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.685864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677474.685870:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.685873:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.685874:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.685875:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.685875:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.685876:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.686603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.686611:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.686614:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.686616:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.686619:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.686619:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.686620:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.686621:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.686623:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.686623:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.686624:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.686628:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.686629:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.686632:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.686632:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.686633:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.686635:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.686635:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.686636:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.686637:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.686641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.686649:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.686651:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.686652:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.686653:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.686655:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.686656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.687075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.687083:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.687086:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.687089:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677474.687091:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.687091:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.687093:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677474.687094:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.687096:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.687097:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.687097:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677474.687101:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.687102:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677474.687105:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.687106:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677474.687106:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677474.687108:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677474.687108:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.687109:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.687110:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677474.687115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.687124:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.687127:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.687128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.687128:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.687130:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.687133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.687351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.687359:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.687362:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.687364:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.687366:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.687366:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.687368:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.687369:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.687370:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.687371:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.687372:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.687375:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.687377:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.687380:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.687380:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.687381:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.687382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.687383:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.687383:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.687384:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.687385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677474.687386:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.687388:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.687389:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.687390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.687390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677474.687391:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.687392:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.687393:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.687394:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.687395:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.687395:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.687398:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.687398:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.687400:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677474.687401:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.687401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.687402:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.687402:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:22.0:1619677474.687402:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.687403:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.687404:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:22.0:1619677474.687404:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.687405:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.687406:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677474.687406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.687408:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.687408:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.687410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.687414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.687416:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.687419:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.687420:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.687420:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.687422:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.687422:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.687424:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.687424:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.687425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.687426:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.687428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.687434:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.687437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.687437:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.687437:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.687439:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.687439:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.687697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.687703:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.687706:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.687707:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.687709:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.687712:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.687713:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677474.687714:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.687715:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.687717:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.687718:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677474.687720:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.687722:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.687724:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.687725:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677474.687726:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.687727:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677474.687728:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.687729:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.687729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677474.687734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.687741:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.687744:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.687745:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.687745:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.687747:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.687749:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.687755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.687762:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.687764:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.687765:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.687765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.687766:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.687767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.687970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.687977:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.687980:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.687981:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.687983:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.687985:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.687987:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.687988:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.687989:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.687990:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.687991:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.687994:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.687996:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.687999:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.688000:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.688000:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.688002:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.688002:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.688003:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.688004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.688008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.688015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.688018:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.688019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.688019:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.688021:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.688024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.688683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.688690:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.688693:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.688694:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.688695:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.688698:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.688700:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.688701:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.688702:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.688703:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.688704:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.688706:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.688708:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.688710:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.688710:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.688711:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.688712:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.688713:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.688714:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.688715:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.688715:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.688719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.688721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.688725:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.688728:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677474.688729:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:24.0:1619677474.688729:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.688729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.688731:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.688731:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677474.688731:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.688732:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677474.688734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.688938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.688945:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.688947:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.688948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.688950:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.688952:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.688954:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.688955:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.688956:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.688957:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.688958:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.688961:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.688962:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.688965:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.688966:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.688966:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.688968:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.688968:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.688969:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.688970:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.688974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.688982:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.688985:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.688986:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.688986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.688988:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.688991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.689408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.689416:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.689419:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.689421:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.689423:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.689424:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.689425:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.689426:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.689427:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.689428:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.689429:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.689432:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.689434:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.689437:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.689438:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.689438:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.689440:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.689440:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.689441:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.689442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.689447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.689455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.689457:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.689458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.689458:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.689460:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.689463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.689571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.689577:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.689579:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.689581:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.689582:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.689584:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.689585:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.689587:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.689588:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.689589:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.689590:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.689593:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.689595:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.689597:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.689598:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.689599:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.689601:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.689601:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.689602:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.689603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.689607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.689615:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.689617:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.689618:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.689620:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.689621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.689622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.689700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.689708:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.689710:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.689711:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.689713:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.689715:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.689717:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.689718:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.689719:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.689720:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.689721:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.689724:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.689726:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.689728:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.689729:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.689730:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.689731:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.689732:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.689733:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.689733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.689737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.689745:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.689747:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.689748:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.689749:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.689751:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.689753:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.689794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.689802:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.689805:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.689807:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.689810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.689811:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.689812:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:12.0:1619677474.689814:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.689815:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.689816:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.689817:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:12.0:1619677474.689820:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.689822:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677474.689824:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.689825:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:12.0:1619677474.689826:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:12.0:1619677474.689827:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:12.0:1619677474.689828:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.689829:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.689830:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677474.689834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.689842:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.689845:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.689846:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.689846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.689848:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.689851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.689855:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.689856:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.689858:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.689860:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.689860:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.689861:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.689861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.689866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.689874:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.689877:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.689879:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.689881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.689882:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.689883:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.689885:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.689886:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.689887:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.689888:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.689891:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.689892:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.689894:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.689895:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.689895:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.689897:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.689897:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.689898:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.689899:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.689903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.689909:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.689912:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.689912:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.689914:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.689916:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.689917:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677474.690125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.690130:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.690133:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.690134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.690136:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.690138:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.690140:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.690141:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.690142:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.690143:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.690144:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.690147:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.690149:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.690151:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.690152:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.690153:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.690154:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.690155:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.690155:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.690156:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.690160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.690167:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.690170:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.690171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.690171:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.690173:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.690175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.690177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.690184:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.690187:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.690188:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.690188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.690189:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.690189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.690234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677474.690241:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.690244:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677474.690247:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.690248:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.690248:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677474.690250:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.690251:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.690252:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.690253:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.690255:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.690258:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.690260:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.690263:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.690264:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.690265:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.690266:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.690267:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.690268:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677474.690269:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677474.690273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.690281:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.690283:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.690284:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.690286:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.690286:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.690289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.690320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.690327:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.690330:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.690331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.690333:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.690336:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.690338:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.690339:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.690340:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.690341:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.690342:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.690346:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.690347:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.690350:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.690351:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.690351:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.690353:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.690354:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.690355:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.690356:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.690357:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.690361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.690363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.690369:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.690370:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.690371:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.690372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.690372:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.690373:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677474.690373:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.690374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677474.690375:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.690378:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.690379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.690385:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.690388:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.690389:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.690389:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.690390:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.690391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.690677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.690685:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.690687:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.690688:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.690690:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.690693:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.690694:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.690695:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.690696:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.690697:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.690698:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.690701:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.690703:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.690705:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.690705:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.690706:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.690707:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.690708:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.690709:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.690709:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.690713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.690720:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.690722:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.690723:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.690723:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.690725:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.690727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.690803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.690811:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.690814:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.690816:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.690818:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.690819:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.690820:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.690821:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.690823:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.690824:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.690825:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.690828:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.690829:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.690831:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.690832:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.690832:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.690834:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.690834:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.690835:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.690836:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.690839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.690846:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.690849:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.690849:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.690851:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.690852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.690853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.690858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.690865:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.690867:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.690868:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.690869:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.690869:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.690870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.691172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.691179:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.691182:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.691184:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.691185:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.691186:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.691187:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.691188:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.691189:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.691190:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.691191:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.691194:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.691195:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.691198:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.691199:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.691200:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.691201:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.691202:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.691203:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.691205:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.691205:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.691208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.691211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.691215:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.691218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.691218:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.691219:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.691219:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.691221:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.691222:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.691222:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.691223:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.691224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.691479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.691487:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.691490:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.691492:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.691495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.691495:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.691496:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.691497:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.691499:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.691500:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.691501:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.691503:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.691505:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.691507:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.691508:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.691508:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.691510:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.691510:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.691511:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.691512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.691516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.691523:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.691525:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.691526:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.691527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.691528:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.691531:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.691565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.691572:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.691575:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.691576:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.691577:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.691580:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.691581:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.691583:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.691584:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.691585:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.691586:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.691588:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.691590:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.691593:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.691594:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.691594:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.691596:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.691596:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.691597:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.691597:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.691602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.691610:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.691613:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.691614:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.691615:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.691617:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.691618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.691619:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.691620:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.691620:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.691620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.691621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.691747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.691756:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.691760:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.691761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.691765:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.691769:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.691771:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.691774:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.691776:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.691778:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.691779:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.691785:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.691788:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.691791:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.691792:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.691793:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.691796:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.691797:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.691798:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.691799:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.691802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.691810:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.691813:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.691814:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.691814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.691816:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.691819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.692046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.692053:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.692056:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.692057:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.692059:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.692062:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.692063:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.692064:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.692065:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.692067:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.692068:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.692071:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.692073:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.692076:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.692077:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.692077:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.692079:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.692079:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.692080:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.692080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.692085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.692092:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.692095:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.692096:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.692097:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.692100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.692101:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.692220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.692228:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.692230:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.692233:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.692234:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.692235:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.692236:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.692237:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.692238:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.692239:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.692240:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.692243:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.692245:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.692246:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.692247:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.692247:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.692249:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.692249:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.692251:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.692253:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.692253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.692254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.692259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.692263:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.692265:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.692265:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.692267:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.692267:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.692268:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.692268:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.692269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.692270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.692271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.692866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.692873:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.692876:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.692877:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.692879:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.692881:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.692882:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.692884:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.692885:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.692886:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.692886:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.692889:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.692891:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.692894:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.692894:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.692895:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.692896:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.692897:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.692898:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.692898:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.692903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.692911:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.692914:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.692915:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.692915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.692918:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.692920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.693289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.693298:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.693301:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.693304:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.693306:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.693306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.693307:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.693309:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.693310:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.693311:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.693312:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.693315:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.693317:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.693320:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.693321:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.693322:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.693323:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.693324:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.693325:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.693325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.693330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.693338:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.693340:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.693341:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.693342:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.693344:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.693346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.693511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.693518:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.693521:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.693522:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.693524:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.693527:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.693528:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677474.693530:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.693531:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.693532:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.693533:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677474.693536:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.693537:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677474.693539:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.693540:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677474.693541:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677474.693542:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677474.693543:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.693543:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.693544:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.693548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.693555:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.693557:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.693558:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.693560:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.693561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.693562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.693640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.693647:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.693649:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.693650:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.693652:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.693653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677474.693654:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.693655:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.693656:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.693658:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.693658:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.693659:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.693661:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677474.693662:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.693663:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677474.693664:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.693665:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.693666:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.693667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677474.693667:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.693667:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677474.693668:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.693669:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:10.0:1619677474.693669:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.693669:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677474.693670:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.693670:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677474.693671:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.693671:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:31.0:1619677474.693672:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.693673:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:25.2:1619677474.693675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677474.693676:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.693677:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.693680:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.693681:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.693681:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.693683:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.693683:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.693683:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.693684:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.693684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.693685:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.693686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.693687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.693688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677474.693689:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.693691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.693693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.693695:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.693698:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.693699:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.693699:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.693700:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:8.0:1619677474.693701:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.693703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677474.693703:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.693704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.693704:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:9.2:1619677474.693705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677474.693705:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.693706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677474.693711:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.693714:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.693715:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.693715:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.693716:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.693717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.693772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.693778:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.693781:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.693781:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677474.693784:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.693786:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.693787:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.693788:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.693790:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.693791:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.693792:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.693795:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.693797:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.693799:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.693800:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.693801:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.693802:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.693803:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.693804:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.693804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.693809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.693817:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.693819:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.693820:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.693822:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.693823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.693824:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.694137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.694143:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.694146:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.694147:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.694149:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.694152:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.694154:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.694155:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.694156:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.694157:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.694158:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.694161:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.694163:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.694165:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.694165:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.694166:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.694168:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.694168:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.694169:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.694170:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.694171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.694174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.694177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.694182:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.694183:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.694184:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.694184:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.694184:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.694185:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.694186:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.694186:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.694187:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677474.694187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:4.2:1619677474.694518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.694525:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.694528:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.694529:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.694531:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.694533:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.694535:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.694536:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.694537:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.694538:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.694539:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.694542:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.694544:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.694547:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.694548:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.694549:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.694550:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.694551:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.694551:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.694552:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.694557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.694565:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.694568:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.694569:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.694570:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.694572:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.694575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.694589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.694597:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.694600:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.694600:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.694602:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.694605:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.694606:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.694608:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.694609:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.694610:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.694611:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.694614:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.694616:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.694619:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.694619:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.694620:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.694622:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.694622:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.694623:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.694623:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677474.694624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:25.2:1619677474.694628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.694630:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.694633:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677474.694634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:6.0:1619677474.694635:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:17.0:1619677474.694636:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.694638:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.694638:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.694639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:17.0:1619677474.694639:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:6.0:1619677474.694639:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.694641:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.694641:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.694642:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.694644:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.694644:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677474.694644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:17.0:1619677474.694648:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.694649:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677474.694651:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.694652:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677474.694653:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677474.694655:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677474.694655:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.694656:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677474.694657:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677474.694661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.694669:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.694672:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.694673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.694673:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.694675:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.694678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:5.2:1619677474.694851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.694856:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.694859:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.694860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.694862:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.694864:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.694866:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.694867:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.694868:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.694869:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.694870:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.694873:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.694875:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.694877:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.694878:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.694878:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.694880:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.694880:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.694881:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.694883:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.694883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.694886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.694888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.694893:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.694893:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.694896:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.694896:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677474.694897:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.694897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.694897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.694897:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.694900:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.694902:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.695186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.695194:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.695197:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.695200:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.695202:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.695202:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.695203:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.695205:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.695206:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.695207:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.695208:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.695211:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.695213:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.695215:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.695216:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.695216:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.695218:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.695218:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.695219:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.695220:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.695223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.695230:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.695232:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.695233:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.695235:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.695236:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.695237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.695394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.695403:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.695405:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.695407:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.695409:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.695410:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.695410:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.695411:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.695412:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.695413:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.695414:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.695417:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.695419:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.695422:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.695423:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.695423:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.695425:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.695425:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.695426:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.695426:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.695432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.695440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.695442:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.695443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.695444:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.695446:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.695448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.695632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.695639:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.695642:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.695643:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.695644:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.695647:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.695648:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.695650:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.695651:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.695652:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.695653:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.695656:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.695657:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.695659:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.695660:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.695661:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.695662:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.695662:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.695664:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.695665:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.695666:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.695668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.695671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.695674:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.695677:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.695677:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.695678:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677474.695678:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.695679:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.695680:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.695681:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.695682:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677474.695682:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.696040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.696047:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.696049:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.696050:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.696051:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.696054:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.696055:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.696056:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.696057:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.696058:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.696059:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.696062:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.696064:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.696066:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.696067:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.696068:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.696069:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.696070:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.696071:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.696071:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.696075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.696084:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.696087:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.696088:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.696088:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.696090:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.696093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.696139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.696147:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.696150:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.696152:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.696153:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.696154:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.696156:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.696157:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.696158:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.696159:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.696160:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.696163:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.696165:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.696167:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.696168:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.696168:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.696170:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.696170:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.696171:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.696172:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.696176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.696183:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.696185:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.696186:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.696187:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.696189:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.696190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.696193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.696199:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.696202:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.696203:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.696203:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.696204:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.696204:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.696869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.696877:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.696879:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.696881:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.696882:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.696883:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.696884:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.696884:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.696886:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.696887:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.696888:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.696890:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.696892:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.696893:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.696894:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.696894:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.696896:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.696896:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.696898:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.696899:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.696899:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.696902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.696906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.696910:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.696912:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.696913:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.696914:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.696914:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.696915:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.696915:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.696916:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.696917:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.696918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.697556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.697564:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.697567:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.697569:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.697571:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.697572:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.697572:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.697573:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.697575:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.697575:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.697576:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.697579:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.697580:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.697583:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.697584:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.697584:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.697586:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.697587:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.697587:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.697588:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.697593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.697602:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.697605:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.697605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.697606:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.697608:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.697610:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.697652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.697660:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.697663:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.697664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.697665:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.697668:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.697669:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.697670:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.697672:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.697672:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.697673:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.697676:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.697678:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.697680:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.697680:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.697681:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.697682:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.697682:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.697683:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.697684:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.697688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.697695:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.697697:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.697698:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.697698:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.697700:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.697702:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.697820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.697826:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.697828:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.697830:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.697831:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.697832:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.697834:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.697835:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.697836:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.697837:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.697838:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.697841:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.697842:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.697844:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.697845:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.697846:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.697847:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.697848:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.697848:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.697849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.697853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.697861:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.697863:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.697864:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.697866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.697866:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.697868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.697870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.697878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.697881:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.697882:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.697882:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.697883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.697884:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.698049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.698057:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.698060:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.698062:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.698063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.698065:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.698066:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.698068:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.698069:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.698070:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.698071:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.698074:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.698076:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.698078:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.698079:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.698079:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.698081:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.698082:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.698083:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.698083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.698086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.698093:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.698095:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.698096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.698096:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.698098:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.698100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.698509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.698517:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.698519:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.698520:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.698522:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.698524:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.698525:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.698526:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.698527:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.698528:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.698529:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.698532:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.698533:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.698535:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.698536:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.698537:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.698538:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.698539:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.698539:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.698540:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.698544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.698551:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.698553:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.698554:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.698555:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.698556:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.698558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.698562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.698568:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.698571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.698571:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.698571:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.698572:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.698573:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.698721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677474.698724:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.698726:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:21.0:1619677474.698729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677474.698729:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.698731:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.698732:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.698733:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.698735:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.698736:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.698737:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.698740:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.698742:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.698744:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.698745:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.698746:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.698747:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.698748:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.698749:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677474.698750:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677474.698751:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:25.2:1619677474.698754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.2:1619677474.698756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.698761:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:6.0:1619677474.698762:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.698764:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:6.0:1619677474.698764:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677474.698765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677474.698765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:6.0:1619677474.698765:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.698766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.698767:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.698769:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.698772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.698779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.698782:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.698782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.698783:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.698784:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.698784:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.698892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.698899:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.698902:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.698903:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.698905:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.698907:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.698908:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.698910:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.698911:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.698912:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.698913:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.698916:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.698917:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.698920:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.698921:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.698921:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.698923:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.698924:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.698925:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.698926:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.698926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.698929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.698932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.698936:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.698939:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.698939:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.698940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.698940:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.698941:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.698942:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.698942:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.698943:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677474.698944:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.699220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.699226:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.699229:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.699230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.699232:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.699234:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.699235:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.699237:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.699238:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.699239:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.699240:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.699244:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.699245:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.699249:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.699249:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.699250:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.699252:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.699252:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.699253:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.699254:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.699258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.699266:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.699269:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.699270:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.699271:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.699273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.699275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.699286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.699293:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.699301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.699301:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.699302:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.699305:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.699306:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.699308:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.699309:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.699310:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.699311:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.699314:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.699316:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.699319:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.699320:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.699320:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.699322:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.699323:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.699324:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.699326:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.699326:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.699328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.699332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.699337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.699339:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.699340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.699341:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.699341:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.699343:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677474.699343:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.699344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.699345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.699346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.699352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.699360:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.699363:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.699365:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.699367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.699368:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.699369:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.699371:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.699372:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.699373:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.699374:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.699377:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.699378:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.699380:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.699381:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.699381:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.699383:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.699383:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.699384:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.699384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.699388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.699395:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.699397:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.699398:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.699400:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.699402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.699403:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.699433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.699440:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.699443:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.699444:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.699445:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.699448:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.699450:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.699451:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.699452:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.699453:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.699454:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.699457:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.699459:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.699462:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.699462:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.699463:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.699464:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.699465:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.699466:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.699466:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.699470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.699478:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.699480:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.699481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.699482:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.699484:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.699487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.700048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.700054:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.700057:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.700058:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.700060:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.700062:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.700064:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.700065:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.700066:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.700067:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.700068:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.700072:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.700073:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.700076:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.700077:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.700077:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.700079:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.700080:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.700080:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.700081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.700085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.700093:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.700096:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.700096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.700097:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.700099:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.700101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.700103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.700110:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.700112:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.700113:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.700114:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.700114:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.700115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.700129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.700137:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.700140:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.700141:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.700143:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.700146:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.700147:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.700148:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.700149:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.700150:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.700152:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.700155:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.700156:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.700159:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.700159:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.700160:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.700162:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.700162:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.700163:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.700163:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.700167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.700174:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.700176:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.700177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.700177:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.700179:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.700181:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.700347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.700354:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.700357:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.700358:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.700359:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.700361:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.700362:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.700363:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.700365:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.700365:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.700366:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.700369:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.700370:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.700372:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.700373:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.700373:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.700375:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.700375:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.700375:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.700376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.700381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.700389:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.700391:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.700392:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.700393:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.700395:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.700396:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.700445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.700453:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.700456:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.700458:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.700459:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.700460:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677474.700461:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.700461:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.700463:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.700464:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.700464:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.700467:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.700469:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.700471:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.700471:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.700472:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.700473:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.700474:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.700475:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.700475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.700479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.700486:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.700488:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.700489:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.700490:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.700492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.700493:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.700761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.700767:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.700770:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.700771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.700773:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.700776:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.700777:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677474.700779:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.700780:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.700781:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.700782:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677474.700786:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.700788:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677474.700791:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.700792:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677474.700792:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677474.700794:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677474.700795:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.700796:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.700797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677474.700800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.700807:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.700810:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.700811:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.700812:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.700813:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.700815:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.700820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.700827:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.700830:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.700831:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.700832:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.700832:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.700833:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.701102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.701109:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.701112:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.701114:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.701116:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.701116:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.701118:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.701119:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.701120:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.701121:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.701122:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.701126:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.701128:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.701131:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.701132:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.701132:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.701134:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.701135:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.701136:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.701136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.701140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.701148:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.701150:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.701151:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.701152:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.701154:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.701157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.701232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.701238:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.701240:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.701242:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.701243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.701245:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.701246:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.701248:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.701249:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.701250:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.701251:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.701254:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.701256:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.701258:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.701259:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.701260:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.701261:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.701262:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.701263:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.701263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.701268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.701277:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.701280:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.701281:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.701281:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.701283:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.701286:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.701688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.701696:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.701699:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.701701:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.701704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.701704:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.701705:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.701707:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.701708:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.701709:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.701710:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.701713:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.701715:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.701717:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.701718:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.701718:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.701720:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.701720:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.701722:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.701723:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.701724:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.701726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.701730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.701735:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.701736:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.701737:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.701739:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.701739:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.701739:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.701740:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.701740:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.701741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.701742:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.701743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.701750:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.701753:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.701753:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.701754:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.701755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.701755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.702353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.702361:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.702363:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.702365:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.702367:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.702368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.702369:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.702369:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.702371:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.702371:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.702372:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.702375:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.702376:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.702380:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.702380:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.702381:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.702383:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.702383:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.702384:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.702384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677474.702390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.702398:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.702400:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.702401:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.702403:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.702404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.702406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.702689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.702697:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.702700:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.702702:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.702704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.702705:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.702706:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.702707:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.702709:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.702710:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.702711:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.702713:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.702715:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.702717:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.702718:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.702718:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.702720:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.702720:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.702721:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.702722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.702726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.702733:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.702735:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.702736:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.702737:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.702738:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.702741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.703167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.703174:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.703177:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.703178:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.703180:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.703182:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.703183:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.703185:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.703186:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.703187:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.703188:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.703191:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.703192:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.703195:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.703196:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.703196:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.703198:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.703198:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.703200:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.703201:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.703201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.703204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.703207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.703212:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.703214:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.703215:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.703216:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677474.703216:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.703216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.703217:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.703218:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677474.703218:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.703221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.703223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.703230:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.703232:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.703233:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.703233:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.703234:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.703235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.703515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.703523:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.703526:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.703529:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.703531:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.703531:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.703533:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677474.703534:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.703535:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.703536:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.703537:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677474.703540:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.703542:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.703545:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.703546:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677474.703546:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677474.703548:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677474.703549:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.703550:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.703551:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.703555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.703566:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.703570:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677474.703571:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.703572:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:9.2:1619677474.703575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:2.0:1619677474.703575:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.703581:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.703583:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.703586:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.703586:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677474.703588:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.703589:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.703590:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677474.703693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.703699:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.703701:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.703702:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.703704:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.703706:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.703707:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.703708:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.703710:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.703711:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.703712:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.703716:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.703717:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.703720:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.703721:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.703722:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.703724:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.703724:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.703726:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.703727:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.703727:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.703730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677474.703732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677474.703737:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:2.0:1619677474.703738:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677474.703739:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.703740:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.703740:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.703741:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677474.703742:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.703742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677474.703743:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.703745:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.703745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.703751:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.703754:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:4.2:1619677474.703754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677474.703755:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.703757:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.703759:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.703760:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.703761:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677474.703762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677474.703762:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677474.703763:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.703763:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.703764:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677474.703765:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.703766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677474.703767:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.703769:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.703770:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677474.703770:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.703771:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.703771:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.703772:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.703772:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.703773:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677474.703773:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:10.0:1619677474.703773:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:18.0:1619677474.703774:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.703774:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.703775:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677474.703775:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.703776:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.703776:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.703777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677474.703777:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.703778:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.703778:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677474.703779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:30.0:1619677474.703780:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.703781:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.703782:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.703783:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.703784:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.703785:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.703785:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677474.703786:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:30.0:1619677474.703788:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.703788:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.703788:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:10.0:1619677474.703789:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.703789:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:2.0:1619677474.703789:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.703790:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677474.703790:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677474.703791:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:2.0:1619677474.703791:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.703792:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.703792:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677474.703793:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.703793:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.703793:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.703793:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677474.703793:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.703795:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.703795:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.703796:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.703797:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.703797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:9.2:1619677474.703801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.703806:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.703808:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:8.0:1619677474.703808:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:22.0:1619677474.703809:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.703809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:8.0:1619677474.703810:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.703811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.703811:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.703812:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.703814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:8.0:1619677474.703814:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.703816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.703971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.703979:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.703981:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.703984:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.703985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.703986:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.703988:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.703989:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.703991:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.703991:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.703993:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.703996:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.703997:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.704000:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.704001:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.704002:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.704004:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.704004:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.704005:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.704005:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.704010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.704017:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.704020:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.704021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.704021:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.704024:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.704027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.704152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.704159:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.704162:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.704163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.704164:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.704167:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.704168:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.704170:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.704171:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.704172:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.704173:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.704176:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.704177:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.704180:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.704181:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.704181:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.704183:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.704184:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.704185:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.704186:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.704187:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.704189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.704192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.704196:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.704199:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.704199:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:22.0:1619677474.704200:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.704200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.704202:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.704202:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.704202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.704204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.704204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.704408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.704416:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.704419:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.704421:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.704424:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.704424:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.704426:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.704427:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.704428:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.704429:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.704430:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.704433:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.704435:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.704437:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.704437:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.704438:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.704439:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.704440:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.704441:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.704441:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.704446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.704453:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.704455:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.704456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.704456:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.704458:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.704460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.704462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.704468:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.704471:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.704471:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.704472:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.704472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.704473:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.704538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.704546:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.704548:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.704549:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.704551:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.704554:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.704555:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677474.704556:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.704557:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.704558:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.704559:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677474.704562:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.704564:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.704566:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.704566:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.704567:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677474.704568:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.704569:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.704570:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.704570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.704574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.704580:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.704582:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.704583:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.704583:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.704585:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.704587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.704804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.704810:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.704813:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.704814:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.704816:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.704818:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.704819:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.704820:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.704822:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.704823:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.704824:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.704827:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.704829:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.704831:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.704832:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.704833:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.704834:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.704835:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.704835:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.704836:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.704841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.704849:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.704852:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.704853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.704853:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.704855:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.704857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.705197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.705205:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.705208:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.705210:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.705212:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.705213:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.705214:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.705216:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.705217:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.705218:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.705219:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.705222:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.705224:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.705227:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.705228:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.705229:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.705230:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.705231:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.705233:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.705234:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.705235:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.705236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.705240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.705244:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.705246:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.705247:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.705248:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.705248:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.705250:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.705251:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.705252:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.705252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.705253:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.705290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.705299:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.705302:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.705304:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.705306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.705307:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.705308:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.705310:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.705311:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.705312:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.705313:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.705316:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.705318:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.705320:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.705321:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.705321:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.705323:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.705323:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.705324:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.705326:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.705326:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.705329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.705332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.705335:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.705338:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.705339:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.705339:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.705340:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.705341:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.705343:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.705343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.705343:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.705344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.705369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.705375:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.705378:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.705378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.705381:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.705383:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.705384:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.705385:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.705387:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.705388:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.705389:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.705392:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.705394:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.705396:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.705397:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.705398:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.705399:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.705400:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.705400:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.705401:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.705405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.705414:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.705416:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.705417:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.705419:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.705420:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.705421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.706043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677474.706050:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.706054:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.706054:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:8.0:1619677474.706056:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.706059:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.706061:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677474.706062:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.706063:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.706064:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.706065:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677474.706069:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.706070:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677474.706073:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.706074:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677474.706075:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677474.706076:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677474.706077:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.706078:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.706078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677474.706082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.706090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.706093:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.706094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.706094:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.706095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:1.0:1619677474.706097:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.706099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677474.706100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677474.706101:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677474.706104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.706104:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.706106:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.706107:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:8.2:1619677474.706107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:20.0:1619677474.706108:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.706109:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.706111:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.706111:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.706114:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.706114:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.706116:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.706117:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.706118:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.706118:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.706118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.706119:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.706119:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.706120:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.706120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677474.706121:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.706122:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.706123:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.706123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.706127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.706135:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.706138:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.706138:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.706140:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.706141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.706142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.706681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.706689:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.706692:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.706694:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.706697:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.706697:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.706698:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.706700:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.706701:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.706702:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.706703:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.706706:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.706708:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.706710:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.706711:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.706711:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.706713:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.706713:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.706715:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.706715:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.706719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.706726:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.706729:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.706729:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.706731:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.706733:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.706734:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.706736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.706744:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.706747:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:4.2:1619677474.706747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:31.0:1619677474.706748:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.706750:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.706750:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.706752:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.706753:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.706754:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677474.706754:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:31.0:1619677474.706755:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.706756:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.706757:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.706758:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:31.0:1619677474.706760:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.706760:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.706762:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.706763:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.706764:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.706764:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:31.0:1619677474.706765:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.706765:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.706765:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.706767:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677474.706767:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.706767:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.706768:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.706768:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000400:00000200:10.0:1619677474.706768:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677474.706769:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:10.0:1619677474.706771:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677474.706773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.706773:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.706775:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.706776:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.706777:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.706778:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.706778:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.706779:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.706780:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677474.706781:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:25.2:1619677474.706784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:6.0:1619677474.706784:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.706785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.706785:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.706787:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.706790:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.706793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:12.0:1619677474.706793:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.706796:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.706797:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.706800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:12.0:1619677474.706800:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.706803:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677474.706803:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677474.706804:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:13.0:1619677474.706804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677474.706804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.706805:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.706806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.707024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.707027:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.707030:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.707032:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.707033:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.707034:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.707036:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.707037:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.707039:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.707040:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.707041:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.707044:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.707046:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.707048:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.707049:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.707049:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.707051:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.707052:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.707053:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.707054:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.707055:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.707058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.707060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.707065:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677474.707066:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677474.707067:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.707068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.707069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677474.707069:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.707070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.707070:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.707072:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.707074:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.707498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677474.707503:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.707506:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677474.707507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677474.707509:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.707511:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.707512:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677474.707513:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.707514:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.707515:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.707516:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677474.707520:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.707521:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677474.707525:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.707525:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677474.707526:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677474.707528:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677474.707529:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.707529:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677474.707530:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:25.2:1619677474.707535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677474.707543:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.707546:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.707548:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.707550:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.707552:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.707554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:4.2:1619677474.708069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677474.708077:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.708080:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677474.708083:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.708085:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677474.708086:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.708087:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.708088:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.708090:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.708091:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.708092:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.708095:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.708097:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.708100:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.708101:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.708101:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.708103:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.708104:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.708105:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.708105:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677474.708110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.708118:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.708121:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.708122:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.708122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.708125:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.708128:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.708358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.708366:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.708369:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.708370:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.708372:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.708375:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.708377:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.708378:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.708379:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.708380:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.708381:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.708384:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.708386:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.708388:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.708389:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.2:1619677474.708389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677474.708390:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.708392:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.708392:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.708393:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.708393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:31.0:1619677474.708398:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677474.708398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.708400:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.708402:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.708405:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.708405:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.708406:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677474.708407:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677474.708408:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.708409:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.708409:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:31.0:1619677474.708410:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.708410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677474.708411:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677474.708411:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.708414:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:31.0:1619677474.708416:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.708417:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.708418:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.708420:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.708421:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.708421:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.708423:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.708423:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.708424:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.708425:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.708428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.708435:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.708438:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.708439:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.708440:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.708442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.708443:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.708602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.708610:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.708612:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.708613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.708615:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.708618:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.708618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:26.0:1619677474.708619:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.708621:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.708622:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.708624:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.708625:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.708626:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.708627:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677474.708629:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.708629:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.708631:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.708631:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.708633:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.708634:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.708635:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.708636:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:10.0:1619677474.708636:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.708637:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677474.708637:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.708638:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.708639:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.708640:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.708641:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.708641:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:10.0:1619677474.708641:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.708642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.708643:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677474.708645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.708645:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.708646:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.708646:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.708648:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.708649:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.708649:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.708649:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.708652:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:25.2:1619677474.708654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.708655:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.708656:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.708657:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.708658:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.708659:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.708660:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.708661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:6.0:1619677474.708661:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.708662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.708662:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.708665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.708810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.708818:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.708821:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.708823:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.708825:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.708826:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.708827:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.708829:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.708830:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.708831:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.708832:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.708835:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.708836:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.708839:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.708840:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.708841:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.708842:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.708843:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.2:1619677474.708843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:31.0:1619677474.708844:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.708845:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.708849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.708849:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.708853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.708853:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677474.708855:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.708856:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677474.708858:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.708859:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:1.0:1619677474.708859:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:22.0:1619677474.708860:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.708860:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.708861:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:1.0:1619677474.708862:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.708863:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.708864:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.708864:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677474.708866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:1.0:1619677474.708867:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.708869:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.708872:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.708872:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.708873:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.708875:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.708875:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.708876:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.708876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.708881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.708889:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.708892:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.708892:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.708894:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.708895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.708896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.709048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.709055:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.709058:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.709061:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.709062:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.709064:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.709066:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.709067:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.709068:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.709069:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.709070:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.709074:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.709075:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.709078:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.709078:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.709079:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.709081:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.709081:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.709082:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.709083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.709086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.709093:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.709096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.709096:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.709097:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.709099:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.709101:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.709393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.709400:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.709403:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.709404:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.709405:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.709409:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.709410:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.709411:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.709412:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.709413:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.709414:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.709417:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.709419:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.709421:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.709422:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.709423:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.709424:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.709425:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.709426:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.709426:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.709431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.709439:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.709442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.709442:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.709443:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.709445:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.709448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.709449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:25.2:1619677474.709450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.709457:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.709457:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.709460:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677474.709460:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.709460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677474.709461:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.709461:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:24.0:1619677474.709462:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.709462:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.709463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:24.0:1619677474.709465:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.709467:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.709468:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.709470:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.709471:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.709472:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.709475:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.709476:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.709479:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.709480:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.709480:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.709482:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.709482:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.709483:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.709484:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.709487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.709495:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.709497:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.709498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.709499:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.709501:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.709503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.709729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.709735:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.709738:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.709739:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.709740:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.709743:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.709745:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.709746:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.709747:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.709748:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.709749:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.709753:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.709755:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.709757:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.709758:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.709758:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.709760:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.709761:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.709761:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.709762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.709766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.709774:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.709777:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.709778:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.709778:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.709780:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.709782:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.709799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.709807:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.709810:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.709812:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.709815:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.709815:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.709817:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.709818:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.709820:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.709821:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.709822:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.709825:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.709827:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.709829:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.709830:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.709831:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.709832:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.709833:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.709834:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.709836:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.709836:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.709838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.709842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.709844:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.709847:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.709848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.709848:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.709849:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.709850:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.709852:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.709852:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.709853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.709854:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.709857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.709863:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.709866:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.709867:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.709867:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.709867:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.709868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.710035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.710041:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.710044:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.710045:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.710047:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.710049:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.710051:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.710052:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.710053:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.710054:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.710055:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.710058:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.710060:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.710062:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.710063:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.710064:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.710066:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.710066:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.710067:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.710067:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.710083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.710090:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.710092:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.710093:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.710093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.710096:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.710097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.710103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.710109:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.710112:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.710113:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.710113:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.710114:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.710115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.710560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.710567:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.710571:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.710573:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.710575:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.710576:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.710577:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.710578:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.710580:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.710581:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.710582:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.710585:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.710587:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.710590:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.710591:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.710591:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.710593:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.710594:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.710595:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.710595:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.710600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.710608:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.710610:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.710611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.710612:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.710614:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.710616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.710741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.710748:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.710751:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.710753:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.710755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.710756:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.710758:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.710759:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.710760:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.710761:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.710762:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.710765:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.710767:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.710770:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.710771:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.710771:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.710773:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.710774:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.710775:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.710775:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.710780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.710788:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.710791:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.710792:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.710794:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.710795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.710797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.711544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.711552:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.711555:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.711557:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.711559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.711559:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.711561:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.711562:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.711563:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.711564:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.711565:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.711568:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.711569:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.711572:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.711572:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.711573:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.711574:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.711575:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.711576:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.711576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.711581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.711588:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.711590:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.711591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.711591:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.711593:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.711595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.711722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677474.711728:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.711731:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677474.711732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677474.711734:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.711736:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.711738:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677474.711739:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.711740:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.711741:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.711742:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677474.711745:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.711747:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.711748:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.711749:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.711750:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.711751:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.711751:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.711752:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677474.711753:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677474.711757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.711765:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.711767:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.711767:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.711768:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.711769:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.711771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.711849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.711855:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.711857:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.711858:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.711860:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.711863:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.711864:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.711865:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.711866:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.711867:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.711868:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.711872:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.711873:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.711875:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.711876:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.711877:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.711878:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.711879:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.711880:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.711880:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.711884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.711892:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.711894:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.711894:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.711895:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.711896:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.711897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.711907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.711915:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.711917:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.711918:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.711918:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.711919:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.711920:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.711951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.711956:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.711959:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.711961:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.711961:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.711964:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.711965:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.711967:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.711968:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.711969:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.711970:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.711973:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.711975:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.711978:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.711979:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.711979:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.711981:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.711982:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.711983:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.711984:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.711984:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677474.711987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.711990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.711993:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677474.711994:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677474.711996:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.711996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677474.711997:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677474.711997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677474.711998:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.712000:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.712000:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.712003:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.712302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.712310:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.712312:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.712314:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.712315:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.712318:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.712319:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.712320:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.712321:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.712322:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.712323:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.712326:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.712328:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.712331:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.712331:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.712332:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.712334:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.712334:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.712335:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.712335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677474.712340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.712348:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.712351:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.712352:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.712352:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.712354:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.712357:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.712503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.712509:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.712512:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.712513:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.712515:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.712517:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.712518:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.712519:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.712520:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.712522:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.712523:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.712525:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.712527:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.712529:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.712530:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.712530:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.712532:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.712532:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.712533:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.712533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.712538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.712544:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.712547:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.712547:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.712548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.712549:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.712551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.712679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.712686:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.712689:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.712692:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.712693:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.712694:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.712696:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.712697:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.712698:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.712699:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.712700:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.712703:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.712705:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.712709:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.712709:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.712710:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.712712:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.712712:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.712714:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.712715:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.712716:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.712718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.712721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.712725:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.712728:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.712729:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.712729:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.712729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.712731:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.712732:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.712733:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.712734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.712734:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.712736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.712742:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.712745:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.712746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.712746:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.712747:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.712747:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.713275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.713278:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.713280:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.713282:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.713283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.713285:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.713287:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.713288:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.713289:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.713290:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.713291:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.713301:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.713302:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.713305:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.713306:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.713306:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.713308:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.713308:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.713309:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.713310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.713315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.713323:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.713326:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.713327:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.713327:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.713329:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.713331:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.713492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.713500:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.713503:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.713504:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.713506:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.713506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:10.0:1619677474.713509:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.713511:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.713513:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.713514:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:10.0:1619677474.713514:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.713515:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.713516:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.713516:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.713518:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.713520:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.713520:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.713522:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677474.713522:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.713523:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.713524:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.713525:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.713526:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.713529:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.713530:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.713530:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.713531:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.713531:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.713534:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.713534:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.713534:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.713535:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.713535:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.713535:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.713536:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677474.713537:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.713537:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.713538:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.713539:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:25.2:1619677474.713540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.713541:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.713541:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.713543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.713547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.713550:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.713552:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.713552:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:22.0:1619677474.713553:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.713553:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677474.713554:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:6.0:1619677474.713554:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.713555:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.713556:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.713557:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677474.713558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.713558:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.713559:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.713559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.713560:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.713561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.713914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.713923:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.713926:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.713928:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.713930:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.713930:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.713932:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.713933:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.713934:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.713935:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.713936:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.713939:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.713941:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.713943:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.713944:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.713944:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.713946:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.713946:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.713947:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.713948:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.713952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.713958:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.713960:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.713961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.713961:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.713963:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.713965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.714344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.714351:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.714354:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.714355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.714356:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.714359:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.714360:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.714361:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.714363:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.714363:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.714364:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.714367:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.714369:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.714371:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.714372:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.714373:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.714374:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.714375:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.714375:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.714376:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.714381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.714389:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.714392:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.714393:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.714393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.714395:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.714398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.714487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.714495:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.714497:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.714498:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.714500:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.714503:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.714504:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.714506:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.714507:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.714508:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.714509:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.714512:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.714513:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.714516:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.714517:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.714517:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.714519:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.714519:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.714520:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.714521:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.714525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.714532:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.714535:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.714536:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.714536:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.714538:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.714541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.714709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.714717:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.714720:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.714722:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.714725:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.714725:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.714727:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.714728:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.714730:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.714731:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.714732:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.714735:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.714736:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.714740:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.714740:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.714741:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.714743:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.714743:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.714744:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.714744:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.714748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.714756:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.714758:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.714760:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.714760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.714762:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.714764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.714771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.714778:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.714781:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.714782:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.714782:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.714783:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.714784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.714805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.714812:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.714814:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.714815:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.714817:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.714819:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.714820:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.714822:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.714823:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.714824:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.714825:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.714828:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.714830:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.714832:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.714833:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.714834:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.714835:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.714836:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.714837:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.714838:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.714839:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.714842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.714844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.714849:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.714849:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.714851:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.714851:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677474.714852:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.714852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.714852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.714854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.714855:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.714857:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.715174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.715182:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.715185:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.715187:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.715189:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.715190:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.715191:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.715193:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.715194:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.715195:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.715196:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.715199:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.715201:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.715203:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.715204:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.715205:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.715206:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.715207:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.715208:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.715210:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.715210:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.715212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.715216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.715218:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.715221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.715221:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.715222:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.715224:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.715224:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.715226:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.715226:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.715227:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.715228:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.715626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.715632:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.715635:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.715636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.715638:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.715640:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.715642:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.715643:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.715644:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.715645:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.715646:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.715650:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.715651:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.715654:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.715655:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.715655:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.715657:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.715657:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.715658:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.715658:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677474.715664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.715672:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.715674:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.715675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.715675:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.715678:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.715680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.715773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.715781:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.715784:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.715786:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.715789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.715789:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.715790:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.715791:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.715793:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.715794:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.715795:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.715798:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.715800:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.715802:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.715803:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.715803:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.715805:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.715806:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.715807:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.715807:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.715811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.715818:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.715820:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.715821:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.715821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.715823:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.715825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.715828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.715834:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.715837:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.715837:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.715838:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.715838:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.715839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.715915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.715921:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.715924:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.715924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.715927:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.715929:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.715930:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.715932:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.715933:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.715934:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.715935:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.715938:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.715940:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.715942:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.715943:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.715944:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.715946:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.715946:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.715947:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677474.715948:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.715949:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.715952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.715954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.715959:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.715961:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.715961:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677474.715962:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677474.715962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677474.715964:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.715965:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.715967:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.715968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.715969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.715974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.715983:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.715986:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.715987:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.715987:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.715988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.715989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.716048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.716055:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.716058:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.716059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.716060:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.716063:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.716064:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677474.716066:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.716067:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.716068:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.716069:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677474.716072:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.716074:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.716076:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.716077:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.716077:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.716079:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.716080:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.716081:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677474.716082:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677474.716082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:4.2:1619677474.716088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.716095:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:9.2:1619677474.716096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.716097:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677474.716098:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677474.716099:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:8.0:1619677474.716100:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.716103:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.716103:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.716104:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.716105:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.716107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.716316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677474.716322:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.716325:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677474.716325:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677474.716328:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677474.716331:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.716332:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677474.716333:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.716334:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.716336:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.716336:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.716340:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677474.716342:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.716344:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.716345:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677474.716346:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.716347:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677474.716348:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.716349:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677474.716349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677474.716364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.716371:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.716373:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.716374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.716375:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.716377:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.716379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.716380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.716388:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.716391:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.716392:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.716394:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.716396:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.716397:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.716398:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.716400:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.716401:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.716401:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.716404:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.716406:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.716409:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.716409:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.716410:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.716411:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.716412:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.716413:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.716413:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.716417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.716425:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.716428:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.716429:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.716430:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.716433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.716434:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.716440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:21.2:1619677474.716447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677474.716448:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.716452:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.716453:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.716454:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.716455:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677474.716456:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677474.716456:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677474.716458:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.716460:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.716463:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.716463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.716464:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.716465:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.716466:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.716467:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.716468:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.716471:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.716473:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.716475:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.716476:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.716477:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.716478:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.716479:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.716480:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.716480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.716484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.716490:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.716493:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.716494:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.716494:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.716496:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.716498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.717045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677474.717051:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.717054:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.717054:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000400:00000200:1.0:1619677474.717056:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.717059:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.717061:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.717062:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.717063:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.717064:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.717065:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.717068:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.717070:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.717072:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.717073:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.717074:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.717075:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.717076:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.717077:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677474.717077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:25.2:1619677474.717091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.717099:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.717102:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.717103:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.717103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.717105:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.717108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.717120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.717126:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.717129:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.717130:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.717132:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.717134:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.717136:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.717137:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.717139:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.717140:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.717141:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.717144:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.717146:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.717149:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.717149:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.717150:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.717152:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.717153:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.717154:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.717154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.717158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.717165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.717167:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.717168:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.717170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:12.0:1619677474.717170:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.717171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.717173:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677474.717178:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.717181:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.717183:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.717184:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.717185:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.717187:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.717188:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.717190:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.717191:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.717192:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.717195:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.717197:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.717200:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.717201:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.717201:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.717203:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.717204:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.717205:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.717205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.717210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.717218:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.717220:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.717221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.717222:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.717224:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.717227:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.717357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.717363:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.717366:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.717367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.717369:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.717372:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.717373:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.717374:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.717376:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.717377:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.717378:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.717380:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.717382:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.717383:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.717384:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.717385:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.717386:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.717387:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.717388:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.717389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.717393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.717401:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.717403:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.717404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.717404:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.717405:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.717407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.717669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677474.717675:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.717678:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677474.717679:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677474.717681:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.717683:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.717685:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.717686:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.717688:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.717688:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.717690:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.717692:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.717694:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.717696:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.717697:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.717698:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.717699:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.717700:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.717700:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677474.717701:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:25.2:1619677474.717706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.717713:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.717715:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.717716:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.717717:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.717719:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.717721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.717766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.717774:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.717777:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.717779:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.717782:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.717782:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.717784:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.717785:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.717787:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.717788:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.717789:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.717792:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.717794:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.717796:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677474.717796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.717797:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.717798:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.717799:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.717800:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.717800:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.717801:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677474.717804:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677474.717805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.717806:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.717807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.717809:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.717811:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.717812:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:10.0:1619677474.717813:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677474.717814:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:10.0:1619677474.717814:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.717815:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.717815:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.717816:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.717817:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.717817:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.717818:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:10.0:1619677474.717819:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.717820:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:10.0:1619677474.717821:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.717823:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.717824:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.717825:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.717826:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.717826:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.717827:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.717828:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677474.717832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.717839:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.717841:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.717842:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.717842:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.717844:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.717846:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.718255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.718263:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.718265:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.718268:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677474.718270:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.718270:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.718271:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677474.718273:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.718274:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.718275:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.718276:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677474.718279:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.718280:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677474.718283:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.718284:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677474.718285:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677474.718286:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677474.718287:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.718288:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677474.718290:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677474.718290:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677474.718292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677474.718298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677474.718299:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.718302:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.718303:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.718303:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677474.718305:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.718306:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:26.0:1619677474.718308:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:12.0:1619677474.718308:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677474.718309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.718310:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.718316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677474.718322:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.718325:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677474.718326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.718326:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.718327:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.718327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677474.718540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.718547:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.718549:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.718550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.718551:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.718553:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.718555:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.718556:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.718557:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.718558:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.718559:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.718562:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.718563:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.718566:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.718567:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.718567:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.718569:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.718570:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.718571:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.718572:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.718572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.718575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.718578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.718584:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.718585:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.718586:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.718587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.718587:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677474.718587:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.718588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.718589:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.718589:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.718592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.718594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.718601:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.718603:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.718604:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.718605:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.718606:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.718606:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.718643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.718646:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.718649:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.718651:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.718652:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.718654:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.718655:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.718657:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.718658:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.718659:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.718660:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.718663:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.718665:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.718667:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.718668:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.718669:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.718670:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.718671:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.718671:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.718672:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.718677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.718685:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.718687:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.718688:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.718689:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.718691:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.718692:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.718748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.718756:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.718758:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.718759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.718761:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.718763:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.718765:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.718766:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.718768:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.718769:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.718770:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.718773:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.718774:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.718777:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.718778:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.718779:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.718780:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.718781:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.718781:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.718782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.718787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.718795:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.718797:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.718798:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.718799:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.718801:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.718803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.718807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.718815:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.718817:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.718818:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.718818:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.718819:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.718820:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.718900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.718907:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.718910:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.718911:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.718913:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.718915:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.718917:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.718918:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.718919:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.718920:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.718921:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.718924:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.718926:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.718928:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.718929:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.718930:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.718931:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.718932:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.718932:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.718933:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.718937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.718945:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.718948:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.718949:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.718949:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.718952:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.718954:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.718959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.718966:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.718968:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.718969:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.718969:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.718970:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.718971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.718984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.718992:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.718995:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.718997:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.718999:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.719000:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.719001:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.719002:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.719004:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.719004:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.719005:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.719008:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.719010:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.719013:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.719014:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.719014:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.719016:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.719017:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.719018:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.719018:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.719022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.719029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.719032:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.719033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.719033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.719035:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.719038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.719202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.719205:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.719208:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.719210:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.719211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.719212:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.719214:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.719215:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.719216:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.719217:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.719218:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.719222:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.719223:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.719226:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.719227:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.2:1619677474.719228:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.719231:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.719232:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.719233:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.719234:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.719234:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.719234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:21.0:1619677474.719235:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677474.719235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.719237:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677474.719238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:21.0:1619677474.719240:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.719241:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.719243:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.719244:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.719245:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:21.0:1619677474.719245:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.719246:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.719247:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.719248:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.719248:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677474.719249:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.719251:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:24.0:1619677474.719252:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.719254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677474.719254:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.719254:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.719255:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.719257:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.719258:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.719258:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.719259:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.719263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.719271:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.719273:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.719274:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.719275:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.719277:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.719279:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.719285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.719293:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.719298:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.719301:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.719302:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.719303:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.719303:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.719574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.719581:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.719584:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.719585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.719587:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.719589:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.719590:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.719592:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.719593:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.719594:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.719595:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.719598:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.719600:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.719601:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.719602:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.719603:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.719604:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.719605:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.719606:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.719606:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.719611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.719619:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.719621:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.719621:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.719622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.719623:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.719624:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.720143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.720149:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.720152:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.720153:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.720155:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.720157:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.720158:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.720160:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.720161:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.720162:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.720163:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.720166:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.720167:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.720170:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.720171:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.720171:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.720173:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.720173:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.720174:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.720175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.720179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.2:1619677474.720186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677474.720187:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.720189:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.720190:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.720190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.720193:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.720194:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.720195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.720197:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.720200:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.720202:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.720203:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.720204:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.720205:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.720207:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.720208:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.720209:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.720212:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.720214:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.720216:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.720217:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.720217:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.720219:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.720219:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677474.720220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677474.720221:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.720222:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.720223:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.720225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677474.720228:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677474.720229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.720230:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.720231:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677474.720233:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:8.0:1619677474.720233:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.720235:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.720235:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.720236:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.720237:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.720237:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.720238:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:8.0:1619677474.720238:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.720239:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677474.720239:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96503265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.720240:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:8.0:1619677474.720240:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.720241:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:8.0:1619677474.720241:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.720242:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:8.0:1619677474.720242:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677474.720244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000400:00000200:8.0:1619677474.720245:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.720247:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.720250:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.720250:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.720251:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677474.720251:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.720253:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.720253:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677474.720254:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.720254:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:8.0:1619677474.720254:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:30.0:1619677474.720255:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:8.0:1619677474.720255:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:30.0:1619677474.720256:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677474.720256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677474.720257:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.720259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.720261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.720266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.720268:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.720268:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.720269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.720270:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:6.0:1619677474.720270:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.720271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.720272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677474.720272:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.720275:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.721082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.721090:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.721093:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.721095:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.721097:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.721098:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.721098:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.721099:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.721101:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.721102:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.721102:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.721106:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.721107:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.721109:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.721110:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.721111:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.721112:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.721113:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.721114:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.721114:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.721119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.721126:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.721128:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:21.2:1619677474.721128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:30.0:1619677474.721129:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.721131:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.721133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.721134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.721135:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.721138:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.721139:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.721141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.721142:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.721143:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.721144:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.721145:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.721146:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.721147:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.721150:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.721151:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.721154:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.721155:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.721156:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.721157:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.721158:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.721160:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.721161:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.721161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.721164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.721167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.721171:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.721174:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.721175:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.721175:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.721175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.721177:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677474.721177:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.721178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.721179:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.721180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.721473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.721481:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.721484:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.721485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.721486:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.721489:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.721490:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.721491:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.721493:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.721493:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.721494:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.721497:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.721499:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.721501:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.721502:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677474.721502:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.721504:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.721504:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.721505:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.721506:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.721507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677474.721510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677474.721513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.721517:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.721519:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677474.721520:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677474.721520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.721520:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.721522:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677474.721522:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.721523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.721524:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.721524:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.721705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.721712:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.721715:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.721716:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.721717:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.721719:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.721721:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.721722:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.721724:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.721724:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.721726:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.721730:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.721731:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.721734:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.721735:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.721736:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.721738:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.721738:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.721739:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.721740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.721744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.721751:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.721754:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.721755:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.721757:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.721759:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.721760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.721891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.721899:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.721902:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.721904:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.721905:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.721906:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.721908:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.721909:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.721910:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.721911:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.721912:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.721915:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.721917:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.721919:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.721920:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.721921:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.721922:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.721923:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.721924:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.721924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.721928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.721934:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.721936:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.721937:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.721937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.721939:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.721941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.721948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.721953:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.721956:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.721957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.721957:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.721957:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.721958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.722539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677474.722547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.722549:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677474.722550:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677474.722551:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.722554:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.722555:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677474.722556:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.722557:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.722558:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.722559:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677474.722562:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.722564:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.722565:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.722566:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.722567:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.722568:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.722569:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.722570:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677474.722570:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677474.722574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.722581:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.722583:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.722583:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.722584:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.722585:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.722587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.722689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.722697:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.722699:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.722701:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.722704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.722704:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.722705:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.722706:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.722707:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.722708:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.722709:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.722712:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.722713:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.722716:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.722716:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.722717:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.722718:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.722719:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.722720:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.722720:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.722724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.722730:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.722732:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.722733:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.722733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.722735:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.722737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.722867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.722873:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.722876:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.722877:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.722879:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.722882:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.722883:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.722884:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.722885:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.722886:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.722887:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.2:1619677474.722888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:2.0:1619677474.722891:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.722892:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.722894:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.722895:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677474.722896:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:2.0:1619677474.722896:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.722897:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.722898:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:31.0:1619677474.722899:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:2.0:1619677474.722899:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677474.722900:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.722900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:31.0:1619677474.722901:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.722903:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.722903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:31.0:1619677474.722904:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.722905:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.722906:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677474.722906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000400:00000200:31.0:1619677474.722908:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.722909:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.722910:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677474.722911:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.722912:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.722912:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000400:00000200:31.0:1619677474.722913:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677474.722913:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.722913:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.722914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:31.0:1619677474.722915:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.722915:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.722916:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.722916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677474.722917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.722918:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.722919:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.722920:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:9.2:1619677474.722921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677474.722922:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.722922:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.722923:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.722924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.722928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677474.722928:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.722931:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.722932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.722932:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.722933:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.722933:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677474.722935:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.722938:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.722939:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.722940:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.722941:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.722944:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.722946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.722953:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.722956:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.722957:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.722957:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.722958:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.722959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.723242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.723246:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.723249:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.723251:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.723252:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.723253:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.723254:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.723256:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.723257:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.723258:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.723259:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.723262:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.723263:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.723266:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.723267:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.723267:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.723269:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.723270:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.723270:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.723271:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.723275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.723281:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.723284:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.723285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.723285:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.723288:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.723290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.723298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.723304:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.723306:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.723307:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.723307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.723308:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.723309:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.723422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.723429:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.723432:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.723433:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.723435:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.723437:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.723439:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.723440:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.723441:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.723442:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.723443:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.723446:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.723448:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.723451:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.723452:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.723452:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.723454:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.723455:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.723455:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.723456:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.723461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.723469:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.723471:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.723472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.723473:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.723475:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.723477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.723505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.723510:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.723512:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.723514:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.723515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.723517:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.723519:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677474.723520:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.723521:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.723522:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.723523:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677474.723527:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.723529:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.723531:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.723532:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677474.723532:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677474.723534:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677474.723535:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.723535:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.723536:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677474.723540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.723548:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.723551:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.723552:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.723552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.723554:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.723557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.723828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.723835:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.723838:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.723839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.723841:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.723844:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.723845:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.723846:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.723848:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.723849:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.723850:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.723853:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.723855:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.723858:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.723859:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.723859:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.723861:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.723861:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.723862:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.723862:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.723867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.723875:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.723877:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.723878:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.723879:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.723881:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.723883:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.724225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.724232:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.724234:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.724235:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.724237:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.724239:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.724241:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.724242:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.724243:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.724244:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.724245:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.724248:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.724250:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.724252:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.724253:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.724254:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.724255:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.724256:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.724256:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.724256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.724261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.724270:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.724273:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.724274:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.724274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.724276:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.724279:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.724469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.724477:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.724479:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.724482:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.724484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.724485:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.724486:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.724487:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.724488:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.724489:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.724490:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.724494:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.724495:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.724497:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.724498:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.724498:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.724499:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.724500:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.724501:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.724503:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.724503:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.724506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.724510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.724514:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.724516:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.724516:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.724518:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.724518:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.724519:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.724519:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.724520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.724521:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.724522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677474.724809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.724814:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.724817:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.724818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.724820:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.724822:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.724823:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.724824:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.724826:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.724827:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.724828:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.724831:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.724833:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.724836:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.724836:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.724837:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.724839:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.724839:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.724840:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.724840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.724845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.724852:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.724854:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.724855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.724856:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.724858:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.724860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.724919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.724927:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.724930:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.724932:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.724933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.724934:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.724936:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.724937:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.724938:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.724939:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.724940:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.724943:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.724945:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.724948:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.724949:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.724949:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.724951:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.724952:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.724953:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.724953:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.724957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.724965:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.724967:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.724968:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.724968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.724970:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.724973:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.725201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.725209:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.725212:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.725213:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.725215:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.725218:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.725220:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.725221:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.725222:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.725223:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.725224:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.725227:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.725229:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.725231:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.725232:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.725233:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.725234:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.725235:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.725236:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.725236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.725240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.725248:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.725250:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.725251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.725252:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.725254:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.725256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.725611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.725614:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.725617:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.725619:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.725620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.725621:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.725623:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.725624:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.725626:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.725626:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.725628:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.725631:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.725633:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.725635:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.725636:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.725637:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.725638:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.725639:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.725639:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.725640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.725645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.725654:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.725656:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.725657:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.725657:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.725659:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.725662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.726312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.726318:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.726320:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.726322:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.726323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.726325:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.726326:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.726328:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.726329:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.726330:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.726331:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.726334:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.726335:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.726337:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.726338:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.726339:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.726340:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.726341:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.726342:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.726343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.726347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.726355:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.726358:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.726359:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.726359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.726361:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.726363:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.726364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.726371:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.726373:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.726374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.726375:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.726376:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.726376:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.726473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.726481:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.726484:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.726486:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.726489:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.726490:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.726491:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.726493:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.726494:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.726495:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.726496:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.726499:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.726501:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.726504:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.726504:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.726505:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.726507:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.726507:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.726508:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.726509:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.726512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.726519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.726522:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.726523:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.726523:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.726525:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.726527:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.726975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677474.726981:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.726984:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677474.726984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677474.726986:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.726990:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.726991:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677474.726993:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.726994:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.726995:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.726996:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677474.726999:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.727001:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677474.727003:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.727004:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677474.727004:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677474.727007:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677474.727007:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.727008:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677474.727008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677474.727013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.727021:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.727023:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677474.727024:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.727025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.727027:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.727030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.727132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.727139:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.727142:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.727145:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.727146:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.727147:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.727149:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.727151:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.727152:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.727153:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.727154:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.727157:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.727159:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.727162:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.727163:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.727164:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.727166:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.727166:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.727167:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.727168:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.727172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.727179:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.727182:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.727183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.727183:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.727186:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.727189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.727415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.727422:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.727425:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.727428:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.727430:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.727431:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.727432:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.727433:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.727435:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.727436:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.727437:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.727440:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.727441:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.727443:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.727444:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.727444:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.727446:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.727447:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.727447:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.727448:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.727452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.727459:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.727462:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.727463:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.727463:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.727465:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.727467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.727470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.727476:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.727479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.727479:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.727479:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.727480:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.727481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.727488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.727495:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.727497:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.727498:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.727500:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.727502:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.727503:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.727505:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.727506:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.727507:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.727508:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.727510:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.727512:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.727514:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.727515:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.727516:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.727517:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.727518:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.727519:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.727520:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.727520:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.727524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.727526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.727531:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677474.727532:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677474.727533:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.727534:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677474.727534:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677474.727535:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677474.727535:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.727536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.727537:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.727540:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.727722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.727730:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.727734:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.727734:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677474.727736:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.727739:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.727741:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.727742:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.727744:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.727744:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.727746:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.727749:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.727750:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.727753:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.727754:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.727754:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.727756:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.727757:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.727757:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.727758:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.727763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.727771:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.727773:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.727774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.727775:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.727777:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.727780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.727783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.727789:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.727792:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.727793:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.727793:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.727794:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.727795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.728059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.728067:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.728070:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.728072:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.728074:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.728075:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.728076:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.728078:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.728079:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.728080:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.728081:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.728084:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.728086:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.728089:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.728089:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.728090:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.728091:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.728092:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.728093:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.728093:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.728097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.728103:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.728106:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.728107:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.728107:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.728109:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.728111:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.728142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.728148:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.728150:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.728151:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.728153:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.728155:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.728157:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.728158:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.728159:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.728160:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.728161:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.728165:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.728166:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.728168:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.728169:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.728169:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.728171:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.728172:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.728173:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.728173:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.728177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.728184:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.728186:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.728187:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.728188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.728189:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.728190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.728497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.728503:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.728506:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.728507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.728509:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.728511:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.728512:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.728514:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.728515:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.728516:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.728517:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.728520:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.728521:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.728524:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.728524:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.728525:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.728527:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.728527:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.728528:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.728528:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.728533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.728542:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.728544:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.728545:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.728547:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.728548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.728549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.728805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.728813:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.728816:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.728819:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.728821:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.728821:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.728823:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.728824:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.728825:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.728826:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.728827:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.728830:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.728832:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.728835:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.728836:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.728836:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.728838:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.728839:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.728841:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.728842:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.728843:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.728845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.728848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.728853:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.728856:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.728856:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.728857:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.728857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.728859:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.728860:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677474.728860:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.728861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.728862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.728871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.728879:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.728882:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.728882:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.728884:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.728887:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.728889:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.728890:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.728891:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.728892:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.728893:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.728896:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.728898:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.728901:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.728901:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.728902:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.728904:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.728904:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.728905:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.728906:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.728907:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677474.728909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.728913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677474.728917:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.728919:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677474.728919:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:12.0:1619677474.728920:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.728921:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.728922:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.728922:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.728923:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677474.728923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.728924:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.729185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.729193:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.729195:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.729196:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.729198:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.729200:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.729201:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.729203:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.729204:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.729205:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.729205:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.729208:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.729210:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.729212:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.729213:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.729214:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.729215:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.729216:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.729216:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.729217:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677474.729231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677474.729232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.729239:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677474.729240:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.729242:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677474.729242:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.729243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677474.729244:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.729244:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.729246:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.729246:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.729246:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.729248:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:6.0:1619677474.729248:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677474.729249:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.729251:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.729252:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.729252:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.729256:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.729257:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.729259:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.729260:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.729261:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.729262:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.729263:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.729264:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.729265:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.729266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.729268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.729271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.729276:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.729278:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.729278:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.729280:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.729280:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.729281:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.729281:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.729282:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.729283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.729284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.729391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.729398:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.729401:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.729402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.729404:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.729406:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.729408:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.729409:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.729410:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.729411:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.729412:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.729415:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.729417:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.729419:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.729420:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.729421:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.729422:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.729423:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.729424:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.729424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.729428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.729436:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.729439:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.729440:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.729440:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.729443:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.729445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677474.729727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.729733:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.729736:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.729737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.729739:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.729741:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.729742:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.729743:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.729744:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.729746:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.729746:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.729750:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.729751:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.729753:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.729754:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.729754:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.729756:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.729756:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.729757:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.729758:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.729759:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.729763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.729765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.729771:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.729772:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.729772:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.729773:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677474.729773:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.729774:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.729775:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.729775:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.729776:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.729776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.730039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.730044:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.730047:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.730048:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.730050:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.730052:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.730053:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.730055:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.730056:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.730057:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.730058:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.730061:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.730063:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.730065:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.730066:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.730067:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.730068:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.730069:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.730070:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.730070:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.730075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.730079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919461bd2400] (20)++
00000800:00000200:22.0:1619677474.730082:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.730085:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.730086:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.730086:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919461bd2400] (21)++
00000800:00000200:6.0:1619677474.730086:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.730088:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.730089:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.22@o2ib3
00000800:00000200:22.0:1619677474.730090:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:21.0:1619677474.730091:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.3.22@o2ib3 : GET - for me
00000800:00000200:20.0:1619677474.730091:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919461bd2400] (22)--
00000400:00000200:21.0:1619677474.730094:0:19935:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.3.22@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:21.0:1619677474.730096:0:19935:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.3.22@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:21.0:1619677474.730097:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919461bd2400] (21)++
00000800:00000200:21.0:1619677474.730099:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919461bd2400] (22)--
00000400:00000200:21.0:1619677474.730100:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.3.22@o2ib3 local destination
00000400:00000200:21.0:1619677474.730104:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.3.22@o2ib3(10.51.3.22@o2ib3:10.51.3.22@o2ib3) : REPLY try# 0
00000800:00000200:21.0:1619677474.730106:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.3.22@o2ib3
00000800:00000200:21.0:1619677474.730108:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9197c81a8c00] -> 10.51.3.22@o2ib3 (2) version: 12
00000800:00000200:21.0:1619677474.730108:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919461bd2400] (21)++
00000800:00000200:21.0:1619677474.730109:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919461bd2400] (22)++
00000800:00000200:21.0:1619677474.730111:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919461bd2400] (23)--
00000800:00000200:21.0:1619677474.730111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919461bd2400] (22)--
00000800:00000200:4.2:1619677474.730118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919461bd2400] (21)++
00000800:00000200:20.0:1619677474.730122:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919461bd2400] (22)++
00000800:00000200:20.0:1619677474.730124:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919461bd2400] (23)--
00000400:00000200:20.0:1619677474.730125:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.730126:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.3.22@o2ib3: REPLY: OK
00000800:00000200:21.0:1619677474.730127:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919461bd2400] (22)--
00000800:00000200:20.0:1619677474.730129:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919461bd2400] (21)--
00000800:00000200:4.2:1619677474.730339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.730347:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.730350:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.730351:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.730353:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.730356:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.730358:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.730359:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.730360:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.730361:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.730362:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.730365:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.730367:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.730370:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.730370:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.730371:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.730373:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.730373:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.730375:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.730376:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.730376:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.730379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.730382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.730387:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.730390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.730390:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.730390:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.730391:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.730392:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.730392:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.730393:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677474.730393:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.730395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.730398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.730400:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.730402:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.730403:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.730403:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.730404:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.730404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.730787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.730792:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.2:1619677474.730794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:21.0:1619677474.730795:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.730797:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.730800:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.730800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677474.730800:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:21.0:1619677474.730801:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.730803:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.730803:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.730804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:21.0:1619677474.730805:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.730806:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.730807:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.730808:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:2.0:1619677474.730810:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.730811:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:21.2:1619677474.730812:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:2.0:1619677474.730813:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.730814:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.730815:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.730816:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677474.730817:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.730820:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677474.730820:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:2.0:1619677474.730821:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.730823:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.0:1619677474.730823:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.730823:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.730824:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677474.730824:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:30.0:1619677474.730825:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.730826:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.730826:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.730827:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677474.730827:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.730828:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.730828:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.730828:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677474.730828:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:30.0:1619677474.730829:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677474.730829:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:30.0:1619677474.730831:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.730831:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677474.730831:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:30.0:1619677474.730832:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677474.730832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677474.730832:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000400:00000200:30.0:1619677474.730833:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.730833:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:30.0:1619677474.730834:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.730834:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677474.730837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:30.0:1619677474.730838:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.730839:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.2:1619677474.730839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:30.0:1619677474.730843:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.730843:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:30.0:1619677474.730844:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.730844:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.730844:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:20.0:1619677474.730845:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:18.0:1619677474.730845:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:30.0:1619677474.730846:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:20.0:1619677474.730846:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.730847:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:26.0:1619677474.730847:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:18.0:1619677474.730847:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:10.0:1619677474.730847:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:30.0:1619677474.730848:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.730848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:26.0:1619677474.730848:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.730848:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.730848:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:21.0:1619677474.730849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:2.0:1619677474.730849:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.0:1619677474.730850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:26.0:1619677474.730851:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.730852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.730854:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677474.730860:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.730862:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.730863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.730864:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.730866:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.730868:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.731490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.731495:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.731498:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.731500:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.731500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.731502:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.731504:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.731505:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.731506:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.731507:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.731508:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.731511:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.731513:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.731516:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.731516:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.731517:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.731519:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.731519:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.731520:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.731522:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.731522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.731526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.731527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.731530:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.731532:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.731533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.731533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677474.731534:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.731537:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.731538:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.731538:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.731540:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.731542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.731835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.731839:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.731842:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.731844:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.731845:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.731846:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.731847:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.731848:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.731850:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.731851:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.731852:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.731855:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.731856:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.731859:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.731860:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.731860:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.731862:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.731863:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.731864:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.731865:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.731865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.731869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.731871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.2:1619677474.731872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677474.731875:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.731878:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677474.731878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:24.0:1619677474.731879:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.731879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.731880:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677474.731881:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.731881:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.731882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.731882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:31.0:1619677474.731883:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677474.731884:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:31.0:1619677474.731886:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.731888:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.731888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.731890:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.731891:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.731892:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.731893:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.731894:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.731897:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.731899:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.731901:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.731902:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.731903:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.731904:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.731905:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.731906:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.731906:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.731910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.731917:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.731920:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.731921:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.731922:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.731924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.731925:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.731929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.731936:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.731939:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.731939:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.731940:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.731941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.731941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.732432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677474.732437:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.732439:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677474.732442:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.732442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677474.732445:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.732446:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.732447:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.732449:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.732450:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.732451:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.732454:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.732456:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.732458:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.732459:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.732459:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.732461:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.732461:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.732462:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:21.0:1619677474.732464:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677474.732464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677474.732468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.732471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.732477:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.732478:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.732479:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.732479:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:21.0:1619677474.732480:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.732480:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677474.732480:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677474.732482:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.732483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.732484:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.732485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:4.2:1619677474.732492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677474.732495:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.732498:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.732499:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.732499:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.732500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:10.0:1619677474.732500:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677474.732501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677474.732503:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.732504:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.732506:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.732508:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.732510:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.732511:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.732512:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.732513:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.732514:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.732518:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.732519:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.732522:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.732523:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.732524:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.732526:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.732526:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.732527:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.732527:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.732532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.732540:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.732543:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.732544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.732544:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.732546:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.732549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.732583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.732591:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.732594:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.732596:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.732598:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.732598:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.732600:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.732601:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.732602:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.732604:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.732604:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.732608:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.732610:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.732613:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.732614:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.732614:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.732616:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.732617:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.732617:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.732618:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.732623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.732630:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.732633:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.732634:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.732634:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.732636:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.732639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.732828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:21.2:1619677474.732830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:12.0:1619677474.732836:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:30.0:1619677474.732837:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677474.732839:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:30.0:1619677474.732840:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:12.0:1619677474.732841:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.732843:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.732844:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:12.0:1619677474.732844:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.732845:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.732845:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677474.732846:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.732847:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.732848:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.732849:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.732849:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.732851:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.732851:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.732852:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.732852:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677474.732853:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677474.732856:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.732858:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.732859:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677474.732860:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.732862:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.732862:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.732863:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.732863:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:12.0:1619677474.732863:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:12.0:1619677474.732864:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677474.732865:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.732865:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677474.732865:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677474.732866:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677474.732866:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:30.0:1619677474.732867:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677474.732867:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.732868:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677474.732871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.2:1619677474.732872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677474.732877:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.732880:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.732880:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.732880:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:31.0:1619677474.732882:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.732882:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.732883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677474.732884:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:17.0:1619677474.732884:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.732885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:17.0:1619677474.732886:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.732889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.732970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.732977:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.732979:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.732980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.732982:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.732985:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.732986:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.732987:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.732989:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.732990:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.732991:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.732994:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.732996:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.732998:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.732998:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.732999:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.733000:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.733001:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.733002:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.733003:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.733007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.733014:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.733017:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.733017:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.733018:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.733019:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.733020:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.733028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.733036:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.733038:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.733039:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.733039:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.733039:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.733040:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.733816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.733824:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.733827:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.733829:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.733831:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.733832:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677474.733832:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.733833:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.733835:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.733836:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.733837:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.733840:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.733842:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.733844:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.733845:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.733846:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.733847:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.733848:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.733849:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.733850:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.733854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.733861:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.733864:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.733865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.733865:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.733867:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.733869:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.734034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.734042:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.734044:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.734045:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.734047:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.734050:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.734051:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.734052:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.734054:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.734054:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.734056:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.734059:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.734060:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.734063:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677474.734063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:26.0:1619677474.734064:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.734064:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.734066:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.734066:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.734067:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.734067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677474.734069:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:25.2:1619677474.734072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.734072:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.734073:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.734075:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.734077:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.734078:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:6.0:1619677474.734079:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677474.734080:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.734081:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.734082:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:1.0:1619677474.734082:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.734083:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.734083:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.734083:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677474.734087:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.734088:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.734090:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677474.734090:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677474.734093:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.734094:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677474.734094:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677474.734096:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677474.734097:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.734097:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.734098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.734102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.734109:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.734111:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.734112:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.734113:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.734115:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.734118:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677474.734223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677474.734231:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.734234:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677474.734236:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.734238:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677474.734238:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.734240:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.734241:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.734242:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.734243:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.734244:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.734248:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.734250:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.734253:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.734253:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.734254:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.734256:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.734256:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.734257:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.734258:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677474.734262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.734270:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.734273:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.734274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.734274:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.734276:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.734279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.734423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.734429:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.734432:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.734433:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.734434:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.734436:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.734437:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.734438:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.734440:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.734441:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.734442:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.734445:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.734447:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.734450:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.734451:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.734451:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.734453:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.734454:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.734455:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.734455:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.734460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.734468:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.734470:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.734471:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.734473:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.734475:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.734476:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.734571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.734579:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.734582:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.734583:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.734585:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.734587:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.734589:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.734590:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.734591:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.734592:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.734594:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.734597:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.734598:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.734601:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.734602:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.734603:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.734604:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.734605:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.734606:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.734606:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.734621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.734629:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.734632:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.734633:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.734633:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.734635:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.734639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.734642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.734649:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.734652:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.734653:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.734653:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.734654:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.734655:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.734700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.734708:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.734711:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.734714:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.734716:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.734717:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.734718:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.734719:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.734721:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.734722:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.734722:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.734725:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.734727:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.734730:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.734731:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.734732:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.734733:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.734734:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.734735:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.734735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.734739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.734747:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.734750:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.734751:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.734751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.734753:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.734756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.734760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.734767:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.734769:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.734770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.734771:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.734772:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.734772:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.734804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.734809:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.734812:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.734813:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.734815:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.734817:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.734819:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.734820:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.734821:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.734822:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.734823:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.734827:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.734828:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.734831:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.734831:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.734832:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.734834:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.734834:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.734836:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.734837:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.734837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.734840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.734842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.734847:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.734847:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.734849:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.734850:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.734850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.734850:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.734850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677474.734852:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.734852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.734855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.734858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.734866:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.734869:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.734871:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.734873:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.734874:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.734875:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.734876:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.734878:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.734879:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.734880:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.734883:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.734885:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.734888:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.734888:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.734889:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.734890:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.734891:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.734892:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.734894:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.734895:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.734896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.734900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.734903:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.734905:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.734906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.734906:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.734908:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.734908:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.734910:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.734910:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.734911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.734912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.735417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.735425:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.735428:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.735429:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.735430:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.735433:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.735434:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.735435:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.735437:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.735438:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.735439:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.735441:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.735443:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.735446:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.735447:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.735447:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.735449:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.735450:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.735450:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.735451:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.735456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.735464:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.735466:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.735467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.735468:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.735470:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.735473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.735744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.2:1619677474.735747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:20.0:1619677474.735748:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.735751:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.735753:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677474.735753:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:21.0:1619677474.735754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.735755:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.735757:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:17.0:1619677474.735757:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:20.0:1619677474.735758:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.735758:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:20.0:1619677474.735760:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.735760:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.735761:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.735762:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.735763:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.735764:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:20.0:1619677474.735766:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:17.0:1619677474.735766:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.735767:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.735769:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.735769:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.735769:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677474.735771:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.735772:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.735773:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:17.0:1619677474.735773:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.735775:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.735775:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.735776:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:17.0:1619677474.735776:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677474.735777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:17.0:1619677474.735777:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.735778:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.735779:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.735780:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:9.2:1619677474.735780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.735781:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.735781:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677474.735782:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677474.735783:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:8.2:1619677474.735786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.2:1619677474.735789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:8.0:1619677474.735792:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.735794:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.735795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.735795:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.735795:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.735796:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677474.735796:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.735797:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677474.735797:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.735797:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.735798:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.735799:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.735799:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.735800:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677474.735801:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677474.735801:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.736500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.736508:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.736511:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.736513:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.736516:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.736516:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.736517:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.736518:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.736520:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.736521:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.736522:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.736525:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.736527:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.736531:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.736531:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.736532:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.736534:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.736534:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.736536:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.736537:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.736538:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.736540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.736544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.736549:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.736551:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.736552:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677474.736552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.736553:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.736554:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.736555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677474.736555:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.736556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.736558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.736716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.736722:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.736725:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.736727:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.736727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.736729:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.736731:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677474.736732:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.736733:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.736734:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.736735:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677474.736738:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.736740:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677474.736743:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.736744:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677474.736744:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677474.736746:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677474.736747:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.736748:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.736749:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.736750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677474.736752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.736755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.736758:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.2:1619677474.736759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677474.736760:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677474.736761:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.736761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.736762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:2.0:1619677474.736762:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.736763:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.736764:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.736765:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.736767:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677474.736769:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.736770:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.736772:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.736774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.736775:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.736777:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.736778:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.736780:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.736781:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.736782:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.736785:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.736786:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.736788:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.736789:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.736790:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.736791:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.736792:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.736793:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.736793:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.736797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.736804:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.736807:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.736807:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.736809:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.736810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.736812:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:9.2:1619677474.736965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677474.736972:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.736974:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677474.736975:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677474.736977:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.106@o2ib7 : GET - for me
00000400:00000200:26.0:1619677474.736979:0:19941:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.106@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:26.0:1619677474.736981:0:19941:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.106@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:26.0:1619677474.736983:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.736984:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677474.736985:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677474.736989:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : REPLY try# 0
00000800:00000200:26.0:1619677474.736990:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677474.736992:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.736992:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.736993:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.736994:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.736994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:9.2:1619677474.736999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.737005:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.737007:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.737008:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.737009:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.737010:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: REPLY: OK
00000800:00000200:10.0:1619677474.737011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677474.737062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.737069:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.737071:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.737072:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.737074:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.737076:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.737078:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.737079:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.737080:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.737082:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.737083:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.737086:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.737087:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.737089:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.737090:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.737091:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.737092:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.737093:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.737094:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.737094:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.737098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.737105:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.737107:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.737107:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.737108:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.737109:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.737110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.737329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.737337:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.737340:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.737341:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.737342:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.737345:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.737347:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.737348:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.737349:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.737351:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.737351:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.737354:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.737357:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.737360:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.737360:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.737361:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.737363:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.737363:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.737364:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.737365:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.737366:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.737369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.737372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.737377:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.737379:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.737380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.737380:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677474.737381:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677474.737381:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.737382:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.737383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.737383:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.737385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.737392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.737400:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.737402:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.737403:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.737403:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.737404:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.737405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.737597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.737605:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.737607:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.737609:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.737612:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.737612:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.737614:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677474.737615:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.737616:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.737617:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.737618:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677474.737621:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.737623:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677474.737626:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.737627:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677474.737627:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677474.737629:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677474.737630:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.737630:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.737631:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.737635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.737644:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.737646:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.737647:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.737648:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.737650:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.737653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.737654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.737661:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.737664:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.737665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.737665:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.737666:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.737666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.737779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.737782:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.737784:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.737786:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.737788:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.737789:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.737790:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.737791:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.737793:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.737794:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.737794:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.737798:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.737799:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.737802:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.737802:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.737803:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.737805:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.737805:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.737806:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.737806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.737812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677474.737817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:17.0:1619677474.737820:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:18.0:1619677474.737823:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:17.0:1619677474.737823:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.737824:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.737824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:18.0:1619677474.737826:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.737826:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:17.0:1619677474.737827:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677474.737829:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.737829:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:18.0:1619677474.737831:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.737832:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:8.2:1619677474.737832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:18.0:1619677474.737833:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.737834:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.737836:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.737836:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.737840:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.737840:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:18.0:1619677474.737841:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677474.737843:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.737843:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677474.737844:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.737844:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:1.0:1619677474.737844:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:18.0:1619677474.737845:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677474.737845:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.737846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677474.737847:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.737847:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.737848:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.737849:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.737854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.737860:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.737863:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.737863:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.737864:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.737865:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.737867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.738093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677474.738100:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.738103:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677474.738104:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677474.738106:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.738109:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.738111:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677474.738112:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.738113:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.738114:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.738115:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677474.738119:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.738121:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.738123:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.738124:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677474.738125:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677474.738127:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677474.738127:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.738128:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677474.738128:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677474.738132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.738140:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.738143:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.738144:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.738144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.738146:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.738149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.738557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.738565:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.738568:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.738570:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.738572:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.738573:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.738573:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.738575:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.738576:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.738577:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.738578:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.738581:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.738582:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.738585:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.738586:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.738586:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.738587:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.738588:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.738589:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.738590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.738594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.738601:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.738603:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.738604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.738604:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.738606:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.738609:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.738612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.738618:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.738620:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.738621:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.738621:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.738622:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.738623:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.738710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.738716:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.738719:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.738720:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.738722:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.738724:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.738726:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677474.738727:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.738728:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.738729:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.738730:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677474.738733:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.738735:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677474.738738:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.738739:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677474.738740:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677474.738741:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677474.738742:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.738744:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677474.738744:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677474.738745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677474.738747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.2:1619677474.738750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:26.0:1619677474.738755:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:2.0:1619677474.738755:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677474.738757:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:26.0:1619677474.738758:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677474.738758:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:2.0:1619677474.738758:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:26.0:1619677474.738759:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.738760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:26.0:1619677474.738762:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.738765:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.738849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.738857:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.738860:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.738862:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.738864:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.738865:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.738866:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.738868:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.738869:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.738870:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.738871:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.738874:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.738876:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.738879:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.738880:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.738881:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.738882:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.738883:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.738884:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.738884:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.738889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.738896:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.738899:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.738900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.738901:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.738903:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.738905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.739231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.739238:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.739240:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.739241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.739243:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.739246:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.739248:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.739249:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.739250:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.739251:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.739252:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.739256:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.739257:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.739260:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.739261:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.739261:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.739263:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.739264:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.739265:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.739265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.739269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.739276:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.739279:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.739280:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.739282:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:30.0:1619677474.739282:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.739285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.739291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.739299:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.739302:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.739303:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.739304:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.739304:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.739306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.739382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.739390:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.739392:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.739393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.739395:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.739398:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.739399:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.739400:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.739402:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.739403:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.739404:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.739407:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.739409:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.739412:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.739413:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.739413:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.739415:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.739415:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.739416:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.739418:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.739418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.739422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.739424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.739430:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.739432:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.739432:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.739433:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677474.739433:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.739433:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.739434:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.739435:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677474.739436:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.739438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.739443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.739450:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.739453:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.739454:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.739454:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.739455:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.739456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.739642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.739650:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.739653:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.739655:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.739657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.739658:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.739659:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.739660:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.739661:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.739663:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.739663:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.739667:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.739668:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.739670:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.739671:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.739671:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.739673:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.739674:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.739675:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.739675:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.739678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.739685:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.739688:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.739689:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.739689:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.739691:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.739693:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.739738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.739745:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.739747:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.739748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.739749:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.739752:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.739753:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.739755:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.739756:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.739757:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.739757:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.739761:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.739762:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.739765:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.739766:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.739766:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.739768:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.739768:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.739769:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.739769:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.739774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.739782:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.739785:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.739785:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.739786:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.739788:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.739791:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.739980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.739987:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.739989:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.739990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.739992:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.739994:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.739995:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.739997:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.739998:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.739999:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.740000:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.740003:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.740005:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.740007:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.740008:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.740009:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.740011:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.740011:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.740012:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.740012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.740017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.740024:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.740026:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.740027:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.740028:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.740030:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.740032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.740339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.740348:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.740351:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.740353:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.740355:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.740355:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.740357:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.740358:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.740359:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.740360:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.740361:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.740365:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.740366:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.740369:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.740370:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.740370:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.740372:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.740372:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.740374:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.740376:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.740376:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.740378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.740382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.740385:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.740388:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.740389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.740389:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.740390:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.740391:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.740393:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.740393:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677474.740393:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.740394:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.740601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.740608:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.740611:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.740613:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.740615:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.740615:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.740617:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.740618:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.740619:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.740620:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.740621:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.740624:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.740625:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.740629:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.740629:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.740630:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.740632:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.740632:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.740633:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.740633:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.740638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.740646:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.740649:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.740649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.740650:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.740652:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.740655:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.740675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.740681:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.740684:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.740684:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.740687:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.740689:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.740690:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.740692:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.740693:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.740694:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.740695:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.740698:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.740700:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.740702:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.740703:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.740704:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.740705:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.740706:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.740706:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.740707:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.740712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.740720:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.740723:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.740723:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.740725:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.740727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.740728:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.741593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.741602:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.741605:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.741607:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.741609:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.741610:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.741611:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.741613:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.741614:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.741615:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.741616:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.741619:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.741621:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.741623:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.741624:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.741625:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.741626:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.741627:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.741628:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.741629:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.741633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.741640:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.741642:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.741643:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.741644:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.741646:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.741647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.742176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.742184:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.742186:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.742187:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.742189:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.742192:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.742193:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.742194:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.742196:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.742196:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.742197:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.742200:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.742202:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.742205:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.742205:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.742206:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.742208:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.742208:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.742209:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.742209:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.742214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.742222:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.742224:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.742225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.742226:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.742228:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.742230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.742812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.742820:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.742823:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.742825:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.742827:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.742827:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.742828:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.742829:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.742830:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.742832:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.742832:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.742836:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.742838:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.742841:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.742842:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.742842:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.742844:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.742844:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.742845:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.742846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.742851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.742859:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.742861:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.742862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.742863:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.742865:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.742867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.743772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677474.743777:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.743780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.743780:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677474.743782:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.743785:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.743786:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677474.743788:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.743789:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.743791:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.743791:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677474.743795:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.743796:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677474.743799:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.743800:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677474.743800:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677474.743802:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.2:1619677474.743803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:6.0:1619677474.743803:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.743804:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.743804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677474.743809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:2.0:1619677474.743810:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.743815:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.743817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:1.0:1619677474.743818:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:2.0:1619677474.743820:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.743821:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.743822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.743822:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.743824:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.743824:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:2.0:1619677474.743826:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677474.743827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:2.0:1619677474.743828:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.743831:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.743832:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.743834:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.743839:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.743842:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.743845:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.743846:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.743847:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.743849:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.743850:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.743851:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.743852:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.743856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.743865:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.743869:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.743870:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.743871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.743873:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.743875:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.743887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.743891:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.743894:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.743896:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.743897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.743898:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.743900:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.743901:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.743902:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.743904:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.743905:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.743908:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.743910:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.743912:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.743913:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.743913:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.743915:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.743916:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.743916:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.743917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.743921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.743928:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.743930:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.743931:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.743933:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.743933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.743935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.744139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.744147:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.744150:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.744153:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.744155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.744156:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.744157:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.744159:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.744160:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.744161:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.744162:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.744165:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.744167:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.744170:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.744170:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.744171:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.744172:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.744173:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.744174:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.744174:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.744178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.744185:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.744188:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.744188:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.744190:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.744192:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.744193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.744197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.744204:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.744207:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.744207:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.744208:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.744208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.744209:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.744223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.744226:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.744229:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.744231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.744231:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.744233:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.744235:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.744236:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.744238:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.744239:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.744240:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.744243:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.744245:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.744247:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.744248:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.2:1619677474.744250:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:20.0:1619677474.744253:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.744254:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.744255:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.744255:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.744256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:2.0:1619677474.744257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677474.744260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:2.0:1619677474.744262:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.744264:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.744266:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.744268:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:2.0:1619677474.744269:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.744270:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.744271:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.744271:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:1.0:1619677474.744271:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:2.0:1619677474.744273:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.744274:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.744276:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:2.0:1619677474.744276:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.744278:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.744279:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.744284:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.744287:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.744289:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.744291:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.744292:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.744300:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:9.2:1619677474.744301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.744302:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.744303:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.744305:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.744307:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.744309:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.744310:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.744311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.744313:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.744315:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.744479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.744487:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.744490:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.744492:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.744495:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.744495:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.744497:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.744498:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.744499:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.744500:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.744501:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.744505:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.744507:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.744510:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.744511:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.744512:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.744514:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.744515:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.744516:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.744518:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.744518:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.744520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.744524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.744528:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.744531:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.744532:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.744532:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.744532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.744534:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677474.744534:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.744535:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.744536:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.744537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.744541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.744549:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.744551:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.744552:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.744552:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.744553:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.744554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.744562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.744571:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.744575:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.744576:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.744580:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.744582:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.744584:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.744587:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.744589:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.744590:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.744592:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.744597:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.744600:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.744603:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.744604:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.744605:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.744607:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.744608:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.744610:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.744611:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.744614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.744622:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.744624:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.744626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.744626:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.744628:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.744630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.744678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.744686:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.744689:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.744690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.744692:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.744694:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.744696:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.744697:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.744698:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.744699:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.744700:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.744703:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.744705:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.744708:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.744709:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.744710:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.744711:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.744712:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.744713:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.744714:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.744714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.744717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.744720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.744724:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.744727:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.744727:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:22.0:1619677474.744728:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.744728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.744729:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.744730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677474.744731:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.744731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.744733:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.744754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.744760:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677474.744761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677474.744762:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.744763:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.744765:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.744767:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.744768:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677474.744769:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:17.0:1619677474.744770:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.744771:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677474.744771:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.744772:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.744772:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:17.0:1619677474.744773:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.744774:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.744777:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.744778:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.744779:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:17.0:1619677474.744779:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.744780:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.744782:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.744783:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.744783:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.744783:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:26.0:1619677474.744784:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.744785:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.744785:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.744785:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.744786:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.744787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677474.744788:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.744790:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677474.744791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.744793:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.744794:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.744794:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.744796:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.744796:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.744797:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.744798:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.744798:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.744800:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.744801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:25.2:1619677474.744802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677474.744802:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.744804:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.744807:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.744809:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.744810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.744810:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.744811:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.744813:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.744815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.744965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.744975:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.744978:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.744982:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.744983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.744986:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.744988:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677474.744991:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.744994:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.744995:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.744997:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677474.745002:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.745005:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677474.745009:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.745010:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:13.0:1619677474.745011:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:13.0:1619677474.745013:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677474.745014:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.745016:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.745017:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677474.745019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.745027:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.745030:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.745031:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.745032:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.745034:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.745037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.745040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.745041:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.745043:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677474.745044:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.745045:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.745045:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.745046:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.745085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.745092:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.745095:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.745097:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.745099:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.745100:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.745102:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.745103:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.745104:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.745105:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.745106:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.745109:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.745111:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.745113:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.745114:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.745114:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.745116:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.745117:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.745118:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.745118:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.745122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.745128:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.745131:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.745132:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.745133:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.745135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.745136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.745300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.745308:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.745310:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.745311:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.745313:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.745315:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.745317:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.745318:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.745319:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.745320:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.745321:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.745324:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.745325:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.745328:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.745329:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.745329:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.745331:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.745331:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.745332:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.745332:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.745337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.745345:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.745347:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.745348:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.745349:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.745351:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.745353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.745363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677474.745371:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.745374:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.745376:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.745377:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.745377:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.745379:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.746028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.746037:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.746039:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.746041:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.746043:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.746044:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.746044:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.746046:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.746047:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.746048:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.746049:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.746052:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.746053:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.746055:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.746056:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.746057:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.746058:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.746059:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.746060:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.746062:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.746062:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.746064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.746068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.746070:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.746073:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.746074:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.746074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.746076:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.746076:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.746078:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677474.746078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.746079:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.746080:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.746538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677474.746544:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.746547:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677474.746548:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677474.746549:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.746552:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.746553:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677474.746555:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.746556:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.746557:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.746558:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677474.746561:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677474.746563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:17.0:1619677474.746563:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677474.746566:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.746566:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677474.746567:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677474.746569:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677474.746569:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:31.0:1619677474.746570:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677474.746571:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677474.746572:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677474.746572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:31.0:1619677474.746573:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.746576:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677474.746576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.2:1619677474.746577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:30.0:1619677474.746578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.746579:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.746580:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.746582:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.746583:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.746583:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000400:00000200:31.0:1619677474.746584:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.746584:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:31.0:1619677474.746585:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.746585:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677474.746585:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:2.0:1619677474.746586:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:1.0:1619677474.746586:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:2.0:1619677474.746587:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.746588:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:31.0:1619677474.746590:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.746590:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.746592:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677474.746593:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:31.0:1619677474.746596:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.746597:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.746598:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.746600:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.746601:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.746602:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.746602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.746607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.746615:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.746617:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.746618:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.746619:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.746621:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.746624:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.746853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.746860:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.746863:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.746865:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.746867:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.746868:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.746870:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.746871:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.746872:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.746873:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.746874:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.746877:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.746879:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.746881:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.746881:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.746882:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.746883:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.746884:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.746885:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.746885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.746889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.746896:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.746899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.746899:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.746900:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.746902:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.746903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.746908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.746913:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.746916:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.746917:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.746917:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.746917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.746918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.746924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.746932:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.746934:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.746935:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.746936:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.746938:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.746940:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.746941:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.746942:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.746943:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.746944:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.746947:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.746949:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.746951:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.746952:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.746953:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.746954:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.746955:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.746955:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.746956:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.746962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.746969:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.746971:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.746972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.746973:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.746975:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.746977:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.746984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.746991:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.746994:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.746994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.746995:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.746996:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.746996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.747093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.747098:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.747100:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.747103:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.747103:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.747105:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.747106:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.747108:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.747109:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.747110:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.747111:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.747114:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.747116:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.747119:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.747119:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.747120:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.747122:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.747122:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.747123:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.747124:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.747128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.747135:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.747137:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.747138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.747138:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.747140:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.747142:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.747145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.747151:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.747154:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.747154:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.747155:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.747156:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.747156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.747952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677474.747959:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677474.747962:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677474.747963:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677474.747965:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.747968:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.747970:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677474.747971:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.747972:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.747973:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.747974:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677474.747977:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.747979:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.747982:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.747983:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677474.747984:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677474.747986:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677474.747986:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677474.747987:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677474.747987:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:25.2:1619677474.747993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.748000:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.748003:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.748004:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.748006:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.748007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.748009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.748020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677474.748027:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.748030:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.748031:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.748032:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.748032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.748033:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.748348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.748356:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.748359:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.748362:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.748364:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.748364:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.748366:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.748367:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.748368:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.748369:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.748370:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.748374:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.748375:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.748379:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.748379:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.748380:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.748382:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.748382:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.748383:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.748384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.748389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.748397:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.748400:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.748401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.748401:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.748403:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.748406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.748549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.748556:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.748559:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.748562:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.748563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.748565:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.748566:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.748567:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.748569:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.748570:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.748570:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.748573:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.748575:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.748577:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.748578:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.748578:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.748580:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.748581:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.748582:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677474.748583:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677474.748584:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677474.748586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.748589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:8.0:1619677474.748592:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.748595:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.748596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.748596:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.748597:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000400:00000200:8.0:1619677474.748598:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.748600:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677474.748600:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:8.0:1619677474.748600:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.748601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:21.2:1619677474.748990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.748998:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.749000:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.749002:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.749004:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.749004:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.749006:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.749007:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.749008:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.749009:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.749010:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.749013:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.749014:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.749016:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.749017:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.749017:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.749019:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.749019:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.749020:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.749021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.749024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.749031:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.749033:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.749034:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.749035:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.749038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.749038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677474.749099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677474.749106:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.749108:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677474.749110:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677474.749111:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.749113:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.749115:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677474.749116:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.749117:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.749118:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.749119:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:24.0:1619677474.749123:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.749124:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677474.749127:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.749128:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677474.749129:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677474.749131:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677474.749131:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.749132:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677474.749133:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677474.749134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:25.2:1619677474.749137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677474.749139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000800:00000200:21.0:1619677474.749144:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.0:1619677474.749146:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:21.0:1619677474.749147:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.749148:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.749148:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000400:00000200:21.0:1619677474.749149:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.749149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677474.749150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:20.0:1619677474.749150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.749152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.749194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.749201:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.749203:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.749204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.749206:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.749209:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.749210:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.749211:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.749212:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.749213:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.749214:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.749217:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.749219:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.749221:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.749222:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.749222:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.749224:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.749224:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.749225:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:21.2:1619677474.749226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:18.0:1619677474.749226:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.749230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677474.749232:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.749235:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.749236:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:2.0:1619677474.749237:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:17.0:1619677474.749238:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.749239:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.749239:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.749240:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.749241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:17.0:1619677474.749242:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.749243:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.749243:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.749245:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.749245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:17.0:1619677474.749246:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.749247:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.749250:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.749252:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:9.2:1619677474.749252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:17.0:1619677474.749255:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.749256:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.749256:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.749258:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:18.0:1619677474.749259:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677474.749259:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.749259:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.749260:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677474.749262:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.749263:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.749263:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.749264:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.2:1619677474.749264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677474.749265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677474.749272:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.749274:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.749275:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.749277:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.749278:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.749279:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.749487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.749495:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.749498:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.749499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.749500:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.749503:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.749505:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.749506:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.749508:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.749509:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677474.749509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:8.0:1619677474.749510:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.749513:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.749515:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.749515:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.749517:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.749518:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677474.749518:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.749519:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.749519:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677474.749520:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.749521:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.749522:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.749522:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677474.749522:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.749523:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677474.749524:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.749525:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.749527:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.749527:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677474.749528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677474.749529:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.749532:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.749534:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.749536:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677474.749536:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.749537:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.749538:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677474.749538:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677474.749539:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.749539:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.749540:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.749540:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.749541:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.749541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.749542:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.749544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.749546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.749553:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.749555:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.749556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.749556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.749558:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.749559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.749616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.749624:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.749627:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.749628:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.749629:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.749632:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.749633:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.749635:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.749637:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.749638:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.749639:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.749642:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.749644:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.749647:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.749648:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.749648:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.749650:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.749650:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.749651:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.749652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.749656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.749663:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.749666:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.749667:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.749667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.749669:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.749672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.750110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677474.750117:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.750119:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677474.750120:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677474.750122:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.750125:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.750126:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677474.750127:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.750128:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.750129:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.750130:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677474.750133:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.750134:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677474.750137:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.750137:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:24.0:1619677474.750138:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677474.750140:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:24.0:1619677474.750140:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.750141:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677474.750141:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:25.2:1619677474.750146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.750153:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.750155:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.750156:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.750158:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.750160:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.750161:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677474.750218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.750223:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.750226:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.750228:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.750229:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.750230:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.750232:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.750233:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.750235:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.750236:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.750237:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.750240:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.750242:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.750244:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.750245:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.750246:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.750248:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.750248:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.750249:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.750249:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.750254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.750262:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.750264:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.750265:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.750265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.750267:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.750270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.750335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06fbb0600] (20)++
00000800:00000200:10.0:1619677474.750343:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (21)++
00000800:00000200:26.0:1619677474.750346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (22)--
00000800:00000200:10.0:1619677474.750346:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000400:00000200:10.0:1619677474.750349:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.750352:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.750354:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677474.750355:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.750357:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.750357:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.750359:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677474.750362:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.750363:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677474.750366:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.750367:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677474.750368:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.2:1619677474.750369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677474.750370:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.750370:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (21)++
00000800:00000200:10.0:1619677474.750371:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (22)--
00000800:00000200:10.0:1619677474.750372:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (21)--
00000800:00000200:9.2:1619677474.750377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677474.750378:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677474.750379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06fbb0600] (20)++
00000800:00000200:30.0:1619677474.750380:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.750382:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.750384:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.750385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.750385:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677474.750385:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:30.0:1619677474.750386:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.750386:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (21)++
00000800:00000200:2.0:1619677474.750387:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:30.0:1619677474.750388:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.750388:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000400:00000200:30.0:1619677474.750389:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677474.750389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.750389:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (22)--
00000400:00000200:2.0:1619677474.750389:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.750390:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677474.750391:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.750392:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.750393:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.750394:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677474.750395:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677474.750396:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.750397:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677474.750398:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.750399:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.750399:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.750400:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.750400:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:26.0:1619677474.750400:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.750402:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.750402:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.750403:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677474.750403:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.750403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06fbb0600] (21)++
00000800:00000200:30.0:1619677474.750404:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677474.750405:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677474.750405:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:25.2:1619677474.750407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677474.750408:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.750409:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677474.750410:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.750412:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677474.750412:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (22)++
00000800:00000200:26.0:1619677474.750413:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:10.0:1619677474.750413:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000800:00000200:31.0:1619677474.750414:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:26.0:1619677474.750414:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:26.0:1619677474.750415:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)--
00000400:00000200:10.0:1619677474.750415:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.750416:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (22)--
00000800:00000200:31.0:1619677474.750417:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.750418:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.750418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:31.0:1619677474.750419:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.750419:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.750420:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.750421:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:10.0:1619677474.750421:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.750422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:10.0:1619677474.750422:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.750423:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.750424:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677474.750426:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:10.0:1619677474.750427:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.750428:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677474.750429:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:4.2:1619677474.750429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06fbb0600] (21)++
00000800:00000200:2.0:1619677474.750429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.750430:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.750431:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677474.750432:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.750432:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.750433:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677474.750435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:10.0:1619677474.750435:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.750436:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (22)++
00000800:00000200:10.0:1619677474.750437:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:26.0:1619677474.750438:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:26.0:1619677474.750439:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000800:00000200:10.0:1619677474.750439:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:10.0:1619677474.750440:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000400:00000200:26.0:1619677474.750441:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.750441:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:10.0:1619677474.750442:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000400:00000200:26.0:1619677474.750444:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.750444:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.750446:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677474.750447:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677474.750447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677474.750449:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677474.750449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000400:00000200:26.0:1619677474.750450:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.750450:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677474.750453:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.750454:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:10.0:1619677474.750454:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.750456:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.750456:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677474.750456:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677474.750456:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.750457:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677474.750458:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:10.0:1619677474.750458:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.750458:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.750459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.750459:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.750460:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677474.750460:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:10.0:1619677474.750460:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.750460:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677474.750461:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000400:00000200:10.0:1619677474.750462:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.750462:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.750462:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.750462:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677474.750463:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (24)++
00000400:00000200:10.0:1619677474.750463:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677474.750465:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:9.2:1619677474.750465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677474.750465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677474.750466:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000400:00000200:10.0:1619677474.750466:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.750468:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:10.0:1619677474.750468:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677474.750469:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:26.0:1619677474.750469:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)--
00000800:00000200:26.0:1619677474.750471:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.750472:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677474.750472:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.750473:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.750473:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677474.750473:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:26.0:1619677474.750474:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.750475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.750475:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.750477:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.750477:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (22)++
00000800:00000200:26.0:1619677474.750478:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:10.0:1619677474.750478:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (23)--
00000800:00000200:26.0:1619677474.750479:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000800:00000200:10.0:1619677474.750479:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)--
00000800:00000200:10.0:1619677474.750480:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677474.750480:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (22)++
00000800:00000200:10.0:1619677474.750481:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000400:00000200:26.0:1619677474.750482:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.750482:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:8.2:1619677474.750482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:26.0:1619677474.750483:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.750483:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.750485:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677474.750485:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.750486:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.750486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677474.750487:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.750487:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.750487:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677474.750489:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677474.750489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:26.0:1619677474.750490:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677474.750490:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:26.0:1619677474.750491:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677474.750492:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.750493:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:1.0:1619677474.750493:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.750494:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.750494:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.750495:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.750495:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.750496:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:26.0:1619677474.750497:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677474.750497:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.750497:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.750499:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:10.0:1619677474.750499:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.750500:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677474.750501:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.750502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677474.750503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:26.0:1619677474.750503:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677474.750504:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.750504:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:30.0:1619677474.750505:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677474.750505:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:10.0:1619677474.750505:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.750507:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.750507:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:10.0:1619677474.750507:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677474.750508:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.750508:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (23)++
00000400:00000200:30.0:1619677474.750509:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.750509:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.750509:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:10.0:1619677474.750509:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.750510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)--
00000800:00000200:10.0:1619677474.750510:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677474.750511:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (22)++
00000800:00000200:10.0:1619677474.750511:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (22)++
00000800:00000200:26.0:1619677474.750512:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000800:00000200:10.0:1619677474.750512:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (23)--
00000400:00000200:30.0:1619677474.750513:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.750513:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:8.2:1619677474.750513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (23)++
00000800:00000200:10.0:1619677474.750514:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06fbb0600] (24)++
00000800:00000200:30.0:1619677474.750515:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.750515:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.750515:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:10.0:1619677474.750516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:10.0:1619677474.750516:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:30.0:1619677474.750518:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.750518:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.750518:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000800:00000200:30.0:1619677474.750519:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.750520:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677474.750520:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677474.750520:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.750521:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677474.750521:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96503139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.750521:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:30.0:1619677474.750522:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:26.0:1619677474.750522:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.750523:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.750523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:26.0:1619677474.750523:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.750523:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:26.0:1619677474.750524:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677474.750524:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.750526:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000400:00000200:20.0:1619677474.750526:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.750526:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.750527:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:26.0:1619677474.750528:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677474.750528:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.750528:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677474.750530:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.750530:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96503138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.750531:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.750532:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.750533:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.750534:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677474.750536:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:25.2:1619677474.750537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.750537:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.750538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.750538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677474.750539:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.750539:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677474.750540:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.2:1619677474.750540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:26.0:1619677474.750541:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.750542:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.750543:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.750543:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677474.750543:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677474.750544:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (24)++
00000800:00000200:26.0:1619677474.750545:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:6.0:1619677474.750545:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.750546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:26.0:1619677474.750546:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:10.0:1619677474.750546:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677474.750546:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:10.0:1619677474.750547:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:10.0:1619677474.750547:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:6.0:1619677474.750547:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.750548:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000800:00000200:22.0:1619677474.750548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.750548:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677474.750548:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:10.0:1619677474.750549:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06fbb0600] (24)--
00000400:00000200:6.0:1619677474.750549:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.750550:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.750550:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677474.750550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)--
00000800:00000200:21.0:1619677474.750551:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:10.0:1619677474.750551:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (22)++
00000800:00000200:9.2:1619677474.750551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677474.750551:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.750552:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.750552:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.750553:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:10.0:1619677474.750553:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000400:00000200:26.0:1619677474.750554:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677474.750554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677474.750554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677474.750555:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.750557:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.750558:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.750559:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677474.750559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677474.750559:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:10.0:1619677474.750561:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.750562:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:26.0:1619677474.750563:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677474.750563:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.750563:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:10.0:1619677474.750564:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.750565:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.750566:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:18.0:1619677474.750566:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.750566:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.750567:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96503136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.750569:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:10.0:1619677474.750569:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.750569:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.750570:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.750570:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677474.750571:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.750573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677474.750573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.750574:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:26.0:1619677474.750574:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.750575:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.750575:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677474.750575:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.750577:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.750577:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.750578:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:25.2:1619677474.750578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677474.750579:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:10.0:1619677474.750579:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.750580:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)--
00000800:00000200:10.0:1619677474.750580:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.750581:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (22)++
00000800:00000200:10.0:1619677474.750581:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.750582:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000800:00000200:25.2:1619677474.750583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.750583:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.750583:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:30.0:1619677474.750584:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:26.0:1619677474.750584:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.750584:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06fbb0600] (24)++
00000800:00000200:10.0:1619677474.750585:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000400:00000200:26.0:1619677474.750586:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.750586:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:30.0:1619677474.750587:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.750587:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:30.0:1619677474.750588:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000400:00000200:26.0:1619677474.750588:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:25.2:1619677474.750588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.750588:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000800:00000200:30.0:1619677474.750589:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.750589:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:26.0:1619677474.750589:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.750590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:26.0:1619677474.750591:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.750591:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.750591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:26.0:1619677474.750592:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677474.750593:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:10.0:1619677474.750594:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.750594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:26.0:1619677474.750595:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.750595:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.750596:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:22.0:1619677474.750597:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.750597:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.750598:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.750598:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:10.0:1619677474.750598:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:13.0:1619677474.750599:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.750600:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.750601:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.750602:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677474.750603:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:10.0:1619677474.750603:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.750604:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.750607:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.750608:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.750608:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:10.0:1619677474.750608:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.750609:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677474.750610:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.750611:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677474.750611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677474.750612:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (24)++
00000800:00000200:26.0:1619677474.750613:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:10.0:1619677474.750613:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.750614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:10.0:1619677474.750614:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.750615:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:10.0:1619677474.750615:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.750616:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000800:00000200:25.2:1619677474.750616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.750617:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:26.0:1619677474.750618:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.750618:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (24)++
00000800:00000200:10.0:1619677474.750618:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:10.0:1619677474.750619:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:10.0:1619677474.750620:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000400:00000200:26.0:1619677474.750621:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.750621:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000400:00000200:26.0:1619677474.750622:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677474.750623:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677474.750623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:12.0:1619677474.750623:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:10.0:1619677474.750623:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.750625:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.750625:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677474.750625:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:26.0:1619677474.750626:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:12.0:1619677474.750626:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.750628:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.750630:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.750630:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677474.750630:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.750632:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:12.0:1619677474.750632:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677474.750633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:10.0:1619677474.750633:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.750633:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677474.750634:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.750634:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:6.0:1619677474.750634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.750635:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:10.0:1619677474.750636:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:6.0:1619677474.750636:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.750637:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.750637:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.750639:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.750640:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.750641:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.750642:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (24)++
00000800:00000200:9.2:1619677474.750642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.750643:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:6.0:1619677474.750643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677474.750644:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:26.0:1619677474.750645:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000400:00000200:10.0:1619677474.750645:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.750646:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:26.0:1619677474.750647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)--
00000800:00000200:10.0:1619677474.750647:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.750648:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.750650:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.750650:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.750651:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.750652:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.750652:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.750653:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.750654:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.750655:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (22)++
00000800:00000200:10.0:1619677474.750655:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.750656:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000800:00000200:10.0:1619677474.750656:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:10.0:1619677474.750657:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06fbb0600] (24)++
00000400:00000200:26.0:1619677474.750658:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.750658:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:10.0:1619677474.750658:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000400:00000200:26.0:1619677474.750659:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.750659:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677474.750660:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000400:00000200:26.0:1619677474.750661:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:10.0:1619677474.750661:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000400:00000200:26.0:1619677474.750662:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.750663:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.750664:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.750665:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677474.750665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:10.0:1619677474.750667:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.750668:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.750669:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.750669:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.750670:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.750671:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677474.750671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:10.0:1619677474.750672:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.750673:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.750674:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.750676:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.750677:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677474.750678:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (23)++
00000400:00000200:10.0:1619677474.750678:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.750679:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (23)++
00000800:00000200:10.0:1619677474.750679:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.750681:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (25)--
00000800:00000200:26.0:1619677474.750682:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (24)++
00000800:00000200:21.0:1619677474.750682:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (25)--
00000800:00000200:10.0:1619677474.750682:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.750683:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:26.0:1619677474.750683:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000400:00000200:21.0:1619677474.750683:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.750683:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.750684:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000400:00000200:21.0:1619677474.750684:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.750684:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677474.750685:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677474.750686:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000800:00000200:10.0:1619677474.750686:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677474.750687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.750687:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (24)++
00000800:00000200:10.0:1619677474.750688:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000400:00000200:26.0:1619677474.750689:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.750689:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:10.0:1619677474.750690:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:25.2:1619677474.750691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.750691:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677474.750692:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.750692:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.750693:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.750693:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.750694:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677474.750694:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000400:00000200:26.0:1619677474.750695:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.750695:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000400:00000200:26.0:1619677474.750697:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.750698:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677474.750698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:26.0:1619677474.750699:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.750699:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:10.0:1619677474.750701:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.750702:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.750702:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.750703:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.750703:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.750703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.750704:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.750706:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.750706:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:10.0:1619677474.750706:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677474.750707:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96503128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.750708:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.750709:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.750709:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.750710:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.750711:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.750711:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:21.0:1619677474.750712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:25.2:1619677474.750713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677474.750715:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.750717:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677474.750718:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.750718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.750719:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.750720:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.750720:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677474.750720:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677474.750721:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.750721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.750722:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.750722:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (24)++
00000800:00000200:22.0:1619677474.750722:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.750722:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677474.750722:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:22.0:1619677474.750723:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.750724:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:22.0:1619677474.750724:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.750724:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.750724:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)++
00000800:00000200:8.2:1619677474.750724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.750725:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:22.0:1619677474.750725:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677474.750726:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677474.750726:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06fbb0600] (25)++
00000800:00000200:10.0:1619677474.750727:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:26.0:1619677474.750728:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (25)++
00000800:00000200:26.0:1619677474.750728:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:10.0:1619677474.750728:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (25)--
00000800:00000200:26.0:1619677474.750729:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:10.0:1619677474.750729:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (24)++
00000800:00000200:25.2:1619677474.750730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.750730:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.17@o2ib3
00000800:00000200:9.2:1619677474.750730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.750731:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)--
00000800:00000200:21.0:1619677474.750731:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.750733:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:10.0:1619677474.750733:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.17@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.750734:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.750734:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.750734:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:4.2:1619677474.750734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06fbb0600] (22)++
00000800:00000200:21.0:1619677474.750735:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.750735:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000400:00000200:26.0:1619677474.750736:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.750736:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:10.0:1619677474.750736:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.750737:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:10.0:1619677474.750737:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677474.750738:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96503127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.750738:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677474.750740:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.750741:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.750741:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677474.750741:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.750742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.750742:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.750743:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.750745:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:6.0:1619677474.750745:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.750746:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)++
00000800:00000200:26.0:1619677474.750747:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06fbb0600] (24)--
00000800:00000200:26.0:1619677474.750748:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (23)--
00000400:00000200:10.0:1619677474.750748:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.17@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.750748:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677474.750749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677474.750750:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (22)--
00000800:00000200:10.0:1619677474.750750:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677474.750752:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.750753:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677474.750753:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.750755:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.750755:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677474.750755:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677474.750755:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677474.750756:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.750757:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677474.750758:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677474.750759:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb0600] (21)++
00000800:00000200:10.0:1619677474.750759:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb0600] (22)--
00000800:00000200:10.0:1619677474.750760:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb0600] (21)--
00000800:00000200:8.2:1619677474.750763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.750771:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.750773:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.750774:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.750775:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.750777:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.750780:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.750848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.750851:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.750854:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.750856:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.750857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.750858:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.750860:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.750861:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.750862:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.750863:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.750865:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.750868:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.750870:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.750872:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.750873:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.750873:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.750875:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.750876:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.750877:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.750877:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.750881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.750889:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.750891:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.750892:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.750892:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.750894:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.750896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.751504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.751512:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.751515:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.751517:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.751520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.751520:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.751521:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.751522:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.751523:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.751524:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.751525:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.751528:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.751530:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.751532:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.751533:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.751534:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.751535:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.751536:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.751537:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.751539:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.751539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.751542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.751545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.751550:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.751551:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.751552:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.751553:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677474.751554:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.751554:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.751555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.751556:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.751557:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.751558:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.751846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.751854:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.751857:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.751858:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.751860:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.751863:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.751864:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.751865:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.751866:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.751867:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.751868:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.751871:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.751872:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.751875:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.751876:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.751877:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.751879:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.751879:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.751880:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.751880:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.751885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.751892:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.751895:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.751896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.751896:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.751898:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.751901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.751958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1000] (20)++
00000800:00000200:1.0:1619677474.751964:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1000] (21)++
00000800:00000200:1.0:1619677474.751967:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.13@o2ib3
00000800:00000200:17.0:1619677474.751968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1000] (22)--
00000400:00000200:1.0:1619677474.751970:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.0.13@o2ib3 : GET - for me
00000400:00000200:1.0:1619677474.751973:0:19931:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.0.13@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1619677474.751975:0:19931:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.0.13@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:1.0:1619677474.751976:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdbb1000] (21)++
00000800:00000200:1.0:1619677474.751977:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdbb1000] (22)--
00000400:00000200:1.0:1619677474.751979:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.0.13@o2ib3 local destination
00000400:00000200:1.0:1619677474.751983:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.0.13@o2ib3(10.51.0.13@o2ib3:10.51.0.13@o2ib3) : REPLY try# 0
00000800:00000200:1.0:1619677474.751985:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.0.13@o2ib3
00000800:00000200:1.0:1619677474.751987:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918ce37f3000] -> 10.51.0.13@o2ib3 (2) version: 12
00000800:00000200:1.0:1619677474.751988:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918cfdbb1000] (21)++
00000800:00000200:1.0:1619677474.751988:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdbb1000] (22)++
00000800:00000200:1.0:1619677474.751990:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918cfdbb1000] (23)--
00000800:00000200:1.0:1619677474.751990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1000] (22)--
00000800:00000200:21.2:1619677474.751995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdbb1000] (21)++
00000800:00000200:17.0:1619677474.752001:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdbb1000] (22)++
00000800:00000200:17.0:1619677474.752002:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdbb1000] (23)--
00000400:00000200:17.0:1619677474.752003:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.752004:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1000] (22)--
00000400:00000200:17.0:1619677474.752005:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.0.13@o2ib3: REPLY: OK
00000800:00000200:17.0:1619677474.752006:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdbb1000] (21)--
00000800:00000200:21.2:1619677474.752213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.752220:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.752223:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.752226:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.752227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.752228:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.752229:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.752230:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.752232:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.752233:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.752234:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.752237:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.752239:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.752241:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.752242:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.752243:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.752244:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.752245:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.752246:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.752246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.752250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.752253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.0:1619677474.752257:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.752260:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:31.0:1619677474.752261:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.752261:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.752261:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.752263:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:8.0:1619677474.752263:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.752265:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.752265:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.752267:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.752268:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.752269:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.752271:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.752272:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.752273:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.752274:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.752277:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.752278:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.752282:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.752282:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.752283:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.752285:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.752285:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.752287:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.752289:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.752289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.752291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.752297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.752299:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.2:1619677474.752300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677474.752302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.752302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.752303:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.752305:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.752305:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.752307:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:2.0:1619677474.752307:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.752308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.752308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.752309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:2.0:1619677474.752310:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:25.2:1619677474.752311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677474.752311:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.752313:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.752315:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.752316:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.752317:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.752319:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:2.0:1619677474.752319:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.752320:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.752321:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677474.752322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.752322:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.752323:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.752324:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.752325:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:2.0:1619677474.752325:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.752326:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.752330:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.752331:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.752331:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.752333:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.752334:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.752335:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.752336:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.752337:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.752339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677474.752342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677474.752347:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677474.752348:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.752350:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:13.0:1619677474.752350:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677474.752351:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:13.0:1619677474.752351:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.752351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677474.752352:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.752354:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.752355:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.752363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677474.752369:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.752372:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.752373:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.752373:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.752374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.752376:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.752456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.752464:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.752466:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.752468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.752469:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.752471:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.752473:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.752474:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.752475:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.752477:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.752478:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677474.752481:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.752482:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.752486:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.752486:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.752487:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677474.752489:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677474.752489:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.752490:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.752490:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677474.752495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.752502:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.752505:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:4.2:1619677474.752505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:18.0:1619677474.752506:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.752506:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.752509:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.752511:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677474.752512:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677474.752514:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.752516:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.752516:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.752518:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.752520:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.752521:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.752522:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.752523:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.752524:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.752528:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.752530:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.752533:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.752534:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.752535:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.752536:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.752537:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.752538:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.752538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.752542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.752549:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.752552:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.752553:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.752555:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.752556:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.752559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.752590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.752595:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.752598:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.752599:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.752601:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.752604:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.752605:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.752606:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.752607:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.752608:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.752609:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.752612:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.752614:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.752615:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.752616:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.752617:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.752619:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.752619:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.752620:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.752620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.752625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.752631:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.752633:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.752634:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.752634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.752635:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.752637:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.753955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.753963:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.753966:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.753968:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.753970:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.753971:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.753972:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.753973:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.753974:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.753975:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.753976:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.753979:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.753981:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.753983:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.753984:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.753985:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.753987:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.753987:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.753988:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.753989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.753993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.754000:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.754002:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.754003:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.754004:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.754006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.754007:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.754104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.754108:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.754110:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.754112:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.754113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.754115:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.754116:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.754118:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.754119:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.754120:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.754121:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.754124:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.754126:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.754128:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.754128:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.754129:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.754131:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.754131:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.754133:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.754134:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.754134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677474.754137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.754141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677474.754145:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.754147:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677474.754147:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:22.0:1619677474.754148:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.754148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.754149:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.754149:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:22.0:1619677474.754150:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.754150:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677474.754152:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.754310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:5.2:1619677474.754315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.754319:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.754321:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.754321:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.754322:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677474.754324:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.754325:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.754326:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.754327:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.754328:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.754330:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.754331:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.754333:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.754334:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.754336:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.754337:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.754338:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.754339:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.754340:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677474.754340:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.754341:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.754344:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677474.754345:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.754345:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.754346:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.754347:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.754349:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.754349:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.754349:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677474.754350:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.754351:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.754351:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677474.754351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.754353:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.754353:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.754353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:1.0:1619677474.754353:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.754354:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.754355:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:9.2:1619677474.754355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.754356:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.754359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.754361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677474.754362:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.754365:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.754366:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.754366:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:8.0:1619677474.754367:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.754367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.754368:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:8.0:1619677474.754369:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.754369:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.754370:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.754370:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.754371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.754371:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.754372:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677474.754373:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.754376:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677474.754379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.754386:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.754388:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.754389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.754390:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.754391:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.754391:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.754405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.754414:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.754416:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.754419:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.754421:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.754421:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.754422:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.754423:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.754425:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.754426:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.754427:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.754430:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.754432:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.754435:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.754436:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.754436:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.754438:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.754438:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.754439:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.754440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.754444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.754451:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.754454:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.754455:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.754455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.754457:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.754459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.754640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677474.754648:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677474.754651:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677474.754653:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.754655:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.754655:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677474.754657:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677474.754658:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.754660:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.754661:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.754662:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677474.754665:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.754666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677474.754667:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.754669:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.754670:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677474.754671:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677474.754673:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677474.754673:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.754674:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.754675:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:30.0:1619677474.754675:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:31.0:1619677474.754676:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677474.754677:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677474.754677:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.754678:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.754679:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.754679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:30.0:1619677474.754680:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677474.754682:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677474.754682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000400:00000200:30.0:1619677474.754683:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.754684:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.754684:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.754686:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:30.0:1619677474.754687:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.754689:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.754689:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:31.0:1619677474.754690:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000800:00000200:30.0:1619677474.754690:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.754691:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:30.0:1619677474.754691:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.754691:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677474.754691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.754691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.754691:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.754692:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677474.754693:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:30.0:1619677474.754693:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:2.0:1619677474.754693:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.754694:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.754694:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.754695:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:2.0:1619677474.754697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677474.754698:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:25.2:1619677474.754699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:9.2:1619677474.754701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.754701:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.754704:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.754704:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.754705:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:2.0:1619677474.754706:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.754707:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677474.754707:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.754708:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000400:00000200:2.0:1619677474.754708:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:18.0:1619677474.754709:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.754709:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.754710:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:18.0:1619677474.754710:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:31.0:1619677474.754711:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.754711:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:2.0:1619677474.754711:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.754712:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:2.0:1619677474.754713:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.754714:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:2.0:1619677474.754714:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.754715:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:2.0:1619677474.754717:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.754719:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.754720:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.754721:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.754722:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.754723:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.754724:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.754725:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.754725:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.754729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.754736:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.754738:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.754739:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.754740:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.754741:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.754742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.754888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.754895:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.754898:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.754898:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.754901:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.754903:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.754905:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.754906:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.754907:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.754908:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.754909:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.754912:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.754914:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.754917:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.754918:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.754918:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.754920:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.754921:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.754921:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.754922:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.754926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.754934:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.754937:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.754937:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.754938:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.754940:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.754943:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.754971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.754979:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.754982:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.754984:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.754986:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.754987:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.754988:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677474.754990:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.754991:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.754992:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.754993:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677474.754996:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.754998:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677474.755001:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.755002:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:13.0:1619677474.755002:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:13.0:1619677474.755004:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677474.755004:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:4.2:1619677474.755005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:13.0:1619677474.755006:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677474.755007:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677474.755008:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677474.755010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677474.755013:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:4.2:1619677474.755013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:10.0:1619677474.755015:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.755016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677474.755018:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:10.0:1619677474.755018:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.755020:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:12.0:1619677474.755021:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:10.0:1619677474.755021:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.755021:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.755022:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.755022:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677474.755023:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:17.0:1619677474.755024:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.755024:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:10.0:1619677474.755024:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677474.755025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:10.0:1619677474.755025:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.755026:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.755027:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.755028:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:10.0:1619677474.755031:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.755033:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.755036:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.755036:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.755037:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.755039:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.755039:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.755040:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.755040:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.755045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.755052:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.755055:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.755056:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.755056:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.755058:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.755061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.755063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.755070:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.755073:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.755073:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.755074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.755075:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.755075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.755840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.755848:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.755851:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.755853:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.755855:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.755856:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.755857:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.755858:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.755860:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.755860:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.755861:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.755865:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.755867:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.755868:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.755869:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.755870:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.755871:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.755872:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.755873:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.755873:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.755877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.755884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.755887:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.755888:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.755889:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.755891:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.755892:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.755921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.755927:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.755930:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.755931:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.755932:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.755935:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.755936:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.755938:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.755939:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.755940:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.755941:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.755945:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.755946:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.755949:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.755950:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.755951:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.755953:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.755953:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.755954:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.755954:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677474.755959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.755966:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.755969:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.755970:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.755970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.755972:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.755974:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.756051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.756059:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.756062:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.756064:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.756066:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.756066:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.756068:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.756069:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.756070:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.756071:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.756072:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677474.756072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:31.0:1619677474.756076:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.756077:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:31.0:1619677474.756078:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.756080:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:31.0:1619677474.756082:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.756082:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:31.0:1619677474.756083:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.756083:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677474.756083:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677474.756085:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:21.0:1619677474.756085:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.756086:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.756087:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.756087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:21.0:1619677474.756087:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.756088:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.756090:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.756090:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677474.756091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677474.756092:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.756095:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.756097:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677474.756098:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.756099:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.756100:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677474.756100:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677474.756101:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677474.756102:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.756102:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.756103:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.756103:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.756104:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000400:00000200:6.0:1619677474.756104:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.756105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.756107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677474.756108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.756115:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.756118:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.756119:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.756119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.756121:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.756124:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.756368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.756374:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.756377:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.756378:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.756380:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.756382:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.756384:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.756385:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.756387:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.756388:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.756388:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.756391:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.756393:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.756396:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.756396:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.756397:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.756399:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.756399:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.756401:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677474.756402:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.756402:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.756406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.756408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.756413:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677474.756414:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677474.756415:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.756415:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677474.756416:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677474.756416:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677474.756417:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.756418:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.756419:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.756421:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.756505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.756510:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.756513:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.756515:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.756515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.756517:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.756519:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.756520:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.756522:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.756523:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.756524:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.756527:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.756529:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.756531:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.756532:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.756533:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.756535:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.756535:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.756536:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.756537:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.756538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.756541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.756543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.756546:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.756548:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.756548:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.756549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.756549:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.756551:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.756552:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.756552:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.756554:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.756557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.756561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.756568:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.756571:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.756572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.756572:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.756573:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.756573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.756603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.756611:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.756614:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.756615:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.756617:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.756620:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.756621:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.756622:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.756624:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.756625:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.756626:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.756628:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.756630:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.756633:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.756634:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.756634:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.756636:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.756637:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.756638:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.756639:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.756639:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.756643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.756645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.756651:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.756652:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.756654:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677474.756654:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677474.756654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.756655:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.756655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.756656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677474.756657:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.756660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.756995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.757003:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.757006:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.757008:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.757010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.757011:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.757012:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.757014:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.757015:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.757016:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.757017:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.757021:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.757022:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.757024:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.757025:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.757025:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.757027:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.757027:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.757029:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.757030:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.757031:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.757032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.757037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.757041:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.757042:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.757043:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.757044:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.757045:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.757046:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.757046:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.757047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.757048:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.757049:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.757049:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.757050:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.757050:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.757051:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.757051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.757156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.757163:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.757165:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.757167:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.757168:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.757170:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.757172:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.757173:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.757175:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.757176:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.757177:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.757180:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.757182:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.757184:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.757185:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.757185:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.757187:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.757188:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.757189:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.757189:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.757193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.757201:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.757203:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.757204:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.757204:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.757206:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.757208:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.757221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.757228:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.757231:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.757232:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.757232:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.757233:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.757233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.757691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.757695:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.757697:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.757700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.757700:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.757702:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.757703:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.757705:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.757706:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.757707:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.757708:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.757712:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.757714:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.757716:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.757717:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.757717:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.757719:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.757720:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.757720:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.757721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.757726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.757734:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.757737:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.757738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.757738:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.757740:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.757742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.758123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.758132:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.758135:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.758137:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.758139:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.758139:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.758141:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.758142:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.758144:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.758145:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.758146:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.758149:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.758151:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.758155:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.758155:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.758156:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.758158:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.758158:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.758159:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.758160:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.758164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.758171:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.758174:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.758175:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.758175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.758177:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.758180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.758184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.758191:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.758194:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.758195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.758195:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.758196:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.758196:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.758279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.758286:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.758289:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.758291:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.758293:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.758302:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.758304:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.758305:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.758307:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.758308:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.758309:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.758312:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.758314:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.758316:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.758317:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.758317:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.758319:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.758319:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.758321:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.758321:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.758325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.758332:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.758335:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.758336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.758336:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.758338:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.758340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677474.759001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.759007:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.759010:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.759011:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.759013:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.759015:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.759017:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.759018:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.759019:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.759020:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.759021:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.759024:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.759026:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.759028:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.759029:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.759030:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.759031:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.759032:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.759033:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.759033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.759038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.759045:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.759048:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.759049:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.759049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.759051:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.759052:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.759056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.759063:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.759066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.759066:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.759067:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.759068:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.759068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.759210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.759217:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.759220:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.759223:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.759224:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.759226:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.759227:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.759228:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.759230:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.759231:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.759232:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.759235:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.759237:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.759239:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.759240:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.759241:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.759242:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.759243:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.759244:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.759244:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.759248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.759255:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.759257:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.759258:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.759259:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.759261:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.759262:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.759341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.759348:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.759351:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.759351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.759354:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.759356:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.759357:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.759359:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.759360:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.759361:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.759362:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.759365:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.759366:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.759368:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.759369:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.759369:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.759370:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.759371:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.759372:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.759373:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.759374:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.759377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.759380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.759386:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.759387:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.759387:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.759388:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:2.0:1619677474.759388:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.759389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.759390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.759390:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.759391:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.759391:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.759439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.759447:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.759450:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.759451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.759452:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.759456:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.759457:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.759459:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.759460:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.759461:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.759462:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.759465:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.759467:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.759470:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.759471:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.759471:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.759473:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.759474:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.759475:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.759477:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.759477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677474.759479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677474.759483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677474.759487:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677474.759490:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677474.759490:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.759491:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.759491:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.759493:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.759493:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.759494:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677474.759494:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.759497:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.759636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.759641:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.759644:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.759645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.759647:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.759649:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.759651:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677474.759652:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.759654:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.759655:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.759656:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677474.759659:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.759661:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677474.759664:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.759665:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677474.759666:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677474.759668:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677474.759669:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.759670:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677474.759671:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677474.759672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:25.2:1619677474.759675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.2:1619677474.759677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.759682:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:12.0:1619677474.759684:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:6.0:1619677474.759684:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677474.759684:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.759685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:12.0:1619677474.759688:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.759689:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.759692:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.759693:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.759696:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.759787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.759795:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.759798:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.759801:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.759802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.759803:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.759803:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.759804:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.759805:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.759807:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.759808:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.759809:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.759810:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677474.759812:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.759813:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677474.759814:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.759814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.759816:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.759817:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.759818:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.759819:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.759819:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.759820:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677474.759820:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.759821:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:10.0:1619677474.759821:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.759822:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.759823:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000400:00000200:10.0:1619677474.759823:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.759824:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:10.0:1619677474.759824:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.759825:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677474.759826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.759828:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.759829:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677474.759830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:10.0:1619677474.759832:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.759833:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.759834:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:10.0:1619677474.759834:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:10.0:1619677474.759836:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (25)--
00000800:00000200:31.0:1619677474.759837:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:10.0:1619677474.759837:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.759838:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:10.0:1619677474.759838:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.759838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.759839:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.759839:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:31.0:1619677474.759840:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:10.0:1619677474.759840:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677474.759841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.759841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:22.0:1619677474.759841:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.759842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.759845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:4.2:1619677474.759845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.759847:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.759849:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.759850:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.759850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.759851:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.759852:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.759853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677474.759855:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.759856:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677474.759856:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.760002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.760007:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.760009:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.760012:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.760012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.760014:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.760016:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.760017:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.760019:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.760020:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.760021:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.760024:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.760025:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.760027:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.760028:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.760029:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.760030:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.760031:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.760032:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.760032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.760037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.760044:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.760046:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.760047:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.760049:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.760050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.760051:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.760157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.760162:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.760165:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.760166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.760168:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.760170:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.760171:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.760172:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.760174:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.760175:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.760176:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.760178:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.760180:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.760182:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.760182:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.760183:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.760185:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.760185:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.760186:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677474.760187:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.760188:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677474.760191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.760193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.760199:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.760200:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677474.760201:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.760201:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.760201:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.760202:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.760203:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.760203:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.760204:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677474.760204:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:4.2:1619677474.760372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.760380:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.760382:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.760383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.760386:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.760388:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.760390:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.760391:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.760393:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.760394:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.760395:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.760398:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.760399:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.760402:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.760403:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.760403:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.760405:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.760405:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.760406:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.760406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.760412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.760420:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.760423:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.760424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.760424:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.760426:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.760429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.760570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677474.760577:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.760580:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677474.760581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677474.760583:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.760586:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.760588:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677474.760589:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.760590:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.760591:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.760592:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677474.760595:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677474.760596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677474.760597:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.760600:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.760600:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677474.760601:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677474.760602:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677474.760603:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:31.0:1619677474.760604:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.760604:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677474.760604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:31.0:1619677474.760607:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677474.760608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:31.0:1619677474.760609:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.760612:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.760612:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.760613:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.760614:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.760616:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.760616:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.760617:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677474.760617:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.760620:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:31.0:1619677474.760621:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.760621:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.760621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677474.760623:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677474.760624:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.760625:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.760626:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.760626:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677474.760627:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677474.760628:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.760628:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.760629:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.760630:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.760634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.760640:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.760643:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.760644:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.760645:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.760647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.760648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.760933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.760941:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.760943:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.760944:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.760946:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.760948:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.760950:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.760951:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.760952:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.760953:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.760954:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.760957:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.760958:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.760961:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.760962:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.760963:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.760964:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.760965:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.760965:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.760966:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.760970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.760978:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.760980:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.760981:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.760981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.760984:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.760986:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.761138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.761143:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.761146:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.761148:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.761148:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.761150:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.761152:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.761153:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.761154:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.761155:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.761156:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.761159:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.761161:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.761163:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.761164:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.761165:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.761166:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.761167:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.761168:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.761168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.761172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.761179:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.761181:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.761182:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.761183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.761185:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.761187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.761193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.761200:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.761202:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.761203:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.761204:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.761205:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.761205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.761212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677474.761219:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.761222:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677474.761223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677474.761225:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.761228:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.761230:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.761231:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.761232:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.761233:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.761234:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677474.761238:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.761239:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677474.761242:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.761243:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.761244:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.761245:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.761246:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.761247:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677474.761247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677474.761252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.761260:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.761263:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.761264:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.761264:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.761266:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.761269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.761275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.761282:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.761284:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.761285:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.761286:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.761287:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.761287:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.761444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.761450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.761452:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.761453:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.761455:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.761456:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.761458:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.761459:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.761460:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.761462:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.761463:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.761466:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.761468:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.761471:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.761472:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.761473:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.761475:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.761475:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.761476:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.761476:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.761482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.761490:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.761492:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.761493:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.761495:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.761497:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.761498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.761538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.761546:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.761548:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.761549:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.761551:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.761554:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.761555:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.761556:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.761558:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.761559:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.761559:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.761562:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.761564:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.761566:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.761567:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.761568:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.761569:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.761570:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.761571:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.761571:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.761575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.761583:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.761585:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.761586:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.761587:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.761590:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.761591:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.762066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.762073:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.762076:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.762078:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.762080:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.762080:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.762081:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.762082:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.762084:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.762084:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.762085:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.762088:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.762090:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.762093:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.762094:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.762094:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.762096:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.762097:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.762098:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.762098:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.762102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.762110:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.762113:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.762114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.762114:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.762116:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.762119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.762183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.762190:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.762193:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.762194:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.762196:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.762199:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.762200:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677474.762202:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.762203:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.762204:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.762205:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677474.762208:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.762209:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677474.762211:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.762212:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677474.762213:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677474.762214:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677474.762215:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.762216:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.762217:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.762217:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677474.762220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.762223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677474.762227:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.762229:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.762230:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.762230:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:31.0:1619677474.762231:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.762232:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:30.0:1619677474.762233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677474.762233:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:31.0:1619677474.762234:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:26.0:1619677474.762234:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.762240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.762246:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.762249:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.762250:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.762251:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.762251:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.762252:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.762641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.762648:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.762651:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.762653:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.762656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.762656:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.762657:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.762658:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.762660:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.762660:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.762661:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.762664:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.762666:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.762668:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.762669:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.762669:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.762671:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.762671:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.762673:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.762674:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.762675:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.762677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.762681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.762684:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.762687:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.762688:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.762688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.762689:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677474.762690:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.762691:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.762692:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.762692:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.762693:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.762873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.762876:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.762879:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.762881:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.762882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.762883:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.762884:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.762886:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.762887:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.762888:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.762889:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.762893:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.762895:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.762897:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.762898:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.762898:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.762900:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.762901:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.762901:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.762902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.762906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.762914:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.762917:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.762918:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.762918:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.762920:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.762922:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677474.763736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.763741:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.763744:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.763745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.763747:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.763749:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.763750:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.763751:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.763752:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.763753:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.763754:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.763757:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.763759:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.763761:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.763762:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.763762:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.763764:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.763765:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.763765:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.763766:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.763771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.763779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.763782:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.763783:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.763783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.763785:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.763787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.763855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.763864:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.763867:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.763869:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.763871:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.763871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.763873:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.763874:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.763875:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.763876:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.763877:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.763881:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.763883:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.763886:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.763886:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.763887:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.763889:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.763889:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.763890:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.763891:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.763895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.763902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.763905:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.763906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.763906:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.763908:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.763911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.763991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.763998:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.764001:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.764003:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.764005:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.764006:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.764007:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.764009:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.764010:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.764011:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.764012:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.764015:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.764017:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.764019:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.764020:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.764020:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.764022:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.764022:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.764023:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.764024:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.764027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.764035:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.764037:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.764038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.764038:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.764040:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.764042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.764048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.764054:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.764056:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.764057:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.764057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.764058:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.764059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.764656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.764665:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.764667:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.764669:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.764672:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.764672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.764673:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.764674:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.764675:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.764677:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.764677:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.764681:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.764682:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.764685:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.764685:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.764686:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.764688:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.764688:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.764689:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677474.764689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677474.764690:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.764694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.764698:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.764700:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.764702:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.764703:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.764703:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.764704:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.764705:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.764706:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.764708:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.764709:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.764710:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.764711:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.764712:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.764713:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.764714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.764715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:8.0:1619677474.764716:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.764718:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.764721:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.764722:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.764722:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.764724:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.764724:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.764725:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.764725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.764730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.764738:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.764741:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.764742:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.764742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.764744:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.764747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.765017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677474.765023:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.765026:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677474.765027:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677474.765029:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.765032:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.765033:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677474.765034:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.765036:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.765037:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.765038:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677474.765041:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.765043:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.765044:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.765045:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.765046:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.765047:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.765048:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.765049:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.765049:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677474.765054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.765062:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.765064:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.765064:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.765065:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.765066:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.765067:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.765122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.765129:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.765132:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.765133:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.765135:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.765137:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.765139:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.765140:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.765141:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.765143:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.765143:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.765147:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.765148:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.765151:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.765152:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.765152:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.765154:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.765155:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.765155:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.765156:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.765160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.765168:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.765170:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.765171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.765172:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.765174:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.765176:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.765178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.765185:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.765188:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.765189:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.765189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.765190:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.765191:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.765610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677474.765613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677474.765619:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.765621:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677474.765622:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.765623:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677474.765624:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.765626:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.765626:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.765628:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.765629:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.765630:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.765632:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.765633:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.765634:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.765636:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677474.765637:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.765637:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.765639:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.765639:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.765639:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.765640:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677474.765643:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.765643:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:31.0:1619677474.765644:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.765644:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.765645:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.765646:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677474.765646:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.765647:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.765648:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.765648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.765649:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:31.0:1619677474.765649:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:30.0:1619677474.765649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.765650:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677474.765651:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:31.0:1619677474.765652:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:9.2:1619677474.765652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.765653:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:31.0:1619677474.765655:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:31.0:1619677474.765655:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:25.2:1619677474.765657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:24.0:1619677474.765659:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.765661:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:20.2:1619677474.765661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000400:00000200:24.0:1619677474.765662:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.765662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.765664:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.765664:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:24.0:1619677474.765667:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:13.0:1619677474.765667:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677474.765667:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.765668:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000400:00000200:13.0:1619677474.765669:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.765671:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:13.0:1619677474.765671:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:30.0:1619677474.765672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:12.0:1619677474.765672:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:30.0:1619677474.765673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:21.2:1619677474.765718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.765725:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.765727:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.765728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.765730:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.765733:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.765735:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.765736:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.765738:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.765739:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.765740:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.765742:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.765744:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.765746:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.765746:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.765747:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.765749:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.765749:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.765750:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.765751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.765755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.765762:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.765765:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.765765:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.765766:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.765767:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.765768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.765785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.765792:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.765795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.765795:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.765797:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.765800:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.765801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677474.765802:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.765803:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.765804:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.765805:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.765806:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677474.765808:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677474.765810:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.765811:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.0:1619677474.765811:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677474.765812:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.765814:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.765814:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.765815:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.765816:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:26.0:1619677474.765817:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.765818:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.765818:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677474.765819:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677474.765819:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.765819:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677474.765820:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.765821:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.765822:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.765823:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.2:1619677474.765824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:26.0:1619677474.765826:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.765827:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.765830:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.765831:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.765831:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.765832:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677474.765833:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.765834:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.765834:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.765835:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677474.765835:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.765836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.765836:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.765838:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.765839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.2:1619677474.765840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:1.0:1619677474.765841:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677474.765847:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.765847:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.765850:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:30.0:1619677474.765851:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677474.765851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.765851:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.765853:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.765854:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.765855:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.765855:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.765856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:30.0:1619677474.765857:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.765858:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.765859:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.765860:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.765861:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.765864:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.765866:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.765868:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.765869:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.765870:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.765871:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.765872:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.765873:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.765874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.765877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.765885:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.765887:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.765888:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.765888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.765890:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.765893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.766369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.766377:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.766379:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.766380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.766382:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.766385:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.766386:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677474.766387:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.766389:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.766390:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.766391:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677474.766394:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.766395:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677474.766398:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677474.766398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677474.766399:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677474.766400:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677474.766402:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.766402:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.766402:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.766403:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677474.766404:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.766407:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.2:1619677474.766408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677474.766408:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.766410:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : GET - for me
00000400:00000200:17.0:1619677474.766412:0:19930:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.2.27@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:17.0:1619677474.766414:0:19930:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.2.27@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:17.0:1619677474.766415:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.766416:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.0:1619677474.766417:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:17.0:1619677474.766418:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.2.27@o2ib3 local destination
00000800:00000200:21.0:1619677474.766420:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.766421:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.766422:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.766422:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.2.27@o2ib3(10.51.2.27@o2ib3:10.51.2.27@o2ib3) : REPLY try# 0
00000800:00000200:20.0:1619677474.766423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:17.0:1619677474.766424:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.2.27@o2ib3
00000800:00000200:21.0:1619677474.766426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677474.766426:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918ce53e8380] -> 10.51.2.27@o2ib3 (2) version: 12
00000800:00000200:17.0:1619677474.766427:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.766427:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.766429:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.766429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:5.2:1619677474.766434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.766439:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.766441:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677474.766442:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.766443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.766444:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.2.27@o2ib3: REPLY: OK
00000800:00000200:1.0:1619677474.766446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677474.766663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.766667:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.766670:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.766672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.766672:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.766674:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.766676:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.766677:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.766678:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.766680:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.766680:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.766684:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.766686:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.766689:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.766689:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.766690:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.766692:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.766693:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.766694:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.766695:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.766696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.766699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.766701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677474.766705:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677474.766706:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677474.766708:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677474.766708:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:24.0:1619677474.766709:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.766709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677474.766709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.766710:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:24.0:1619677474.766712:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.766714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.766781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.766786:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.766788:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.766791:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.766791:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.766794:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.766795:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.766797:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.766798:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.766799:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.766800:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.766804:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.766805:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.766808:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.766809:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.766810:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.766811:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.766812:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.766813:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.766814:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.766815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677474.766817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.766820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.766823:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677474.766825:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677474.766826:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.766826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.766827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677474.766828:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.766829:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.766831:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.766832:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.766834:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.767538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.767546:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.767549:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.767551:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.767554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.767554:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.767555:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.767556:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.767558:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.767558:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.767559:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.767563:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.767564:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.767567:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.767568:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.767569:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.767570:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.767571:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.767572:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.767574:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.767574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.767577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.767581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.767585:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.767588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.767588:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.767589:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.767589:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.767591:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677474.767591:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.767592:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.767593:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.767594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.767931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.767938:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.767941:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.767941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.767944:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.767945:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.767947:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.767948:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.767949:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.767950:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.767951:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.767955:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.767956:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.767958:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.767959:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.767959:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.767960:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.767961:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.767962:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.767962:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.767967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.767973:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.767976:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.767976:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.767977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.767978:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.767979:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.768571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.768579:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.768581:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.768584:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.768586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.768587:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.768588:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.768590:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.768591:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.768592:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.768593:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.768596:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.768598:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.768601:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.768602:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.768602:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.768604:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.768605:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.768606:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.768608:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.768608:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.768610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.768614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.768617:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.768620:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.768621:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.768621:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.768622:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.768623:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.768625:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677474.768625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.768626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.768627:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.768738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.768740:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.768743:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.768745:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.768746:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.768748:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.768749:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.768751:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.768752:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.768753:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.768754:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.768757:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.768759:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.768761:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.768762:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.768763:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.768764:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.768765:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.768766:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.768766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.768771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.768779:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.768781:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.768782:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.768782:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.768785:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.768787:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677474.768889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.768895:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.768897:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.768898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.768900:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.768901:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.768903:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.768904:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.768905:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.768906:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.768907:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.768911:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.768912:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.768915:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.768916:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.768917:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.768918:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.768919:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.768919:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.768919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.768924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.768932:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.768934:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.768935:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.768936:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.768938:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.768940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.768945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.768952:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.768954:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.768955:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.768956:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.768957:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.768957:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.769246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677474.769254:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.769256:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677474.769257:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677474.769259:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.769262:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.769264:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677474.769265:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.769266:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.769267:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.769268:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677474.769272:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.769274:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677474.769277:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.769278:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677474.769278:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677474.769280:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677474.769280:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.769281:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677474.769281:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677474.769286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.769293:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.769298:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.769301:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.769302:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.769304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677474.769304:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.769307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.769311:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677474.769312:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677474.769312:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.769314:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677474.769315:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677474.769316:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677474.769316:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.769317:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.769317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:30.0:1619677474.769318:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.769320:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.769320:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.769321:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.769323:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.769324:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.769325:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.769326:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.769329:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.769331:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.769333:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.769334:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.769334:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.769335:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.769336:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.769337:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.769338:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.769342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.769349:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.769351:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.769352:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.769353:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.769355:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.769356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.769463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.769470:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.769473:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.769474:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.769476:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.769479:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.769480:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677474.769482:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.769483:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.769484:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.769485:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677474.769488:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.769489:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677474.769491:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.769492:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677474.769492:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677474.769494:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677474.769494:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.769495:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.769495:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.769499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.769506:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.769509:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.769509:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.769511:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.769513:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.769513:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.769518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.769524:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.769527:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.769528:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.769528:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.769529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.769530:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.769559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.769566:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.769569:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.769570:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.769571:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.769574:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.769576:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.769577:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.769578:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.769579:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.769580:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.769583:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.769584:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.769587:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.769588:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.769588:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.769590:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.769590:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.769592:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.769593:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.769593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.2:1619677474.769594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:25.2:1619677474.769596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.769599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677474.769601:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.769603:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677474.769604:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677474.769605:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.769606:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.769606:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:31.0:1619677474.769607:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.769607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.769608:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677474.769608:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677474.769608:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.769609:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677474.769609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.769610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.769610:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.769611:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.769613:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.769613:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677474.769614:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.769615:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.769618:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.769619:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.769623:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.769623:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.769624:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.769626:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.769626:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.769627:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.769628:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.769632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.769639:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.769642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.769642:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.769643:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.769644:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.769647:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.769650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.769657:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.769660:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.769661:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.769661:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.769662:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.769663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.769959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.769966:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.769969:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.769969:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.769971:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.769973:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.769975:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.769976:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.769977:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.769978:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.769979:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.769983:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.769985:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.769988:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.769989:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.769990:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.769992:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.769992:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.769993:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.769994:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.769998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.770004:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.770007:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.770008:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.770010:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.770011:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.770013:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.770333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.770341:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.770343:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.770345:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.770348:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.770348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.770349:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.770350:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.770352:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.770353:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.770354:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.770356:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.770358:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.770360:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.770361:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.770362:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.770363:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.770364:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.770365:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.770365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.770370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.770377:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.770379:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.770380:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.770381:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.770383:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.770384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.770467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.770475:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.770478:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.770478:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.770480:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.770483:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.770484:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.770485:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.770486:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.770487:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.770488:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.770491:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.770492:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.770495:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.770496:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.770496:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.770498:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.770499:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.770499:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.770500:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.770504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.770512:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.770515:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.770516:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.770516:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.770518:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.770521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.770529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.770535:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.770538:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.770539:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.770541:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.770543:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.770545:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.770546:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.770547:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.770548:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.770549:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.770553:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.770554:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.770557:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.770557:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.770558:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.770560:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.770560:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.770561:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.770562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.770578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.770586:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.770588:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.770589:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.770590:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.770591:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.770593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.770599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.770606:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.770609:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.770610:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.770611:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.770611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.770612:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.770614:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.770620:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.770623:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.770624:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.770626:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.770629:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.770630:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.770632:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.770633:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.770634:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.770635:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.770638:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.770640:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.770642:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.770642:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.770643:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.770645:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.770646:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.770647:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.770647:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.770651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.770658:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.770660:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.770661:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.770661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.770663:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.770664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.770724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.770731:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.770734:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.770735:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.770737:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.770740:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.770741:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.770742:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.770743:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.770744:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.770745:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.770748:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.770750:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.770753:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.770754:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.770754:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.770756:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.770756:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.770757:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.770758:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.770762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.770769:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.770772:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.770773:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.770773:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.770775:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.770778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.770806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.770814:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.770816:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.770817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.770819:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.770821:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.770823:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.770824:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.770825:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.770826:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.770827:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.770830:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.770831:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.770834:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.770835:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.770835:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.770837:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.770837:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.770838:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.770838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.770843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.770851:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.770853:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.770854:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.770854:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.770856:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.770860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.770864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.770871:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.770874:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.770875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.770875:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.770876:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.770876:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.770968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.770972:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.770975:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.770977:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.770978:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.770979:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.770980:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.770982:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.770983:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.770984:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.770985:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.770989:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.770990:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.770993:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.770994:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.770994:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.770996:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.770997:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.770997:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.770998:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.771002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.771008:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.771011:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.771012:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.771012:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.771015:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.771017:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.771087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.771095:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.771097:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.771100:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.771101:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.771102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.771102:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.771104:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.771105:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.771106:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.771107:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.771110:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.771112:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.771114:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.771115:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.771116:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.771117:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.771118:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.771119:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.771119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.771123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.771130:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.771132:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.771133:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.771135:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.771136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.771137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.771191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.771198:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.771201:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.771203:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.771205:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.771205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.771206:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.771207:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.771208:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.771209:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.771210:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.771213:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.771215:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.771218:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.771218:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.771219:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.771221:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.771221:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.771222:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.771222:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.771227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.771234:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.771237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.771237:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.771238:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.771240:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.771242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.772043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677474.772049:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.772052:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677474.772053:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677474.772055:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.772058:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.772059:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677474.772061:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.772062:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.772063:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.772064:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677474.772067:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.772069:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677474.772072:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.772073:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677474.772074:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677474.772076:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677474.772076:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.772077:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677474.772079:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677474.772079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:9.2:1619677474.772082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:5.2:1619677474.772087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:8.0:1619677474.772090:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.772092:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677474.772092:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:24.0:1619677474.772093:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.772094:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.772094:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677474.772095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:8.0:1619677474.772096:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.772096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.0:1619677474.772099:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.772101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.772107:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.772109:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.772110:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.772110:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.772111:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.772112:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.772690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.772698:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.772701:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.772702:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.772704:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.772707:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.772708:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677474.772709:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.772711:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.772712:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.772713:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677474.772716:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.772718:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677474.772721:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.772722:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677474.772722:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677474.772724:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677474.772724:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.772725:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.772725:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.772731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.772739:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.772742:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.772743:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.772744:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.772745:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.772749:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.772752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.772759:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.772762:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.772763:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.772763:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.772764:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.772765:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.773272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.773281:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.773284:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.773286:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.773289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.773289:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.773291:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.773292:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.773297:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.773298:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.773299:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.773302:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.773303:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.773307:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.773307:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.773308:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.773310:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.773310:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.773311:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.773312:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.773316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.773325:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.773328:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.773329:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.773329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.773331:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.773334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.773430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.773438:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.773441:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.773443:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.773445:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.773446:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.773447:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.773448:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.773449:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.773450:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.773451:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.773454:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.773456:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.773457:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.773458:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.773459:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.773460:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.773461:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.773462:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.773464:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.773464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.773466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.773470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.773474:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.773476:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.773476:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.773478:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.773478:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.773479:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.773479:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.773480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.773481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.773482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677474.773654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.773660:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.773663:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.773664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.773666:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.773669:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.773670:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.773671:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.773673:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.773674:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.773675:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.773678:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.773680:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.773683:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.773683:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.773684:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.773686:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.773686:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.773686:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.773687:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.773692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.773699:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.773701:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.773702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.773703:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.773705:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.773707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.774196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.774204:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.774207:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.774209:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.774211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.774212:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.774214:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677474.774215:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.774216:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.774217:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.774218:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677474.774222:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.774223:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677474.774226:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.774227:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:13.0:1619677474.774228:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:13.0:1619677474.774229:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677474.774230:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.774230:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.774231:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677474.774236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.774244:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.774247:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.774248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.774248:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.774250:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.774253:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.774561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.774567:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.774570:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.774571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.774573:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.774575:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.774576:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.774578:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.774579:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.774580:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.774581:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.774585:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.774587:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.774590:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.774591:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.774592:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.774593:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.774594:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.774595:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.774596:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.774597:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.774600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677474.774603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677474.774608:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677474.774609:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677474.774610:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677474.774611:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:12.0:1619677474.774611:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.774612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.774613:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677474.774613:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.774615:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.774616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.774729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.774738:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.774741:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.774743:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.774745:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.774745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.774746:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.774748:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.774749:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.774750:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.774751:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.774754:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.774756:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.774758:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.774759:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.774759:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.774761:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.774761:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.774762:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.774763:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.774766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.774773:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.774775:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.774776:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.774778:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.774779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.774780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.774785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.774792:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.774794:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.774795:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.774795:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.774796:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.774797:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.774943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.774951:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.774954:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.774954:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.774956:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.774959:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.774961:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.774962:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.774963:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.774964:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.774965:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.774967:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.774969:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.774972:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.774973:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.774973:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.774975:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.774975:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.774976:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.774976:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.774981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.774989:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.774992:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.774993:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.774993:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.774995:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.774998:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.775008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.775015:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.775018:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.775020:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.775022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.775022:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.775023:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.775024:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.775026:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.775027:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.775028:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.775031:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.775033:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.775036:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.775036:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.775037:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.775039:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.775039:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.775041:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.775042:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.775043:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.775045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.775048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.775052:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.775055:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.775055:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.775056:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.775056:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.775058:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.775059:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.775059:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.775060:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.775061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.775065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.775071:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.775074:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.775075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.775075:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.775076:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.775077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.775098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677474.775106:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.775109:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677474.775111:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.775113:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677474.775114:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.775115:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677474.775117:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.775118:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.775119:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.775120:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677474.775123:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.775124:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677474.775128:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.775128:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677474.775129:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.775131:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677474.775132:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.775133:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677474.775133:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677474.775137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.775145:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.775148:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677474.775149:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.775150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.775151:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.775155:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677474.775211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.775216:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.775219:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.775221:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.775221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.775224:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.775225:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.775227:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.775228:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.775229:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.775230:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.775234:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.775235:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.775238:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.775238:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.775239:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.775241:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.775241:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677474.775241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.0:1619677474.775242:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.775243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.775247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.775249:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.775252:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.775253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677474.775254:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:26.0:1619677474.775255:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.775257:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:26.0:1619677474.775258:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.775258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.775258:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.775260:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.775261:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.775262:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.775263:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.775263:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:26.0:1619677474.775264:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.775265:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.775268:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.775270:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.775273:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.775274:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.775274:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.775276:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.775277:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.775278:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.775278:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.775282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.775290:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.775292:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.775293:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.775300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.775302:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.775304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.775612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.775619:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.775622:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.775623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.775625:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.775628:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.775629:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.775631:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.775632:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.775633:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.775634:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.775637:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.775638:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.775641:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.775642:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.775642:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.775644:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.775645:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.775645:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.775647:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.775647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.775651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.775653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.775658:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.775660:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.775661:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677474.775662:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677474.775662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.775662:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.775662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:26.0:1619677474.775663:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.775663:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.775664:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.775667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.775670:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.775673:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.775675:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.775677:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.775679:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.775681:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.775682:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.775683:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.775684:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.775685:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.775688:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.775690:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.775692:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.775693:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.775693:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.775695:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.775696:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.775696:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.775697:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.775701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.775708:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.775710:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.775711:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.775711:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.775713:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.775716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.775720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:9.2:1619677474.775720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.775726:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677474.775727:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677474.775729:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.775730:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677474.775730:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:8.0:1619677474.775730:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.775731:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.775731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:2.0:1619677474.775731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.775733:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.775735:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.775737:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.775738:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.775739:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677474.775740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677474.775740:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.775741:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.775745:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.775746:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677474.775748:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677474.775748:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.775749:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.775749:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677474.775751:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677474.775751:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.775752:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677474.775753:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.775753:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.775753:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:31.0:1619677474.775755:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.775755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.775757:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:9.2:1619677474.775757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:31.0:1619677474.775758:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.775759:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.775760:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.775761:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677474.775764:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:31.0:1619677474.775765:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.775766:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677474.775766:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.775767:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.775768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677474.775769:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.775770:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.775770:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:2.0:1619677474.775770:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.775772:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.775772:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677474.775772:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677474.775773:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.775774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.775778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.775784:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.775787:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.775787:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.775789:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.775791:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.775791:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.2:1619677474.775912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.775918:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.775920:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.775921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.775923:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.775925:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.775926:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.775927:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.775929:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.775930:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.775931:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.775934:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.775936:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.775937:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.775938:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.775939:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.775940:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.775941:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.775942:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.775942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.775946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.775954:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.775955:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.775956:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.775957:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.775958:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.775959:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.776217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.776224:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.776227:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.776228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.776229:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.776232:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.776234:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.776235:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.776237:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.776237:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.776239:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.776242:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.776243:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.776246:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.776247:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.776248:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.776250:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.776250:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.776251:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.776251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.776256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.776263:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.776267:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.776268:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.776268:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.776270:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.776273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.776947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677474.776950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.776955:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.776957:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677474.776959:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677474.776959:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.776960:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.776961:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.776962:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.776962:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.776963:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.776965:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.776966:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.776967:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.776969:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.776970:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.776970:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.776972:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.776972:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.776973:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.776974:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.776974:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.776978:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.776978:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.776979:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.776979:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.776979:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.776981:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.776981:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.776981:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.776982:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.776982:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.776983:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.776983:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.776984:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.776984:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677474.776985:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.776986:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.776986:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.776987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.776988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:9.2:1619677474.776991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.776997:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.776998:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677474.776999:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.777000:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677474.777000:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.777001:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:22.0:1619677474.777001:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.777001:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.777003:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.777004:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.777006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.777008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.777749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677474.777754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:26.0:1619677474.777757:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.777759:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.777760:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.777762:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.777764:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:26.0:1619677474.777765:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.777767:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.777768:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677474.777768:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:26.0:1619677474.777769:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.777771:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.777772:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677474.777773:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.777774:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:26.0:1619677474.777775:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.777777:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.777779:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677474.777779:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.777780:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.777781:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:13.0:1619677474.777781:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:26.0:1619677474.777782:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.777783:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.777783:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677474.777783:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96503015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.777784:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:13.0:1619677474.777786:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.777788:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677474.777789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:13.0:1619677474.777790:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677474.777796:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.777797:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.777798:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.777799:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:12.0:1619677474.777800:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.777802:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677474.777802:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.777804:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677474.777804:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677474.777805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.777807:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677474.777807:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.777808:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.777808:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.777809:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.777810:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:12.0:1619677474.777810:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.777811:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:25.2:1619677474.777812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677474.777812:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.777813:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677474.777819:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.777822:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.777823:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.777824:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.777826:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.777827:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:5.2:1619677474.778355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.778361:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.778364:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.778365:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.778367:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.778369:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.778370:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.778372:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.778373:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.778374:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.778375:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.778379:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.778381:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.778383:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.778384:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.778385:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.778387:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.778387:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.778388:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.778388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.778393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.778401:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.778404:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.778405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.778405:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.778407:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.778409:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.778412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:25.2:1619677474.778413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677474.778420:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.778420:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.778423:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677474.778423:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.778423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.778424:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000400:00000200:31.0:1619677474.778425:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.778425:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.778426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.778427:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.778428:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.778429:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677474.778431:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.778432:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.778433:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.778434:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677474.778437:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.778439:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677474.778442:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.778443:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677474.778443:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677474.778445:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677474.778446:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.778447:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.778447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677474.778452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.778460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.778463:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.778464:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.778464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.778466:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.778469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.778487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.778495:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.778498:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.778500:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.778502:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.778502:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.778504:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.778505:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.778507:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.778508:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.778508:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.778511:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.778513:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.778515:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.778516:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.778516:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.778518:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.778518:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.778520:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.778521:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.778522:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.778523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.778527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.778532:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.778533:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.778534:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.778535:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677474.778536:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.778536:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.778537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.778538:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.778539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.778540:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.779092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.779100:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.779103:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.779104:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.779105:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.779107:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.779109:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.779110:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.779112:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.779112:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.779114:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.779117:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.779119:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.779122:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.779123:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.779123:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.779125:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.779126:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.779127:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.779127:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.779131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.779138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.779141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.779142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.779143:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.779145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.779146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.779548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.779556:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.779559:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.779561:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.779563:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.779564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.779565:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.779566:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.779567:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.779568:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.779569:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.779573:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.779574:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.779577:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.779577:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.779578:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.779580:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.779580:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.779581:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.779582:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.779586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.779593:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.779596:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.779596:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.779598:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.779600:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.779601:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.779786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.779792:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.779794:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.779796:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.779797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.779799:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.779800:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.779801:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.779803:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.779804:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.779805:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.779808:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.779810:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.779812:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.779813:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.779813:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.779815:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.779816:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.779817:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.779818:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.779819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.779822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.779824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.779829:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677474.779830:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677474.779831:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.779832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.779832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677474.779833:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.779834:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.779834:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.779836:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.779838:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.779839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.779847:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.779849:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.779850:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.779850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.779851:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.779852:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.779920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.779927:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.779930:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.779931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.779932:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.779935:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.779937:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.779938:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96503008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.779939:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.779940:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.779941:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.779944:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.779946:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.779949:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.779950:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.779950:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.779952:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.779952:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.779954:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.779955:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.779955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.779958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.779961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.779966:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.779968:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.779968:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.779969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.779969:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.779970:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.779971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.779972:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677474.779972:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.779975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.780135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.780140:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.780144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.780144:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677474.780147:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.780149:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.780150:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.780151:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.780152:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.780153:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.780154:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.780157:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.780159:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.780161:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.780162:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.780163:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.780164:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.780165:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.780165:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.780166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.780170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.780178:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.780180:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.780181:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.780183:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.780183:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.780185:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.780399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.780407:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.780409:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.780410:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.780412:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.780415:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.780416:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.780418:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.780419:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.780420:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.780421:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.780424:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.780426:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.780429:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.780430:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.780430:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.780432:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.780432:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.780433:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.780433:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.780439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.2:1619677474.780439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.780447:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.780447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.780449:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:31.0:1619677474.780450:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:6.0:1619677474.780450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.780451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.780453:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.780453:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.780455:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.780455:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.780456:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677474.780457:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:31.0:1619677474.780458:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.780459:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.780460:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.780461:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.780464:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.780466:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.780469:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.780470:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.780470:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.780472:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.780473:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.780474:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.780474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.780478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.780485:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.780488:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.780489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.780489:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.780491:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.780494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.780500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.780507:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.780510:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.780511:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.780511:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.780512:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.780513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.780576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.780584:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.780586:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.780587:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.780589:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.780592:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.780594:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.780595:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.780596:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.780597:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.780598:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.780601:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.780603:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.780605:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.780606:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.780607:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.780608:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.780609:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.780610:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.780611:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.780611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.780614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.780617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.780622:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.780624:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.780624:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.780625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.780626:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677474.780626:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.780627:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.780627:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.780628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.780630:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.781262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.781267:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.781270:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.781271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.781273:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.781276:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.781277:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.781278:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96503003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.781280:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.781281:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.781282:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.781285:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.781287:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.781288:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.781289:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.781290:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.781291:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.781292:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.781297:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:9.2:1619677474.781298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.781298:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.781301:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.781303:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.2:1619677474.781304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.781305:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.781306:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.781307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.781307:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.781308:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677474.781309:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.781309:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677474.781310:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.781310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:21.2:1619677474.781720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.781728:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.781732:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.781734:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.781736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.781737:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.781738:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.781740:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96503002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.781741:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.781742:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.781743:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.781746:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.781748:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.781751:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.781752:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.2:1619677474.781752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677474.781753:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.781754:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.781755:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677474.781755:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677474.781756:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.781758:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.781758:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:20.0:1619677474.781758:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.781760:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677474.781761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.781761:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.781762:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.781764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:20.0:1619677474.781764:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.781765:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.781766:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.781767:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.781768:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677474.781769:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.781772:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.781772:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:20.0:1619677474.781772:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.781773:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.781773:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677474.781773:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.781774:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.781775:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677474.781775:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.781776:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.0:1619677474.781776:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.781777:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.781777:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677474.781778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677474.781779:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.781780:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.781781:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.781781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677474.781782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc5000] (20)++
00000800:00000200:4.2:1619677474.781783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:8.2:1619677474.781786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677474.781787:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc5000] (21)++
00000800:00000200:6.0:1619677474.781790:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.54@o2ib3
00000800:00000200:22.0:1619677474.781791:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (22)--
00000800:00000200:10.0:1619677474.781791:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.0:1619677474.781793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:6.0:1619677474.781793:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.54@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.781794:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.781795:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.0:1619677474.781795:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.781796:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.781796:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.781797:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.781798:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.781798:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:20.0:1619677474.781799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:6.0:1619677474.781799:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96503000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.781800:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.781801:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:10.0:1619677474.781801:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677474.781801:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.781802:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.781803:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.781804:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677474.781805:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.781806:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.781807:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677474.781810:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.781812:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:6.0:1619677474.781815:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.781817:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.781817:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677474.781818:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.781819:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.781820:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677474.781820:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.781821:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677474.781821:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677474.781822:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:6.0:1619677474.781822:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.781824:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.781824:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:6.0:1619677474.781824:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:6.0:1619677474.781825:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc5000] (21)++
00000800:00000200:9.2:1619677474.781826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677474.781826:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919874bc5000] (22)++
00000800:00000200:6.0:1619677474.781827:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc5000] (23)--
00000800:00000200:6.0:1619677474.781828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (22)--
00000800:00000200:9.2:1619677474.781829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677474.781831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.2:1619677474.781833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc5000] (21)++
00000800:00000200:26.0:1619677474.781836:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.781837:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.781837:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:26.0:1619677474.781838:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.781838:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc5000] (22)++
00000800:00000200:10.0:1619677474.781838:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.781839:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.781840:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.781840:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919874bc5000] (23)--
00000800:00000200:22.0:1619677474.781841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (22)--
00000400:00000200:10.0:1619677474.781841:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.781842:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:22.0:1619677474.781842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (21)--
00000800:00000200:26.0:1619677474.781844:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.781844:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.781845:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.781845:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.781846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677474.781846:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677474.781847:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.2:1619677474.781894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc5000] (20)++
00000800:00000200:6.0:1619677474.781899:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc5000] (21)++
00000800:00000200:6.0:1619677474.781902:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.54@o2ib3
00000800:00000200:22.0:1619677474.781903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (22)--
00000400:00000200:6.0:1619677474.781905:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.54@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.781907:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.781909:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677474.781910:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.781911:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.781912:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.781913:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677474.781916:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.781917:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.781919:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.781919:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.781920:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.781921:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.781922:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc5000] (21)++
00000800:00000200:6.0:1619677474.781922:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc5000] (22)--
00000800:00000200:21.2:1619677474.781923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677474.781923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (21)--
00000800:00000200:25.2:1619677474.781927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.781931:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.781933:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677474.781934:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677474.781935:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.781936:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.781937:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.781938:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.781938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677474.781938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.781939:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.781940:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.781941:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.781941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677474.781943:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.781944:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.781945:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.781948:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.781950:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.781952:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.781953:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.781953:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.781953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc5000] (20)++
00000800:00000200:31.0:1619677474.781955:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.781955:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.781956:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.781957:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677474.781958:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc5000] (21)++
00000800:00000200:9.2:1619677474.781960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677474.781961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (22)--
00000800:00000200:6.0:1619677474.781961:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.54@o2ib3
00000400:00000200:6.0:1619677474.781963:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.54@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.781965:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.781967:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:6.0:1619677474.781967:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:6.0:1619677474.781968:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.781969:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.781970:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:6.0:1619677474.781970:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.781971:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.781971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:6.0:1619677474.781971:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677474.781974:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.781975:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.781977:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 344 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.781978:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677474.781980:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.781981:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677474.781981:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677474.781983:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677474.781984:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc5000] (21)++
00000800:00000200:6.0:1619677474.781985:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc5000] (22)--
00000800:00000200:6.0:1619677474.781985:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (21)--
00000800:00000200:9.2:1619677474.781989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.781996:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.781998:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.781999:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.781999:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.782001:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.782003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc5000] (20)++
00000800:00000200:8.0:1619677474.782004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677474.782008:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc5000] (21)++
00000800:00000200:22.0:1619677474.782010:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.54@o2ib3
00000800:00000200:6.0:1619677474.782011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (22)--
00000400:00000200:22.0:1619677474.782013:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.54@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.782016:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.782017:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677474.782018:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.782019:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.782020:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.782021:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677474.782024:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.782025:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677474.782028:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.782029:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:22.0:1619677474.782029:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.782030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc5000] (21)++
00000800:00000200:22.0:1619677474.782031:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677474.782031:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc5000] (22)++
00000800:00000200:22.0:1619677474.782033:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc5000] (23)--
00000800:00000200:22.0:1619677474.782033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (22)--
00000800:00000200:22.0:1619677474.782034:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc5000] (21)++
00000800:00000200:22.0:1619677474.782036:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.54@o2ib3
00000800:00000200:9.2:1619677474.782036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677474.782036:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc5000] (22)++
00000800:00000200:6.0:1619677474.782037:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.54@o2ib3
00000400:00000200:22.0:1619677474.782038:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.54@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.782040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.782040:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.54@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.782041:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677474.782042:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.782043:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:22.0:1619677474.782044:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.782044:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.782045:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:22.0:1619677474.782045:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:8.0:1619677474.782046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:6.0:1619677474.782046:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.782047:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.782049:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677474.782050:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.782050:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96502993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.782051:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.782052:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.782053:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:6.0:1619677474.782053:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.782054:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677474.782056:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677474.782058:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677474.782058:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.782059:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677474.782060:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:22.0:1619677474.782060:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:22.0:1619677474.782062:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:22.0:1619677474.782063:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc5000] (23)++
00000800:00000200:6.0:1619677474.782063:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.782064:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919874bc5000] (24)++
00000800:00000200:6.0:1619677474.782064:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:6.0:1619677474.782065:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:22.0:1619677474.782066:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc5000] (25)--
00000800:00000200:22.0:1619677474.782066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (24)--
00000800:00000200:22.0:1619677474.782067:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc5000] (23)++
00000800:00000200:6.0:1619677474.782067:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:9.2:1619677474.782068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677474.782068:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc5000] (23)++
00000800:00000200:22.0:1619677474.782069:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.54@o2ib3
00000800:00000200:6.0:1619677474.782069:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc5000] (25)--
00000800:00000200:6.0:1619677474.782069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (24)--
00000800:00000200:6.0:1619677474.782070:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc5000] (23)++
00000400:00000200:22.0:1619677474.782071:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.54@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.782071:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919874bc5000] (24)--
00000800:00000200:6.0:1619677474.782072:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (23)--
00000800:00000200:25.2:1619677474.782073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:6.0:1619677474.782073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (22)--
00000800:00000200:26.0:1619677474.782074:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:22.0:1619677474.782075:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.782076:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:26.0:1619677474.782077:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:22.0:1619677474.782077:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96502992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.782078:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.782079:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.782079:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.782079:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677474.782080:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:22.0:1619677474.782081:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677474.782082:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.782083:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677474.782084:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.782085:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:22.0:1619677474.782085:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:13.0:1619677474.782086:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.782087:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677474.782087:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:22.0:1619677474.782089:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.782090:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677474.782090:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677474.782090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc5000] (21)++
00000800:00000200:13.0:1619677474.782090:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:22.0:1619677474.782092:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677474.782093:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc5000] (22)++
00000800:00000200:22.0:1619677474.782094:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc5000] (23)--
00000800:00000200:22.0:1619677474.782095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (22)--
00000800:00000200:22.0:1619677474.782096:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc5000] (21)++
00000800:00000200:8.2:1619677474.782098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.782099:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.54@o2ib3
00000800:00000200:6.0:1619677474.782100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (22)--
00000400:00000200:22.0:1619677474.782102:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.54@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.782105:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.782107:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677474.782107:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.782107:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:22.0:1619677474.782109:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677474.782109:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:22.0:1619677474.782110:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677474.782110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:22.0:1619677474.782111:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677474.782111:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.782113:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.782115:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.782116:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.782118:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677474.782120:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.782120:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:22.0:1619677474.782121:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677474.782122:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:22.0:1619677474.782123:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc5000] (21)++
00000800:00000200:22.0:1619677474.782124:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc5000] (22)--
00000800:00000200:22.0:1619677474.782124:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc5000] (21)--
00000800:00000200:8.2:1619677474.782129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.782136:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.782139:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.782140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.782140:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.782142:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.782144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.782206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677474.782214:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.782216:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677474.782217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677474.782219:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.782222:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.782224:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677474.782225:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.782226:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.782227:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.782228:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677474.782232:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.782233:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677474.782237:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.782237:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677474.782238:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677474.782240:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677474.782240:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.782241:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.782241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677474.782245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.782253:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.782256:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.782257:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.782257:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.782259:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.782262:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.782312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.782319:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.782322:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.782323:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.782325:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.782328:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.782329:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.782331:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.782332:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.782333:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.782334:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.782337:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.782339:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.782341:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.782342:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.782343:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.782344:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.782345:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.782345:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.782346:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.782350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.782358:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.782361:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.782362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.782362:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.782364:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.782367:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.782368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.782376:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.782379:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.782379:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.782380:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.782381:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677474.782381:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.782576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.782580:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.782583:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.782586:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.782587:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.782588:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.782590:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.782591:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.782592:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.782593:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.782594:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.782598:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.782600:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.782603:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.782604:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.782604:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.782606:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.782607:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.782607:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.782608:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.782613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.782621:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.782624:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.782625:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.782627:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.782628:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.782630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.782632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.782639:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.782641:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.782642:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.782643:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.782644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.782645:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.782686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.782694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.782697:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.782697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.782699:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.782701:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.782703:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.782704:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.782705:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.782706:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.782707:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.782710:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.782711:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.782713:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.782714:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.782714:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.782716:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.782716:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.782717:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.782717:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.782722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.782729:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.782731:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.782732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.782732:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.782734:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.782735:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.782745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.782752:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.782754:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.782755:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.782755:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.782756:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.782756:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.783007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.783012:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.783015:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.783016:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.783017:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.783020:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.783021:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.783022:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.783024:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.783025:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.783026:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.783029:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.783031:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.783033:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.783034:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.783035:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.783037:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.783037:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.783039:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.783040:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.783040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.783043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.783045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.783051:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.783051:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.783052:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.783053:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.783054:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.783054:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.783055:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.783055:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.783057:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.783059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.783365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.783374:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.783376:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.783379:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.783381:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.783381:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.783382:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.783383:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.783385:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.783386:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.783387:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.783390:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.783392:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.783395:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.783395:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.783396:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.783398:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.783398:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.783399:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.783400:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.783404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.783411:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.783414:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.783414:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.783415:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.783416:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.783419:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.783565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.783573:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.783575:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.783577:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677474.783579:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.783580:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.783581:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677474.783582:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.783583:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.783584:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.783585:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677474.783588:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.783590:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677474.783593:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.783594:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677474.783595:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677474.783597:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677474.783597:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.783598:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.783599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.783603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.783611:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.783614:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.783615:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.783615:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.783618:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.783620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677474.783658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677474.783664:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.783667:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677474.783669:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.783669:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677474.783672:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.783674:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677474.783675:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.783676:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.783678:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.783679:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677474.783682:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.783684:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.783685:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.783686:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.783687:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.783689:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.783689:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.783690:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677474.783691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.783695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.783702:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.783705:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.783705:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.783707:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.783708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.783709:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.783861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.783870:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.783872:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.783874:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.783876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.783877:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.783878:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.783879:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.783881:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.783882:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.783883:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.783886:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.783888:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.783890:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.783891:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.783892:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.783893:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.783894:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.783895:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.783897:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.783897:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.783899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.783903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.783907:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.783908:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.783909:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.783911:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.783911:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.783912:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.783912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.783913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.783914:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.783915:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.783926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.2:1619677474.783928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.783933:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.783935:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.783936:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.783936:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.783937:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.783937:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.783938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.783938:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677474.783939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.783940:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.783941:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.783943:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.783944:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.783945:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.783946:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.783947:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.783950:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.783952:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.783955:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.783956:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.783956:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.783958:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.783958:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.783959:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.783959:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.783964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.783972:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.783975:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.783976:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.783976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.783978:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.783981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.783983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.783990:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.783993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.783993:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.783994:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.783995:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.783996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.784740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.784749:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.784752:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.784754:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.784756:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.784757:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.784758:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.784759:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.784760:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.784761:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.784761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:31.0:1619677474.784762:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.784766:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.784767:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.784769:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.784769:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677474.784770:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.784770:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.784771:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677474.784772:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.784772:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.784772:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677474.784773:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677474.784773:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.784774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:24.0:1619677474.784776:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.784778:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.784779:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96502979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.784780:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.784781:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.784782:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.784785:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.784786:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677474.784789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.784789:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.784790:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.784790:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.784792:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.784792:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.784793:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.784794:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677474.784796:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:25.2:1619677474.784798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.784799:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.784799:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.784801:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.784803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.784804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.784806:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.784808:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.784809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.784810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.784812:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.784815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.784822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba2600] (20)++
00000800:00000200:18.0:1619677474.784829:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba2600] (21)++
00000800:00000200:18.0:1619677474.784832:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.26@o2ib3
00000800:00000200:2.0:1619677474.784832:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba2600] (22)--
00000400:00000200:18.0:1619677474.784835:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.26@o2ib3 : GET - for me
00000400:00000200:18.0:1619677474.784838:0:19933:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.26@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:18.0:1619677474.784840:0:19933:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.26@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:18.0:1619677474.784842:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba2600] (21)++
00000800:00000200:18.0:1619677474.784843:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba2600] (22)--
00000400:00000200:18.0:1619677474.784844:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.26@o2ib3 local destination
00000400:00000200:18.0:1619677474.784848:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.26@o2ib3(10.51.5.26@o2ib3:10.51.5.26@o2ib3) : REPLY try# 0
00000800:00000200:18.0:1619677474.784850:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.26@o2ib3
00000800:00000200:18.0:1619677474.784851:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91907d36ce80] -> 10.51.5.26@o2ib3 (2) version: 12
00000800:00000200:18.0:1619677474.784852:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919066ba2600] (21)++
00000800:00000200:18.0:1619677474.784853:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba2600] (22)++
00000800:00000200:18.0:1619677474.784854:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919066ba2600] (23)--
00000800:00000200:18.0:1619677474.784854:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba2600] (22)--
00000800:00000200:20.2:1619677474.784859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba2600] (21)++
00000800:00000200:2.0:1619677474.784865:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba2600] (22)++
00000800:00000200:2.0:1619677474.784867:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba2600] (23)--
00000400:00000200:2.0:1619677474.784867:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.784869:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba2600] (22)--
00000400:00000200:2.0:1619677474.784869:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.26@o2ib3: REPLY: OK
00000800:00000200:2.0:1619677474.784871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba2600] (21)--
00000800:00000200:21.2:1619677474.785268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677474.785274:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.785277:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677474.785278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677474.785280:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.785282:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.785283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:6.0:1619677474.785284:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677474.785285:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.785287:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.785288:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.785289:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.785290:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677474.785292:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.785294:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677474.785300:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.785301:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:6.0:1619677474.785303:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.785304:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.785304:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677474.785304:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:10.0:1619677474.785306:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.785306:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.785307:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677474.785307:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000400:00000200:10.0:1619677474.785308:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.785308:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.785308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:10.0:1619677474.785310:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.785311:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.785312:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677474.785313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:10.0:1619677474.785316:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.785318:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.785321:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.785321:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.785322:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.785322:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:26.0:1619677474.785323:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.785323:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:26.0:1619677474.785325:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.785325:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.785327:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.785328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.785328:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.785329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677474.785330:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:25.2:1619677474.785331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.785339:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.785341:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.785342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.785343:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.785344:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.785348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.785410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.785418:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.785421:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.785422:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.785423:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.785426:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.785428:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.785429:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.785430:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.785431:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.785432:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.785435:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.785436:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.785439:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.785440:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.785440:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.785442:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.785442:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.785443:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.785443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.785448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.785455:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.785458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.785458:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.785459:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.785461:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.785464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.785670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.785677:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.785680:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.785682:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.785684:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.785685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.785685:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.785687:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.785688:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.785689:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.785690:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.785693:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.785694:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.785697:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.785698:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.785699:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.785701:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.785701:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.785702:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.785702:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.785706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.785714:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.785716:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.785717:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.785718:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.785720:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.785723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.786093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.786099:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.786101:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.786103:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.786104:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.786106:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.786107:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.786108:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.786110:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.786111:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.786112:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.786115:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.786117:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.786119:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.786120:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.786121:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.786122:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.786123:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.786124:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.786124:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.786129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.786138:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.786140:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.786141:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.786141:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.786143:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.786146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.786638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.786644:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.786648:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.786648:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.786650:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.786652:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.786654:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.786655:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.786656:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.786657:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.786658:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.786661:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.786662:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.786664:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.786665:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.786666:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.786667:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.786668:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.786669:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.786669:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.786674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.786682:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.786684:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.786685:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.786685:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.786686:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.786687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.786689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.786695:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.786697:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.786698:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.786699:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.786701:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.786703:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.786704:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.786705:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.786706:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.786706:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.786709:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.786711:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.786713:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.786713:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.786714:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.786715:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.786716:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.786716:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.786717:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.786733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.786742:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.786744:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.786745:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.786745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.786747:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.786749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677474.787701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.787704:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.787707:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.787709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.787709:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.787711:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.787713:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.787714:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.787716:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.787717:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.787718:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677474.787720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:20.0:1619677474.787721:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.787723:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.787725:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.787726:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.787726:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:31.0:1619677474.787728:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677474.787728:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.787728:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.787729:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:5.2:1619677474.787729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677474.787730:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677474.787731:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.787733:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.787735:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677474.787736:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.787736:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.787738:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677474.787738:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.787739:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677474.787740:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.787741:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.787741:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.787743:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.787743:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.787744:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677474.787745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:1.0:1619677474.787745:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.787746:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96502969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.787747:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.787748:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677474.787749:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.787750:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.787751:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.787753:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:31.0:1619677474.787754:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677474.787754:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.787755:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.787755:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:31.0:1619677474.787756:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677474.787756:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.787756:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:21.0:1619677474.787757:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.787758:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.787758:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677474.787758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:31.0:1619677474.787759:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:1.0:1619677474.787759:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.787760:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677474.787760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677474.787760:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.787760:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.787763:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.787763:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677474.787764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.787764:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.787764:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.787768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.787775:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.787776:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.787776:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.787777:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.787777:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.787778:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.787780:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.787780:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.787783:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.787783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.787784:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.787784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.787785:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.787786:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.787786:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:25.2:1619677474.787788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.787788:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.787791:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.787792:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.787794:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.787795:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.787795:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.787795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.787796:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.787859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677474.787867:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.787870:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677474.787870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677474.787873:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.787876:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.787878:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677474.787879:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.787880:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.787881:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.787882:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677474.787886:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.787888:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677474.787891:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.787892:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677474.787892:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.787894:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.787895:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.787896:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:26.0:1619677474.787897:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:26.0:1619677474.787897:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:8.2:1619677474.787900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677474.787903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:17.0:1619677474.787908:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.787910:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:10.0:1619677474.787910:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000400:00000200:17.0:1619677474.787911:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.787911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677474.787912:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000400:00000200:17.0:1619677474.787913:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.787913:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677474.787914:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:17.0:1619677474.787917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.787920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.787926:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.787929:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.787930:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.787930:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.787931:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.787932:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.788036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.788044:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.788047:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.788050:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.788051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.788052:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.788054:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.788055:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.788056:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.788057:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.788058:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.788061:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.788062:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.788064:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.788065:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.788065:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.788067:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.788068:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.788069:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.788069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.788073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.788079:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.788082:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.788082:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.788084:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.788086:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.788087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.788374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.788380:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.788383:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.788383:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.788385:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.788387:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.788389:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.788390:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.788392:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.788392:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.788393:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.788396:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.788398:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.788401:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.788402:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.788402:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.788404:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.788405:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.788405:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.788406:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.788410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.788418:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.788421:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.788421:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.788423:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.788424:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.788426:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.788432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.788439:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.788442:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.788443:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.788443:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.788444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.788445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.788886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.788894:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.788897:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.788899:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.788901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.788901:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.788903:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.788904:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.788906:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.788907:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.788908:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.788911:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.788913:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.788915:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.788916:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.788916:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.788918:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.788918:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.788919:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.788920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.788924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.788931:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.788933:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.788934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.788934:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.788936:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.788938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.788943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.788949:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.788951:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.788952:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.788952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.788953:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.788954:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.789098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.789105:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.789107:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.789108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.789109:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.789112:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.789113:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.789114:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.789115:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.789116:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.789117:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.789120:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.789121:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.789124:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.789125:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.789126:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.789128:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.789128:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.789129:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.789129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.789134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.789142:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.789145:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.789145:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.789146:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.789148:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.789151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.789449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.789457:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.789460:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.789462:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.789465:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.789465:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.789466:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.789467:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.789468:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.789469:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.789470:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.789473:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.789475:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.789478:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.789478:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.789479:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.789480:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.789481:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.789482:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.789484:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.789484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.789487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.789491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.789495:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.789497:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.789499:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.789499:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.789499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.789501:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677474.789501:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.789502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.789503:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.789504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.789652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.789660:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.789663:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.789666:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.789668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.789669:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.789670:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.789671:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.789672:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677474.789672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:30.0:1619677474.789673:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.789674:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.789678:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.789679:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677474.789680:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677474.789681:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.789682:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.789682:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677474.789682:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.789683:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677474.789684:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.789684:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.789685:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:26.0:1619677474.789685:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.789687:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.789687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:26.0:1619677474.789688:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.789689:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:25.2:1619677474.789689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:26.0:1619677474.789690:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.789692:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.789692:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.789693:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677474.789693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677474.789696:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.789697:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:26.0:1619677474.789698:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.789699:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.789700:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.789701:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677474.789701:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.789702:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.789702:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.789702:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677474.789702:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.789702:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677474.789703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677474.789704:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.789704:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677474.789705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.789705:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677474.789705:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.789706:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.789710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.789717:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.789720:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.789721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.789721:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.789723:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.789726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.790056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.790064:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.790066:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.790067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.790069:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.790072:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.790074:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.790075:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.790076:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.790077:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.790078:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.790081:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.790083:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.790085:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.790086:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.790087:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.790088:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.790089:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.790090:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.790092:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.790092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.790094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.790098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.790102:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.790104:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.790105:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.790105:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677474.790105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.790107:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.790107:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.790108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.790109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.790110:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.790215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.790222:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.790225:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.790225:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.790227:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.790230:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.790231:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.790233:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.790234:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.790235:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.790236:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.790239:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.790240:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.790243:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.790244:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.790244:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.790246:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.790246:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.790247:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.790247:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.790252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.790259:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.790262:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.790263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.790263:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.790265:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.790268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.790270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.790277:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.790279:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.790280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.790281:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.790282:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.790282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.791498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677474.791501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.791506:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.791508:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677474.791509:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:31.0:1619677474.791510:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.791511:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.791512:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.791513:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.791513:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.791515:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.791516:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.791517:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.791518:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.791521:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677474.791521:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.791522:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677474.791523:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.791523:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.791524:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.791525:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.791526:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.791529:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.791529:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:30.0:1619677474.791529:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.791530:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.791531:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.791531:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.791532:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.791533:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.791534:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.791535:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.791535:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.791535:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.791536:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.791536:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.791538:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.791538:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:9.2:1619677474.791538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.791539:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.791540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.791544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.791545:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.791547:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.791548:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.791548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.791550:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.791552:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.791553:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677474.791554:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.791555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.791556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.791558:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.791561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.791714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.791721:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.791724:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.791725:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.791727:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.791729:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.791731:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.791732:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.791734:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.791735:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.791736:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.791740:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.791742:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.791745:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.791746:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.791746:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.791748:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.791749:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.791750:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.791750:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.791755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.791762:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.791765:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.791767:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.791770:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.791771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.791774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677474.791938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.791944:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.791947:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.791949:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.791950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.791951:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.791953:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.791954:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.791956:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.791957:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.791958:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.791962:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.791964:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.791966:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.791967:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.791968:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.791970:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.791970:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.791972:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.791973:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.791974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.791977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.791978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.791981:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.791983:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.791984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.791985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677474.791986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.791989:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.791990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.791990:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.791992:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.791995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.791998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.792006:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.792010:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.792012:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.792015:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.792018:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.792021:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.792023:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.792025:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.792027:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.792028:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.792033:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.792036:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.792038:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.792040:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.792041:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.792043:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.792044:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.792045:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.792046:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.792050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.792059:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.792062:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.792064:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.792065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.792067:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.792069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.792105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.792112:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.792116:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.792118:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.792120:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.792123:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.792125:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.792126:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.792129:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.792130:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.792131:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.792136:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.792138:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.792140:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.792141:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.792142:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.792144:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.792145:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.792146:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.792148:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.792148:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.792151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.792154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.792162:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.792163:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.792164:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677474.792164:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.792165:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.792166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.792167:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.792168:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.792169:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677474.792170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677474.792425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.792431:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.792434:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.792435:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.792437:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.792439:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.792440:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.792441:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.792443:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.792444:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.792445:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.792448:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.792450:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.792453:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.792454:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.792454:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.792456:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.792457:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.792457:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.792458:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.792464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.792473:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.792475:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.792477:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.792477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.792479:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.792481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.792797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.792801:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.792804:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.792805:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.792806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.792808:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.792809:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.792810:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.792811:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.792812:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.792813:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.792816:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.792818:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.792820:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.792821:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.792821:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.792823:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.792823:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.792824:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.792825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.792829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.792837:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.792840:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.792840:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.792842:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.792842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.792844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.793087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.793095:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.793098:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.793100:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.793103:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.793104:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.793105:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.793106:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.793107:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.793108:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.793109:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.793112:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.793114:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.793117:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.793117:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.793118:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.793120:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.793120:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.793121:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.793122:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.793125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.793132:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.793134:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.793135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.793135:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.793137:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.793139:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.793271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.793279:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.793282:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.793283:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.793286:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.793286:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.793287:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.793289:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.793290:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.793291:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.793292:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.793299:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.793301:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.793304:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.793305:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.793305:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.793307:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.793307:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.793308:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.793309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.793313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.793322:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.793325:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.793326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.793326:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.793328:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.793331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.793600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.793603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.793605:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.793607:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.793608:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.793610:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.793611:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.793612:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.793614:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.793615:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.793616:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.793619:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.793620:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.793622:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.793623:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.793624:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.793625:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.793626:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.793627:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.793628:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.793632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.793639:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.793642:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.793642:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.793644:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.793645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.793646:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.793951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.793959:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.793962:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.793964:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.793966:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.793967:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.793968:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.793969:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.793971:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.793972:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.793973:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.793976:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.793978:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.793980:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.793981:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.793982:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.793983:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.793984:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.793985:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.793985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.793989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677474.793992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677474.793996:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.793999:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.793999:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.793999:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:31.0:1619677474.794001:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.794002:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:31.0:1619677474.794003:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.794003:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:30.0:1619677474.794004:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:26.0:1619677474.794005:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.794008:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.794008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000400:00000200:26.0:1619677474.794009:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.794010:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.794011:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.794012:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.794013:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:31.0:1619677474.794014:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:26.0:1619677474.794016:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.794017:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.794017:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.794018:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:26.0:1619677474.794018:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677474.794019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.794019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:26.0:1619677474.794020:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.794021:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.794022:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.794023:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.794024:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.794025:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.794026:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.794027:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677474.794029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.794032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677474.794037:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.794039:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677474.794039:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:13.0:1619677474.794040:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.794041:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.794041:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.794042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.794043:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677474.794044:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.794045:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.794332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.794339:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.794342:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.794343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.794344:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.794348:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.794349:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.794350:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.794351:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.794352:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.794353:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.794356:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.794358:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.794361:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.794361:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.794362:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.794364:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.794364:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.794365:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.794365:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.794370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.794379:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.794382:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.794382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.794383:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.794385:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.794387:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.794499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.794507:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.794510:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.794510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.794512:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.794515:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.794516:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.794518:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.794519:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.794520:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.794521:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.794524:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.794525:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.794528:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.794529:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.794529:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.794531:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.794532:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.794533:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.794534:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.794534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.794537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.794540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.794545:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.794547:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.794547:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.794548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.794549:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677474.794549:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.794550:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677474.794550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677474.794551:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.794553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.795464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677474.795469:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.795472:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677474.795473:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677474.795476:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.795477:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.795479:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677474.795480:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.795481:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.795483:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.795483:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677474.795487:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.795489:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677474.795491:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.795492:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677474.795493:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677474.795494:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677474.795495:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.795495:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.795496:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677474.795501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.795507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677474.795512:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:4.2:1619677474.795512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.0:1619677474.795515:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.795516:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.795516:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677474.795518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.795518:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.795519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.795519:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677474.795519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677474.795521:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:10.0:1619677474.795522:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.795522:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.795523:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.795525:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.795526:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.795527:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.795528:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.795529:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.795529:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677474.795530:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.795530:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.795531:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.795532:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.795533:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.795533:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.795535:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677474.795537:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.795538:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.795540:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.795541:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677474.795543:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.795543:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.795544:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.795544:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.795544:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.795545:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677474.795546:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.795547:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.795547:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.795547:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.795547:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:10.0:1619677474.795548:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677474.795548:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.795549:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.795553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.2:1619677474.795553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677474.795561:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677474.795561:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:22.0:1619677474.795564:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677474.795564:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:6.0:1619677474.795565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677474.795565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:22.0:1619677474.795566:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.795566:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.795569:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.795569:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.795571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.795572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677474.795574:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.795576:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.795578:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.795579:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.795579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.795580:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.795581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.795597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.795605:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.795607:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.795609:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.795611:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.795612:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.795613:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.795614:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.795615:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.795616:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.795617:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.795621:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.795622:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.795625:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.795626:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.795627:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.795628:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.795629:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.795630:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.795630:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.795634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.795642:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.795645:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.795646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.795646:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.795648:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.795651:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.795730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.795737:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.795739:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.795740:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.795742:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.795745:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.795747:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677474.795748:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.795749:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.795750:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.795751:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677474.795754:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.795756:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677474.795759:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.795760:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677474.795760:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.795762:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.795762:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.795763:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.795763:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.795768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.795775:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.795778:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.795779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.795779:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.795781:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.795784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.795917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.795926:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.795930:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.795934:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.795936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.795937:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.795940:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.795942:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.795944:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.795945:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.795947:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.795952:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.795955:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.795958:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.795959:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.795960:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.795962:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.795963:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.795965:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.795967:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:9.2:1619677474.795967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.795968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.795973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.795974:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.795976:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.795977:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.795977:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.795979:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.795982:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.795983:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.795986:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.795987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.795988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.796737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.796743:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.796746:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.796747:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.796749:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.796751:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.796752:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.796753:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.796755:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.796756:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.796757:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.796760:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.796762:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.796764:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.796764:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.796765:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.796766:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.796767:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.796768:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.796768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.796773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.796780:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.796782:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.796783:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.796783:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.796785:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.796786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.796792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.796799:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.796802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:2.0:1619677474.796802:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.796803:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.796803:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.796803:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.796804:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677474.796808:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.796811:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.796812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.796814:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.796815:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.796817:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677474.796818:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.796819:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.796820:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.796821:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:17.0:1619677474.796825:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.796827:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677474.796830:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.796831:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677474.796832:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677474.796833:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677474.796834:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.796835:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.796835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677474.796839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.796847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.796850:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.796851:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.796853:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.796854:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.796855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677474.797058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.797065:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.797068:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.797069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.797071:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.797073:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.797074:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.797075:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.797077:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.797078:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.797079:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.797082:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.797083:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.797086:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.797087:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.797088:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.797090:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.797091:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.797091:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.797092:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.797093:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.797096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677474.797098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677474.797102:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677474.797104:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677474.797105:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.797106:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.797106:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.797106:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.797107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677474.797108:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.797109:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.797111:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.797116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677474.797123:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.797125:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.797126:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.797127:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.797127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.797129:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.797357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.797364:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.797367:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.797369:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.797371:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.797371:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.797373:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.797374:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.797375:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.797376:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.797377:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.797380:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.797382:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.797385:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.797386:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.797387:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.797388:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.797389:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.797390:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.797392:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.797392:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.797395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.797399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.797403:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.797406:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.797406:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.797407:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.797407:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.797409:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677474.797409:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.797410:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.797411:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.797412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.797595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.797601:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.797604:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.797605:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.797607:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.797609:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.797611:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.797612:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.797614:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.797614:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.797616:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.797618:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.797620:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.797623:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.797623:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.797624:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.797626:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.797626:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.797627:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.797627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.797632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.797639:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.797642:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.797643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.797643:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.797645:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.797647:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.797650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.797657:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.797659:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.797660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.797661:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.797662:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.797662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.797823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.797831:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.797834:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.797836:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.797838:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.797839:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.797840:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.797842:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.797843:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.797844:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.797845:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.797848:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.797850:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.797853:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.797853:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.797854:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.797855:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.797856:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.797857:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.797857:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.797861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.797868:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.797870:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.797871:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.797873:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.797874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.797876:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.797930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677474.797935:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.797938:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677474.797939:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677474.797941:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.797943:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.797944:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677474.797946:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.797947:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.797948:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.797949:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.797952:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.797954:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.797957:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.797958:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677474.797958:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.797960:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677474.797961:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.797961:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677474.797962:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677474.797966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.797973:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.797976:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.797977:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.797977:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.797979:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.797982:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.798413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.798421:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.798424:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.798426:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.798428:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.798429:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.798429:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.798430:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.798431:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.798432:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.798433:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.798437:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.798438:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.2:1619677474.798438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677474.798440:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.798441:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.798442:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.798443:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.798444:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.798444:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677474.798445:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.798445:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677474.798446:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.798449:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.798449:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677474.798450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677474.798451:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.798452:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.798453:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.798454:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.798455:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.798456:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677474.798457:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.798459:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:21.0:1619677474.798459:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.798460:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.798462:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.798462:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677474.798463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.0:1619677474.798464:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.798465:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.0:1619677474.798465:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.798466:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.798468:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.798468:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.798469:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.798470:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.798474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.798482:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.798485:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.798486:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.798486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:21.0:1619677474.798487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.798488:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.798491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677474.798493:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.798496:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.798496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.798498:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.798500:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.798502:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.798503:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.798505:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.798506:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.798507:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.798510:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.798512:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.798514:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.798515:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.798516:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.798518:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.798518:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.798519:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.798520:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677474.798524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.798532:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.798535:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.798536:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.798536:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.798538:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.798541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.798979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.798987:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.798990:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.798992:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.798993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.798994:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.798995:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.798997:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.798998:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.798999:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.799000:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.799003:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.799004:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.799006:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.799007:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.799008:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.799009:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.799010:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.799010:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.799011:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.799014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:25.2:1619677474.799015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.799021:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677474.799022:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.799024:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:10.0:1619677474.799024:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:31.0:1619677474.799025:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.799025:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:31.0:1619677474.799027:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.799027:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.799028:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.799029:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:10.0:1619677474.799030:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.799032:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.799033:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.799034:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.799035:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.799036:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.799039:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.799040:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.799043:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.799043:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.799044:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.799046:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.799046:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.799047:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.799047:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.799051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.799059:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.799061:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.799062:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.799064:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.799066:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.799066:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.799288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.799295:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.799299:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.799302:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677474.799304:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.799306:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.799308:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.799309:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.799310:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.799311:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.799312:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.799315:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.799316:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.799319:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.799320:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.799320:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.799322:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.799323:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.799324:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.799324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.799329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.799337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.799339:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.799340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.799340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.799343:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.799346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.799452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:4.2:1619677474.799458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:30.0:1619677474.799460:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.799463:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.799465:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.799465:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:31.0:1619677474.799467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.799467:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.799468:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:30.0:1619677474.799469:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677474.799469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:30.0:1619677474.799471:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.799472:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.799472:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.799473:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.799474:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.799474:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.799476:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:30.0:1619677474.799478:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677474.799478:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96502923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.799479:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.799481:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.799481:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.799482:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:30.0:1619677474.799484:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.799485:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:8.0:1619677474.799485:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.799486:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.799487:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.799487:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677474.799488:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.799489:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.799489:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.0:1619677474.799489:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.799490:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677474.799491:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677474.799492:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677474.799492:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:25.2:1619677474.799493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.799494:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:8.0:1619677474.799495:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677474.799496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:9.2:1619677474.799498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677474.799500:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:4.2:1619677474.799501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000800:00000200:6.0:1619677474.799503:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.799504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.799504:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:6.0:1619677474.799504:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.799506:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.799507:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.799508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677474.799508:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000400:00000200:10.0:1619677474.799508:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.799509:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677474.799510:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000400:00000200:10.0:1619677474.799510:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.799511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:25.2:1619677474.799512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.799512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:10.0:1619677474.799513:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:22.0:1619677474.799519:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.799522:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.799523:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.799523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.799524:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.799525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.799637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.799642:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.799645:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.799647:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.799648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.799650:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.799651:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.799653:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.799654:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.799655:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.799656:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.799659:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.799661:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.799663:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.799664:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.799665:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.799666:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.799667:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.799668:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.799669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.799672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.799680:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.799682:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.799683:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.799685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.799685:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.799687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.799693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.799700:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.799703:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.799704:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.799704:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.799705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.799706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.799736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.799743:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.799746:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.799747:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.799749:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.799751:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.799753:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.799754:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.799755:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.799756:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.799757:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.799760:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.799762:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.799765:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.799765:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.799766:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.799768:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.799768:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.799769:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.799769:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.799776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.799783:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.799786:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.799787:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.799787:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.799789:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.799792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.799920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.799927:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.799930:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.799931:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.799933:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.799935:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.799937:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.799938:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.799939:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.799940:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.799941:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.799944:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.799946:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.799948:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.799949:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.799950:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.799951:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.799951:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.799952:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.799952:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.799957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.799964:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.799967:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.799967:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.799969:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.799971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.799972:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677474.800065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677474.800068:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.800070:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:21.0:1619677474.800073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677474.800073:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.800074:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.800076:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677474.800077:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.800079:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.800080:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.800081:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677474.800084:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.800085:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677474.800088:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.800089:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:20.0:1619677474.800090:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677474.800092:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677474.800092:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.800093:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677474.800094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:9.2:1619677474.800098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.800104:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.800107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.800107:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.800108:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.800110:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.800113:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.800318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.800325:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.800328:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.800328:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.800330:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.800332:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.800334:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.800335:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.800336:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.800337:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.800338:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.800342:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.800343:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.800346:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.800347:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.800347:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.800349:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.800349:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.800350:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.800352:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.800352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.800358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.800365:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:25.2:1619677474.800366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.800367:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.800368:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.800369:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.800374:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.800377:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.800377:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.800378:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.800380:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.800383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.800614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.800620:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.800623:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.800624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.800626:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.800629:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.800630:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.800631:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.800633:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.800634:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.800635:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.800638:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.800640:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.800642:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.800643:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.800644:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.800645:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.800646:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.800647:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.800648:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.800649:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.800652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.800654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.800659:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677474.800661:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.2:1619677474.800661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:17.0:1619677474.800661:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.800662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677474.800663:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677474.800663:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:21.0:1619677474.800665:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.800666:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.800667:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:18.0:1619677474.800668:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:21.0:1619677474.800669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677474.800671:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.800672:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.800674:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.800678:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.800679:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.800681:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.800682:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.800683:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.800684:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.800687:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.800689:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.800692:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.800693:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.800693:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.800695:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.800696:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.800697:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.800697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677474.800701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.800709:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.800711:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.800712:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.800714:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.800716:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.800717:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.800749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.800755:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.800757:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.800758:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.800760:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.800763:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.800764:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.800765:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.800766:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.800767:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.800768:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.800771:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.800773:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.800774:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.800774:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.800775:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.800777:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.800777:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.800778:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677474.800779:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.800779:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677474.800783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.800785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.800791:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.800792:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:21.2:1619677474.800793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:18.0:1619677474.800793:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.800793:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.800793:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.800794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.800795:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.800795:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.800796:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677474.800797:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:1.0:1619677474.800798:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.800801:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677474.800802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677474.800804:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677474.800806:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.800808:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677474.800809:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.800810:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.800812:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.800812:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677474.800816:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.800817:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677474.800820:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.800820:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677474.800821:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.800823:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.800823:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677474.800824:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677474.800825:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677474.800826:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:25.2:1619677474.800829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.800831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:6.0:1619677474.800836:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677474.800837:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:17.0:1619677474.800839:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:6.0:1619677474.800839:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.800840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.800840:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:6.0:1619677474.800840:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.800841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000400:00000200:6.0:1619677474.800843:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.800845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.801232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677474.801240:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.801242:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677474.801243:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677474.801246:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.801248:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.801250:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677474.801251:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.801253:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.801254:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.801255:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677474.801258:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.801260:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.801262:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.801263:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677474.801263:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677474.801265:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677474.801265:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.801266:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677474.801266:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.2:1619677474.801280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.801288:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.801291:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.801291:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.801296:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:31.0:1619677474.801298:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.801301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.801386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.801394:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.801397:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.801399:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.801401:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.801401:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.801403:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.801404:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.801405:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.801406:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.801407:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.801410:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.801411:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.801415:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.801416:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.801416:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.801418:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.801418:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.801419:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.801419:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.801425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.801432:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.801435:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.801436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.801437:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.801439:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.801441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.801446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.801453:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.801455:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.801456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.801457:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.801458:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.801458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.801511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.801517:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.801520:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.801521:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.801522:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.801524:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.801526:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.801527:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.801528:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.801529:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.801530:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.801533:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.801535:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.801536:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.801537:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.801538:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.801539:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.801540:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.801541:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.801541:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.801545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.801552:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.801554:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.801555:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.801555:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.801556:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.801558:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.801643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.801649:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.801651:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.801652:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.801653:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.801654:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.801655:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.801656:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.801657:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.801658:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.801659:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:4.2:1619677474.801660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677474.801661:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.801663:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.801664:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.801664:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.801665:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.801666:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.801667:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.801667:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677474.801667:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677474.801668:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:24.0:1619677474.801670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.801670:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:9.2:1619677474.801672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:8.0:1619677474.801672:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.801675:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.801676:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.801677:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96502909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.801679:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.801679:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:8.0:1619677474.801680:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.801681:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.801681:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.801682:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.801683:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.801684:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.801685:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.801686:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677474.801688:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.801691:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.801692:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.801693:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.801694:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.801695:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.801696:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.801698:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.801699:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677474.801701:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677474.801704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677474.801709:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677474.801711:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677474.801712:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.801713:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.801713:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.801714:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.801715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.801716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677474.801716:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.801718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.802165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.802169:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.802172:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.802174:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.802175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.802176:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.802178:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.802179:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.802180:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.802181:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.802182:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.802186:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.802188:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.802191:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.802192:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.802192:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.802194:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.802195:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.802196:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.802196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.802210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.802218:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.802220:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.802221:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.802223:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.802224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.802227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.802314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.802322:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.802325:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.802327:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.802329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.802329:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.802331:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.802332:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.802333:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.802334:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.802335:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.802339:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.802340:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.802343:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.802344:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.802344:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.802346:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.802346:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.802348:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:5.2:1619677474.802348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677474.802349:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.802350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.802352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677474.802354:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677474.802355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677474.802357:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.802358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.0:1619677474.802359:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677474.802360:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.802362:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.802362:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:31.0:1619677474.802363:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677474.802363:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.802363:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.802364:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677474.802366:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.802366:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677474.802366:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.802366:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.802367:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:17.0:1619677474.802367:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.802368:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.802369:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677474.802369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.802372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:17.0:1619677474.802373:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.802375:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:24.0:1619677474.802377:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677474.802377:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.802378:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.802378:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677474.802380:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:17.0:1619677474.802380:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677474.802381:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677474.802381:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677474.802381:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.802382:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677474.802382:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:24.0:1619677474.802383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677474.802383:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.802384:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.802387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.802388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.802393:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.802394:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.802396:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.802396:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677474.802397:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.802397:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.802398:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.802398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.802400:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.802402:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.802522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677474.802530:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.802533:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677474.802533:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677474.802536:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.802539:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.802540:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.802542:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.802543:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.802544:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.802545:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.802549:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.802550:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.802554:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.802554:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.802555:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.802557:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.802557:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677474.802558:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677474.802558:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677474.802563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.802572:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.802574:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.802575:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.802577:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.802577:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.802580:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.802582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.802590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.802592:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.802593:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.802594:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.802595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.802596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.802677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677474.802684:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.802687:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677474.802688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677474.802690:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677474.802693:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.802694:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.802695:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.802697:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.802698:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.802699:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.802702:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.802703:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.802706:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.802707:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.802707:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.802709:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.802709:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.802710:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677474.802711:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677474.802715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.802722:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.802725:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.802726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.802726:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.802728:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.802731:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.802763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.802770:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.802773:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.802774:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.2:1619677474.802774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000400:00000200:10.0:1619677474.802776:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.802778:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.802780:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:10.0:1619677474.802780:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677474.802781:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.802783:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:10.0:1619677474.802783:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.802783:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:10.0:1619677474.802784:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.802785:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677474.802787:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.802789:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.802790:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.802791:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677474.802792:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677474.802792:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:17.0:1619677474.802793:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.802794:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.802795:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677474.802796:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.802796:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677474.802797:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:10.0:1619677474.802798:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:17.0:1619677474.802799:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.802801:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.802801:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.802802:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.802803:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:25.2:1619677474.802804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677474.802804:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.802804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:17.0:1619677474.802805:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677474.802806:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677474.802807:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677474.802808:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.802808:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677474.802809:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677474.802810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:4.2:1619677474.802810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:12.0:1619677474.802811:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:9.2:1619677474.802813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:12.0:1619677474.802814:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.802815:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.802817:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.802818:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.802819:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:13.0:1619677474.802819:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.802820:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:26.0:1619677474.802821:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.802821:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.802822:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:26.0:1619677474.802823:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:25.2:1619677474.802823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677474.802823:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.802824:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.802826:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.802829:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:13.0:1619677474.802830:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677474.802833:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:9.2:1619677474.802833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:13.0:1619677474.802834:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677474.802834:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677474.802835:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677474.802836:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:26.0:1619677474.802839:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.802842:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.802843:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.802843:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.802844:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.802845:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.802884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.802890:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677474.802892:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677474.802893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.802894:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:21.2:1619677474.803152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.803160:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.803162:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.803165:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.803166:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.803167:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.803169:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.803170:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.803171:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.803172:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.803173:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.803177:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.803178:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.803181:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.803181:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.803182:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.803184:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.803184:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.803185:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.803186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.803190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.803197:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.803199:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.803200:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.803201:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.803203:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.803204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.803206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.803212:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.803215:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.803216:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.803216:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.803217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.803217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.803303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.803311:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.803313:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.803315:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.803317:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.803318:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.803318:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.803320:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.803321:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.803322:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.803323:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.803325:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.803327:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.803330:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.803331:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.803331:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.803333:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.803334:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.803334:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.803335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.803340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.803348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.803351:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.803352:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.803352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.803355:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.803357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.803513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677474.803518:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.803521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.803521:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677474.803523:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.803526:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.803528:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:6.0:1619677474.803529:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.803531:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.803532:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.803533:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677474.803536:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.803538:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677474.803541:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.803542:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:6.0:1619677474.803542:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677474.803545:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:6.0:1619677474.803546:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.803546:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.803547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677474.803551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.803558:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.803561:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.803562:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.803562:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.803564:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.803567:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677474.803587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677474.803593:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.803596:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677474.803596:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.803597:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.803598:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677474.803598:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677474.803651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.803658:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.803661:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.803662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.803664:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.803666:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.803667:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.803669:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.803670:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.803671:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677474.803672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677474.803672:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.803675:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.803677:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.803679:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.803680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677474.803680:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.803681:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.803682:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:30.0:1619677474.803683:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:18.0:1619677474.803683:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.803684:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.803684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677474.803685:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.803687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.803688:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677474.803689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:30.0:1619677474.803690:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.803691:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.803693:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.803693:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.803695:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.803698:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.803698:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:30.0:1619677474.803701:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677474.803701:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.803702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.803702:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.803704:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677474.803704:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.803705:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.803706:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.803708:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.803708:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677474.803708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677474.803710:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.803710:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.803710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:9.2:1619677474.803714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677474.803718:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.803721:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:17.0:1619677474.803721:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.803722:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.803722:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.803723:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:17.0:1619677474.803723:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:24.0:1619677474.803724:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.803724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677474.803724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.803727:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.803729:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.804155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.804162:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.804165:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.804167:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.804169:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.804169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.804171:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.804172:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.804173:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.804174:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.804174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677474.804175:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.804178:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.804180:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.804181:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677474.804182:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.804183:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.804183:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.804184:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677474.804185:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677474.804185:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677474.804186:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677474.804186:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.804187:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.804187:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677474.804189:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.804190:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:9.2:1619677474.804190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:10.0:1619677474.804191:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.804193:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.804194:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.804194:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677474.804197:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:10.0:1619677474.804198:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.804199:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.804200:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.804200:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677474.804200:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.804202:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.804203:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.804204:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.804204:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677474.804204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677474.804206:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.804206:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.804207:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.804208:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.804212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.804220:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.804222:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.804223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.804224:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.804226:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.804229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.804316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.804319:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.804322:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.804324:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.804325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.804326:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.804328:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.804329:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.804331:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.804332:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.804333:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.804336:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.804337:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.804340:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.804340:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.804341:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.804343:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.804343:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.804344:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.804345:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.804346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.804349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.804351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.804356:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.804358:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677474.804358:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677474.804359:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.804360:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677474.804360:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.804361:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.804362:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.804364:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.804366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.804444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.804452:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.804455:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.804457:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.804459:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.804460:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.804461:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.804463:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.804464:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.804465:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.804466:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.804469:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.804471:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.804474:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.804475:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.804476:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.804477:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.804478:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.804479:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.804481:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.804482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.804484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.804487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.804493:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.804495:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.804496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.804496:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.804497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.804498:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.804499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.804499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:6.0:1619677474.804499:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.804501:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.805139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.805146:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.805148:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.805149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.805151:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.805154:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.805155:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677474.805157:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.805158:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.805159:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.805160:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677474.805163:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.805165:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.805166:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.805167:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.805167:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.805169:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.805169:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.805170:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.805170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677474.805174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.805181:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.805182:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.805183:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.805184:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.805185:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.805186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.805193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.805199:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.805201:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677474.805201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:24.0:1619677474.805202:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.805203:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.805203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.805204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677474.805207:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.805210:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677474.805211:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677474.805213:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677474.805215:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.805216:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677474.805218:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.805219:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.805220:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.805221:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.805225:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677474.805226:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.805229:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.805230:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677474.805230:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.805232:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677474.805233:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.805233:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677474.805234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677474.805238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.805244:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.805247:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.805248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.805248:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.805250:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.805253:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.805563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.805570:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.805573:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.805574:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.805575:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.805578:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.805579:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.805580:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.805581:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.805582:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.805583:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.805587:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.805589:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.805592:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.805592:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.805593:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.805595:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.805595:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.805596:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.805596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.805601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.805610:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.805613:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.805614:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.805614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.805617:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.805619:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.805621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.805628:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.805631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.805631:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.805632:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.805633:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.805633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.805800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.805808:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.805811:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.805812:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.805813:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.805817:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.805818:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.805820:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.805821:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.805822:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.805823:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.805826:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.805828:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.805831:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.805832:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.805832:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.805834:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.805835:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.805835:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.805836:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.805840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.805847:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.805850:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.805851:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.805851:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.805853:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.805856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.806001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.806009:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.806012:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.806015:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.806017:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.806018:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.806019:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.806021:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.806022:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677474.806022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:31.0:1619677474.806023:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.806024:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.806027:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.806029:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.806030:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.806031:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677474.806032:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.806033:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.806033:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677474.806033:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.806035:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677474.806035:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.806036:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.806037:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.806037:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:30.0:1619677474.806037:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.806038:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.806039:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.806039:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.806040:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.806041:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:25.2:1619677474.806041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677474.806044:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.806045:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.806048:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.806049:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.806049:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.806049:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.806051:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.806051:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677474.806051:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.806052:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.806052:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.806053:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.806054:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.806055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677474.806055:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.806056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.806057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.806060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.806063:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.806065:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.806066:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.806067:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.806068:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677474.806069:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.806070:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.806071:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677474.806071:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.806073:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.806171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.806177:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.806179:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.806181:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.806182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.806184:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.806185:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.806187:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.806188:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.806189:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.806190:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.806193:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.806195:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.806196:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.806197:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.806198:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.806200:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.806200:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.806201:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.806202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.806223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.806231:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.806233:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.806234:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.806236:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.806237:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.806238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.806581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.806588:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.806591:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.806592:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.806594:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677474.806597:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.806598:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.806599:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.806601:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.806602:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.806603:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.806606:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677474.806607:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.806610:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.806610:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.806611:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.806613:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.806613:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.806614:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.806615:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.806619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.806627:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.806630:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.806630:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.806632:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.806634:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.806636:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.806639:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.806640:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.806641:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.806642:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.806643:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.806872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.806878:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.806880:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.806882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.806883:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.806886:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.806887:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.806888:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.806890:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.806891:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.806892:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.806895:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.806897:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.806898:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.806899:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.806899:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.806901:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.806901:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.806902:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.806903:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.806907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.806914:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.806917:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.806917:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.806918:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.806919:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.806920:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.806929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.806936:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.806938:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.806939:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.806940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.806940:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.806940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.807009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.807015:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.807018:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.807019:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.807021:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.807024:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.807026:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.807027:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.807028:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.807029:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.807030:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.807034:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.807036:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.807038:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.807039:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.807040:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.807041:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.807042:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.807043:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.807043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.807048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.807055:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.807058:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.807059:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.807059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.807061:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.807063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.807067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.807073:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.807076:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.807077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.807077:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.807078:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.807079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.807679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.807687:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.807690:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.807692:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.807693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:30.0:1619677474.807694:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.807695:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.807697:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.807698:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.807698:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:31.0:1619677474.807699:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.807700:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.807702:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.807702:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.807704:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.807704:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:31.0:1619677474.807706:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677474.807707:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.807709:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.807709:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.807711:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.807712:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677474.807713:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677474.807713:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677474.807714:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:21.0:1619677474.807714:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.807715:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.807716:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.807717:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.807718:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000400:00000200:21.0:1619677474.807718:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.807719:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677474.807720:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677474.807723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.807723:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.807723:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.807724:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:21.0:1619677474.807726:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:21.0:1619677474.807727:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.807727:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.807728:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677474.807731:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677474.807734:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677474.807734:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000400:00000200:22.0:1619677474.807735:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.807735:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677474.807736:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.807738:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.807738:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.807741:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.807741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.807742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.807746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.807754:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.807756:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.807757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.807757:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.807759:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.807759:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.808019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.808027:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.808030:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.808032:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.808034:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.808035:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.808037:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.808038:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.808039:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.808041:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.808041:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.808045:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.808047:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.808050:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.808051:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.808052:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.808053:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.808054:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.808055:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.808055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.808060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.808068:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.808070:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.808071:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.808073:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.808074:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.808076:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.808418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.808425:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.808428:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.808429:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.808431:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.808434:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.808435:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.808437:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.808438:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.808439:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.808440:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.808443:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.808445:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.808448:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.808448:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.808449:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.808451:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.808451:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.808452:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.808452:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.808457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.808466:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.808468:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.808469:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.808471:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.808472:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.808474:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.808559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.808566:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.808569:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.808569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.808572:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.808573:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.808574:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.808576:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.808577:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.808578:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.808579:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.808582:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.808583:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.808585:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.808586:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.808586:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.808588:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.808589:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.808590:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.808590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.808595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.808601:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.808604:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.808604:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.808605:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.808606:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.808607:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.808611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.808619:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.808622:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.808624:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.808627:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.808627:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.808628:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:4.2:1619677474.808628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000400:00000200:31.0:1619677474.808630:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.808631:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.808632:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.808633:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.808636:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.808636:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.808637:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000800:00000200:31.0:1619677474.808638:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.808639:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.808640:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.808641:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.808642:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:30.0:1619677474.808642:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.808643:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:30.0:1619677474.808643:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.808645:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.808646:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.808647:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.808647:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.808648:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.808649:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.808649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.808650:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677474.808651:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.808652:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.808652:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.808655:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.808655:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:30.0:1619677474.808656:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.808657:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.808658:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677474.808658:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:30.0:1619677474.808659:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000400:00000200:24.0:1619677474.808659:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.808659:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.808660:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.808660:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000400:00000200:24.0:1619677474.808661:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.808664:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.2:1619677474.808664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:9.2:1619677474.808670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.0:1619677474.808672:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.808675:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.808676:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.808676:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677474.808678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.808678:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.808679:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.808679:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.808680:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.808681:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.808681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677474.808682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.808931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.808936:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.808938:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.808940:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.808941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.808943:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.808944:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.808946:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.808947:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.808948:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.808949:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.808952:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.808954:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.808956:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.808957:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.808957:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.808959:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.808959:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.808960:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.808962:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.808962:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677474.808965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.808968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.808973:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.808974:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.808975:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:20.0:1619677474.808975:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.808976:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.808976:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.808977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.808977:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.808978:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677474.808979:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.809016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677474.809020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:10.0:1619677474.809024:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:18.0:1619677474.809026:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677474.809027:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.809028:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677474.809029:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.809030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:10.0:1619677474.809031:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.809033:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.809035:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.809035:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.809037:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677474.809038:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677474.809039:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.809040:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.809041:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.809042:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.809042:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.809044:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677474.809044:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.809045:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677474.809049:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.809051:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.809052:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677474.809054:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677474.809055:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.809056:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677474.809057:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677474.809057:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.809057:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.809058:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:18.0:1619677474.809059:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677474.809059:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.809060:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:10.0:1619677474.809060:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:18.0:1619677474.809061:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:10.0:1619677474.809061:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.809062:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.809062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677474.809064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:25.2:1619677474.809066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677474.809072:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.809074:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.809075:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.809075:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:8.0:1619677474.809075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.809077:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:24.0:1619677474.809078:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.809078:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.809079:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.809080:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677474.809081:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.809083:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.809107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.809114:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.809117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.809117:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.809119:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.809122:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.809124:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677474.809125:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.809127:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.809128:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.809129:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677474.809132:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.809134:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677474.809137:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.809138:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677474.809138:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677474.809140:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677474.809140:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.809141:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.809142:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.809146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.809155:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.809157:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.809158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.809159:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.809161:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.809164:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.809182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.809187:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.809190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.809190:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677474.809192:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.809194:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.809196:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.809196:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.809198:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.809199:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.809200:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.809203:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.809205:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.809208:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.809208:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.809209:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.809210:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.809211:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.809212:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.809212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.809216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.809224:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.809227:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.809227:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.809229:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.809230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.809231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.809235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.809243:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.809246:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.2:1619677474.809248:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.0:1619677474.809249:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.809250:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.809251:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.809252:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677474.809254:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.809257:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.809258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.809260:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.809262:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.809264:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.809265:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.809267:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.809268:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.809269:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.809273:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.809274:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.809278:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.809279:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.809279:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.809281:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.809282:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.809283:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.809283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.809287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.809294:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.809301:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.809302:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.809302:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.809304:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.809308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.809313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.809319:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.809323:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.809323:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.809324:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.809325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.809326:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.809403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.809411:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.809414:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.809416:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.809418:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.809418:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.809420:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.809421:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.809422:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.809423:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.809424:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.809428:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.809429:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.809432:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.809433:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.809433:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.809435:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.809435:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.809436:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.809437:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.809440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.809447:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.809449:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.809450:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.809451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.809452:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.809454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.809755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.809762:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.809765:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.809767:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.809769:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.809770:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.809772:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.809773:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.809774:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.809775:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.809776:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.809779:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.809780:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.809784:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.809784:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.809785:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.809786:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.809787:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.809788:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.809788:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.809793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.809800:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.809803:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.809804:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.809804:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.809807:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.809810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.809966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677474.809971:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.809973:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.809975:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.809976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.809978:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.809979:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.809981:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.809982:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.809983:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.809984:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.809987:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.809989:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.809991:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.809992:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.809993:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.809994:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.809995:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.809995:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677474.809996:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677474.810000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.810008:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.810010:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.810011:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.810011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.810013:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.810015:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.810110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677474.810116:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.810119:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677474.810120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677474.810122:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.810124:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.810126:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677474.810127:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.810129:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.810129:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.810131:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677474.810133:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.810135:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677474.810137:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.810138:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677474.810139:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677474.810141:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677474.810141:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.810142:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677474.810142:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677474.810147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.810155:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.810157:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.810158:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.810159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.810160:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.810163:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.810366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.810374:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.810377:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.810378:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.810379:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.810382:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.810384:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.810385:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.810386:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.810387:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.810388:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.810392:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.810394:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.810397:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.810397:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.810398:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.810400:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.810400:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.810401:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.810401:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.810406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.810415:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.810417:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.810418:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.810418:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.810421:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.810424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.810608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.810615:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.810618:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.810619:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.810621:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.810624:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.810625:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.810627:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.810628:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.810629:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.810630:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.810632:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.810634:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.810637:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.810637:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.810638:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.810640:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.810640:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.810641:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.810641:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.810645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.810653:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.810655:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.810656:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.810656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.810659:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.810661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.810664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.810671:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.810674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.810674:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.810675:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.810676:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.810676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.810725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.810734:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.810736:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.810739:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.810741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.810742:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.810743:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.810744:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.810746:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.810747:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.810748:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.810751:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.810753:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.810756:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.810757:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.810757:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.810759:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.810760:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.810761:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.810763:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.810763:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.810766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.810769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.810773:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.810776:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677474.810777:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677474.810777:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.810778:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.810779:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.810779:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.810780:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.810781:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.810781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.810785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.810793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.810795:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.810796:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.810797:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.810797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.810798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.810839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.810845:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.810848:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.810849:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.810850:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.810853:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.810855:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.810856:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.810858:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.810859:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.810860:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.810863:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.810864:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.810866:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.810867:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.810868:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.810869:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.810870:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.810871:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.810871:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.810875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.810882:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.810884:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.810885:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.810885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.810886:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.810888:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.811446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.811454:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.811457:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.811459:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.811462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.811463:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.811465:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.811466:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.811467:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.811468:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.811469:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.811472:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.811474:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.811477:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.811478:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.811479:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.811480:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.811481:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.811482:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.811484:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.811484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.811488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.811491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.811496:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.811499:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.811499:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.811500:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.811500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.811502:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.811502:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677474.811502:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.811504:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.811505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.811784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.811790:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.811793:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.811794:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.811796:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.811798:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.811799:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.811801:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.811802:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.811803:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.811804:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.811808:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.811809:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.811812:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.811813:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.811813:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.811815:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.811815:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.811816:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.811816:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.811821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.811828:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.811831:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.811832:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.811832:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.811834:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.811836:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.812052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.812058:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.812060:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.812063:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.812063:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.812065:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.812066:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.812068:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.812069:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.812070:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.812071:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.812075:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.812076:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.812079:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.812080:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.812081:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.812082:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.812083:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.812083:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.812084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.812088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.812095:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.812098:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.812099:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.812099:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.812101:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.812104:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.812242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.812249:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.812252:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.812254:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.812256:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.812256:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.812258:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.812259:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.812261:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.812261:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.812263:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.812266:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.812268:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.812270:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.812271:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.812271:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.812273:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.812274:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.812274:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.812275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.812279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.812285:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.812288:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.812289:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.812290:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.812292:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.812293:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.812320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.812326:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.812329:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.812330:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.812331:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.812334:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.812335:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.812336:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.812338:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.812339:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.812340:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.812343:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.812345:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.812347:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.812347:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.812348:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.812349:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.812350:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.812351:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677474.812352:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.812353:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677474.812356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.812359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.812365:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.812366:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.812366:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000400:00000200:18.0:1619677474.812367:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.812367:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677474.812367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.812368:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.812369:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.812369:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677474.812370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.812992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.813000:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.813002:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.813005:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.813006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.813007:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.813008:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.813010:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.813011:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.813012:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.813013:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.813016:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.813018:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.813020:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.813021:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.813022:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.813023:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.813024:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.813025:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.813025:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.813029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.813036:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.813038:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.813039:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.813041:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.813041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:30.0:1619677474.813042:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.813043:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.0:1619677474.813048:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.813051:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677474.813052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677474.813054:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.813056:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.813058:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677474.813059:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.813060:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.813061:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.813062:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677474.813065:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.813067:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677474.813070:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.813070:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677474.813071:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677474.813073:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677474.813073:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.813074:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.813074:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677474.813089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.813098:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.813100:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.813101:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.813101:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.813103:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.813106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.813112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.813119:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.813122:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.813123:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.813123:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.813124:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.813124:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.813313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.813320:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.813323:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.813324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.813326:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.813329:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.813330:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.813331:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.813333:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.813334:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.813334:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.813337:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.813339:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.813342:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.813342:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.813343:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.813345:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.813345:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.813347:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.813348:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.813348:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.813352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.813354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.813360:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.813361:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.813362:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.813363:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.813363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.813364:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.813365:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.813365:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677474.813366:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.813368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.813503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.813511:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.813514:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.813517:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.813519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.813519:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.813520:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.813521:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.813523:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.813524:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.813525:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.813528:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.813530:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.813533:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.813534:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.813534:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.813536:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.813536:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.813537:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.813538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.813542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.813549:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.813552:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.813553:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.813553:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.813555:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.813558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.813737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.813745:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.813748:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.813750:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.813751:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.813753:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.813754:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.813755:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.813757:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.813758:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.813759:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.813762:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.813763:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.813765:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.813766:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.813767:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.813768:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.813769:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.813770:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.813770:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.813774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.813781:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.813783:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.813784:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.813786:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.813787:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.813788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.813792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.813796:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.813799:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.813801:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.813802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.813804:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.813805:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.813807:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.813808:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.813810:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.813811:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.813814:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.813815:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.813818:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.813819:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.813819:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.813821:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.813822:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.813823:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.813825:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.813825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.813827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.813830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677474.813834:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677474.813836:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677474.813837:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.813838:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.813838:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:8.0:1619677474.813838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.813839:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.813839:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:24.0:1619677474.813840:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.813843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.814610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.814618:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.814620:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.814621:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.814623:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.814625:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.814627:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.814628:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.814629:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.814630:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.814631:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.814634:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.814636:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.814639:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.814640:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.814640:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.814642:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.814642:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.814644:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.814645:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.814645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.814649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.814651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.814657:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.814658:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.814660:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.814661:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.814661:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677474.814661:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.814661:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.814663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.814663:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.814666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.814694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.814700:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.814702:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.814704:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.814705:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.814707:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.814709:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.814710:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.814711:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.814712:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.814713:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.2:1619677474.814713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000400:00000200:21.0:1619677474.814717:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.814718:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.814720:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.814721:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.814722:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.814722:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.814722:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.814724:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.814724:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677474.814724:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.115@o2ib7 : GET - for me
00000800:00000200:21.0:1619677474.814725:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.814726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:20.0:1619677474.814726:0:19934:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.115@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:21.0:1619677474.814727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.814728:0:19934:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.115@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:20.0:1619677474.814730:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.814730:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677474.814730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:20.0:1619677474.814732:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677474.814735:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : REPLY try# 0
00000800:00000200:20.0:1619677474.814737:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.814738:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.814738:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677474.814739:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.814740:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (22)++
00000800:00000200:17.0:1619677474.814740:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:20.0:1619677474.814741:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (23)--
00000400:00000200:17.0:1619677474.814741:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.814741:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.814742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:17.0:1619677474.814743:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.814746:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.814747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.814755:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.814757:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.814758:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.814759:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: REPLY: OK
00000800:00000200:20.0:1619677474.814760:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.814762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.815074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.815080:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.815083:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.815084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.815086:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.815089:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.815090:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.815091:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.815093:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.815094:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.815095:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.815099:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.815100:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.815103:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.815104:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.815105:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.815107:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.815107:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.815108:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.815109:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.815112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.815120:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.815123:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.815124:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.815125:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.815128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.815129:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.815237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.815245:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.815248:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.815250:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.815252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.815252:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.815254:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.815255:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.815256:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.815257:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.815258:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.815262:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.815263:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.815267:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.815267:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.815268:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.815270:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.815270:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.815271:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.815272:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.815276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.815284:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.815286:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.815287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.815288:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.815290:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.815292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.815297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.815305:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.815307:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.815308:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.815309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.815310:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.815310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.815352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.815360:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.815362:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.815365:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.815367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.815367:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.815369:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677474.815370:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.815371:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.815372:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.815373:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:12.0:1619677474.815377:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.815378:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677474.815382:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.815383:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:12.0:1619677474.815383:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:12.0:1619677474.815385:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:12.0:1619677474.815386:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.815387:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677474.815388:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677474.815389:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677474.815392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677474.815394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:4.2:1619677474.815399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677474.815400:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:13.0:1619677474.815402:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:1.0:1619677474.815403:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.815404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677474.815404:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:1.0:1619677474.815404:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677474.815405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.815406:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:1.0:1619677474.815406:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.815407:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.815409:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677474.815409:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677474.815410:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.815412:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.815415:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.815417:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.815418:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.815419:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.815420:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.815421:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.815425:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.815426:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.815429:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.815430:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.815430:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.815432:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.815432:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.815433:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.815433:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.815438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.815446:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.815449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.815449:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.815450:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.815452:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.815455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.816437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.816443:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.816446:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.816446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.816448:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.816451:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.816452:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.816453:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.816455:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.816456:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.816457:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.816460:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.816462:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.816464:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.816465:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.816466:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.816467:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.816468:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.816468:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.816469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.816474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.816482:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.816485:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.816486:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.816486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.816488:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.816490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.816492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.816499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.816501:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.816502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.816503:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.816504:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.816504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.816958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.816968:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.816972:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.816975:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.816976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.816979:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.816981:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.816983:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.816986:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.816987:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.816989:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.816995:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.816997:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.817000:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.817002:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.817003:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.817005:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.817006:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.817008:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.817010:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.817011:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.817012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.817016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.817021:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.817022:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.817023:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.817025:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.817026:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.817027:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.817028:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.817028:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.817029:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.817029:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.817030:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.817031:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677474.817031:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.817032:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.817033:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.817136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.817143:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.817146:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.817147:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.817149:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.817151:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.817153:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.817154:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.817155:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.817157:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.817158:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.817161:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.817163:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.817164:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.817165:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.817166:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.817167:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.817168:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.817169:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.817169:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.817173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.817180:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.817182:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.817183:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.817184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.817185:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.817187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.817190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677474.817195:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:9.2:1619677474.817195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:22.0:1619677474.817198:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677474.817199:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677474.817201:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.817202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000400:00000200:22.0:1619677474.817204:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.817205:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:18.0:1619677474.817206:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:22.0:1619677474.817207:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677474.817208:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.817208:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000400:00000200:22.0:1619677474.817209:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.817209:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677474.817209:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.817209:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.817210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.817210:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000400:00000200:22.0:1619677474.817211:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.817211:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677474.817212:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.817214:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.817215:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.817218:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:2.0:1619677474.817218:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.817219:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:22.0:1619677474.817220:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.817221:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677474.817221:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:2.0:1619677474.817221:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.817222:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.817223:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.817223:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.817224:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000400:00000200:2.0:1619677474.817224:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677474.817225:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677474.817225:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:2.0:1619677474.817227:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677474.817229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.817229:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.817231:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.817233:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.817233:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.817235:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.817236:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.817237:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.817238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677474.817239:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.817240:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.817241:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.817241:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.817242:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.817242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.817246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:18.0:1619677474.817247:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.817249:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.817249:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.817249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000400:00000200:18.0:1619677474.817250:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.817252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.817252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677474.817253:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.817254:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.817255:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.817255:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.817257:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.817257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.817369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.817376:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.817379:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.817381:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.817384:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.817385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.817385:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.817386:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.817388:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.817388:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.817389:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.817392:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.817394:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.817397:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.817398:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.817399:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.817400:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.817401:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.817402:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.817402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.817407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.817415:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.817418:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.817419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.817419:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.817421:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.817424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.817663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.817673:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.817677:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.817680:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.817680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.817684:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.817686:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.817688:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.817690:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.817692:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.817693:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.817698:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.817701:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.817704:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.817705:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.817706:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.817708:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.817709:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.817710:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.817711:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.817714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.817720:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.817723:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.817724:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.817725:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.817727:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.817729:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.818214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.818221:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.818223:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677474.818224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677474.818225:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.818227:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.818227:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.818228:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.818230:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.818231:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.818232:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.818232:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.818232:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.818234:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.818235:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677474.818236:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.818237:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.818238:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.818240:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.818240:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:10.0:1619677474.818240:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.818241:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.818242:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:10.0:1619677474.818242:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677474.818243:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.818243:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677474.818243:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.818244:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677474.818244:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.818245:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.818246:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677474.818248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677474.818249:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.818250:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.818253:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.818254:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.818254:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677474.818255:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:10.0:1619677474.818256:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:30.0:1619677474.818257:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.818257:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.818257:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.818257:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.818258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677474.818259:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.818261:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.818262:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.818262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.818270:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.818272:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.818273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.818274:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.818276:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.818278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.818350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.818354:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.818358:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.818360:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.818361:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.818362:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.818364:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.818365:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.818366:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.818367:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.818368:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.818372:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.818373:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.818375:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.818376:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.818377:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.818378:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.818379:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.818379:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.818380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.818385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.818393:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.818395:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.818396:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.818397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.818398:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.818401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.818990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.818998:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.819001:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.819002:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.819004:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.819006:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.819008:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.819009:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.819010:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.819011:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.819012:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.819015:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.819017:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.819019:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.819020:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.819021:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.819022:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.819023:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.819023:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.819024:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.819028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.819035:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.819038:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.819038:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.819040:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.819042:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.819043:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.819168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.819175:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.819178:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.819180:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.819182:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.819183:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.819183:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.819185:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.819186:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.819187:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.819188:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.819191:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.819192:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.819195:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.819196:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.819196:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.819198:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.819199:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.819200:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.819202:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.819202:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.819204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.819208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.819211:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.819214:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.819215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.819216:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.819216:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.819218:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.819219:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.819220:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.819220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.819221:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.819225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.819232:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.819234:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.819235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.819235:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.819237:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.819237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.819405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.819413:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.819416:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.819417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.819418:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.819421:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.819423:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.819424:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.819425:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.819426:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.819427:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.819430:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.819432:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.819435:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.819435:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.819436:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.819438:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.819438:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.819439:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.819439:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.819444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.819452:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.819455:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.819455:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.819456:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.819458:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.819460:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.820222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.820229:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.820232:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.820233:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.820234:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.820237:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.820238:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.820240:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.820241:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.820242:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.820243:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.820246:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.820247:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.820250:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.820251:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.820251:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.820253:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.820254:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.820255:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.820256:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.820256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.820259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.820262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.820266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.2:1619677474.820268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677474.820269:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.820269:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.820270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.820270:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.820272:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.820272:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.820272:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.820274:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.820275:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.820276:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.820279:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.820281:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.820283:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.820283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.820285:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.820286:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.820287:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.820288:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.820289:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.820292:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.820297:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.820299:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.820300:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.820301:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.820302:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.820303:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.820304:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.820304:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.820308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.820315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.820317:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.820318:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.820319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.820320:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.820323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.820533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.820538:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.820541:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.820543:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.820544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.820545:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.820546:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.820548:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.820549:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.820550:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.820551:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.820554:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.820556:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.820558:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.820559:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.820560:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.820561:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.820562:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.820562:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.820563:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.820567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.820576:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.820578:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.820579:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.820580:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.820581:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.820584:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.821154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.821161:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:5.2:1619677474.821164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:2.0:1619677474.821164:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.821165:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.821166:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.821168:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.821169:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677474.821169:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.821171:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.821172:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:2.0:1619677474.821172:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.821173:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:2.0:1619677474.821174:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.821175:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677474.821176:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.821179:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.821181:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:17.0:1619677474.821182:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.821183:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677474.821184:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.821184:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000400:00000200:17.0:1619677474.821185:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.821185:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:17.0:1619677474.821186:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.821187:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.821187:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.821188:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677474.821188:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.821189:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.821190:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.821190:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677474.821191:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.821192:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.2:1619677474.821193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.821195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677474.821195:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.821196:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.821197:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.821199:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.821200:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:18.0:1619677474.821201:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677474.821202:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.821202:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677474.821203:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:17.0:1619677474.821203:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.821203:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.821203:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.821204:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677474.821204:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.821205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677474.821205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677474.821205:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.821206:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:5.2:1619677474.821208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.821208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677474.821213:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.821214:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.821216:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.821216:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677474.821217:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.821217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.821217:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.821219:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.821219:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.821221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.821264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.821272:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.821275:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.821277:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.821279:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.821279:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.821281:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.821282:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.821284:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.821285:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.821286:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.821289:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.821291:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.821299:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.821299:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.821300:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.821302:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.821302:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.821303:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.821304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.821319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.821327:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.821330:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.821330:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.821331:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.821333:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.821335:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.821392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.821396:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.821398:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.821400:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.821401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.821403:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.821405:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.821406:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.821407:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.821408:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.821409:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.821413:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.821415:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.821418:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.821418:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.821419:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.821421:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.821421:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.821422:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.821423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.821427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.821434:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.821437:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.821438:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.821440:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.821441:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.821443:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.821871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.821878:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.821882:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.821883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.821885:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.821887:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.821889:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.821890:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.821892:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.821893:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.821894:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.821897:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.821899:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.821901:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.821902:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.821902:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.821904:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.821904:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.821906:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.821906:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.821910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.821917:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.821919:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.821920:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.821920:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.821921:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.821922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.822105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.822111:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.822114:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.822115:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.822116:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.822119:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.822120:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.822122:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.822123:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.822124:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.822125:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.822129:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.822130:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.822133:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.822134:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.822134:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.822136:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.822137:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.822137:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.822138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.822143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.822151:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.822153:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.822154:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.822156:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.822157:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.822158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.822195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.822203:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.822206:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.822208:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.822210:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.822211:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.822213:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.822214:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.822215:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.822216:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.822218:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.822221:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.822222:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.822225:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.822226:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.822226:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.822228:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.822229:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.822230:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.822232:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.822232:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.822234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.822238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.822240:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.822242:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.822243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.822243:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.822245:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.822246:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677474.822247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.822248:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.822249:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.822250:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.822253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.822259:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.822262:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.822263:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.822263:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.822263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.822264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.822876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.822884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.822887:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.822889:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.822891:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.822892:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.822892:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.822894:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.822895:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.822896:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.822897:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.822900:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.822902:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.822904:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.822905:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.822905:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.822907:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.822907:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.822908:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.822909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.822913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.822920:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.822923:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.822924:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.822925:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.822927:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.822928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.822959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677474.822967:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677474.822970:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677474.822971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677474.822974:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.822976:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.822978:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677474.822979:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.822981:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.822982:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.822983:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677474.822986:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.822988:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.822991:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.822992:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677474.822992:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677474.822994:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677474.822994:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677474.822995:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (22)++
00000800:00000200:10.0:1619677474.822996:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:10.0:1619677474.822997:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:25.2:1619677474.823000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677474.823002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (21)++
00000800:00000200:20.0:1619677474.823008:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677474.823010:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000800:00000200:20.0:1619677474.823011:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:26.0:1619677474.823012:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000400:00000200:20.0:1619677474.823012:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.823013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:20.0:1619677474.823013:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.823013:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:21.0:1619677474.823015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.823016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.823025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677474.823028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:4.2:1619677474.823029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:26.0:1619677474.823032:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.823035:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:21.0:1619677474.823035:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:31.0:1619677474.823036:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:10.0:1619677474.823036:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.0:1619677474.823037:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:26.0:1619677474.823038:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.823039:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:21.0:1619677474.823040:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.823040:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:26.0:1619677474.823041:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.823042:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.823043:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:26.0:1619677474.823043:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:21.0:1619677474.823043:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.823044:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.823044:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.823045:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.823046:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.823046:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.823047:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.823048:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.823048:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.823049:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:21.0:1619677474.823049:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.823050:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.823050:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.823051:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.823052:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.823053:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.823055:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677474.823056:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.823058:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.0:1619677474.823059:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.823061:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.823061:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.823062:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:21.0:1619677474.823062:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.823063:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.823063:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677474.823063:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.823064:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677474.823065:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.823065:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677474.823066:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.823066:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.823066:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.0:1619677474.823066:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.823066:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:31.0:1619677474.823067:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.823067:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:31.0:1619677474.823068:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.823068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:31.0:1619677474.823069:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.823070:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.823071:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.823071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.823072:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.823072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:25.2:1619677474.823075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.823080:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677474.823081:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:8.0:1619677474.823082:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.823083:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677474.823083:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:8.0:1619677474.823083:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.823085:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.823085:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:8.0:1619677474.823086:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.823086:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.823087:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.823087:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.823089:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677474.823089:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.823091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.823092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.823092:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:9.2:1619677474.823093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677474.823093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.823096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.823100:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.823102:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.823103:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677474.823103:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.823103:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.823104:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.823105:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677474.823106:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.823107:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.823107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.823108:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.823109:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.824311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.824319:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.824322:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.824324:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.824325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677474.824327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.824327:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.824329:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.824330:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.824331:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.824332:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677474.824333:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.824333:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.824335:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.824336:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677474.824337:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.824340:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:8.0:1619677474.824341:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.824342:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.824343:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.824343:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:8.0:1619677474.824344:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.824345:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.824345:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677474.824345:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.824346:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677474.824346:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96502815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.824347:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:8.0:1619677474.824347:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.824348:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.824349:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677474.824351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:8.0:1619677474.824353:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.824355:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.824358:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.824359:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677474.824359:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677474.824360:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.824360:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.824360:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:24.0:1619677474.824362:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.824362:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.824363:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.824364:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.824364:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.824365:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677474.824365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.824370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.824378:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.824380:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.824381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.824382:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.824384:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.824386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.825140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.825148:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.825151:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.825153:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.825154:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.825155:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.825157:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.825158:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.825159:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.825161:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.825161:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.825165:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.825167:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.825170:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.825171:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.825171:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.825173:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.825173:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.825174:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.825175:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.825180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.825187:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.825189:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.825190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.825191:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.825193:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.825196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.825340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.825346:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.825349:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.825350:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.825352:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.825354:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.825355:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.825356:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.825357:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.825359:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.825360:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.825362:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.825364:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.825366:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.825366:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.825367:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.825369:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.825369:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.825370:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.825371:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.825372:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.825376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.825379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.825385:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.825386:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.825386:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.825386:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.825387:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.825388:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.825388:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.825389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.825389:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677474.825390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:5.2:1619677474.825442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677474.825448:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.825451:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677474.825452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677474.825454:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677474.825456:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.825458:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677474.825459:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.825461:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.825462:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.825462:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677474.825466:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677474.825467:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677474.825470:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.825471:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677474.825472:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677474.825474:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677474.825474:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677474.825475:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677474.825475:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677474.825479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.825487:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.825489:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.825490:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.825491:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.825493:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: ACK: OK
00000800:00000200:10.0:1619677474.825497:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677474.825611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.825619:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.825621:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.825622:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.825624:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.825626:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.825628:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.825629:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.825630:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.825631:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.825632:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.825635:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.825637:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.825640:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.825640:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.825641:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.825643:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.825643:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.825643:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.825644:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.825649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.825657:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.825659:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.825660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.825661:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.825663:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.825665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.825927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.825935:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.825940:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.825941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.825945:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.825949:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.825951:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677474.825953:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.825955:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.825957:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.825958:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677474.825963:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.825966:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.825969:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.825970:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.825970:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.825972:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.825973:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.825974:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.825975:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.825978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.825985:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.825989:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.825990:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.825991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.825993:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.825995:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:5.2:1619677474.826018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.826025:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.826027:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.826028:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.826031:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.826033:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.826035:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.826036:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.826037:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.826038:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.826039:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.826042:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.826044:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.826046:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.826047:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.826048:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.826050:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.826050:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.826051:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.826051:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.826056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.826063:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.826066:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.826067:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.826067:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.826069:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.826071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.826523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.826532:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.826535:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.826538:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.826540:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.826541:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.826542:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.826543:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.826545:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.826546:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.826547:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.826550:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.826552:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.826554:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.826555:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.826555:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.826557:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.826557:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.826558:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.826559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.826563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.826570:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.826573:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.826574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.826574:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.826576:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.826578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.826742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.826748:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.826751:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.826752:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.826754:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.826756:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.826757:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.826759:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.826760:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.826761:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.826762:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.826766:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.826768:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.826771:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.826772:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.826773:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.826775:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.826775:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.826776:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.826777:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.826781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.826788:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.826791:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.826792:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.826794:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.826794:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.826797:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.826907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.826911:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.826913:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.826915:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.826917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.826917:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.826919:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.826920:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.826921:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.826922:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.826923:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.826926:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.826928:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.826931:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.826931:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.826932:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.826934:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.826935:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.826936:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.826936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.826940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.826942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:24.0:1619677474.826946:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.826948:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677474.826949:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.826949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.826950:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.826952:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:24.0:1619677474.826952:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:30.0:1619677474.826955:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.826955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.826956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.826957:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.826959:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.826960:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.826961:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.826962:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.826963:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.826967:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677474.826967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677474.826968:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.826971:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.826972:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.826972:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677474.826973:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.826974:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.826975:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677474.826975:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677474.826977:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000400:00000200:21.0:1619677474.826977:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.826977:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677474.826978:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.826979:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.826980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677474.826980:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.826981:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.826982:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677474.826984:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:21.0:1619677474.826988:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.826989:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.826989:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677474.826990:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677474.826992:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.826992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.826992:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.826993:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.826994:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.826995:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677474.826995:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.826996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.826996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677474.826996:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.826998:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.826998:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.826999:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.826999:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.827001:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.827002:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.827003:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.827004:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.827005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677474.827006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:8.2:1619677474.827007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.827009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.827012:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:22.0:1619677474.827013:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.827014:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.827015:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677474.827016:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.827016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.827016:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677474.827017:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.827017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.827018:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:22.0:1619677474.827019:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.827019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:17.0:1619677474.827019:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.827020:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.827021:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.827024:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.827027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.827035:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.827037:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.827038:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.827039:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.827039:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.827040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.827147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.827154:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.827157:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.827157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.827160:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.827162:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.827163:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.827165:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.827166:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.827167:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.827168:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.827171:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.827172:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.827174:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.827175:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.827175:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.827176:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.827177:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.827178:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.827178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.827182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.827189:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.827191:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.827192:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.827193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.827194:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.827195:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.827208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.827215:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.827218:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.827218:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.827219:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.827219:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.827220:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:25.2:1619677474.827309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.827316:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.827319:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.827320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.827320:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.827321:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.827321:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.827457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.827465:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.827468:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.827470:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.827472:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.827473:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.827474:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.827476:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.827477:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.827478:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.827479:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.827482:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.827484:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.827486:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.827486:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.827487:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.827489:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.827489:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.827491:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.827492:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.827492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.827495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.827498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.827503:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.827505:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.827505:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.827506:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.827507:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677474.827507:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.827508:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.827509:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.827510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.827511:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.827512:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.827512:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.827512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.827513:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.827514:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.828092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.828101:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.828103:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.828105:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.828106:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.828107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.828108:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.828109:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.828110:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.828111:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.828112:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.828114:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.828116:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.828117:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.828118:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.828118:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.828120:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.828120:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.828121:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.828122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.828126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.828132:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.828135:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.828135:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.828137:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.828138:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.828140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.828334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677474.828340:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.828342:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677474.828344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677474.828345:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.828347:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.828349:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677474.828350:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.828352:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.828352:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.828354:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677474.828356:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.828358:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677474.828361:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.828362:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677474.828363:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677474.828364:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677474.828365:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.828366:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.828367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677474.828372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.828380:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.828382:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.828383:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.828385:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.828386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.828387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.828683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.828689:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.828692:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.828693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.828695:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.828697:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.828699:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.828700:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.828701:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.828702:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.828703:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.828706:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.828708:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.828710:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.828711:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.828712:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.828713:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.828714:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.828715:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.828715:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677474.828720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.828729:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.828731:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.828732:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.828732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.828734:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.828737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.829187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.829196:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.829198:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.829201:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.829203:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.829203:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.829204:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.829206:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.829207:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.829208:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.829209:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.829213:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.829214:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.829218:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.829218:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.829219:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.829221:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.829221:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.829222:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.829223:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.829227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.829234:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.829237:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.829238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.829238:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.829240:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.829243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.829254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.829262:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.829264:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.829265:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.829267:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.829269:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.829271:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.829272:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.829274:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.829275:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.829275:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.829278:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.829280:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.829283:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.829283:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.829284:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.829286:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.829286:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.829287:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.829288:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.829292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.829300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.829303:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.829304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.829304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.829306:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.829308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.830444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.830452:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.830454:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.830456:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.830457:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.830459:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.830460:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.830462:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.830463:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.830464:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.830465:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.830468:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.830469:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.830472:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.830473:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.830473:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.830475:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.830475:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.830476:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.830476:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.830481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.830489:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.830492:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.830493:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.830493:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.830495:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.830497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.830500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.830507:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.830510:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.830511:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.830511:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.830512:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.830513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.830750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.830758:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.830760:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.830761:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.830763:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.830766:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.830767:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.830768:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.830770:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.830771:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.830771:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.830774:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.830776:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.830778:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.830779:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.830779:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.830781:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.830782:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.830783:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.830783:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.830787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.830797:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.830800:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.830801:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.830804:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.830805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.830807:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.830963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.830975:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.830979:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.830983:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.830985:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.830986:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.830989:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.830991:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.830993:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.830994:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.830996:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.831001:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.831004:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.831008:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.831009:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.831010:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.831012:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.831013:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.831015:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.831016:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.831019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.831028:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.831031:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.831032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.831033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.831036:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.831039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.831164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.831174:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.831178:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.831179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.831183:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.831187:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.831189:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.831191:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.831193:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.831195:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.831196:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.831201:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.831204:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.831207:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.831208:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.831209:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.831211:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.831212:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.831213:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:21.2:1619677474.831214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677474.831215:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677474.831216:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677474.831218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.831221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:31.0:1619677474.831224:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.831228:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677474.831229:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:31.0:1619677474.831231:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.831231:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677474.831231:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.831232:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.831233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:18.0:1619677474.831233:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677474.831234:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:31.0:1619677474.831235:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.831236:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.831236:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.831237:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677474.831238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:2.0:1619677474.831239:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:31.0:1619677474.831240:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.831242:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.831244:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.831245:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.831251:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.831253:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.831257:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.831258:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.831259:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.831261:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.831262:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.831264:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.831265:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.831268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.831277:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.831280:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.831281:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.831284:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.831286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.831288:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.831988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.831997:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.832000:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.832001:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.832003:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.832004:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.832004:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.832005:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.832007:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.832007:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.832008:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.832011:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.832013:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.832015:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.832015:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.832016:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.832017:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.832018:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.832018:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.832019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.832023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.832030:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.832033:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.832034:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.832035:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.832037:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.832037:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:5.2:1619677474.832054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.832061:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.832064:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.832065:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.832068:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.832071:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.832072:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.832074:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.832075:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.832076:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.832077:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.832080:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.832081:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.832084:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.832085:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.832085:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.832087:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.832087:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.832088:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.832089:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.832093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.832101:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.832104:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.832105:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.832105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.832107:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.832109:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.832242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.832248:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.832251:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.832252:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.832255:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.832256:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.832257:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.832259:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.832260:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.832261:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.832262:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.832265:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.832266:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.832268:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.832269:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.832269:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.832271:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.832272:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.832272:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.832273:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.832277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.832284:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.832286:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.832287:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.832288:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.832289:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.832290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.832354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.832362:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.832364:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.832365:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.832367:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.832370:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.832371:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.832372:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.832373:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.832374:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.832375:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.832378:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.832380:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.832383:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.832384:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.832384:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.832386:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.832387:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.832388:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.832388:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.832393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.832400:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.832403:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.832404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.832404:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.832407:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.832409:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.832539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.832547:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.832550:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.832552:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.832553:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.832554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.832555:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.832556:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.832557:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.832558:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.832559:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.832562:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.832564:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.832566:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.832566:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.832567:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.832568:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.832569:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.832571:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.832572:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.832573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.832574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.832578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.832583:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.832584:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.832585:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.832586:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677474.832587:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.832587:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.832587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.832589:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.832590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.832591:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.832592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.832598:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.832601:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.832602:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.832603:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.832603:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.832604:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.832749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.832754:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.832757:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.832759:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.832759:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.832762:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.832763:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.832764:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.832766:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.832767:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.832768:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.832771:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.832773:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.832775:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.832776:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.832776:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.832778:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.832778:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.832780:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.832780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.832786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.832794:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.832796:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.832797:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.832799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.832799:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.832801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.832929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.832936:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.832939:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.832939:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.832941:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.832943:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.832945:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.832946:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.832947:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.832948:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.832949:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.832953:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.832955:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.832958:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.832959:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.832959:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.832961:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.832962:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.832963:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.832963:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.832967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.832975:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.832977:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.832978:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.832980:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.832982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.832983:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.832984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:21.2:1619677474.832988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:13.0:1619677474.832991:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.832994:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.832995:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.832995:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:30.0:1619677474.832996:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:13.0:1619677474.832996:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.832997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677474.832998:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.833001:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.833003:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.833003:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.833004:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.833006:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.833007:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.833008:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.833009:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.833013:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.833014:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.833018:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.833019:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.833019:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.833021:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.833022:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.833022:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.833023:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.833027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.833035:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.833038:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.833038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.833039:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.833041:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.833044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.833626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:24.0:1619677474.833634:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677474.833637:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.3.3@o2ib3
00000800:00000200:8.0:1619677474.833638:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000400:00000200:24.0:1619677474.833640:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : REPLY - routed
00000800:00000200:24.0:1619677474.833643:0:19938:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:24.0:1619677474.833647:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677474.833648:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (22)++
00000800:00000200:24.0:1619677474.833649:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (23)--
00000800:00000200:24.0:1619677474.833649:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:4.2:1619677474.833656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677474.833663:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:8.0:1619677474.833665:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:8.0:1619677474.833666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:21.2:1619677474.833926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.833934:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.833937:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.833939:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.833942:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.833942:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.833943:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.833944:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.833946:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.833947:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.833948:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.833951:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.833953:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.833956:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.833957:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.833957:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.833959:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.833959:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.833960:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.833961:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.833965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.833972:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.833974:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.833975:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.833975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.833977:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.833980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.834047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.834053:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.834056:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.834057:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.834059:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.834061:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.834063:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.834064:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.834065:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.834067:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.834068:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:4.2:1619677474.834068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677474.834071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:17.0:1619677474.834071:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.834072:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.834075:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.834075:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.834076:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677474.834076:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677474.834077:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677474.834078:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677474.834078:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.834079:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677474.834079:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.834079:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677474.834080:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677474.834080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:20.0:1619677474.834082:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677474.834082:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.834083:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677474.834085:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:21.0:1619677474.834086:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.834087:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.834088:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.834089:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.834090:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.834090:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.834091:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.834092:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.834093:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:8.0:1619677474.834093:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96502780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.834094:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.834095:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.834095:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.834096:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677474.834096:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677474.834096:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.834098:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.834099:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.834099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.834100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.834102:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.834102:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:8.0:1619677474.834102:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.834103:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.834105:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.834105:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677474.834105:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.834106:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.834107:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677474.834108:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.834108:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.834109:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.834110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.2:1619677474.834111:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677474.834117:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.834117:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.834118:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:8.0:1619677474.834119:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677474.834119:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:24.0:1619677474.834120:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.3.3@o2ib3
00000800:00000200:8.0:1619677474.834120:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.834121:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677474.834121:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:24.0:1619677474.834122:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39c600] (23)--
00000800:00000200:8.0:1619677474.834122:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (23)++
00000800:00000200:25.2:1619677474.834123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:24.0:1619677474.834123:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.834123:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677474.834123:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.3.3@o2ib3
00000400:00000200:1.0:1619677474.834123:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.834125:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:4.2:1619677474.834126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677474.834126:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.834127:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96502779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.834127:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : REPLY - routed
00000800:00000200:1.0:1619677474.834128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:24.0:1619677474.834129:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.834130:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.834131:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:6.0:1619677474.834131:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.834133:0:19939:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:6.0:1619677474.834133:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:24.0:1619677474.834134:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:22.0:1619677474.834134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.834135:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.834137:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:6.0:1619677474.834137:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.834138:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39c600] (23)++
00000800:00000200:24.0:1619677474.834139:0:19938:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:6.0:1619677474.834139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.834140:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (24)++
00000800:00000200:8.0:1619677474.834140:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (25)--
00000800:00000200:8.0:1619677474.834141:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (24)--
00000800:00000200:8.0:1619677474.834142:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:8.0:1619677474.834143:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.834145:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.834145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.834146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677474.834147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (22)++
00000800:00000200:24.0:1619677474.834153:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.834154:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677474.834155:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677474.834155:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (23)++
00000800:00000200:8.0:1619677474.834155:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (24)--
00000800:00000200:8.0:1619677474.834156:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:24.0:1619677474.834157:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677474.834158:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (22)++
00000800:00000200:24.0:1619677474.834159:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (23)--
00000800:00000200:24.0:1619677474.834159:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:25.2:1619677474.834171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.834177:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.834180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.834180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:25.2:1619677474.834212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.834219:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.834222:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[2] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.834225:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.834226:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:30.0:1619677474.834240:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.834240:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:4.2:1619677474.834536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677474.834544:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:8.0:1619677474.834547:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.3.3@o2ib3
00000800:00000200:24.0:1619677474.834548:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:8.0:1619677474.834549:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39c600] (22)--
00000400:00000200:8.0:1619677474.834550:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.834552:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677474.834553:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96502778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.834554:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.834555:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.834556:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677474.834559:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:8.0:1619677474.834561:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.834562:0:19939:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:8.0:1619677474.834571:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.834572:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677474.834572:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:8.0:1619677474.834573:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:8.0:1619677474.834574:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677474.834575:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000800:00000200:8.0:1619677474.834575:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:25.2:1619677474.834808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.834817:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:31.0:1619677474.834821:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (24)--
00000400:00000200:31.0:1619677474.834822:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.834823:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: REPLY: OK
00000800:00000200:30.0:1619677474.834824:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:30.0:1619677474.834825:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (24)--
00000800:00000200:31.0:1619677474.834826:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.834827:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:25.2:1619677474.834852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.834858:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.834861:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.834865:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:30.0:1619677474.834865:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.834886:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.834887:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.2:1619677474.834921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.834929:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.834931:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.834934:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.834935:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.834936:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.834938:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.834939:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.834940:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.834941:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.834942:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.834945:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.834946:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.834950:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.834950:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.834951:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.834953:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.834953:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.834954:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.834956:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.834956:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.834962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.834970:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.834972:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.834973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.834974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.835041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.835048:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.835051:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.835052:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.835052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.835054:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.835057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.835062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.835069:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.835071:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.835072:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.835072:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.835074:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.835074:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.835415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.835423:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.835427:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.835428:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.835429:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: REPLY: OK
00000800:00000200:30.0:1619677474.835430:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:4.2:1619677474.835430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.835432:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:26.0:1619677474.835438:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.835440:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.835441:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.835443:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.835446:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.835447:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.835448:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.835450:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.835451:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.835451:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.835454:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.835456:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.835458:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.835459:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.835460:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.835461:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.835462:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.835462:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.835463:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.835468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.835476:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.835479:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.835480:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.835480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.835482:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.835485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.835514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.835522:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.835525:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.835527:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.835529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.835530:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.835532:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.835533:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.835534:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.835535:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.835536:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.835539:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.835541:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.835544:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.835545:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.835545:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.835547:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.835548:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.835549:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.835549:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.835581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.835589:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.835591:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.835592:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.835594:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.835595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.835597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.836329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.836337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.836340:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.836342:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.836344:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.836344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.836346:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.836347:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.836348:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.836349:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.836350:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.836353:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.836355:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.836356:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.836357:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.836358:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.836359:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.836360:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.836360:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.836361:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.836365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.836372:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.836374:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.836375:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.836377:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.836378:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.836380:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.836485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.836490:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.836492:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.836495:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.836495:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.836497:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.836499:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.836500:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.836502:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.836502:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.836504:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.836506:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.836508:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.836511:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.836511:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.836512:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.836514:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.836514:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.836516:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.836517:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.836517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677474.836520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.836522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.836526:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677474.836527:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.836529:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.836529:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677474.836529:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677474.836530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677474.836530:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.836531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.836532:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.836535:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.836713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.836721:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.836724:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.836725:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.836727:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.836730:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.836732:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.836733:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.836734:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.836735:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.836736:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.836739:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.836741:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.836743:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.836744:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.836745:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.836746:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.836747:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.836748:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.836749:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.836750:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.836752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.836756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.836760:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.836763:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.836763:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.836763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.836764:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.836766:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.836766:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677474.836766:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.836767:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.836769:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.836794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.836801:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.836804:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.836805:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.836806:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.836808:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.836810:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.836811:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.836812:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.836813:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.836815:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.836818:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.836820:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.836822:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.836823:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.836823:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.836825:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.836825:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.836826:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.836827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.836831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.836839:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.836842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.836842:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.836843:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.836845:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.836846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.836851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.836858:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.836860:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.836861:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.836861:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.836862:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.836863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.837547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.837555:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.837558:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.837560:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677474.837562:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.837562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.837564:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.837565:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.837567:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.837568:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.837569:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.837572:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677474.837574:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.837577:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.837578:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.837578:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.837580:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.837581:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.837582:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.837582:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.837589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.837593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677474.837596:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.837599:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.837600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.837600:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.837601:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677474.837602:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.837603:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677474.837605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677474.837607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.837607:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.837610:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.837611:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.837612:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.837614:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.837615:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.837616:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.837619:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.837620:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.837622:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.837623:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.837623:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.837625:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.837625:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.837627:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.837628:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.837629:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.837630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.837634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.837639:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.837640:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.837641:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.837642:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.837642:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.837643:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.837643:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.837644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.837645:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.837646:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.838147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.838153:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.838156:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.838157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.838159:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.838161:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.838162:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.838164:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.838165:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.838166:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.838167:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.838171:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.838172:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.838175:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.838176:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.838176:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.838178:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.838179:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.838179:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.838180:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.838185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.838193:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.838196:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.838197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.838197:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.838199:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.838201:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.838480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.838488:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.838491:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.838492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.838494:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.838497:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.838498:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.838499:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.838501:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.838502:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.838503:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.838506:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.838508:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.838511:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.838512:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.838512:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.838514:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.838515:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.838515:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.838516:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.838520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.838528:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.838531:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.838532:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.838532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.838534:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.838537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.838539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.838546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.838548:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.838549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.838549:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.838550:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.838551:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.838673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.838681:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.838684:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.838686:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.838689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.838689:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.838690:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.838691:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.838693:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.838693:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.838694:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.838698:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.838700:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.838701:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.838702:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.838703:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.838704:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.838705:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.838706:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.838706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.838711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.838718:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.838721:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.838721:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.838723:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.838724:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.838725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.839569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.839577:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.839580:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.839582:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.839583:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.839584:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.839585:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.839585:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.839587:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.839587:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.839588:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.839591:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.839593:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.839596:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.839597:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.839598:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.839599:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.839600:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.839600:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.839601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.839606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.839615:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.839617:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.839618:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.839619:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.839621:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.839623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.839626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.839627:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.839630:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.839631:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.839631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.839632:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.839632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.839952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.839960:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.839963:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.839966:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.839967:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.839969:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.839971:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.839972:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.839973:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.839974:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.839975:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.839978:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.839979:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.839981:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.839982:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.839983:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.839984:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.839985:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.839986:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.839986:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.839990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.839997:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.840000:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.840001:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.840001:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.840003:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.840005:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.840107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.840115:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.840118:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.840120:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.840122:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.840122:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.840123:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.840124:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.840126:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.840126:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.840127:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.840130:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.840132:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.840133:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.840134:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.840135:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.840136:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.840137:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.840137:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.840138:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.840142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.840149:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.840151:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.840152:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.840153:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.840155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.840156:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.840159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.840165:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.840168:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.840168:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.840169:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.840169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.840170:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.840307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677474.840314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677474.840315:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.840317:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:20.0:1619677474.840317:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677474.840318:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677474.840319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677474.840321:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.840321:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.840323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677474.840324:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.840324:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.840326:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677474.840326:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:20.0:1619677474.840326:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.840327:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.840328:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677474.840329:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.840329:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677474.840330:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.840330:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.840330:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677474.840331:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.840332:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.840332:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.840333:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.840334:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677474.840335:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.840337:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.840337:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.840338:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96502760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.840339:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.840339:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.840340:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.840341:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.840342:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677474.840342:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.840342:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677474.840343:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.840344:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.840344:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.840345:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677474.840346:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.840346:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677474.840346:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.840346:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.840347:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.840348:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677474.840348:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.840348:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.840349:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677474.840349:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.840350:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.840350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677474.840351:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677474.840352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.840352:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.840353:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:4.2:1619677474.840354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677474.840355:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:8.2:1619677474.840355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677474.840356:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.840358:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.840358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.840360:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:10.0:1619677474.840361:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.840363:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677474.840363:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:26.0:1619677474.840364:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.840364:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:17.0:1619677474.840364:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:10.0:1619677474.840364:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.840364:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.840365:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:22.0:1619677474.840366:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.840366:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.840367:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:22.0:1619677474.840368:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.840368:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.840368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.840370:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.840370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.840371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.840371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677474.840372:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.840503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39c000] (20)++
00000800:00000200:18.0:1619677474.840511:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c000] (21)++
00000800:00000200:18.0:1619677474.840514:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.42@o2ib3
00000800:00000200:2.0:1619677474.840514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c000] (22)--
00000400:00000200:18.0:1619677474.840517:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.1.42@o2ib3 : GET - for me
00000400:00000200:18.0:1619677474.840521:0:19933:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.1.42@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:18.0:1619677474.840523:0:19933:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.1.42@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:18.0:1619677474.840524:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39c000] (21)++
00000800:00000200:18.0:1619677474.840526:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39c000] (22)++
00000800:00000200:18.0:1619677474.840528:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39c000] (23)--
00000400:00000200:18.0:1619677474.840530:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.1.42@o2ib3 local destination
00000800:00000200:20.2:1619677474.840534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39c000] (22)++
00000400:00000200:18.0:1619677474.840534:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.1.42@o2ib3(10.51.1.42@o2ib3:10.51.1.42@o2ib3) : REPLY try# 0
00000800:00000200:18.0:1619677474.840537:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.1.42@o2ib3
00000800:00000200:18.0:1619677474.840538:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c610f000] -> 10.51.1.42@o2ib3 (2) version: 12
00000800:00000200:2.0:1619677474.840539:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c000] (23)++
00000800:00000200:18.0:1619677474.840540:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91907d39c000] (23)++
00000800:00000200:18.0:1619677474.840540:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39c000] (25)++
00000800:00000200:2.0:1619677474.840540:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39c000] (24)--
00000800:00000200:18.0:1619677474.840541:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91907d39c000] (25)--
00000800:00000200:2.0:1619677474.840541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c000] (24)--
00000800:00000200:18.0:1619677474.840542:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c000] (22)--
00000800:00000200:2.0:1619677474.840542:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c000] (24)--
00000800:00000200:20.2:1619677474.840547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39c000] (21)++
00000800:00000200:2.0:1619677474.840553:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39c000] (22)++
00000800:00000200:2.0:1619677474.840555:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39c000] (23)--
00000400:00000200:2.0:1619677474.840555:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.840556:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c000] (22)--
00000400:00000200:2.0:1619677474.840557:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.1.42@o2ib3: REPLY: OK
00000800:00000200:2.0:1619677474.840559:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39c000] (21)--
00000800:00000200:20.2:1619677474.840922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677474.840927:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.840930:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.840932:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.840932:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.840935:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.840936:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:21.0:1619677474.840938:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.840939:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.840940:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.840941:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677474.840944:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.840946:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.0:1619677474.840950:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.840951:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677474.840951:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677474.840953:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677474.840954:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.840955:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677474.840955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677474.840959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677474.840967:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.840970:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.840971:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.840972:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.840974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.840975:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677474.841164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.841171:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.841174:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.841174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.841176:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.841178:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.841180:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677474.841181:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.841182:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.841184:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.841185:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677474.841188:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.841190:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677474.841193:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.841194:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677474.841194:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677474.841196:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677474.841197:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.841198:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677474.841199:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677474.841199:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677474.841202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.2:1619677474.841204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677474.841209:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:2.0:1619677474.841210:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:10.0:1619677474.841212:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:2.0:1619677474.841212:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:26.0:1619677474.841213:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.841213:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.841213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.841214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:10.0:1619677474.841215:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.841219:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.841264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.841272:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.2:1619677474.841274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.841275:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.841277:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.841279:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.841280:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.841281:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677474.841282:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677474.841282:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.841283:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.841283:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.841284:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.841285:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.841288:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.841289:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.841290:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.841291:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677474.841291:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.841292:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.841299:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.841299:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.841300:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.841303:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.841304:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.841305:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.841305:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.841305:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.841307:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.841307:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.841307:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.841308:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.841308:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.841308:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.841309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677474.841310:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.841310:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.841310:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.841311:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.841312:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.841316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.841323:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:25.2:1619677474.841324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.841326:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.841327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.841327:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.841329:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.841331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.841332:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.841335:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.841336:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.841336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677474.841336:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.841338:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.841341:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677474.841342:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.841345:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.841346:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.841346:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.841347:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.841347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.841597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.841604:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.841607:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.841608:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.841608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:26.0:1619677474.841610:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.841612:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:26.0:1619677474.841613:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.841615:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677474.841615:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:26.0:1619677474.841617:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.841618:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.841618:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.841618:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.841619:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.841620:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677474.841620:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.841622:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.841623:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.841625:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677474.841625:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.841627:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.841628:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677474.841628:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.841631:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.841632:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.841632:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:20.0:1619677474.841632:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.841634:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677474.841634:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.841635:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.841635:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.841636:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677474.841637:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.841638:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.841638:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677474.841640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677474.841640:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.841640:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.841641:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.841641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.841645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.841651:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.841653:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.841654:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.841655:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.841655:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.841656:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.841657:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.841658:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.841660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.841660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.841661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.841661:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.841668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.841675:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.841678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.841678:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.841679:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.841680:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.841681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.842018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:5.2:1619677474.842023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:18.0:1619677474.842024:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.842027:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.842027:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:1.0:1619677474.842029:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:18.0:1619677474.842030:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.842032:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.842032:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.842033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677474.842034:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.842035:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.842035:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.842037:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.842038:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.842038:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.842039:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:1.0:1619677474.842039:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.842040:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.842042:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.842043:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.842044:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677474.842045:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677474.842047:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.842049:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677474.842049:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.842050:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.842050:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.842052:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.842052:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:1.0:1619677474.842052:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677474.842053:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.842054:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:1.0:1619677474.842054:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.842055:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.842056:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.842057:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:9.2:1619677474.842058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:1.0:1619677474.842058:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.842059:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.842060:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.842061:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.842063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677474.842065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:5.2:1619677474.842066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677474.842067:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.842068:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.842069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.842070:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.842071:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.842072:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677474.842072:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677474.842074:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.842074:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677474.842075:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.842075:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.842075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.842076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.842077:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.842080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.842656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.842664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.842667:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.842670:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.842672:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.842672:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.842674:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.842675:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.842677:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.842678:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.842679:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.842682:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.842684:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.842688:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.842688:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.842689:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.842691:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.842691:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.842693:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677474.842694:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677474.842695:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677474.842697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.842701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:18.0:1619677474.842706:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:31.0:1619677474.842709:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:18.0:1619677474.842709:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.842710:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.842710:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677474.842712:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000400:00000200:18.0:1619677474.842712:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.842713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.842714:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:18.0:1619677474.842716:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.842717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.842724:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.842727:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.842728:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.842728:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.842729:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.842730:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.843006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.843015:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.843017:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.843020:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.843021:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.843023:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.843024:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.843025:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.843027:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.843028:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.843028:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.843032:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.843033:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.843035:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.843036:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.843036:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.843038:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.843038:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.843040:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.843041:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.843042:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.843043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.843047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.843052:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.843053:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.843054:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.843055:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.843056:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.843057:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.843057:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.843058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.843059:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.843060:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.843292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.843297:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.843300:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.843302:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.843303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.843305:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.843306:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.843308:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.843309:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.843310:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.843311:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.843314:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.843315:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.843318:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.843319:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.843320:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.843321:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.843322:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.843323:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.843324:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:21.2:1619677474.843335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:9.2:1619677474.843338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.843343:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.843346:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:8.0:1619677474.843346:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:31.0:1619677474.843348:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.843348:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.843349:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:31.0:1619677474.843350:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.843350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:8.0:1619677474.843350:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.843352:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.843352:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:31.0:1619677474.843353:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.843355:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.843355:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.843355:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:31.0:1619677474.843356:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.843359:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.843361:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.843364:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.843365:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.843366:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.843368:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.843368:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.843369:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.843370:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.843371:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.843374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.843376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.843382:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677474.843384:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677474.843385:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.843386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.843386:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.843387:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.843388:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.843389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:6.0:1619677474.843389:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.843391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.843455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.843459:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.843462:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.843464:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.843465:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.843467:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.843468:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.843469:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.843471:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.843472:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.843473:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.843476:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.843478:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.843479:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.843480:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.843481:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.843482:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.843482:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.843483:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.843484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.843488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.843496:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.843498:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.843499:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.843500:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.843501:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.843503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.843508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.843516:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.843519:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.843520:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.843521:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.843521:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.843522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.843819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.843826:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.843829:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.843832:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.843833:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.843835:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.843836:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.843837:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.843838:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.843840:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.843841:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.843844:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.843846:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.843848:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.843848:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.843849:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.843850:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.843851:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.843852:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.843853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.843856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.843863:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.843865:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.843866:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.843868:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.843869:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.843870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.844262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.844268:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.844271:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.844272:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.844274:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.844276:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.844277:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.844279:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.844280:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.844281:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.844282:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.844285:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.844286:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.844289:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.844290:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.844291:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.844293:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:25.2:1619677474.844299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677474.844301:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.844302:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.844303:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.844303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677474.844307:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.2:1619677474.844309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.844310:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.844311:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.844313:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.844314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.844315:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677474.844316:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:2.0:1619677474.844317:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.844318:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.844319:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.2:1619677474.844414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.844421:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.844424:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.844425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.844427:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.844429:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.844431:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.844432:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.844433:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.844434:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.844435:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.844438:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.844439:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.844442:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.844442:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.844443:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.844445:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.844445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.844446:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.2:1619677474.844447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677474.844447:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.844448:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.844452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.844453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:31.0:1619677474.844455:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.844457:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677474.844458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677474.844459:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:31.0:1619677474.844460:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.844460:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000400:00000200:31.0:1619677474.844461:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.844461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:30.0:1619677474.844462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677474.844462:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677474.844462:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:31.0:1619677474.844463:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677474.844463:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.844465:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.844465:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.844466:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.844466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:31.0:1619677474.844467:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.844467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:31.0:1619677474.844468:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.844472:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.844474:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.844476:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.844477:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.844477:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.844479:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.844480:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.844481:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.844481:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.844485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.844492:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.844495:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.844496:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.844497:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.844499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.844500:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.844648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.844656:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.844658:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.844660:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.844662:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.844663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.844664:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677474.844665:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.844666:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.844667:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.844668:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677474.844671:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.844673:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677474.844676:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.844676:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677474.844677:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.844678:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677474.844679:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.844680:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.844680:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.844685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.844693:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.844696:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.844697:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.844697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.844700:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.844703:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.844833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.844842:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.844844:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.844845:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.844847:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.844850:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.844851:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.844853:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.844854:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.844855:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.844856:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.844859:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.844861:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.844864:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.844865:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.844865:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.844867:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.844867:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.844868:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.844868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.844873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.844881:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.844884:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.844885:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.844885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.844887:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.844890:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.845083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677474.845088:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.845091:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677474.845092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677474.845094:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.845096:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.845098:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677474.845099:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.845100:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.845102:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.845102:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677474.845106:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.845108:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677474.845110:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.845111:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677474.845112:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677474.845114:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677474.845114:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.845116:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677474.845117:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677474.845117:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:8.2:1619677474.845120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677474.845122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.845127:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677474.845129:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:17.0:1619677474.845129:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677474.845130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677474.845131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:17.0:1619677474.845131:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.845132:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.845132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.845135:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.845137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.845177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.845186:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.845188:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.845189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.845192:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.845194:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.845196:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.845197:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.845198:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.845200:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.845200:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.845204:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.845206:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.845208:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.845209:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.845210:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.845211:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.845212:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.845213:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.845213:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.845218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.845226:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.845228:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.845229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.845230:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.845232:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.845234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.845327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.845333:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.845335:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.845337:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.845338:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.845341:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.845342:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677474.845343:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.845345:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.845346:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.845347:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677474.845350:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.845352:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677474.845355:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.845355:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677474.845356:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677474.845358:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677474.845358:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.845359:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.845359:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.845364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.845372:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.845375:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677474.845375:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.845376:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.845378:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.845381:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.845398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.845404:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.845407:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.845407:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.845409:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.845412:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.845413:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677474.845415:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.845416:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.845417:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.845418:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.845422:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.845423:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677474.845426:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.845427:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.845428:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.845429:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677474.845430:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.845431:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.845431:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677474.845436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.845443:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.845446:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.845447:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.845449:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.845450:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.845452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.845456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.845462:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.845465:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.845466:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.845467:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.845468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.845469:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.845721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.845727:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.845729:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.845732:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.845732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.845733:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.845735:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.845736:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.845738:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.845739:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.845740:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.845743:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.845745:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.845747:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.845747:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.845748:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.845750:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.845750:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.845751:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.845752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.845756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.845763:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.845766:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.845767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.845767:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.845769:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.845771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.845773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.845780:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.845782:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.845783:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.845783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.845784:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.845785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.845851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.845859:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.845862:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.845864:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.845866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.845867:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.845869:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.845870:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.845871:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.845872:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.845873:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.845877:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.845879:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.845882:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.845883:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.845884:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.845886:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.845886:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.845887:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.845888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.845891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.845899:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.845901:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.845902:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.845902:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.845904:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.845908:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.846438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.846445:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.846448:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.846449:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.846450:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.846453:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.846454:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.846456:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.846457:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.846458:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.846459:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.846462:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.846464:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.846467:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.846467:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.846468:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.846469:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.846470:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.846471:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.846471:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.846476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.846484:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.846487:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.846488:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.846488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.846490:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.846493:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.846654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.846657:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.846660:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.846662:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.846663:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.846665:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.846666:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.846668:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.846669:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.846670:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.846671:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.846674:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.846676:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.846678:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.846679:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.846680:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.846681:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.846682:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.846683:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.846683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.846688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.846697:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.846699:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.846700:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.846700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.846702:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.846705:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.846707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677474.846715:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.846717:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.846718:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.846718:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.846720:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.846720:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677474.846824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.846829:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.846832:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.846833:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.846834:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.846836:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.846838:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.846839:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.846840:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.846841:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.846842:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.846845:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.846847:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.846850:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.846850:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.846851:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.846853:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.846853:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.846853:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.846854:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.846859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.846866:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.846868:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.846869:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.846869:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.846871:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.846873:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.846877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.846884:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.846887:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.846888:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.846888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.846889:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.846889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.847140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.847149:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.847152:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.847154:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.847156:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.847157:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.847159:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.847160:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.847161:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.847163:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.847164:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.847167:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.847169:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.847172:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.847173:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.847174:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.847175:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.847176:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.847177:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.847178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.847181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.847189:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.847191:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.847192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.847193:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.847194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677474.847195:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.847198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.847201:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.847204:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.847206:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.847208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.847209:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.847211:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.847212:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.847213:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.847214:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.847215:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.847218:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.847219:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.847222:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.847222:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.847223:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.847224:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.847225:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.847226:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.847226:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.847230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.847236:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.847239:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.847240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.847240:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.847242:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.847244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.848216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.848224:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.848227:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.848229:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.848231:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.848231:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.848232:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.848233:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.848235:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.848235:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.848236:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.848239:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.848241:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.848243:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.848243:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.848244:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.848245:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.848246:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.848247:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.848249:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.848249:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.848252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.848256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.848261:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.848262:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.848263:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.848264:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.848264:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.848265:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.848265:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.848266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.848267:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.848268:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.848814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.848822:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.848825:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.848827:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.848829:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.848830:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.848831:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.848832:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.848833:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.848834:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.848835:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.848838:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.848839:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.848842:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.848843:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.848843:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.848845:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.848846:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.848846:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.848847:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.848852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.848861:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.848863:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.848864:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.848866:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.848867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.848869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.848875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.848883:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.848885:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.848886:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.848887:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.848888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.848889:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.849283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.849290:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.849297:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677474.849298:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.849300:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.849303:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.849304:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.849306:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.849307:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.849308:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.849309:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.849312:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.849314:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.849317:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.849318:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.849318:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.849320:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.849321:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.849322:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.849324:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.849324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.849331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:25.2:1619677474.849337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.849339:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.849341:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.849342:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.849343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.849346:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.849349:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.849349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.849350:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.849352:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.849355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.849605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.849613:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.849616:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.849618:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.849620:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.849621:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.849623:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.849624:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.849625:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.849626:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.849627:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.849630:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.849632:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.849635:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.849636:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.849636:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.849638:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.849638:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.849639:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.849640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.849645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.849653:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.849656:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.849657:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.849658:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.849659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.849662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.849684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.849689:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.849691:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.849693:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.849694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.849696:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.849697:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.849698:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.849700:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.849701:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.849702:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.849705:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.849706:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.849708:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.849709:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.849710:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.849711:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.849712:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.849713:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.849714:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.849715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677474.849717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.849720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.849724:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677474.849725:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.849726:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.849726:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.849727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.849727:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.849728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.849728:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.849730:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.849732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.849739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.849746:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.849748:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.849749:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.849749:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.849750:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.849751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.849835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.849842:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.849845:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.849846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.849848:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.849851:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.849852:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.849854:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.849855:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.849856:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.849857:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.849860:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.849861:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.849865:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.849865:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.849866:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.849868:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.849868:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.849869:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.849871:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.849871:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.849874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.849877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.849881:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.849883:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.849884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.849884:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:6.0:1619677474.849885:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.849886:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.849887:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677474.849887:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.849888:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.849890:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.849937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.849942:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.849945:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.849945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.849947:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.849950:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.849952:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677474.849953:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.849955:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.849956:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.849957:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677474.849960:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.849962:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677474.849965:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.849966:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677474.849966:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677474.849969:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677474.849969:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.849971:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677474.849971:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677474.849972:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:25.2:1619677474.849975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677474.849977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:30.0:1619677474.849981:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:6.0:1619677474.849982:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:30.0:1619677474.849984:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:6.0:1619677474.849984:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000400:00000200:30.0:1619677474.849985:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.849985:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.849986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:30.0:1619677474.849987:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.849988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.849990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.850034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677474.850041:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.850043:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677474.850045:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677474.850047:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.850049:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.850051:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.850052:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.850054:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.850055:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.850056:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.850059:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.850060:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.850062:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.850063:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.850064:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.850065:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.850066:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677474.850067:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677474.850067:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.850071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.850079:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.850081:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.850081:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.850082:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.850083:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.850084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.850376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.850384:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.850387:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.850387:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.850390:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.850393:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.850394:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677474.850396:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.850397:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.850398:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.850399:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677474.850403:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.850405:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677474.850408:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.850409:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677474.850409:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677474.850411:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677474.850412:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.850412:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.850413:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.850418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.850426:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.850429:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.850430:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.850430:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.850433:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.850436:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.850520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:4.2:1619677474.850521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677474.850525:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.850528:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:10.0:1619677474.850528:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677474.850528:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677474.850531:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.850532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.850532:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677474.850533:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.850534:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677474.850535:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.850536:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.850537:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.850538:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.850538:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.850539:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677474.850540:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.850542:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.850543:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.850543:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.850544:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677474.850545:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677474.850545:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.850547:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.850548:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.850549:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.850550:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:10.0:1619677474.850550:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.850551:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.850552:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:10.0:1619677474.850552:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.850553:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677474.850554:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:10.0:1619677474.850555:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.850556:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.850556:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.2:1619677474.850556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677474.850558:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.850558:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677474.850559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:10.0:1619677474.850559:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.850560:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.850565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.850565:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.850566:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677474.850566:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.850567:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:1.0:1619677474.850567:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.850568:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677474.850569:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.850569:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.850570:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677474.850571:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677474.850573:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.850576:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.850577:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.850577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.850580:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.850583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.850787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.850793:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.850796:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.850797:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.850798:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.850800:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.850802:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.850803:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.850804:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.850805:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.850806:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.850810:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.850812:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.850815:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.850816:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.850817:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.850819:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.850820:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.850820:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.850821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.850825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.850835:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.850839:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.850840:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.850843:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:13.0:1619677474.850844:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.850848:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.850876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.850884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.850887:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.850889:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.850891:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.850891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.850893:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.850894:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.850895:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.850896:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.850897:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.850900:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.850902:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.850905:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.850906:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.850907:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.850908:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.850909:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.850910:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.850910:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.850914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.850922:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.850925:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.850925:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.850927:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.850929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.850930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.851003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.851011:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.851014:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.851016:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.851017:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.851020:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.851021:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.851022:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.851024:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.851025:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.851026:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.851029:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.851030:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.851034:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.851034:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.851035:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.851037:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.851037:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.851038:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.851039:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.851043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.851050:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.851053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.851053:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.851054:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.851056:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.851059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.851261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.851269:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.851271:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.851272:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.851274:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.851277:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.851279:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.851280:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.851281:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.851283:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.851283:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.851287:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.851289:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.851291:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.851292:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.851293:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.851299:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.851300:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.851301:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.851301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.851306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.851314:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.851317:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.851317:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.851318:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.851320:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.851323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.851468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.851475:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.851478:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.851479:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.851482:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.851484:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.851485:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.851487:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.851488:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.851489:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.851490:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.851493:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.851495:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.851496:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.851497:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.851498:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.851499:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.851499:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.851500:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.851501:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.851505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.851512:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.851513:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.851514:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.851515:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.851516:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.851517:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.851579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.851586:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.851588:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.851589:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.851591:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.851593:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.851594:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.851596:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.851597:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.851598:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.851599:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.851602:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.851603:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.851606:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.851607:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.851607:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.851609:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.851609:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.851610:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.851611:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.851615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.851623:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.851625:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.851626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.851626:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.851628:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.851630:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.852851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.852857:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.2:1619677474.852860:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677474.852862:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.852864:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.852866:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677474.852868:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:21.0:1619677474.852868:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.852870:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677474.852871:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:21.0:1619677474.852872:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.852873:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.852875:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.852876:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.852876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677474.852876:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.852879:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.852880:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677474.852881:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.852882:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.852883:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.852884:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.852885:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.852886:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677474.852886:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.852887:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.852887:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.852889:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:31.0:1619677474.852890:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.852890:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.852890:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.852891:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:31.0:1619677474.852892:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.852894:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.852895:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.852896:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:9.2:1619677474.852896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.852897:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.852898:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.852899:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.852901:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.852901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.852903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677474.852903:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.852905:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.852906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.852906:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.852907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677474.852908:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.852911:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.852912:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.852913:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.852914:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.852915:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.852915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.852916:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.852916:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.852917:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.852918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.852919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.852920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677474.852920:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.852923:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.852924:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.852924:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.852925:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.852925:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.853197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.853205:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.853208:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.853210:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.853212:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.853212:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.853214:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.853215:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.853216:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.853217:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.853218:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.853221:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.853223:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.853226:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.853226:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.853227:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.853228:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.853229:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.853230:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.853230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.853235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.853243:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.853246:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.853247:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.853247:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.853249:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.853252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.853253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.853260:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.853263:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.853264:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.853264:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.853265:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.853266:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.853423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.853431:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.853434:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.853436:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.853439:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.853439:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.853440:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.853442:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.853443:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.853444:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.853445:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.853448:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.853450:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.853451:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.853452:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.853453:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.853454:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.853455:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.853456:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.853456:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.853460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.853467:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.853470:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.853471:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.853472:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.853474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.853475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.854066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.854074:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.854076:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.854078:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.854080:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.854081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.854082:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.854082:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.854084:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.854085:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.854085:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.854088:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.854090:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.854092:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.854093:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.854094:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.854095:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.854096:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.854096:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.854097:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.854101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.854108:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.854110:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.854111:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.854111:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.854113:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.854115:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.854718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.854725:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.854727:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.854729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.854730:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.854732:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.854733:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.854734:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.854735:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.854736:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.854737:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.854740:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.854742:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.854744:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.854745:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.854746:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.854748:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.854748:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.854749:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.854749:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.854765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.854773:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.854776:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.854777:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.854777:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.854779:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.854782:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.854875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.854881:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.854884:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.854885:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.854887:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.854889:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.854891:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.854892:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.854893:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.854895:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.854895:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.854899:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.854900:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.854903:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.854904:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.854905:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.854907:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.854907:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.854908:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.854908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.854913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.854921:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.854924:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.854925:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.854927:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.854928:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.854930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.854965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.854973:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.854976:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.854979:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.854981:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.854981:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.854983:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.854985:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.854986:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.854987:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.854988:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.854992:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.854993:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.854997:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.854998:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.854999:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.855000:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.855001:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.855002:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.855003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.855006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.855015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.855018:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.855019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.855019:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.855022:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.855025:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.855124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.855131:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.855134:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.855137:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.855138:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.855139:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.855141:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.855142:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.855143:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.855144:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.855145:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.855149:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.855150:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.855154:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.855155:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.855156:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.855158:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.855158:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.855159:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.855159:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.855164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.855172:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.855175:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.855176:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.855178:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.855179:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.855181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.855260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:4.2:1619677474.855260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.855267:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677474.855268:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677474.855270:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:18.0:1619677474.855271:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.855271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677474.855273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:18.0:1619677474.855274:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.855275:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.855276:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.855278:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677474.855279:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.855279:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.855280:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.855282:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.855282:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.855283:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:20.0:1619677474.855284:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.855287:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.855287:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677474.855289:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.855290:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.855290:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.855293:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.855293:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.855300:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.855301:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.855302:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.855303:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.855304:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:20.0:1619677474.855305:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677474.855308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.0:1619677474.855309:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.855313:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.855315:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.855316:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677474.855316:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:20.0:1619677474.855318:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677474.855318:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:20.0:1619677474.855319:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:12.0:1619677474.855319:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.855321:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:12.0:1619677474.855321:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.855322:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:20.0:1619677474.855323:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:25.2:1619677474.855324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.855324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:12.0:1619677474.855324:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.855330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677474.855332:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.855334:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.855336:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.855337:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.855337:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677474.855339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.855339:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.855340:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677474.855341:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.0:1619677474.855341:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.855347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.855354:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.855357:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.855358:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.855358:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.855359:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.855360:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.855370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.855379:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.855381:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.855382:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.855384:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.855387:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.855389:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.855390:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.855392:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.855393:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.855394:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.855397:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.855398:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.855401:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.855402:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.855403:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.855404:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.855405:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.855405:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.855406:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.855411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.855420:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.855422:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.855424:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.855424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.855425:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.855428:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.856197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.856204:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.856207:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.856208:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.856209:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.856212:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.856214:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.856215:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.856216:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.856217:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.856218:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.856220:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.856222:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.856224:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.856225:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.856225:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.856227:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.856228:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.856229:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.856230:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.856230:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.856233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.856236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.856241:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.856243:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.856243:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.856244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.856245:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677474.856245:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.856246:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.856247:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.856247:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.856250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.856340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.856346:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.856349:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.856350:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.856352:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.856354:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.856355:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.856356:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.856358:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.856358:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.856360:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677474.856361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:17.0:1619677474.856363:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.856364:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.856367:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677474.856367:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.856367:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.856368:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.856370:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:17.0:1619677474.856370:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.856370:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.856371:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.856371:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.856371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.856373:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.856375:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.856376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677474.856377:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.856378:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.856380:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.856382:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.856383:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.856385:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.856386:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.856387:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.856387:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.856389:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.856390:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.856391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.856393:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.856393:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.856394:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.856394:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.856395:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.856395:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.856396:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.856397:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.856397:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.856397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.856398:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.856398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677474.856399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.856403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.856410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.856413:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.856413:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.856415:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.856417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.856418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.856436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.856444:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.856447:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.856450:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.856452:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.856453:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.856454:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.856455:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.856457:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.856458:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.856459:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.856462:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.856464:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.856467:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.856468:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.856469:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.856471:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.856471:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.856473:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.856475:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.856475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.856477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.856481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.856485:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.856488:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677474.856489:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677474.856489:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.856491:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.856491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.856491:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.856492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.856493:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.856494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.856495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.856503:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.856505:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.856506:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.856507:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.856507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.856509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.856590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.856595:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.856597:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.856599:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.856600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.856602:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.856604:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.856605:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.856607:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.856607:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.856608:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.856611:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.856613:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.856616:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.856617:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.856617:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.856619:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.856620:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.856620:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.856621:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.856625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.856632:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.856635:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.856636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.856637:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.856638:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.856641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.857024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.857032:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.857034:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.857037:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.857038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.857040:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.857041:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.857042:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.857044:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.857045:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.857046:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.857049:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.857051:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.857054:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.857055:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.857056:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.857057:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.857058:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.857059:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.857061:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.857061:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.857064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.857067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.857072:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.857074:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.857075:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677474.857075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.857076:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.857077:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.857078:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.857079:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.857079:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.857080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.857084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.857091:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.857094:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.857095:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.857095:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.857096:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.857097:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.857297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.857303:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.857305:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.857307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.857308:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.857310:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.857312:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.857313:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.857314:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.857315:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.857316:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.857319:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.857321:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.857323:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.857323:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.857324:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.857326:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.857326:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.857327:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.857327:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.857333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.857341:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.857343:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.857344:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.857345:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.857347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.857348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.857351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677474.857358:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.857361:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.857362:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.857363:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.857363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.857364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.857889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.857897:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.857900:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.857902:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.857905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.857905:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.857907:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.857908:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.857909:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.857910:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.857911:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.857915:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.857917:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.857919:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.857919:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.857920:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.857921:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.857922:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.857923:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.857924:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.857927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.857934:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.857937:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.857937:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.857939:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.857941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.857941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:25.2:1619677474.857950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.857957:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.857960:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.857960:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.857961:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.857962:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.857962:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.858386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.858392:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.858395:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.858396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.858398:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.858400:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.858401:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.858403:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.858404:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.858405:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.858406:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.858409:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.858410:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.858413:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.858414:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.858414:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.858416:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.858417:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.858418:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.858419:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.858420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.858423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.858425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.858430:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.858432:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.858432:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.858432:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.858433:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.858434:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.858435:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.858437:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.2:1619677474.858437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:21.0:1619677474.858439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.858440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677474.858443:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.858446:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.858447:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.858449:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.858452:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.858453:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.858455:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.858456:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.858458:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.858459:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.858461:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.858463:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.858465:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.858465:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.858466:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.858467:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.858468:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.858469:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.858470:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.858470:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.858473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.858476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.858482:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.858483:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.858484:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.858484:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.858484:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.858484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.858485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.858486:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.858486:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677474.858487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.858635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.858644:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.858647:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.858649:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.858651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.858651:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.858652:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.858654:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.858655:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.858656:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.858657:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.858660:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.858662:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.858664:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.858665:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.858665:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.858667:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.858667:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.858668:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.858669:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.858672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.858680:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.858682:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.858683:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.858684:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.858686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.858687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.858744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.858751:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.858753:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.858755:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.858757:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.858758:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.858758:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.858759:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.858760:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.858761:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.858762:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.858765:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.858767:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.858770:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.858771:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.858771:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.858773:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.858773:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.858774:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.858774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.858779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.858787:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.858789:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.858790:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.858790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.858793:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.858796:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.859303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (20)++
00000800:00000200:21.0:1619677474.859309:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)++
00000800:00000200:21.0:1619677474.859311:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.4.72@o2ib3
00000400:00000200:21.0:1619677474.859314:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.72@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.859314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000800:00000200:21.2:1619677474.859316:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:21.0:1619677474.859320:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.859321:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677474.859322:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677474.859323:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.859324:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.859325:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:21.0:1619677474.859325:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.859326:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677474.859326:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.859329:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.859331:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.859331:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.859332:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.0:1619677474.859333:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677474.859334:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.859335:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.859336:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.859337:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.859337:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.859338:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:21.0:1619677474.859338:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:22.0:1619677474.859341:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.859341:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677474.859342:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1600] (21)++
00000800:00000200:21.0:1619677474.859342:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1600] (22)--
00000800:00000200:22.0:1619677474.859343:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.859343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)--
00000800:00000200:22.0:1619677474.859345:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.859346:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.859346:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677474.859347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677474.859348:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.859348:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.859349:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.859350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:4.2:1619677474.859351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (20)++
00000800:00000200:25.2:1619677474.859354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:13.0:1619677474.859355:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:20.0:1619677474.859357:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)++
00000800:00000200:13.0:1619677474.859358:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:20.0:1619677474.859359:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.72@o2ib3
00000400:00000200:13.0:1619677474.859359:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.859362:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:20.0:1619677474.859362:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.72@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677474.859362:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.859362:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677474.859363:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.859364:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.859365:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.859366:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.859367:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:20.0:1619677474.859369:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677474.859370:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.859371:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.859373:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.859373:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677474.859374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.859375:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000800:00000200:21.0:1619677474.859376:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:20.0:1619677474.859377:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.859379:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677474.859381:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.859382:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:20.0:1619677474.859383:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677474.859385:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677474.859385:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1600] (21)++
00000800:00000200:20.0:1619677474.859386:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1600] (22)--
00000800:00000200:20.0:1619677474.859387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)--
00000800:00000200:25.2:1619677474.859391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.859392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (20)++
00000800:00000200:21.0:1619677474.859398:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)++
00000800:00000200:6.0:1619677474.859398:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:21.0:1619677474.859400:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.72@o2ib3
00000800:00000200:6.0:1619677474.859401:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.859402:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.859402:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.859403:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.72@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.859403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000400:00000200:6.0:1619677474.859404:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.859405:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.859407:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:21.0:1619677474.859408:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:21.0:1619677474.859410:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.859411:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.859412:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.859413:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:21.0:1619677474.859416:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.859417:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.0:1619677474.859420:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.859420:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:21.0:1619677474.859421:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677474.859422:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.0:1619677474.859423:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1600] (21)++
00000800:00000200:21.0:1619677474.859424:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1600] (22)--
00000800:00000200:21.0:1619677474.859425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)--
00000800:00000200:8.2:1619677474.859430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.859438:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.859441:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.859442:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.859442:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.859443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (20)++
00000400:00000200:1.0:1619677474.859444:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.859447:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)++
00000800:00000200:1.0:1619677474.859447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677474.859450:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.72@o2ib3
00000800:00000200:21.0:1619677474.859453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000400:00000200:20.0:1619677474.859453:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.72@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.859454:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.859456:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677474.859457:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.859459:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.859460:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.859461:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677474.859464:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.859466:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:20.0:1619677474.859469:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.859469:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677474.859470:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677474.859472:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677474.859472:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1600] (21)++
00000800:00000200:20.0:1619677474.859474:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e84c1600] (22)++
00000800:00000200:20.0:1619677474.859475:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1600] (23)--
00000800:00000200:20.0:1619677474.859476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000800:00000200:25.2:1619677474.859478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:4.2:1619677474.859481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (21)++
00000800:00000200:12.0:1619677474.859484:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:21.0:1619677474.859486:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)++
00000800:00000200:12.0:1619677474.859487:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:21.0:1619677474.859488:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e84c1600] (23)--
00000400:00000200:12.0:1619677474.859488:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.859489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000400:00000200:12.0:1619677474.859489:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.859490:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)--
00000800:00000200:13.0:1619677474.859490:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.859492:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:4.2:1619677474.859574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.859582:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.859585:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.859586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.859587:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.859590:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.859591:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.859593:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.859594:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.859595:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.859596:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.859599:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.859600:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.859603:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.859604:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.859605:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.859606:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.859607:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.859607:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.859608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.859612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.859620:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.859623:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.859624:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.859624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.859626:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.859629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.859828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.859836:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.859839:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.859840:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.859842:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.859845:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.859847:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.859848:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.859849:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.859850:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.859851:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.859854:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.859856:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.859859:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.859859:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.859860:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.859861:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.859862:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.859863:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.859864:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.859864:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677474.859867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.859870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677474.859874:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.859876:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.859877:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.859877:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:13.0:1619677474.859878:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.859879:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:12.0:1619677474.859880:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.859880:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:13.0:1619677474.859881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677474.859881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.859884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677474.859891:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.859893:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.859894:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.859895:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.859895:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.859896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.860052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.860060:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.860062:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.860064:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.860067:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.860068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.860069:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.860070:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.860071:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.860072:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.860073:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.860076:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.860078:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.860080:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.860081:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.860081:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.860083:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.860083:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.860084:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.860084:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.860088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.860095:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.860097:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.860098:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.860098:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.860100:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.860102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.860186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.860194:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.860196:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.860197:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.860199:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.860201:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.860202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:26.0:1619677474.860203:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.860204:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.860205:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.860206:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.860207:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.860207:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.860210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.860210:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:26.0:1619677474.860211:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.860213:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:6.0:1619677474.860215:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.860217:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.860217:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:6.0:1619677474.860217:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.860218:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677474.860218:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677474.860220:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.860220:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677474.860220:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.860221:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:6.0:1619677474.860221:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677474.860222:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677474.860222:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677474.860223:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677474.860223:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:25.2:1619677474.860226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677474.860226:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677474.860228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.860229:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.860232:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.860232:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.860233:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.860234:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.860235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.860235:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.860236:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.860236:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:22.0:1619677474.860237:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.860237:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677474.860238:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.860238:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677474.860238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677474.860239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677474.860239:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.860239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:25.2:1619677474.860241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.860242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.860249:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.860251:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.860252:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.860254:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.860254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.860257:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.860423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.860426:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.860428:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.860430:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.860431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.860433:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.860434:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.860436:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.860437:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.860438:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.860439:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.860442:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.860444:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.860446:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.860447:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.860447:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.860449:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.860449:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.860451:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.860452:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.860453:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.860455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.860458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.860464:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.860465:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.860465:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.860465:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.860466:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.860466:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:21.0:1619677474.860467:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.860468:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.860468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.860470:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.860472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.860480:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.860482:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.860483:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.860485:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.860488:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.860490:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.860491:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.860492:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.860493:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.860494:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.860497:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.860499:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.860501:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.860502:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.860503:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.860505:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.860505:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.860506:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.860506:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.860511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.860519:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.860522:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.860523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.860523:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.860525:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.860528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.860850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.860858:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.860861:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.860863:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.860865:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.860865:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.860867:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.860868:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.860870:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.860870:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.860871:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.860875:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.860876:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.860879:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.860880:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.860881:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.860882:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.860883:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.860884:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.860884:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.860888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.860896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.860898:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.860899:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.860899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.860901:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.860904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.860906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.860913:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.860915:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.860916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.860916:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.860917:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.860918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.861049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.861056:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.861058:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.861059:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.861062:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.861063:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.861065:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.861066:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.861067:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.861068:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.861069:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.861072:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.861074:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.861076:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.861077:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.861078:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.861079:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.861080:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.861081:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.861082:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.861082:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.861085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.861087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.861092:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.861093:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.861094:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677474.861095:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.861095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677474.861096:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.861096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.861097:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.861099:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.861101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.861232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.861240:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.861243:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.861245:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.861247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.861248:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.861249:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.861251:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.861252:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.861253:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.861254:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.861257:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.861259:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.861262:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677474.861262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677474.861263:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.861263:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.861265:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.861266:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.861267:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:2.0:1619677474.861268:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677474.861269:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.861269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677474.861271:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.861272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.861273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:2.0:1619677474.861274:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:21.2:1619677474.861275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677474.861276:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.861277:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.861278:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.861280:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.861281:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:2.0:1619677474.861281:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.861282:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677474.861283:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677474.861283:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.861284:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.861285:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.861286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677474.861286:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.861287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677474.861287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:2.0:1619677474.861287:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.861289:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.861290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.861292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:2.0:1619677474.861292:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.861293:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.861298:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:2.0:1619677474.861299:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677474.861301:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:2.0:1619677474.861301:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.861301:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.861302:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.861303:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677474.861304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.861305:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:25.2:1619677474.861307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:21.0:1619677474.861307:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677474.861307:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677474.861309:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.861310:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:6.0:1619677474.861310:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.861311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677474.861312:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.861313:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.861314:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:21.0:1619677474.861314:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.861315:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.861316:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677474.861316:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.861317:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:6.0:1619677474.861317:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:31.0:1619677474.861318:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.861319:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96502671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.861321:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000400:00000200:21.0:1619677474.861321:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:6.0:1619677474.861321:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.861322:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:6.0:1619677474.861322:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.861323:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:6.0:1619677474.861323:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.861325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:6.0:1619677474.861327:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.861329:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.861330:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.861331:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677474.861331:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.861332:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677474.861332:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.861333:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.861334:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.861334:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.861335:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677474.861335:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.861335:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677474.861336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677474.861336:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.861337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:9.2:1619677474.861340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:25.2:1619677474.861341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677474.861348:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677474.861348:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:24.0:1619677474.861350:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677474.861351:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677474.861351:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.861352:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.861352:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.861354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.861354:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.861356:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.861357:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677474.861358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:25.2:1619677474.861366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677474.861374:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.861376:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.861377:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.861378:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.861378:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.861380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.861952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677474.861960:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.861963:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677474.861965:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.861967:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677474.861968:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.861970:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677474.861971:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.861972:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.861973:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.861974:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677474.861978:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.861980:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677474.861982:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.861983:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:30.0:1619677474.861983:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:30.0:1619677474.861985:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:30.0:1619677474.861986:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.861987:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.861987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677474.861991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.861998:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.862000:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.862001:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.862001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.862003:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.862006:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.862203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.862209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.862212:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.862212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.862215:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.862217:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.862218:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.862220:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.862221:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.862222:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.862223:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.862227:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.862228:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.862231:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.862232:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.862232:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.862234:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.862235:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.862236:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.862237:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.862238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.862240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.862243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.862248:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.862250:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.862250:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.862250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.862251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.862253:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.862253:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.862255:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.862256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.862257:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.862261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677474.862268:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.862271:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677474.862272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677474.862274:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.862276:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.862278:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677474.862279:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.862281:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.862281:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.862283:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677474.862286:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.862288:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677474.862290:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.862291:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:8.0:1619677474.862292:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:25.2:1619677474.862300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:8.0:1619677474.862300:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.0:1619677474.862301:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.862302:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.862302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:13.0:1619677474.862308:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.862310:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.862310:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.862312:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.862314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.862315:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.862538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.862547:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.862549:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.862551:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.862554:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.862554:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.862555:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.862556:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.862558:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.862558:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.862559:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.862562:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.862564:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.2:1619677474.862565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.862566:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.862567:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.862567:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.862569:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.862569:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.862570:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.862571:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.862572:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.862575:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:9.2:1619677474.862575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:31.0:1619677474.862577:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.862579:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.862579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.862580:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.862581:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.862581:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:31.0:1619677474.862582:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.862582:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.862583:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.862583:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.862584:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.862585:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:31.0:1619677474.862587:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.862587:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.862589:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.862589:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.862592:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.862593:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.862593:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:9.2:1619677474.862594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.862595:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.862595:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.862596:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.862596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.862601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.862601:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.862603:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.862604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.862604:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.862605:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.862606:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677474.862609:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.862612:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.862612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.862613:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.862615:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.862618:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.862663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.862670:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.862673:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.862674:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.862676:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.862679:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.862680:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677474.862682:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.862684:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.862685:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.862686:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677474.862689:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.862691:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677474.862693:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.862694:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677474.862695:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677474.862696:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677474.862697:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.862698:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.862699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677474.862702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.862710:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.862712:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.862713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.862713:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.862715:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.862718:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.862877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.862880:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.862882:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.862885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.862885:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.862887:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.862888:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.862890:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.862891:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.862892:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.862893:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.862896:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.862898:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.862901:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.862901:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.862902:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.862904:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.862904:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.862905:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.862906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.862910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.862918:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.862921:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.862922:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.862922:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.862924:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.862926:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.863124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.863129:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.863132:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.863133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.863135:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.863137:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.863138:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.863140:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.863141:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.863142:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.863143:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.863146:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.863148:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.863150:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.863151:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.863151:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.863153:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.863153:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.863154:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.863155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.863159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.863166:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.863169:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.863169:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.863171:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.863172:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.863173:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.863894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.863902:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.863905:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.863906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.863908:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.863910:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.863912:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.863913:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.863914:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.863916:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.863917:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.863920:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.863922:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.863925:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.863925:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.863926:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.863928:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.863928:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.863929:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.863930:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.863931:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677474.863934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.863937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677474.863942:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.863945:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:6.0:1619677474.863945:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.863946:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.863946:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.863947:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.863948:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:6.0:1619677474.863948:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.863949:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677474.863951:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.864180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.864185:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.864187:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.864188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.864190:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.864192:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.864193:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.864194:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.864195:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.864196:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.864197:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.864200:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.864202:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.864204:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.864205:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.864205:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.864207:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.864207:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.864208:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.864208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.864213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.864221:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.864223:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.864224:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.864226:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.864227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.864228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.864592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.864601:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.864604:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.864606:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.864609:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.864609:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.864611:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.864612:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.864613:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.864615:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.864616:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.864619:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.864621:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.864624:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.864625:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.864626:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.864627:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.864628:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.864630:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.864631:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.864632:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.864634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.864637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.864642:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.864645:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677474.864645:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:4.2:1619677474.864645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:22.0:1619677474.864646:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.864646:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.864648:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677474.864648:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.864649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.864650:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.864651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.864652:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:6.0:1619677474.864654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.864655:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677474.864656:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:6.0:1619677474.864656:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.864658:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.864659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677474.864659:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.864660:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.864661:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677474.864662:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.864662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677474.864664:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677474.864665:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.864666:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.864667:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.864668:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677474.864671:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.864673:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677474.864676:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.864677:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677474.864677:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677474.864679:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677474.864679:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677474.864680:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677474.864680:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:25.2:1619677474.864685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.864692:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.864695:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.864695:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.864697:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.864698:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.864700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.864706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.864713:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.864716:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.864717:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.864718:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.864718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.864719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.864995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.865003:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.865006:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.865007:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.865008:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.865011:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.865011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:24.0:1619677474.865013:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.865014:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.865015:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.865016:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.865017:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.865018:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677474.865020:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.865021:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.865021:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677474.865022:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.865023:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.865025:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.865025:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:26.0:1619677474.865026:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.865026:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677474.865027:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.865028:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.865028:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.865028:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.865029:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.865029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677474.865030:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.865030:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.865031:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677474.865033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677474.865034:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.865036:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.865039:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.865039:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.865040:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677474.865041:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:26.0:1619677474.865042:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (25)--
00000800:00000200:26.0:1619677474.865043:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.865043:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.865044:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.865044:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:22.0:1619677474.865045:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.865045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:25.2:1619677474.865048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000400:00000200:22.0:1619677474.865048:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.865051:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.865053:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.2:1619677474.865054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.865055:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.865056:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.865056:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.865057:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:22.0:1619677474.865058:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.865059:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.865060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677474.865061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.865061:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.865063:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.865065:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.865066:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.865067:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.865068:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.865069:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.865072:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.865074:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.865077:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.865078:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.865078:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.865080:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.865080:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.865081:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.865082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.865086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.865093:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.865095:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.865096:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.865098:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.865099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.865100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.865251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.865256:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.865258:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.865261:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.865261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.865263:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.865265:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.865267:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.865268:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.865269:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.865270:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.865273:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.865275:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.865277:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.865278:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.865279:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.865281:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.865282:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.865282:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.865283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.865287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.865294:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.865302:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.865303:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.865303:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:31.0:1619677474.865304:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.865308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.865324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.865332:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.865334:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.865335:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.865337:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.865340:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.865342:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.865343:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.865344:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.865345:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.865346:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.865349:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.865350:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.865353:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.865354:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.865355:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.865356:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.865357:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.865358:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.865358:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.865363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.865371:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.865374:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.865375:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.865375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.865378:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.865380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.865383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.865390:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.865393:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.865393:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.865394:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.865395:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.865395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.865670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.865677:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.865680:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.865680:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677474.865682:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.865684:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.865686:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.865687:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.865689:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.865690:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.865691:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.865693:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.865695:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.865698:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.865699:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.865700:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.865701:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.865702:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.865702:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.865703:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.865708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.865716:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.865718:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.865719:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.865720:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.865721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.865723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.865728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.865736:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.865738:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.865739:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.865740:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.865740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.865741:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.865756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.865763:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.865766:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.865768:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.865770:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.865771:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.865772:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.865774:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.865775:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.865776:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.865777:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.865781:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.865783:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.865785:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.865786:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:5.2:1619677474.865786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677474.865787:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.865788:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.865789:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.865790:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.865790:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677474.865792:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.865794:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:9.2:1619677474.865794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677474.865795:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.865797:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.865799:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.865800:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677474.865801:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.865802:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.865803:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.865803:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.865804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:17.0:1619677474.865804:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.865804:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.865806:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:8.0:1619677474.865806:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:17.0:1619677474.865809:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.865809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677474.865811:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.865814:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.865815:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.865815:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.865817:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.865817:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.865818:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.865819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.865823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.865831:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.865833:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.865834:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.865834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.865836:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.865838:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.865915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.865920:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.865922:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.865923:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.865925:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.865927:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.865928:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.865929:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.865930:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.865932:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.865932:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.865935:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.865937:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.865939:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.865940:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.865941:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.865942:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.865943:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.865944:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.865945:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.865945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.865948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.865950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.865956:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.865957:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677474.865957:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.865958:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.865959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.865960:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.865960:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.865962:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.865963:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.865964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.866103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.866111:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.866114:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.866116:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.866118:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.866119:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.866121:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677474.866122:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.866123:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.866125:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.866126:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677474.866129:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.866130:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677474.866133:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.866133:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677474.866135:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677474.866136:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677474.866137:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.866138:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.866138:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.866142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.866148:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.866151:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.866152:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.866152:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677474.866154:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.866156:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677474.866163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677474.866169:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.2:1619677474.866171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:10.0:1619677474.866172:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677474.866173:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.866173:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.866174:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.866174:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677474.866176:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.866179:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677474.866180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677474.866182:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.866184:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.866186:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677474.866187:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.866188:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.866189:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.866190:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677474.866194:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.866195:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677474.866198:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.866198:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677474.866199:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677474.866201:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677474.866201:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677474.866202:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.866202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677474.866207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.866215:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.866218:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.866219:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.866219:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.866221:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.866224:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.866228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.866236:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.866239:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.866240:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.866240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.866241:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.866241:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.866332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.866340:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.866342:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.866345:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.866347:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.866347:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.866348:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.866350:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.866351:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.866352:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.866353:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.866356:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.866358:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.866361:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.866362:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.866362:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.866364:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.866365:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.866366:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.866366:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.866371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.866379:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.866382:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.866383:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.866383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.866386:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.866388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.866615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.866622:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.866625:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.866627:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.866629:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.866630:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.866631:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.866633:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.866634:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.866635:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.866636:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.866639:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.866641:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.866643:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.866644:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.866644:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.866646:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.866646:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.866647:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.866648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.866651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.866658:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.866661:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.866661:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.866663:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.866665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.866666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.866842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.866849:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.866852:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.866852:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.866854:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.866857:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.866858:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677474.866859:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.866860:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.866861:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.866862:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677474.866865:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.866866:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.866868:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.866868:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.866869:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.866870:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.866871:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.866871:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.866872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.866876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.866882:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.866883:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.866884:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.866885:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.866886:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.866887:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677474.866935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.866941:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.866944:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.866945:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.866947:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.866949:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.866950:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:21.2:1619677474.866951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:2.0:1619677474.866952:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.866954:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.866955:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.866956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:2.0:1619677474.866956:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677474.866959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.866959:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:2.0:1619677474.866960:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.866962:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.866963:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:6.0:1619677474.866965:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.866965:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677474.866966:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677474.866966:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.866966:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677474.866968:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.866968:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.866968:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677474.866969:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.866969:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:6.0:1619677474.866970:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.866970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:6.0:1619677474.866971:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:9.2:1619677474.866974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677474.866974:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.866976:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.866978:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.866979:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.866979:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677474.866981:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:6.0:1619677474.866981:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.866981:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.866982:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677474.866983:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:6.0:1619677474.866983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:18.0:1619677474.866984:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.866985:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.866986:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.866987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677474.866988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677474.866994:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.866997:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.866997:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.866999:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.867000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.867001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.867831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.867837:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.867840:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.867841:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.867842:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.867845:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.867846:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.867847:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.867848:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.867849:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.867850:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.867853:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.867854:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.867856:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.867857:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.867857:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.867859:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.867859:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.867860:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.867860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.867866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677474.867867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:21.0:1619677474.867874:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677474.867875:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:21.0:1619677474.867876:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.867877:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.867878:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:21.0:1619677474.867879:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.867880:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.867880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.867881:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677474.867882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677474.867883:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.867884:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.867885:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677474.867885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000400:00000200:30.0:1619677474.867886:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.867888:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.867888:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.867892:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.867892:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677474.867893:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677474.867895:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.867896:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.867896:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.867896:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677474.867897:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.0:1619677474.867897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677474.867898:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677474.867898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.867899:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.867900:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.867901:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.867901:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677474.867905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.867912:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.867915:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.867916:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.867916:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.867918:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.867920:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.867939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.867945:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.867948:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:21.2:1619677474.867949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677474.867949:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.867951:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.867953:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.867954:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.867956:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.867957:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677474.867957:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.867958:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.867959:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:31.0:1619677474.867960:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.867963:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.867963:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677474.867964:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677474.867965:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.867966:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:31.0:1619677474.867967:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.867968:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.867969:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.867970:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.867970:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.867970:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:31.0:1619677474.867971:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.867972:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:2.0:1619677474.867972:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.867973:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.867974:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.867975:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:31.0:1619677474.867976:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.867976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677474.867978:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677474.867978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677474.867981:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677474.867981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677474.867982:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.867983:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.867985:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.867985:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.867987:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677474.867987:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:31.0:1619677474.867988:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:18.0:1619677474.867988:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677474.867989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677474.867989:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677474.867990:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.867991:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.867991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:1.0:1619677474.867991:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.867992:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.867993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677474.867993:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:21.2:1619677474.867994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677474.867996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677474.867999:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.868002:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677474.868002:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.868003:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.868005:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.868005:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677474.868005:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.868006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.868007:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.868008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.868305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.868313:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.868316:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.868318:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.868320:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.868321:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.868323:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.868324:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.868325:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.868326:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.868328:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.868331:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.868333:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.868336:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.868337:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.868337:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.868339:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.868340:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.868340:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.868341:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.868346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.868354:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.868357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.868357:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.868358:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.868360:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.868363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.868366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.868367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.868370:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.868370:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.868371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.868371:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.868372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.868675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.868680:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.868683:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.868684:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.868686:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.868687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.868689:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.868690:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.868691:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.868692:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.868693:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.868697:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.868698:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.868701:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.868702:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.868702:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.868704:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.868704:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.868705:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.868705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.868710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.868718:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.868720:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.868720:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.868722:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.868723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.868724:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.868837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.868840:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.868843:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.868845:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.868845:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.868848:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.868849:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.868851:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.868852:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.868853:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.868854:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.868857:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.868859:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.868862:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.868863:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.868863:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.868865:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.868866:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.868867:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.868867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.868872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.868882:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.868885:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.868887:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.868887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.868891:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.868893:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.869315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677474.869322:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.869325:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.869326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.869328:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.869330:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.869331:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677474.869332:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.869333:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.869334:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.869335:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677474.869338:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.869340:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677474.869342:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.869343:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677474.869344:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677474.869345:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677474.869346:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.869346:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.869347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677474.869352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.869356:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677474.869359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:8.0:1619677474.869359:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:8.0:1619677474.869359:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.869362:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.869364:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:24.2:1619677474.869372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677474.869375:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677474.869377:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677474.869378:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.869378:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677474.869379:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.869379:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677474.869670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.869676:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.869679:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.869680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.869682:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.869684:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.869685:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.869686:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.869688:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.869689:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.869690:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.869693:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.869695:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.869697:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.869698:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.869698:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.869700:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.869700:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.869701:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.869702:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.869703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.869707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.869708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.869713:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.869714:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.869715:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677474.869716:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.869717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677474.869717:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:21.0:1619677474.869718:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.869719:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677474.869720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:21.0:1619677474.869720:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.869721:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.869721:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.869722:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.869723:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.869724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.869725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:6.0:1619677474.869726:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677474.869727:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.869728:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.869729:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.869730:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677474.869733:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.869734:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677474.869737:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.869737:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:6.0:1619677474.869738:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:6.0:1619677474.869740:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677474.869740:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.869741:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.869741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677474.869746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.869753:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.869756:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.869757:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.869758:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.869760:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.869761:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.869804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.869811:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.869814:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.869815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.869817:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.869819:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.869821:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.869822:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.869824:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.869825:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.869826:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677474.869826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:8.0:1619677474.869829:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.869831:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.869833:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677474.869834:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.869835:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.869835:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.869836:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.869837:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677474.869837:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677474.869838:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677474.869838:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.869839:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.869840:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000400:00000200:10.0:1619677474.869841:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.869841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677474.869842:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:25.2:1619677474.869843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677474.869843:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.869844:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.869845:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.869846:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677474.869846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677474.869849:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.869850:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.869851:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.869853:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.869853:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677474.869853:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.869853:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.869854:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.869854:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.869855:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:10.0:1619677474.869855:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.869856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677474.869856:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.869856:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.869857:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677474.869857:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.869858:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.869858:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.869859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:25.2:1619677474.869862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.869870:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.869872:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.869873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.869873:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.869875:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.869877:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.870022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.870030:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.870034:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.870036:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.870038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.870039:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.870040:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.870041:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.870042:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.870043:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.870044:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.870047:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.870049:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.870052:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.870052:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.870053:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.870055:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.870055:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.870056:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.870057:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.870061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.870068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.870070:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.870071:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.870071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.870074:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.870076:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.870544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.870549:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.870551:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.870552:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.870554:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.870556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.870558:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.870559:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.870560:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.870561:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.870562:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.870565:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.870567:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.870569:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.870570:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.870571:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.870572:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.870573:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.870573:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.870574:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.870579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.870587:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.870589:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.870590:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.870591:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.870593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.870594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.870637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677474.870643:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.870645:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677474.870647:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.870648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677474.870649:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.870651:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.870652:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.870654:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.870655:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.870656:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.870659:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.870661:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.870663:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.870664:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.870664:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.870666:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.870667:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.870667:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677474.870668:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.870673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.870681:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.870683:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.870684:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.870686:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.870687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.870688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.870853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.870862:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.870865:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.870867:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.870869:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.870870:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.870871:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.870873:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.870874:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.870875:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.870876:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.870879:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.870880:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.870883:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.870884:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.870884:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.870886:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.870886:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.870888:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.870888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.870891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.870898:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.870901:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.870902:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.870902:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.870904:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.870906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.871379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.871385:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.871388:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.871389:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.871391:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.871393:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.871394:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.871396:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.871397:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.871398:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.871399:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.871402:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.871403:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.871406:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.871407:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.871407:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.871409:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.871409:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.871410:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.871411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.871416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.871424:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.871426:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.871427:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.871428:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.871430:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.871431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.871433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.871440:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.871442:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.871443:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.871444:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.871445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.871446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.872261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:4.2:1619677474.872264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:31.0:1619677474.872270:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:26.0:1619677474.872272:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677474.872273:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.872275:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677474.872275:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.872275:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677474.872277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:26.0:1619677474.872277:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.872278:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.872279:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.872280:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.872281:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.872282:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677474.872283:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.872283:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.872284:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.872285:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.872285:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.872286:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.872287:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677474.872289:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.872291:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.872299:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.872300:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.872301:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.872301:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677474.872302:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.872304:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.872304:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:26.0:1619677474.872304:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.872305:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677474.872306:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:26.0:1619677474.872306:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.872307:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:31.0:1619677474.872308:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.872308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:26.0:1619677474.872308:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.872309:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.872309:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.872310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677474.872314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.2:1619677474.872315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.872321:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.872323:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677474.872324:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677474.872324:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:6.0:1619677474.872324:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.872325:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.872326:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.872327:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.872327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677474.872327:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.872328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:22.0:1619677474.872328:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.872330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.872331:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.872332:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.872332:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.872333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.872333:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.872334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.872334:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.872335:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.872339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.872345:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.872348:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.872349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.872351:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.872353:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.872354:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.872356:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.872357:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.872358:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.872359:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.872363:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.872365:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.872367:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.872368:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.872368:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.872370:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.872370:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.872371:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.872372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.872376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.872384:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.872387:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.872388:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.872388:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.872390:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.872392:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.872398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.872403:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.872405:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.872406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.872408:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.872410:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.872411:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.872412:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.872414:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.872415:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.872416:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.872419:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.872421:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.872424:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.872424:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.872425:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.872427:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.872427:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.872428:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.872428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.872433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.872440:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.872442:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.872443:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.872445:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.872446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.872447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.872540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.872548:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.872551:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.872553:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.872555:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.872556:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.872557:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.872559:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.872560:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.872561:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.872562:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.872565:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.872567:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.872569:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.872570:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.872571:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.872572:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.872573:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.872574:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.872574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.872578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.872585:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.872587:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.872588:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.872588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.872590:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.872592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.872597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.872603:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.872606:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.872606:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.872606:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.872607:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.872608:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.872613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.872621:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.872624:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.872626:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677474.872628:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.872628:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.872630:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677474.872631:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.872633:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.872634:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.872634:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677474.872638:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.872639:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677474.872641:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.872642:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677474.872642:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677474.872644:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677474.872644:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.872645:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.872645:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677474.872650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.872657:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.872659:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.872660:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.872662:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.872663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.872664:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.873179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.873184:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.873187:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.873188:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.873190:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.873193:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.873194:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.873196:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.873197:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.873198:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.873199:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.873202:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.873204:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.873207:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.873207:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.873208:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.873210:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.873210:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.873211:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.873211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.873216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.873224:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.873226:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.873227:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.873229:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.873229:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.873232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.873245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.873250:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.873253:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.873254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.873256:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677474.873257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:22.0:1619677474.873258:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.873259:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.873260:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.873262:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.873263:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.873264:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677474.873264:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.873266:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677474.873267:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.873267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:22.0:1619677474.873269:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:2.0:1619677474.873270:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:22.0:1619677474.873271:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.873272:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:2.0:1619677474.873272:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.873273:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:2.0:1619677474.873273:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677474.873274:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.873275:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:2.0:1619677474.873275:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.873276:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:2.0:1619677474.873276:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.873277:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:2.0:1619677474.873277:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.873278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:2.0:1619677474.873278:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:25.2:1619677474.873281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:2.0:1619677474.873282:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:21.2:1619677474.873283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677474.873283:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.873286:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.873287:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:20.0:1619677474.873288:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.873288:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:2.0:1619677474.873288:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.0:1619677474.873290:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.873290:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:2.0:1619677474.873290:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.873290:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677474.873291:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.873291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.873291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:2.0:1619677474.873291:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.873292:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.0:1619677474.873294:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:25.2:1619677474.873297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:20.0:1619677474.873299:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.873301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:12.0:1619677474.873305:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.873308:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.873309:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.873311:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.873312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.873314:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.873623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.873631:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.873633:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.873636:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.873638:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.873638:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.873639:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.873641:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.873642:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.873643:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.873644:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.873647:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.873648:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.873651:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.873652:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.873652:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.873654:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.873654:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.873655:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.873656:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.873660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.873667:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.873669:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.873670:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.873671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.873672:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.873674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.874092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.874097:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.874100:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.874101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.874104:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.874106:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.874107:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.874108:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.874110:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.874111:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.874112:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.874115:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.874116:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.874119:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.874119:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.874120:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.874121:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.874122:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.874123:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.874123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.874128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.874135:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.874138:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.874138:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.874140:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.874141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.874142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.874223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.874226:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.874229:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.874230:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.874231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.874232:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.874234:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.874235:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.874236:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.874237:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.874238:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.874242:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.874244:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.874246:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.874247:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.874248:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.874250:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:4.2:1619677474.874250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:20.0:1619677474.874251:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.874252:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.874253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:13.0:1619677474.874257:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:8.2:1619677474.874257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:13.0:1619677474.874260:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.874262:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677474.874264:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.874264:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:1.0:1619677474.874265:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:13.0:1619677474.874266:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677474.874267:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.874268:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.874268:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.874269:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.874270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:13.0:1619677474.874270:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.874271:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677474.874272:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:13.0:1619677474.874275:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.874277:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.874278:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.874279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:13.0:1619677474.874280:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.874281:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677474.874282:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:13.0:1619677474.874284:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677474.874284:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.874285:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.874286:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677474.874287:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.874289:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:8.2:1619677474.874289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.874291:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.874291:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.874292:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.874292:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.874300:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677474.874301:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.874302:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.874302:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.874304:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.874307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.874310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677474.874318:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.874320:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.874321:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.874322:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677474.874322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.874323:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.874640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.874648:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.874652:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.874653:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.874655:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.874658:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.874660:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.874661:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.874662:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.874663:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.874664:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.874667:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.874669:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.874671:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.874672:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.874673:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.874674:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.874675:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.874676:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.874676:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.874681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.874690:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.874693:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.874694:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.874694:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.874696:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.874699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.874709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.874716:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.874719:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.874720:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.874722:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.874724:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.874726:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.874727:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.874728:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.874729:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.874730:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.874733:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.874735:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:4.2:1619677474.874735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:18.0:1619677474.874736:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.874737:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.874737:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.874739:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.874740:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:21.0:1619677474.874741:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:18.0:1619677474.874741:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.874741:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.0:1619677474.874743:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:9.2:1619677474.874745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:21.0:1619677474.874746:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.874746:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.874748:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.874750:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.874751:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.874752:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.874753:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677474.874753:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:21.0:1619677474.874754:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.874755:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.874756:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.874757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:21.0:1619677474.874759:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677474.874759:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.874761:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.874761:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677474.874764:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.874765:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.874766:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.874768:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.874768:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.874769:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.874770:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.874774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.874782:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.874785:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.874786:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.874788:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.874790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677474.874790:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:25.2:1619677474.874798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.874804:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.874807:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.874808:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.874809:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.874809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.874811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.875132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.875137:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.875140:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.875141:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.875143:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.875145:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.875146:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.875147:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.875149:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.875150:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.875151:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.875154:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.875156:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.875158:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.875159:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.875160:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.875161:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.875162:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.875162:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.875163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.875168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.875175:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.875177:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.875178:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.875180:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.875181:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.875182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.875183:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.875184:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.875184:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.875185:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (24)--
00000800:00000200:20.0:1619677474.875185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.875186:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.875187:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.875733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.875741:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.875744:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.875746:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.875748:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.875749:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.875750:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.875752:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.875753:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.875754:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.875755:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.875759:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.875760:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.875764:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.875765:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.875765:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.875767:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.875768:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.875769:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.875770:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.875774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.875782:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.875785:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.875786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.875786:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.875789:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.875792:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.875984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.875993:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.875996:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.875997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.875999:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.876000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:8.0:1619677474.876002:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.876004:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.876005:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.876006:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677474.876006:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.876007:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.876009:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:8.0:1619677474.876009:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.876010:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:8.0:1619677474.876012:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.876014:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.876015:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.876017:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.876017:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.876018:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.876018:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677474.876019:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.876020:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.876020:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.876021:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.876021:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677474.876022:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677474.876022:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.876022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.876023:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:25.2:1619677474.876026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677474.876026:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.876028:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.876031:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.876032:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:4.2:1619677474.876032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:22.0:1619677474.876033:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.876034:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.876035:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.876035:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.876036:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:6.0:1619677474.876036:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.876037:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.876037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:6.0:1619677474.876038:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.876039:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.876040:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:25.2:1619677474.876041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.876042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677474.876043:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.876043:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.876045:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.876048:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.876048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:26.0:1619677474.876050:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:21.0:1619677474.876050:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:26.0:1619677474.876051:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.876051:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.876053:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.876053:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.876053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677474.876054:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.876055:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.0:1619677474.876055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677474.876058:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.876060:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.876062:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.876063:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.876064:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.876065:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.876066:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.876066:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.876067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.876071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.876079:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.876081:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.876082:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.876084:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.876086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.876087:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.876090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677474.876097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.876099:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.876100:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.876101:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.876101:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.876102:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.876262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.876269:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.876272:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.876274:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.876277:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.876277:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.876279:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.876280:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.876281:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.876282:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.876283:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.876286:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.876287:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.876289:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.876290:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.876291:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.876292:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.876297:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.876298:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:9.2:1619677474.876298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.876300:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.876300:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.876305:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.2:1619677474.876306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677474.876308:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.876309:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.876309:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.876310:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.876313:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677474.876315:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.876317:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.876318:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.876319:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.876593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.876600:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.876603:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.876604:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.876605:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.876608:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.876609:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.876611:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.876612:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.876613:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.876614:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.876617:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.876618:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.876621:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.876622:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.876622:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.876624:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.876625:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.876626:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.876627:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.876628:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.876631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.876633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.876638:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677474.876640:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677474.876641:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677474.876642:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677474.876642:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.876642:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.876643:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.876644:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677474.876644:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.876647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.876652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.876659:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.876662:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.876663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.876663:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.876664:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.876664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.876861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.876866:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.876868:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.876869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.876871:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.876873:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.876874:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.876875:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.876877:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.876878:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.876879:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.876882:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.876884:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.876886:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.876887:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.876887:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.876889:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.876889:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.876890:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.876891:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.876892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.876895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.876897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.876901:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.876903:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.876903:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.876904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.876905:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.876905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.876906:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.876907:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.876909:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.876909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677474.877111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.877118:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.877121:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.877122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.877124:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.877126:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.877127:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.877129:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.877130:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.877131:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.877132:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.877135:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.877137:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.877139:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.877140:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.877141:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.877142:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.877143:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.877144:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.877144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.877148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.877156:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.877158:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.877159:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.877159:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.877162:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.877164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.877293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6ca00] (20)++
00000800:00000200:6.0:1619677474.877300:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6ca00] (21)++
00000800:00000200:6.0:1619677474.877302:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.14@o2ib3
00000800:00000200:22.0:1619677474.877303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6ca00] (22)--
00000400:00000200:6.0:1619677474.877305:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.14@o2ib3 : ACK - routed
00000400:00000200:6.0:1619677474.877307:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.877308:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677474.877309:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.877310:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.877311:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.877312:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677474.877315:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677474.877317:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677474.877320:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.877321:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:6.0:1619677474.877322:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:6.0:1619677474.877324:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:6.0:1619677474.877325:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6ca00] (21)++
00000800:00000200:6.0:1619677474.877325:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6ca00] (22)--
00000800:00000200:6.0:1619677474.877326:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6ca00] (21)--
00000800:00000200:25.2:1619677474.877331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677474.877339:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.877341:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.877342:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.877344:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.877346:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.877347:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677474.877843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.877851:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.877854:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.877857:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.877859:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.877859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.877860:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.877862:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.877863:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.877864:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.877865:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.877868:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.877869:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.877871:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.877872:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.877873:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.877874:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.877875:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.877876:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.877876:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.877880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.877886:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.877889:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.877890:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.877891:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.877893:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.877894:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.877903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.2:1619677474.877905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:22.0:1619677474.877908:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.877911:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:18.0:1619677474.877912:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:6.0:1619677474.877912:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.877914:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.877915:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.877916:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:22.0:1619677474.877917:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.877918:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677474.877918:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.877920:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.877920:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.877922:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.877922:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677474.877923:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.877923:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.877925:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677474.877925:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.877927:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.877928:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677474.877930:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677474.877932:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.877933:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677474.877934:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677474.877935:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.877936:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.877937:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677474.877937:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.877938:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677474.877938:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:22.0:1619677474.877939:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.877939:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677474.877939:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677474.877940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:18.0:1619677474.877940:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.877941:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.877942:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.877943:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677474.877944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.2:1619677474.877947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677474.877952:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.877954:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.877955:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.877956:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:21.0:1619677474.877957:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.877957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677474.877958:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:21.0:1619677474.877959:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:17.0:1619677474.877960:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.877960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.877962:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.877964:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.878115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.878123:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.878126:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.878128:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.878130:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.878130:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.878132:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.878134:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.878135:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.878136:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.878137:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.878141:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.878143:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.878147:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.878147:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.878148:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.878150:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.878151:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.878152:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.878152:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.878156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.878164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.878166:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.878167:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.878168:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.878170:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.878173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.878174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6ca00] (20)++
00000800:00000200:25.2:1619677474.878176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.878181:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6ca00] (21)++
00000800:00000200:22.0:1619677474.878183:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.14@o2ib3
00000800:00000200:6.0:1619677474.878184:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.878185:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.878186:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.14@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677474.878186:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.878188:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.878188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.878189:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.878189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6ca00] (22)--
00000800:00000200:6.0:1619677474.878190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:22.0:1619677474.878191:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:22.0:1619677474.878192:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.878194:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.878194:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.878196:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677474.878199:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677474.878200:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677474.878203:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.878204:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:22.0:1619677474.878205:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:22.0:1619677474.878206:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:22.0:1619677474.878207:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6ca00] (21)++
00000800:00000200:22.0:1619677474.878207:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6ca00] (22)--
00000800:00000200:22.0:1619677474.878208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6ca00] (21)--
00000800:00000200:25.2:1619677474.878212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677474.878219:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677474.878222:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677474.878223:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.878224:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.878226:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677474.878227:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:25.2:1619677474.878233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:12.0:1619677474.878239:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677474.878242:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677474.878243:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.878244:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677474.878244:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.878246:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677474.878471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.878479:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.878482:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.878484:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.878487:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.878487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.878488:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.878490:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.878491:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.878492:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.878493:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.878497:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.878498:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.878500:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.878501:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.878502:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.878503:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.878504:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.878505:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.878505:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.878510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.878517:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.878519:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.878520:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.878521:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.878523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.878524:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.878528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.878535:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.878537:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.878538:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.878539:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.878539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.878540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.878767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.878773:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.878776:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.878777:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.2:1619677474.878777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:6.0:1619677474.878779:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.878781:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.878783:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.878784:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.878785:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.878785:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677474.878786:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.878788:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677474.878789:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.878791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677474.878791:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.878793:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:2.0:1619677474.878794:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677474.878795:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.878796:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.878797:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.878798:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:2.0:1619677474.878798:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.878799:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.878800:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.878800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:2.0:1619677474.878801:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.878803:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677474.878805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:2.0:1619677474.878805:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.878807:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.878808:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.878812:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.878814:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:2.0:1619677474.878814:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677474.878815:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.878817:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.878817:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.878818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.878819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677474.878821:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.878822:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:25.2:1619677474.878823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677474.878823:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.878826:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.878827:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.878828:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.878829:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.0:1619677474.878831:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677474.878832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.878833:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.878834:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.878835:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.878835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.878836:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.878838:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.878841:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.878842:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.878844:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.878845:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.878847:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.879456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677474.879465:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.879468:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.879468:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:8.0:1619677474.879470:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.879473:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.879475:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677474.879476:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.879477:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.879478:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.879479:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677474.879482:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.879484:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 984 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677474.879487:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.879488:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.0:1619677474.879489:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677474.879490:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677474.879491:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.879492:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.879492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:25.2:1619677474.879497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.879505:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.879508:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.879508:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.879510:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.879511:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.879513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.879674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.879683:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.879686:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.879688:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.879690:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.879691:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677474.879691:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.879693:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.879694:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.879695:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.879696:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.879699:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.879700:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.879702:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.879703:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.879704:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.879705:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.879706:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.879707:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.879707:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.879711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.879717:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.879720:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.879721:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.879721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.879723:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.879725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.879911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.879918:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.879921:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.879923:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.879925:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.879925:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.879926:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.879928:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.879929:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.879930:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.879931:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.879934:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.879935:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.879938:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.879939:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.879939:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.879941:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.879942:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.879942:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.879943:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.879947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.879955:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.879958:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.879959:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.879959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.879962:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.879965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.880234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.880243:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.880245:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.880246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.880248:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.880251:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.880252:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.880254:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.880255:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.880256:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.880257:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.880260:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.880262:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.880265:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.880266:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.880266:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.880268:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.880269:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.880270:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.880271:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.880271:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.880274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.880277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.880282:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.880285:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.880285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.880285:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.880286:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.880287:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.880288:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.880288:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677474.880288:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.880291:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.880331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.880336:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.880338:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.880339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.880341:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.880343:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.880344:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.880345:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.880346:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.880347:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.880348:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.880351:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.880352:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.880355:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.880355:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.880356:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.880358:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.880358:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.880359:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.880359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.880364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.880372:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.880374:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.880375:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.880376:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.880377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.880378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.880380:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.880385:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.880388:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.880390:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.880390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.880392:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.880394:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.880395:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.880396:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.880397:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.880398:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.880402:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.880403:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.880406:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.880407:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.880408:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.880409:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.880410:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.880411:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.880412:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.880413:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.880416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.880418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.880421:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.880423:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.880423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.880424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677474.880424:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:4.2:1619677474.880426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677474.880427:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677474.880428:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.880429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.880430:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.880433:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677474.880434:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.880437:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.880438:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.880440:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.880442:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.880444:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.880445:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.880447:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.880448:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.880449:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.880452:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.880454:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.880457:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.880457:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.880458:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.880460:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.880460:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.880461:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.880461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.880465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.880473:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.880476:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.880476:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.880477:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.880479:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.880482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.880845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.880853:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.880856:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.880858:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.880860:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.880861:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.880862:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677474.880864:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.880865:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.880866:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.880867:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677474.880870:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.880872:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677474.880874:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.880875:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:13.0:1619677474.880876:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:13.0:1619677474.880878:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677474.880878:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.880880:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677474.880881:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677474.880882:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677474.880884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677474.880888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677474.880893:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.880895:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:12.0:1619677474.880895:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:18.0:1619677474.880896:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.880896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677474.880897:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677474.880898:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:18.0:1619677474.880899:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.880899:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677474.880902:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.881046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.881055:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.881058:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.881060:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.881062:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.881063:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.881065:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.881066:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.881067:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.881069:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.881069:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.881073:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.881074:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.881078:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.881079:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.881079:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.881081:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.881082:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.881083:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.881084:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.881085:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.881087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.881091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.881095:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.881097:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677474.881098:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677474.881098:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.881100:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.881101:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.881101:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.881102:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.881103:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.881103:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.881255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677474.881263:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.881265:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677474.881266:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677474.881268:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.881271:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.881273:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677474.881274:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.881275:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.881276:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.881277:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677474.881280:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.881282:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677474.881285:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.881285:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:24.0:1619677474.881286:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677474.881288:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677474.881288:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.881290:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:24.0:1619677474.881291:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677474.881292:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.2:1619677474.881293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.881299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000800:00000200:21.0:1619677474.881303:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.881306:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.881307:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.881307:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000400:00000200:21.0:1619677474.881308:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.881309:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677474.881309:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677474.881310:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:21.0:1619677474.881311:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677474.881311:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:4.2:1619677474.881427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.881434:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.881437:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.881438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.881440:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.881443:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.881443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677474.881444:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.881446:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.881447:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.881448:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.881449:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.881450:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677474.881452:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.881453:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677474.881453:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.881454:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.881456:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.881456:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.881457:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.881457:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677474.881458:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.881459:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.881459:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677474.881460:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677474.881460:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.881460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677474.881461:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.881462:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.881463:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.881463:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677474.881465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677474.881466:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.881468:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.881471:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.881472:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.881472:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.881474:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.881474:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677474.881474:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.881475:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.881475:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.881477:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.881477:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.881478:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.881480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677474.881480:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.881483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.881485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.881487:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.881488:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.881488:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677474.881489:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.881490:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.881491:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.881491:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.881492:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.881492:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.881493:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.881494:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677474.881494:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.881877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.881883:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.881886:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.881887:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.881888:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.881891:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.881892:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.881894:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.881895:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.881896:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.881897:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.881900:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.881901:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.881904:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.881905:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.881905:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.881907:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.881907:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.881909:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.881909:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.881910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.881913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.881915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.881920:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.881921:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.881923:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.881923:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677474.881924:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.881924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.881924:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.881925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.881926:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.881928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.881939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.881947:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.881950:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.881953:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.881955:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.881956:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.881957:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.881959:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.881960:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.881961:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.881962:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.881966:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.881967:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.881971:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.881971:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.881972:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.881974:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.881974:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.881976:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.881977:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.881978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.881980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.881983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.881987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.881990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.881991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.881992:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.881992:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.881994:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677474.881994:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.881995:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.881996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.881997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.882512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.882521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.882524:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.882526:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.882528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.882529:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.882530:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.882531:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.882533:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.882534:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.882535:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.882537:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.882539:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.882541:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.882541:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.882542:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.882544:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.882544:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.882545:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.882545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.882550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.882557:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.882560:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.882560:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.882562:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.882564:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.882565:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.882742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.882748:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.882751:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.882752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.882753:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.882755:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.882757:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.882758:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.882759:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.882760:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.882761:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.882764:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.882766:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.882768:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.882769:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.882769:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.882771:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.882771:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.882773:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.882774:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.882774:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.882777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.882779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677474.882784:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.882785:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.882787:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.882787:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.882787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.882788:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.882788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.882789:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.882790:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.882792:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.882979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.882988:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.882992:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.882996:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.882997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.883000:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.883002:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677474.883005:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.883007:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.883009:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.883010:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677474.883016:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.883019:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677474.883023:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.883025:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677474.883026:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677474.883028:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677474.883029:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.883030:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.883031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.883034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.883043:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.883045:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.883047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.883047:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.883049:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.883052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:2.0:1619677474.883052:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.0:1619677474.883059:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.883062:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677474.883063:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677474.883065:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.883068:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.883069:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677474.883071:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.883072:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.883073:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.883074:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677474.883077:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.883078:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677474.883081:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.883082:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.0:1619677474.883083:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677474.883085:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677474.883085:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677474.883086:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677474.883087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:25.2:1619677474.883091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.883093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.883101:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.883101:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677474.883104:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677474.883104:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.883105:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.883106:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.883106:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.883107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677474.883108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.883108:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.883109:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:21.0:1619677474.883109:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677474.883110:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677474.883111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000400:00000200:30.0:1619677474.883112:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.883112:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.883114:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.883117:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.883118:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677474.883119:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.883121:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.883121:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.883122:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.883122:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677474.883122:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.883122:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.883123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677474.883124:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.883124:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.883124:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677474.883125:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.883126:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.883129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.883136:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.883139:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.883140:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.883141:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.883143:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.883144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.883192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.883195:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.883197:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.883199:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.883200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.883201:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.883203:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.883205:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.883206:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.883207:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.883208:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.883211:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.883213:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.883216:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.883217:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.883217:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.883219:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.883219:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.883220:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.883221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.883225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.883232:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.883235:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.883235:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.883237:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.883238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.883239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.883243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.883251:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.883253:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.883254:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.883255:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.883256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677474.883256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.2:1619677474.884005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.884013:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.884016:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.884018:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.884021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.884021:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.884022:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.884023:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.884025:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.884026:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.884027:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.884031:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.884032:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.884035:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.884036:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.884037:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.884038:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.884039:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.884040:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.884040:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.884045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.884054:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.884057:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.884057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.884058:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.884060:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.884063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.884246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.884250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.884252:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.884254:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.884255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.884256:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.884258:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.884259:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.884260:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.884261:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.884262:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.884266:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.884268:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.884271:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.884271:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.884272:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.884274:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.884275:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.884275:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.884276:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.884280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.884287:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.884290:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.884291:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.884292:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.884293:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.884299:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.884395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.884403:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.884406:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.884408:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.884411:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.884411:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.884412:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.884413:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.884415:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.884415:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.884416:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.884419:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.884421:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.884423:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.884424:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.884425:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.884426:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.884427:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.884428:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.884428:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.884433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.884440:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.884442:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.884443:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.884444:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.884445:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.884448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.884451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.884457:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.884459:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.884460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.884460:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.884461:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.884461:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.884768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677474.884775:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.884778:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677474.884779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677474.884780:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.884783:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.884784:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677474.884785:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.884786:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.884787:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.884788:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677474.884791:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.884793:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677474.884795:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.884796:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677474.884796:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677474.884798:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677474.884798:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.884799:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677474.884801:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677474.884801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:9.2:1619677474.884803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677474.884807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:10.0:1619677474.884810:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.884813:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.884813:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.884814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677474.884814:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000400:00000200:10.0:1619677474.884815:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.884816:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677474.884816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677474.884817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:10.0:1619677474.884818:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677474.884946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.884954:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.2:1619677474.884955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:18.0:1619677474.884957:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.884957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.884960:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677474.884961:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:18.0:1619677474.884962:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.884963:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:17.0:1619677474.884964:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:18.0:1619677474.884965:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.884965:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:18.0:1619677474.884967:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.884968:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.884968:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.884969:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.884972:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.884974:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:17.0:1619677474.884974:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.884976:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:18.0:1619677474.884977:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.884978:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.884978:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:17.0:1619677474.884978:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.884979:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677474.884980:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.884980:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:17.0:1619677474.884980:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677474.884981:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:17.0:1619677474.884981:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:18.0:1619677474.884982:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:17.0:1619677474.884985:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677474.884986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.884987:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.884990:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.884991:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.884992:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.884994:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.884996:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.884996:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.884997:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.884997:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677474.884998:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000400:00000200:1.0:1619677474.884998:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.884999:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.885000:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.2:1619677474.885000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:1.0:1619677474.885000:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677474.885002:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.885006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:21.0:1619677474.885011:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.885012:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677474.885013:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677474.885014:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000400:00000200:21.0:1619677474.885015:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.885015:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.885015:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:21.0:1619677474.885016:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.885017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.885020:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.885573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.885580:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.885583:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.885584:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.885586:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.885588:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.885590:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.885591:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.885592:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.885593:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.885594:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.885598:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.885599:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.885602:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.885603:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.885604:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.885606:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.885606:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.885607:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.885607:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.885614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.885622:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.885625:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.885626:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.885626:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.885629:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.885631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.885783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.885791:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.885793:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.885795:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.885796:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.885799:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.885800:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.885801:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.885802:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.885803:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.885804:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.885807:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.885809:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.885811:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.885812:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.885813:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.885814:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.885815:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.885815:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.885817:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.885817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.885820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.885823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.885827:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.885830:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.885830:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:22.0:1619677474.885831:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.885831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.885832:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.885833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.885833:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.885833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677474.885835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.885839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.885846:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.885848:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.885849:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.885849:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.885850:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.885851:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.885911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.885920:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.885923:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.885925:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.885928:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.885928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.885929:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.885930:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.885932:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.885933:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.885934:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.885936:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.885938:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.885940:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.885941:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.885941:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.885943:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.885944:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.885945:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.885947:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.885947:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.885949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.885953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677474.885954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677474.885955:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.0:1619677474.885957:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677474.885957:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.885958:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.885958:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.885959:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677474.885961:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677474.885961:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.885961:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.885963:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677474.885963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677474.885963:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.885964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677474.885964:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.885965:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677474.885966:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.885967:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.885969:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.885970:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.885971:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.885974:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.885976:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.885979:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.885979:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.885980:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.885982:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.885982:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.885983:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.885984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.885988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.885996:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.885999:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.886000:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.886000:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.886002:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.886005:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.886029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.886035:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.886038:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.886039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.886040:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.886043:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.886045:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.886046:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.886047:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.886048:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.886050:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.886053:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.886054:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.886056:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.886057:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.886058:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.886059:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.886060:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.886061:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.886061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.886065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.886073:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.886074:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.886075:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.886076:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.886077:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.886078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.886090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.886097:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.886100:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.886102:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.886104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.886105:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.886106:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.886107:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.886109:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.886110:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.886111:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.886114:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.886115:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.886118:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.886119:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.886120:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.886121:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.886122:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.886123:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.886123:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.886127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:5.2:1619677474.886131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677474.886135:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677474.886136:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.886138:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.886139:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.886139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677474.886139:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.886140:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677474.886142:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.886143:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.886145:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:1.0:1619677474.886145:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.886147:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.886148:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.886149:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.886150:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.886151:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.886155:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.886156:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.886159:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.886160:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.886160:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.886162:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.886163:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.886163:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.886164:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.886168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.886175:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.886178:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.886179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.886179:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.886181:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.886183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.886186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.886192:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.886195:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.886196:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.886196:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.886197:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.886198:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.886299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.886306:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.886309:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.886310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.886312:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.886315:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.886316:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.886318:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.886319:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.886320:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.886321:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.886324:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.886326:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.886329:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.886329:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.886330:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.886332:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.886332:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.886333:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.886333:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.886338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.886345:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.886348:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.886349:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.886349:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.886351:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.886354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.886527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.886532:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.886535:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.886536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.886538:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.886540:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.886541:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.886542:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.886544:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.886545:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.886546:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.886549:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.886551:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.886553:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.886554:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.886555:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.886557:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.886557:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.886558:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.886558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.886563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.886571:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.886573:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.886574:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.886576:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.886576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.886578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.887160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.887167:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.887170:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.887171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.887172:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.887176:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.887177:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677474.887179:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.887180:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.887181:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.887182:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677474.887186:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.887187:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677474.887190:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.887191:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677474.887192:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677474.887193:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677474.887194:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.887195:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.887195:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677474.887199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.887208:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.887211:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.887212:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.887212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.887215:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.887218:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.887332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.887340:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.887343:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.887345:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.887348:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.887348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.887350:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.887351:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.887352:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.887353:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.887354:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.887358:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.887359:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.887361:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.887362:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.887363:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.887364:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.887365:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.887366:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.887366:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.887371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.887377:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.887380:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.887381:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.887382:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.887384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.887385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.887820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.887827:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.887830:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.887832:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.887834:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.887834:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.887835:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.887836:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.887837:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.887838:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.887839:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.887842:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.887843:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.887847:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.887847:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.887848:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.887850:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.887850:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.887851:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.887851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.887856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.887863:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.887866:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.887867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.887867:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.887869:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.887872:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.888066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.888074:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.888077:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.888079:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.888080:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.888081:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.888083:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.888084:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.888085:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.888086:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.888087:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.888090:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.888091:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.888093:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.888094:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.888094:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.888095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677474.888096:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.888096:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.888097:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.888097:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.888100:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677474.888101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.888102:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.888103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.888105:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.888107:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.888108:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:22.0:1619677474.888109:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.888110:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.888111:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:22.0:1619677474.888111:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.888112:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.888112:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.888114:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.888114:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.888115:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.888116:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:22.0:1619677474.888117:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677474.888119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677474.888119:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.888122:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.888123:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.888123:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.888125:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.888125:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677474.888126:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.888126:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.888127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677474.888129:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.888130:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.888130:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.888131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:25.2:1619677474.888131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677474.888132:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.0:1619677474.888139:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.888141:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.888142:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.888144:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.888145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.888146:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.888413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.888421:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.888424:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.888424:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.888427:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.888429:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.888431:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.888432:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.888433:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.888434:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.888435:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.888439:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.888440:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.888443:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.888444:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.888445:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.888446:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.888447:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.888447:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.888448:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.888453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.888462:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.888465:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.888466:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.888469:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.888472:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.888473:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.888612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.888619:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.888622:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.888623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.888625:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.888627:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.888629:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.888630:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.888631:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.888632:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.888633:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.888636:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.888637:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.888640:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.888641:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.888641:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.888643:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.888643:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.888644:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.888644:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.888659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.888667:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.888670:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.888671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.888671:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.888673:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.888676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.888777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677474.888782:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.888785:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677474.888787:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.888788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677474.888789:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.888791:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677474.888792:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.888794:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.888795:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.888796:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677474.888798:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.888800:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.888802:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.888803:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.888804:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.888806:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.888806:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677474.888807:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677474.888808:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677474.888813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.888820:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.888823:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.888823:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.888825:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.888826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.888827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.888832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677474.888836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.888840:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.888842:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.888844:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.888844:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000400:00000200:21.0:1619677474.888846:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.888846:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677474.888847:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677474.888847:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.888847:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.888848:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677474.888849:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.888849:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.888850:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.888850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:21.0:1619677474.888851:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.888851:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.888854:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.888855:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.888857:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.888858:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.888858:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.888860:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.888860:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.888861:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.888862:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.888866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.888874:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.888876:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.888877:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.888878:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.888879:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.888881:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.889221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677474.889228:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.889231:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677474.889232:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677474.889234:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677474.889236:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.889237:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.889239:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.889240:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.889241:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.889242:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.889245:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.889247:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.889250:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.889250:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.889251:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.889253:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.889253:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.889254:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677474.889256:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677474.889256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:25.2:1619677474.889259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.889262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:22.0:1619677474.889266:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.889269:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:22.0:1619677474.889269:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.889270:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.889270:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.889271:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677474.889272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:22.0:1619677474.889272:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.889273:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677474.889276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.889281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.889288:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.889290:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.889291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.889291:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.889292:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.889293:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.890060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.890068:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.890071:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.890073:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.890074:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.890075:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.890076:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.890077:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.890079:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.890080:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.890081:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.890084:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.890086:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.890089:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.890089:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.890090:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.890092:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.890092:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.890094:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.890095:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.890096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.890098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.890102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.890106:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.890109:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.890110:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.890110:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.890110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.890112:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.890113:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.890113:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.890114:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.890115:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.890371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.890378:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.890381:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.890382:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.890383:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.890386:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.890388:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.890389:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.890390:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.890391:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.890392:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.890395:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.890397:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.890399:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.890400:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.890401:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.890402:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.890403:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.890404:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.890404:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.890409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.890417:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.890420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.890420:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.890421:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.890423:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.890425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.890465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.890470:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.890472:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.890473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.890475:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.890477:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.890478:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.890479:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.890481:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.890482:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.890483:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.890487:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.890488:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.890491:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.890491:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.890492:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.890494:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.890494:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.890495:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.890495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.890500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.890507:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.890509:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.890510:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.890512:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.890513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.890514:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.890609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.890616:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.890619:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.890620:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.890622:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.890625:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.890626:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.890628:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.890629:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.890630:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.890631:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.890634:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.890636:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.890639:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.890640:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.890641:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.890642:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.890643:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.890644:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.890644:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.890648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.890656:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.890659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.890659:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.890660:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.890662:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.890664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.890803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677474.890810:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.890813:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.890814:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677474.890815:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.890818:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.890820:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.890821:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.890822:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.890823:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.890824:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.890827:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.890828:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.890831:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.890832:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.890832:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.890834:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.890835:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.890835:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677474.890836:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:8.2:1619677474.890841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.890849:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.890852:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.890853:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.890854:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.890855:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.890858:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.890872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677474.890879:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.890882:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.890882:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677474.890884:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.890887:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.890889:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677474.890890:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.890891:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.890892:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.890892:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677474.890895:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.890897:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677474.890899:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.890900:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677474.890900:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:10.0:1619677474.890902:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677474.890902:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677474.890903:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677474.890905:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677474.890905:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:25.2:1619677474.890907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:4.2:1619677474.890911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:12.0:1619677474.890915:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:26.0:1619677474.890917:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:12.0:1619677474.890917:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.890918:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.890919:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.890920:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677474.890920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:12.0:1619677474.890921:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677474.890921:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:13.0:1619677474.890922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:5.2:1619677474.890932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.890939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.890941:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.890942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.890944:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.890946:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.890948:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.890949:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.890950:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.890951:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.890952:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.890956:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.890957:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.890960:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.890960:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.890961:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.890962:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.890963:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.890963:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.890964:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.890968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.890976:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.890978:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.890979:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.890979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.890981:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.890983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.891203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.891209:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.891211:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.891213:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.891214:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.891216:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.891218:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.891219:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.891220:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.891221:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.891223:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.891226:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.891228:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.891231:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.891232:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.891232:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.891234:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.891235:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.891236:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.891237:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.891237:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677474.891243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.891248:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:25.2:1619677474.891250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.891250:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.891251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.891252:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677474.891257:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.891259:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.891260:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.891262:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.891263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.891264:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.891676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.891683:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.891686:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.891687:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.891689:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.891692:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.891693:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.891694:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.891696:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.891697:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.891698:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.891700:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.891702:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.891705:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.891706:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.891706:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.891708:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.891709:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.891709:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.891710:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.891715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.891723:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.891726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.891726:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.891727:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.891729:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.891732:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.891808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.891816:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.891819:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.891821:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.891824:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.891824:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.891825:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.891827:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.891828:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.891829:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.891830:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.2:1619677474.891830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:31.0:1619677474.891833:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.891835:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.891837:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.891838:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.891838:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.891838:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.891839:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677474.891840:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.891840:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.891841:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.891841:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.891842:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677474.891843:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.891843:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.891845:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:25.2:1619677474.891845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677474.891846:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.891847:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.891848:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.891849:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.891852:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:30.0:1619677474.891852:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.891853:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.891853:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.891853:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.891855:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.891856:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.891856:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.891857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.891857:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.891858:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.891859:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.891860:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.891861:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.891861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.891865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.891872:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.891875:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.891876:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.891876:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.891878:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.891881:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.891882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.891889:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.891892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.891892:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.891893:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.891895:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.891895:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.891999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.892004:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.892006:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.892007:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.892009:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.892011:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.892012:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.892013:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.892015:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.892016:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.892017:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.892020:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.892022:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.892025:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.892026:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.892026:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.892028:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.892028:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.892030:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.892030:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.892031:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.892034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.892036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.892041:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.892043:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.892043:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.892044:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.892045:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.892045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677474.892046:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.892048:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.892049:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.892050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.892052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.892060:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.892062:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.892063:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.892064:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.892064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.892066:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.892322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.892329:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.892332:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.892333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.892335:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.892338:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.892339:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677474.892340:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.892342:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.892343:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.892344:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677474.892347:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.892349:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.892350:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.892351:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.892352:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.892353:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.892354:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.892354:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.892355:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677474.892370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.892376:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.892379:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.892379:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.892380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.892381:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.892383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.892504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.892512:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.892515:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.892517:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.892519:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.892520:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.892521:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.892522:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.892523:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.892524:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.892525:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.892528:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.892529:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.892531:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.892532:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.892532:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.892533:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.892534:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.892535:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.892537:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.892537:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.892539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.892542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:30.0:1619677474.892545:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.892548:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:22.0:1619677474.892548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:21.2:1619677474.892548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677474.892549:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.892551:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:30.0:1619677474.892552:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.892552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:31.0:1619677474.892553:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.892554:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:22.0:1619677474.892554:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.892555:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.892557:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:22.0:1619677474.892557:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.892558:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677474.892558:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:31.0:1619677474.892559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677474.892559:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.892561:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.892561:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.892562:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677474.892565:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.892567:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677474.892570:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.892571:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:22.0:1619677474.892571:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677474.892573:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:22.0:1619677474.892574:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.892574:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.892575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677474.892579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.892587:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.892589:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.892590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.892591:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.892593:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.892596:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677474.892600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.892606:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.892609:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677474.892610:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.892610:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.892611:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.892611:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677474.893162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.893168:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.893170:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.893172:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.893173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.893175:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.893176:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.893177:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.893179:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.893180:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.893181:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.893184:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.893185:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.893187:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.893188:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.893188:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.893190:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.893190:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.893191:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.893193:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.893193:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.893196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.893199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.893205:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.893206:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.893207:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.893207:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677474.893208:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.893208:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.893209:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.893210:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.893211:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.893212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677474.893536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.893542:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.893544:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.893545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.893548:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.893550:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.893552:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.893553:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.893554:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.893555:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.893556:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.893559:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.893561:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.893563:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.893564:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.893564:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.893566:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.893567:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.893567:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.893568:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.893573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.893581:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.893583:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.893584:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.893585:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.893587:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.893588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.893882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.893890:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.893893:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.893895:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.893897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.893898:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.893899:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.893901:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.893902:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.893903:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.893904:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.893907:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.893909:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.893912:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.893913:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.893914:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.893915:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.893916:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.893917:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.893918:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.893922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.893929:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.893932:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.893933:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.893933:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.893935:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.893939:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.893943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.893949:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.893951:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.893954:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.893954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.893955:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.893956:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677474.893958:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.893959:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.893960:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.893961:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677474.893965:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.893966:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.893969:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.893970:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677474.893971:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.893972:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677474.893973:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.893974:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.893976:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677474.893976:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677474.893978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.893981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677474.893985:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677474.893985:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.893987:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:30.0:1619677474.893988:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677474.893988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677474.893989:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.893989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677474.893991:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.893992:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.893994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.894019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677474.894027:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.894030:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677474.894031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677474.894032:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677474.894034:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.894035:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677474.894037:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.894038:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.894039:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.894040:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677474.894043:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677474.894044:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.894046:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.894046:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.894047:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.894048:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.894048:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.894049:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.894050:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677474.894064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.894070:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.894072:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.894073:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.894074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.894074:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.894076:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.894099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.894106:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.894108:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.894108:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.894109:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.894109:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.894110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.894284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.894292:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.894299:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677474.894301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:2.0:1619677474.894301:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:2.0:1619677474.894306:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.894307:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.894310:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:2.0:1619677474.894310:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.894311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:2.0:1619677474.894312:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:17.0:1619677474.894313:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.894314:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.894316:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.894316:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.894317:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:2.0:1619677474.894318:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.894319:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.894320:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:17.0:1619677474.894321:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.894323:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.894324:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:2.0:1619677474.894327:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677474.894329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677474.894330:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677474.894332:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:17.0:1619677474.894332:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.894334:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677474.894334:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677474.894335:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.894336:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677474.894337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.894337:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.894337:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.894338:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677474.894338:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677474.894339:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.894339:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:20.0:1619677474.894340:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.894340:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.894340:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:17.0:1619677474.894341:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:20.0:1619677474.894342:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:17.0:1619677474.894342:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677474.894342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677474.894342:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000400:00000200:20.0:1619677474.894343:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96502524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.894343:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000400:00000200:20.0:1619677474.894344:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.894344:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677474.894345:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.894346:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:9.2:1619677474.894346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.894349:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:9.2:1619677474.894353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:20.0:1619677474.894355:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.894357:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:18.0:1619677474.894358:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:20.0:1619677474.894359:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.894360:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:18.0:1619677474.894360:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.894360:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.0:1619677474.894361:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:18.0:1619677474.894362:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.894362:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:20.0:1619677474.894363:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.894363:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:2.0:1619677474.894363:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677474.894364:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.894365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:18.0:1619677474.894365:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.894365:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.894367:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.894368:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.894368:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677474.894369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677474.894370:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:2.0:1619677474.894371:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.894372:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.894374:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.894374:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677474.894375:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.894376:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.0:1619677474.894377:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:18.0:1619677474.894377:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.894378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677474.894378:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677474.894379:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.894380:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.894380:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.894380:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000400:00000200:21.0:1619677474.894381:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.894381:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.0:1619677474.894382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:2.0:1619677474.894383:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:21.0:1619677474.894384:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:2.0:1619677474.894384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677474.894388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.894395:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.894398:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.894399:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.894399:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.894400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.894401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.895194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.895203:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.895205:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.895206:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.895208:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.895211:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.895212:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.895214:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.895215:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.895216:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.895217:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.895220:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.895222:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.895224:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.895225:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.895226:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.895228:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.895228:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.895229:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.895229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.895234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.895242:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.895245:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.895246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.895246:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.895249:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.895251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.895432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.895440:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.895443:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.895443:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677474.895445:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.895448:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.895450:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.895451:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.895452:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.895453:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.895454:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.895458:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.895459:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.895462:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.895462:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.895463:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.895465:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.895465:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.895466:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677474.895467:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.895468:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677474.895474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.895481:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:25.2:1619677474.895482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.895483:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.895484:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.895485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677474.895489:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.895492:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.895493:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.895493:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.895495:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.895498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.895507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.895513:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.895516:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.895517:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.895517:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.895518:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.895519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.895549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.895555:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.895559:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.895560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.895561:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.895563:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.895565:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.895566:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.895568:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.895568:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.895570:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.895572:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.895574:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.895576:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.895577:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.895578:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.895579:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.895580:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.895581:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.895581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.895596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.895603:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.895606:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.895607:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.895607:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.895609:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.895611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.895806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.895814:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.895817:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.895820:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.895822:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.895822:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.895824:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.895825:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.895827:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.895828:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.895829:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.895832:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.895834:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.895838:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.895838:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.895839:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.895841:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.895841:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.895842:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.895843:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.895847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.895854:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.895857:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.895858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.895858:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.895860:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.895863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.895932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.895936:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.895939:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.895940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.895942:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.895944:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.895946:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.895947:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.895948:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.895949:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.895950:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.895954:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.895955:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.895958:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.895959:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.895959:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.895961:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.895962:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.895962:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.895963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.895967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.895974:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.895977:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.895977:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.895979:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.895980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.895981:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.896281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.896288:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.896291:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.896292:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.896297:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.896299:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.896300:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.896302:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.896303:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.896304:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.896305:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677474.896309:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.896311:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677474.896314:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.896315:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.896315:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677474.896317:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.896318:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.896319:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.896319:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.896324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.896331:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.896334:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.896335:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.896336:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.896338:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.896340:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.896341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677474.896348:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.896351:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:4.2:1619677474.896351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:31.0:1619677474.896352:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.896353:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677474.896353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.896355:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:13.0:1619677474.896359:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.896362:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.896364:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.896366:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.896367:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.896368:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677474.896369:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.896371:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.896372:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.896373:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:13.0:1619677474.896376:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.896378:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:13.0:1619677474.896381:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.896382:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677474.896383:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677474.896384:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677474.896385:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.896386:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.896387:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.896391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.896399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.896402:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.896403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.896403:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.896405:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.896409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.896552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.896560:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.896563:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.896565:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.896567:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.896568:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.896569:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.896570:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.896571:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.896572:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.896573:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.896576:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.896578:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.896580:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.896580:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.896581:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.896582:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.896583:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.896584:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.896584:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.896588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.896595:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.896597:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.896598:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.896599:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.896601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.896602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.896603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.896611:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.896613:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.896614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.896616:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.896619:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.896620:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.896621:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.896622:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.896623:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.896624:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.896627:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.896629:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.896631:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.896632:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.896633:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.896634:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.896635:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.896635:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.896636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.896640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.896648:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.896651:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.896652:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.896652:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.896654:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.896657:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.896660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.896667:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.896670:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.896670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.896671:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.896672:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.896672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.897234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.897242:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.897245:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.897247:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.897249:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.897249:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.897250:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.897251:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.897253:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.897254:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.897255:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.897257:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.897259:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.897261:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.897261:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.897262:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.897263:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.897264:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.897265:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.897265:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.897269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.897276:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.897278:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.897279:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.897280:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.897282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.897283:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.897288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.897295:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.897302:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.897303:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.897304:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.897304:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.897305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.897435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.897442:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.897445:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.897445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.897448:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.897450:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.897451:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.897453:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.897454:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.897455:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.897456:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.897459:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.897461:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.2:1619677474.897463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:17.0:1619677474.897464:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.897465:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.897465:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.897467:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.897468:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.897468:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.897469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:6.0:1619677474.897469:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.897472:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.897473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.2:1619677474.897474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677474.897474:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.897477:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.897478:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.897479:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.897480:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.897481:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.897482:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:6.0:1619677474.897482:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.897484:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.897485:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.897485:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677474.897487:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.897487:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.897489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:6.0:1619677474.897490:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.897491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677474.897491:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:4.2:1619677474.897491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:6.0:1619677474.897492:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.897493:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.897494:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.897494:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.897495:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677474.897495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677474.897498:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:25.2:1619677474.897500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677474.897500:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.897502:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.897503:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.897504:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.897504:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.897504:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.897505:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.897505:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.897507:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:20.0:1619677474.897507:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.897508:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.897509:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.897513:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.897514:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.897515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.897515:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.897517:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.897518:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.897518:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.897520:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.897521:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.897522:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.897523:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.897523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677474.897526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.897528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677474.897533:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.897534:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677474.897536:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677474.897536:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:22.0:1619677474.897537:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.897537:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677474.897537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.897538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:22.0:1619677474.897539:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.897541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.897947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.897955:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.897958:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.897960:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.897962:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.897963:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.897964:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.897966:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.897967:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.897968:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.897969:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.897972:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.897974:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.897977:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.897978:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.897979:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.897981:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.897981:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.897983:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.897984:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.897985:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.897987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.897991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677474.897994:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.897997:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.897997:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.897998:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.897999:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677474.898000:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.898001:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.898002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.898003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.898003:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.898386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.898394:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.898397:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.898398:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.898400:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.898403:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.898404:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677474.898406:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.898407:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.898408:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.898409:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.898412:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.898414:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.898416:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.898417:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677474.898418:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.898419:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.898420:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.898420:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.898421:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.898426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.898433:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.898436:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.898436:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.898438:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.898441:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.898442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.898491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.898498:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.898501:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.898502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.898503:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677474.898506:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.898507:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.898508:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.898509:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.898510:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.898511:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.898513:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677474.898515:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.898517:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.898518:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.898519:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.898520:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.898521:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.898522:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.898522:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677474.898527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.898536:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.898538:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.898539:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.898540:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.898541:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.898543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.898936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.898938:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.898941:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.898943:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.898944:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.898945:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.898946:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.898948:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.898949:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.898950:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.898951:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.898954:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.898955:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.898958:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.898958:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.898959:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.898960:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.898961:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.898962:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.898962:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.898966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.898968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.898974:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677474.898975:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:22.0:1619677474.898976:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.0:1619677474.898977:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:6.0:1619677474.898977:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677474.898978:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.898980:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.898980:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.898980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:22.0:1619677474.898981:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.898982:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:22.0:1619677474.898983:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.898984:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.898985:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.898986:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.898987:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.898990:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.898992:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.898994:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.898995:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.898996:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.898997:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.898998:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.898999:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.899000:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.899001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.899003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.899006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677474.899010:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.899011:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.899013:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.899013:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.899014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.899014:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.899014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.899015:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.899016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.899018:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.899023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677474.899030:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.899033:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.899034:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.899035:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.899035:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.899036:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.899281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.899288:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.899291:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.899292:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.899300:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.899303:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.899304:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677474.899305:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.899306:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.899307:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.899308:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677474.899311:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.899313:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677474.899315:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.899316:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677474.899317:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677474.899318:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677474.899318:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.899320:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677474.899321:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677474.899322:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677474.899324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677474.899328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677474.899332:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.899335:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.899335:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.899336:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:30.0:1619677474.899337:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.899338:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.899338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:30.0:1619677474.899339:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.899340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:26.0:1619677474.899340:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.2:1619677474.899452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.899460:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.899462:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.899464:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.899467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.899467:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.899468:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.899469:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.899470:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.899471:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.899472:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.899475:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.899477:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.899480:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.899481:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.899482:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.899483:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.899484:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.899485:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.899485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.899490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.899498:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.899500:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.899501:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.899503:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.899503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.899506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.899678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.899685:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.899688:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.899691:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.899692:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.899694:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.899695:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.899697:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.899698:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.899699:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.899700:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.899703:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.899705:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.899708:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.899709:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.899709:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.899711:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.899711:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.899712:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.899713:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.899717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.899725:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.899728:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.899729:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.899729:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.899731:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.899734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.899758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.899763:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.899766:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.899766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.899768:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.899771:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.899772:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677474.899773:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.899774:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.899775:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.899776:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677474.899780:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.899781:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677474.899784:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.899785:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:22.0:1619677474.899786:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677474.899787:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:22.0:1619677474.899788:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.899788:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.899789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677474.899793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.899800:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.899803:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.899804:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.899804:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.899806:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.899809:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.900098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:4.2:1619677474.900102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.900107:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677474.900109:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677474.900111:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677474.900112:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.900112:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677474.900114:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.900115:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.900116:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677474.900117:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.900119:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.900120:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.900121:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677474.900123:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.900125:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.900125:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.900126:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.900127:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 280 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677474.900128:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.900129:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.900129:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.900133:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677474.900134:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.900135:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677474.900135:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677474.900136:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.900138:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677474.900138:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677474.900139:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677474.900139:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.900140:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677474.900140:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.900141:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:10.0:1619677474.900141:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.900142:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.900143:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.900143:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.900144:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:9.2:1619677474.900144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677474.900146:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.900146:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.900149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.900151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677474.900152:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.900155:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.900156:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.900156:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677474.900156:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.900158:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.900159:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.900159:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.900160:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.900160:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.900161:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.900162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:18.0:1619677474.900162:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:6.0:1619677474.900162:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.900163:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677474.900165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.900168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.900175:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.900178:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.900179:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.900179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.900180:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.900180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.900198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.900200:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.900203:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.900205:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.900206:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.900207:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.900209:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.900210:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.900211:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.900213:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.900213:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.900217:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.900219:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.900221:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.900222:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.900222:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.900224:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.900225:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.900225:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.900226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.900230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.900238:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.900240:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.900240:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.900242:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.900243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.900244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.900399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.900407:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.900410:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.900411:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.900413:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.900416:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.900417:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.900419:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.900420:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.900421:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.900422:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.900425:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.900427:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.900430:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.900430:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.900431:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.900433:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.900433:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.900434:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.900434:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:5.2:1619677474.900439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677474.900440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.900446:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677474.900448:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677474.900449:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.900450:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.900450:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.900452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.900452:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.900452:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.900454:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.900454:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.900456:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677474.900457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:1.0:1619677474.900458:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.900459:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.900460:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.900461:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.900464:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.900465:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.900468:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.900469:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.900469:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.900471:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.900471:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.900473:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.900474:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.900475:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.900477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.900479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.900484:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.900485:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.900487:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.900487:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677474.900488:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.900488:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.900488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.900490:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.900490:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.900492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.900811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677474.900819:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677474.900822:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677474.900824:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.900827:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677474.900828:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.900829:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.900831:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.900832:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.900833:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677474.900833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000400:00000200:30.0:1619677474.900834:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.900838:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.900839:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:1.0:1619677474.900839:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.900842:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.900842:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:30.0:1619677474.900843:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.900844:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.900844:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:1.0:1619677474.900845:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.900846:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.900847:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000400:00000200:1.0:1619677474.900847:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.900848:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677474.900848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000400:00000200:1.0:1619677474.900849:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677474.900850:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.900851:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677474.900852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:1.0:1619677474.900852:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.900853:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:1.0:1619677474.900856:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.900858:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677474.900861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:1.0:1619677474.900861:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.900862:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:2.0:1619677474.900863:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:1.0:1619677474.900863:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:18.0:1619677474.900865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.900865:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.900865:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677474.900865:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.900866:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:2.0:1619677474.900867:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.900867:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:2.0:1619677474.900870:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:25.2:1619677474.900871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677474.900878:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.900881:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677474.900882:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.900884:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.900887:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677474.900889:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677474.900997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.901003:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.901006:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.901006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.901008:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.901011:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.901012:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.901013:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.901015:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.901016:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.901017:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.901020:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.901022:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.901023:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.901024:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.901025:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.901027:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.901028:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.901029:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.901029:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.901033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.901040:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.901042:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.901043:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.901044:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.901045:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.901046:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.901052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.901060:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.901063:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.901064:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.901064:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.901065:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.901065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.901376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.901382:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.901385:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.901385:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.901387:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.901390:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.901391:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.901392:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.901394:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.901395:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.901396:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.901399:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.901401:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.901403:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.901404:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.901404:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.901406:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.901407:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.901407:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.901408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.901413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.901420:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.901422:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.901423:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.901425:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.901426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.901427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.901468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.901476:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.901479:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.901481:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.901483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.901484:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.901486:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.901487:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.901488:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.901489:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.901490:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.901494:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.901496:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.901498:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.901499:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.901499:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.901501:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.901501:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.901503:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.901505:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.901505:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.901507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.901511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.901515:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.901517:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.901517:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.901519:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.901519:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.901521:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.901521:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.901522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.901523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.901524:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.901535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.901543:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.901545:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.901547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.901548:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.901551:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.901553:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.901554:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.901555:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.901556:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.901557:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.901560:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.901562:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.901564:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.901565:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.901566:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.901568:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.901568:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.901569:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.901570:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.901571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.901574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.901576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.901581:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677474.901583:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.901584:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.901584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.901585:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.901585:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.901586:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677474.901587:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.901587:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.901590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.901847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.901854:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.901857:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.901858:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.901860:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.901860:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.901861:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.901863:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.901864:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.901865:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.901866:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.901869:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.901871:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.901874:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.901875:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.901875:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.901877:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.901877:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.901878:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.901879:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.901883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.901890:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.901893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.901893:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.901894:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.901896:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.901899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.902172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.902179:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.902182:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.902184:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.902186:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.902187:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.902188:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.902190:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.902191:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.902192:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.902193:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.902196:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.902197:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.902199:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.902200:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.902200:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.902202:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.902202:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.902203:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.902204:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.902208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.902214:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.902216:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.902217:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.902219:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.902220:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.902221:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.902224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.902231:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.902233:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.902234:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.902235:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.902235:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.902236:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.902527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.902533:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.902536:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.902537:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.902539:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.902541:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.902542:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.902543:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.902545:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.902546:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.902547:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.902550:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.902552:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.902555:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.902555:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.902556:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.902558:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.902558:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.902559:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.902559:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.902565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.902573:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.902576:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.902577:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.902577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.902579:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.902582:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.902583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.902590:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.902593:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.902594:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.902594:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.902595:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.902596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.902789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.902795:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.902797:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:4.2:1619677474.902797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:21.0:1619677474.902799:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677474.902801:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.902801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.902801:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:21.0:1619677474.902803:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677474.902803:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.902805:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.902806:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.902807:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.902808:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.902811:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677474.902813:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677474.902814:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.902815:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.902816:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.902817:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.902818:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.902819:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.902820:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.902823:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.902824:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:20.0:1619677474.902824:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.902825:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.902826:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677474.902827:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.902827:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.902828:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.902829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.0:1619677474.902829:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.902830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.902830:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.902831:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.902833:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.902833:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:9.2:1619677474.902833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.902834:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.902835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.902839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677474.902839:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.902842:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.902843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.902844:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.902846:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:24.0:1619677474.902846:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.902849:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:24.0:1619677474.902849:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:30.0:1619677474.902850:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.902851:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677474.902853:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.902854:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.902948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.902954:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.902957:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.902958:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.902960:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.902962:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.902963:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.902964:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.902966:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.902967:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.902968:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.902971:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.902973:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.902976:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.902976:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.902977:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.902978:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.902979:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.902980:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.902980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.902985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.902992:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.902994:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.902995:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.902996:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.902997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.902999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.903125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677474.903132:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.903135:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677474.903137:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.903139:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677474.903140:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.903141:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677474.903143:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.903144:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.903145:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.903146:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677474.903150:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.903152:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677474.903155:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.903156:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677474.903156:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.903158:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677474.903159:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.903160:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.903161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.903164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.903172:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.903175:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.903176:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.903177:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.903179:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.903182:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.903610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.903618:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.903621:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.903623:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.903625:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.903626:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.903627:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.903628:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.903630:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.903631:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.903632:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.903635:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.903636:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.903639:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.903640:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.903640:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.903642:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.903643:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.903644:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.903644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.903651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.903659:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.903662:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.903663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.903663:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.903666:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.903668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.904449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.904454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.904457:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.904458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.904460:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.904462:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.904463:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.904464:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.904466:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.904467:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.904468:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.904472:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.904473:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.904476:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.904476:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.904477:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.904479:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.904479:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.904480:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.904480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.904486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.904494:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.904496:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.904497:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.904499:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.904500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.904501:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.904520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.904525:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.904528:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.904530:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.904530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.904533:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.904534:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.904535:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.904537:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.904538:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.904539:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.904542:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.904544:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.904546:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.904547:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.904548:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.904549:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.904550:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.904551:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.904552:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.904553:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.904556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.904558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.904561:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.904563:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.904564:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.904565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677474.904565:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.904567:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.904569:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.904569:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.904571:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.904573:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.904887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677474.904896:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.904899:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677474.904901:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.904903:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677474.904904:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.904905:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677474.904907:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.904908:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.904909:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.904910:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677474.904913:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.904915:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677474.904917:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.904918:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677474.904919:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.904920:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677474.904921:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:4.2:1619677474.904921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677474.904922:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677474.904923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:26.0:1619677474.904928:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:9.2:1619677474.904928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677474.904931:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.904932:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.904934:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.904936:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:26.0:1619677474.904937:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.904938:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.904939:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677474.904939:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:26.0:1619677474.904940:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.904941:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677474.904941:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.904941:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.904942:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677474.904943:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:26.0:1619677474.904946:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.904947:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677474.904948:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.904951:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.904951:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.904952:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.904954:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.904954:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.904955:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.904955:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.904960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.904967:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.904970:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.904971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.904971:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.904973:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.904976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.904978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.904985:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.904988:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.904989:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.904989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.904990:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.904991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.905154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.905161:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.905164:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.905165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.905166:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.905169:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.905171:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.905172:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.905174:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.905175:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.905175:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.905178:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.905180:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.905182:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.905183:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.905183:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.905185:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.905186:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.905187:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.905187:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.905191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.905198:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.905201:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.905202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.905202:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.905205:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.905208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.905211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.905218:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.905220:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.905221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.905222:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.905222:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.905223:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677474.905623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.905629:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.905632:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.905633:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.905634:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.905637:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.905639:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.905640:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.905641:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.905642:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.905643:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.905646:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.905647:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.905650:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.905651:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.905651:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.905653:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.905653:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.905654:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.905654:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.905660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.905668:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.905670:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.905671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.905672:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.905674:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.905676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.905956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.905964:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.905967:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.905969:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.905971:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.905973:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.905974:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.905975:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.905977:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.905978:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.905979:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.905982:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.905984:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.905987:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.905988:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.905989:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.905990:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.905991:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.905993:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.905994:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.905995:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.905997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.906000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.906004:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.906007:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677474.906008:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.906008:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.906008:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.906010:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.906011:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.906012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.906012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.906014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.906062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.906069:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.906073:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.906075:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.906076:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.906077:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.906079:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.906080:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.906081:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.906082:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.906083:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.906086:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.906088:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.906090:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.906090:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.906091:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.906093:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.906093:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.906094:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.906094:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.906098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.906105:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.906107:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.906108:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.906110:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.906111:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.906112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.906198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677474.906204:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.906207:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677474.906207:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677474.906209:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677474.906212:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.906213:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.906215:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.906216:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.906217:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.906218:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.906221:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677474.906223:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.906225:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.906226:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.906227:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.906228:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.906229:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.906229:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677474.906230:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:25.2:1619677474.906234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.906242:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.906244:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.906245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.906245:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.906247:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.906250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.906440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.906446:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.906449:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.906450:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.906451:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.906454:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.906455:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677474.906456:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.906458:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.906459:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.906459:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677474.906462:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.906464:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677474.906467:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.906467:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677474.906468:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677474.906469:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677474.906470:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.906470:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.906471:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.906487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.906495:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.906497:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.906498:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.906498:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.906500:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.906503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.906526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677474.906533:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.906536:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677474.906537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677474.906538:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.906542:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.906543:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677474.906544:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.906545:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.906546:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.906547:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677474.906550:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.906551:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.906553:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.906554:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677474.906554:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.906556:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677474.906556:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677474.906557:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677474.906557:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677474.906561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.906568:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.906569:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.906570:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.906571:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.906571:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.906573:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.906674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.906682:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.906684:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.906686:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.906688:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.906689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.906689:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.906690:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.906691:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.906692:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.906693:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.906696:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.906698:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.906699:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.906700:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.906701:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.906702:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.906703:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.906703:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.906704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.906708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.906715:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.906717:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.906718:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.906720:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.906721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.906722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.906751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.906758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677474.906760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:24.0:1619677474.906761:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.906762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.906763:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.906765:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.906766:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677474.906766:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677474.906767:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.906769:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.906769:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.906770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677474.906771:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.906772:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.906772:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.906776:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.906777:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.906779:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.906780:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:24.0:1619677474.906781:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.906781:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:22.0:1619677474.906781:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.906782:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677474.906783:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.906784:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.906784:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677474.906784:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.906785:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.906785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677474.906785:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.906788:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.906789:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:25.2:1619677474.906790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.906792:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.906793:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.906794:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.906796:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.906797:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.906798:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.906799:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.906800:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.906800:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.906801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.906801:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.906802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.906802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.906802:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.906805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.906808:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677474.906809:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.906809:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.906810:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677474.906811:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.906811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.906812:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.906812:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.906814:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.906815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.906816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.906822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677474.906829:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.906832:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.906833:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.906834:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.906834:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.906835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.907136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.907141:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.907143:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.907145:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.907146:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.907148:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.907149:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.907151:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.907152:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.907153:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.907154:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.907157:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.907159:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.907162:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.907163:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.907163:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.907165:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.907166:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.907166:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.907167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.907183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.907189:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.907192:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.907193:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.907193:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.907195:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.907198:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.907282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.907290:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.907292:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677474.907297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.907299:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.907299:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.907300:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677474.907301:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.907302:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.907303:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.907304:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.907305:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.907306:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.907307:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.907311:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.907313:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.907314:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.907315:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.907316:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.907316:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:31.0:1619677474.907317:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.907317:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.907318:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677474.907319:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.907320:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.907321:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.907321:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.907322:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677474.907324:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.907324:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:25.2:1619677474.907324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.907325:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.907326:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.907326:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.907328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.907329:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.907329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.907330:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.907330:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.907330:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.907330:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.907332:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677474.907332:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.907332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.907334:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.907334:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.907334:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.907335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677474.907336:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.907337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:25.2:1619677474.907338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.907339:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.907340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.907341:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.907346:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.907349:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.907350:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.907350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.907353:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.907355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.907359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.907365:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.907368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.907368:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.907369:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.907370:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.907370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.907675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677474.907680:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.907682:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.907683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677474.907685:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.907687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.907689:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677474.907690:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.907691:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.907692:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.907693:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677474.907697:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.907698:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677474.907701:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.907702:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:22.0:1619677474.907703:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677474.907705:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:22.0:1619677474.907705:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.907706:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677474.907707:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677474.907707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677474.907711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.2:1619677474.907712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.907717:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:26.0:1619677474.907718:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:6.0:1619677474.907719:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677474.907720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:26.0:1619677474.907721:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:6.0:1619677474.907721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:26.0:1619677474.907722:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.907722:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.907724:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.907727:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.907972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677474.907979:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.907982:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677474.907982:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677474.907985:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677474.907987:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.907989:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677474.907990:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.907991:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.907992:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.907993:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.907997:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677474.907998:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677474.908001:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.908002:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677474.908003:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.908004:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677474.908005:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677474.908006:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677474.908006:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677474.908010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.908017:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.908020:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.908021:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.908021:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.908023:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.908026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.908031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.908037:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.908039:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.908040:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.908040:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.908042:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.908042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.908241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.908244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.908247:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.908249:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.908250:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.908252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.908253:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.908255:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.908256:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.908257:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.908258:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.908262:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.908264:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.908266:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.908267:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.908267:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.908269:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.908269:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.908270:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.908271:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.908275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677474.908279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:1.0:1619677474.908283:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677474.908284:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:1.0:1619677474.908286:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:22.0:1619677474.908287:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:17.0:1619677474.908287:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.908287:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.908288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:1.0:1619677474.908290:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.908291:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.908293:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.908294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:8.2:1619677474.908297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:22.0:1619677474.908301:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.908302:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:17.0:1619677474.908302:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.908303:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:22.0:1619677474.908304:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.908304:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.908305:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677474.908306:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.908306:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:22.0:1619677474.908307:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677474.908307:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.908307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:22.0:1619677474.908308:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677474.908308:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.908308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.908309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:17.0:1619677474.908311:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.908312:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.908313:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677474.908314:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:17.0:1619677474.908314:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.908316:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677474.908316:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677474.908317:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:17.0:1619677474.908317:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.908318:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:17.0:1619677474.908318:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.908319:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:17.0:1619677474.908322:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.908323:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.908323:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:17.0:1619677474.908323:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.908324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.908326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677474.908326:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.908327:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.908327:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.908329:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.908330:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.908331:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677474.908332:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677474.908333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:21.0:1619677474.908334:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:25.2:1619677474.908335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.908337:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.908337:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.908339:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:20.0:1619677474.908340:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677474.908343:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.908343:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.908345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677474.908345:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677474.908345:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:6.0:1619677474.908346:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.908347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.908348:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.908348:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677474.908348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677474.908349:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677474.908351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.908760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677474.908766:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.908770:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677474.908770:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677474.908772:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.908774:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.908776:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677474.908777:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.908779:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.908780:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.908781:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677474.908784:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.908786:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677474.908789:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.908790:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677474.908791:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677474.908792:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677474.908793:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677474.908794:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677474.908794:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677474.908798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.908806:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.908808:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.908809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677474.908810:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.908812:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.908814:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677474.908938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.908944:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.908947:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.908948:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.908950:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.908953:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.908954:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.908955:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.908957:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.908958:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.908958:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.908961:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.908963:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.908966:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.908967:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.908967:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.908969:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.908969:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.908970:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.908971:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.908972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677474.908975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677474.908977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677474.908984:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677474.908984:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.908986:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.908987:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:13.0:1619677474.908988:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677474.908988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:13.0:1619677474.908989:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.908992:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.908994:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.908997:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677474.909144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.909150:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.909153:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.909153:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:2.0:1619677474.909155:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.909158:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.909160:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.909161:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.909162:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.909163:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.909165:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.909168:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.909170:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.909173:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.909174:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.909174:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.909176:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.909177:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.909178:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.909178:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.909182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.909191:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.909194:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.909195:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.909198:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.909199:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.909201:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.909202:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.909203:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.909203:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.909204:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.909205:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.909281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.909289:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.909292:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.909299:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.909299:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.909302:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.909303:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677474.909305:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.909307:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.909308:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.909309:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677474.909312:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.909314:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.909317:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.909318:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677474.909318:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.909320:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677474.909321:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.909322:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.909322:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.909327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.909335:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.909338:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.909339:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.909339:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.909342:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.909345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.909602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.909608:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.909611:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.909612:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.909614:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.909617:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.909618:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:4.2:1619677474.909618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677474.909619:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.909621:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.909622:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.909623:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677474.909626:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677474.909626:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.909628:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:18.0:1619677474.909628:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677474.909629:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677474.909630:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677474.909631:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677474.909631:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.909631:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.909633:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.909633:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677474.909634:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.909634:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677474.909635:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:18.0:1619677474.909635:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:26.0:1619677474.909637:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.909638:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.909639:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677474.909639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:26.0:1619677474.909640:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.909643:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.909645:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677474.909646:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:26.0:1619677474.909647:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.909648:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:2.0:1619677474.909648:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:26.0:1619677474.909649:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:18.0:1619677474.909649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.909649:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.909651:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.909651:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677474.909651:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.909652:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.909652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677474.909653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:25.2:1619677474.909657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.909664:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.909667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.909667:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.909668:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.909670:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.909673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.909826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.909831:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.909834:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.909835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.909837:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.909839:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.909840:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.909841:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.909842:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.909843:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.909844:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.909847:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.909848:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.909851:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.909851:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.909852:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.909854:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.909854:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.909855:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.909855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.909860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.909867:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.909870:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.909871:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.909872:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.909874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.909875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.910083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677474.910089:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.910092:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677474.910093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677474.910094:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.910097:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.910098:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677474.910100:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.910101:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.910102:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.910103:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677474.910106:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.910107:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677474.910111:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.910111:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677474.910112:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677474.910113:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677474.910114:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.910114:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.910115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677474.910119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.910126:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.910129:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.910130:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.910130:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.910132:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.910135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.910214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.910221:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.910224:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.910224:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.910227:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.910230:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.910231:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.910232:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.910233:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.910235:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.910235:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.910239:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.910240:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.910243:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.910244:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.910244:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.910246:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.910246:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.910247:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.910247:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.910252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.910260:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.910262:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.910263:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.910264:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.910266:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.910269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.910523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.910529:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.910532:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.910533:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.910535:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.910537:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.910539:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.910540:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.910541:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.910543:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.910544:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.910547:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.910548:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.910551:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.910552:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.910552:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.910554:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.910554:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.910554:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.910555:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.910560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.910568:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.910571:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.910572:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.910572:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.910575:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.910577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.910579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.910588:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.910591:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.910593:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.910596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.910596:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.910598:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.910599:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.910600:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.910602:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.910603:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.910605:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.910607:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.910610:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.910611:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.910612:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.910614:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.910614:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.910615:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.910616:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.910620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.910627:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.910629:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.910630:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.910631:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.910633:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.910636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.910639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.910646:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.910648:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.910649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.910649:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.910651:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.910651:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.911183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.911190:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.911193:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.911196:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.911197:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.911198:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.911200:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.911201:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.911202:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.911203:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.911204:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.911207:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.911209:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.911210:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.911211:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.911212:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.911213:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.911214:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.911215:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.911215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.911219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.911225:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.911228:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.911229:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.911230:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.911232:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.911234:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.911237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.911243:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.911246:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.911247:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.911248:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.911248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.911249:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.911503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.911509:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.911512:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.911514:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.911515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.911517:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.911518:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.911519:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.911521:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.911522:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.911523:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.911526:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.911528:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.911530:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.911531:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.911532:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.911533:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.911534:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.911534:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677474.911534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.0:1619677474.911535:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.911540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.911541:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.911544:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.911545:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.911547:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.911548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:8.0:1619677474.911549:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.911550:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.911551:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.911551:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.911552:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96502447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.911552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.911554:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:8.0:1619677474.911554:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.911556:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.911557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:8.0:1619677474.911557:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.911560:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.911562:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.911565:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.911566:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.911567:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.911569:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.911569:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.911570:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.911571:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.911575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.911582:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.911584:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.911585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.911585:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.911588:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.911590:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.911941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.911944:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.911946:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.911949:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.911949:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.911951:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.911952:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.911954:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.911955:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.911956:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.911957:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.911960:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.911962:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.911964:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.911964:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.911965:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.911966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677474.911967:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.911967:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.911968:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.911969:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.911973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677474.911975:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.911977:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.911980:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.911981:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:31.0:1619677474.911982:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.911982:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.911983:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:21.0:1619677474.911983:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:30.0:1619677474.911984:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.911984:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.911986:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.911986:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.911986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:30.0:1619677474.911987:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.911988:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677474.911988:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:30.0:1619677474.911992:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677474.911993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:30.0:1619677474.911994:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.911996:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.911997:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.911998:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.911999:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.912000:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.912001:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.912001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.912001:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.912004:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677474.912005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677474.912005:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.912006:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.912006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.912008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677474.912011:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.912014:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.912015:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.912017:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.912018:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.912019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.912159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.912165:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.912168:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.912168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.912170:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.912173:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.912174:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.912176:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.912177:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.912178:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.912179:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.912183:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.912184:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.912187:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.912188:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.912188:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.912190:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.912190:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.912191:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.912191:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.912196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.912203:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.912205:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.912206:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.912208:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.912209:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.912210:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.912594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.912602:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.912605:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.912607:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.912609:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.912609:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.912610:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.912611:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.912613:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.912614:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.912615:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.912618:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.912620:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.912624:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.912625:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.912625:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.912627:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.912628:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.912629:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.912631:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.912631:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.2:1619677474.912634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.912638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677474.912642:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.912644:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677474.912646:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:20.0:1619677474.912646:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.912647:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.912648:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:21.0:1619677474.912648:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.912649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.912650:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.0:1619677474.912651:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.912653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.912661:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.912663:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.912664:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.912665:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.912666:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.912666:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.913116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.913124:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.913126:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.913129:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.913130:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.913132:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.913133:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.913135:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.913136:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.913137:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.913138:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.913141:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.913143:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.913146:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.913147:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.913148:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.913150:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.913150:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.913151:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.913152:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.913155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.913163:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.913165:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.913166:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.913167:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.913169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.913171:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.913689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.913695:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.913697:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.913698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.913700:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.913702:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.913703:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.913704:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.913706:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.913707:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.913708:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.913711:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.913713:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.913715:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.913716:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.913717:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.913718:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.913719:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.913721:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.913722:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.913722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.913725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.913728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.913733:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.913733:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.913735:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677474.913735:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677474.913736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.913736:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.913736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.913738:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.913739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.913740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.913745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677474.913752:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.913755:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.913756:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.913756:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.913757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.913758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.914054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.914063:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.914067:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.914068:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.914072:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.914075:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.914077:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.914080:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.914082:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.914083:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.914085:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.914090:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.914094:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.914098:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.914100:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.914101:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.914104:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.914105:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.914106:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677474.914108:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.914109:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.914110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677474.914114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:12.0:1619677474.914119:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.914121:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.914122:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.914122:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:12.0:1619677474.914124:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:2.0:1619677474.914125:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:13.0:1619677474.914126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.914127:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677474.914127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.914129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.2:1619677474.914178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.914186:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.914189:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.914191:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.914193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.914194:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.914195:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.914196:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.914198:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.914199:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.914200:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.914204:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.914206:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.914209:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.914210:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.914210:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.914212:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.914213:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.914214:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.914215:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.914219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.914228:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.914230:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.914231:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.914232:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.914234:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.914237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.914444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.914452:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.914455:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.914455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.914458:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.914461:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.914462:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:5.2:1619677474.914462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000400:00000200:26.0:1619677474.914464:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.914465:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.914466:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.914467:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.914468:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000400:00000200:26.0:1619677474.914470:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.914471:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.2:1619677474.914471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:1.0:1619677474.914471:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677474.914472:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:26.0:1619677474.914475:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677474.914475:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677474.914476:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.914476:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:1.0:1619677474.914477:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.914478:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.914478:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677474.914479:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:26.0:1619677474.914479:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677474.914479:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677474.914480:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677474.914481:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.914481:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677474.914482:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:1.0:1619677474.914482:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.914483:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.914484:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.914485:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677474.914485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677474.914487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:4.2:1619677474.914487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677474.914488:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677474.914489:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.914490:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:1.0:1619677474.914490:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.914491:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96502436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.914493:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.914493:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.914493:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:31.0:1619677474.914494:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.914494:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:1.0:1619677474.914494:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:31.0:1619677474.914495:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:6.0:1619677474.914495:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.914496:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:1.0:1619677474.914496:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677474.914497:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677474.914497:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.914497:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677474.914497:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:22.0:1619677474.914498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.914498:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677474.914498:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000400:00000200:31.0:1619677474.914499:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.914499:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.914501:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:6.0:1619677474.914501:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.914503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.914504:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.914504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.914505:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:25.2:1619677474.914505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677474.914506:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677474.914507:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:31.0:1619677474.914508:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677474.914509:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:31.0:1619677474.914511:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:31.0:1619677474.914511:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:24.0:1619677474.914512:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:25.2:1619677474.914513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:22.0:1619677474.914513:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677474.914515:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.914516:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.914516:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677474.914516:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677474.914517:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:20.2:1619677474.914517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:6.0:1619677474.914517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:24.0:1619677474.914518:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.914518:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.914519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:13.0:1619677474.914521:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:24.0:1619677474.914522:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.914524:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:13.0:1619677474.914524:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677474.914524:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.914526:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.914526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.914527:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:30.0:1619677474.914528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:13.0:1619677474.914528:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:30.0:1619677474.914529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:12.0:1619677474.914529:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677474.914533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.914536:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.914536:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.914537:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.914538:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.914539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.915117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.915125:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.915127:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.915128:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.915130:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.915133:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.915134:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.915135:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.915136:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.915137:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.915138:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.915141:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.915143:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.915146:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.915146:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.915147:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.915149:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.915149:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.915150:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.915150:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.915157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.915163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:6.0:1619677474.915164:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.915167:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.915168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.915168:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:6.0:1619677474.915168:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.915170:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.915171:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.915173:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.915173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677474.915173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:21.0:1619677474.915175:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.915176:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.915178:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.915179:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.915180:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.915181:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:25.2:1619677474.915183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000400:00000200:21.0:1619677474.915184:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.915186:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.915189:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.915190:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.915190:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.915190:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677474.915192:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.915192:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677474.915193:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677474.915193:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677474.915193:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.915194:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677474.915195:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.915195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677474.915195:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.915196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.915198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.915201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677474.915204:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677474.915205:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.2:1619677474.915206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:24.0:1619677474.915207:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677474.915207:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.915208:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677474.915208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.915209:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.915210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:24.0:1619677474.915211:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.915211:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677474.915213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677474.915214:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.915215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.915216:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.915218:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.915220:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.915221:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.915222:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.915223:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677474.915223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:6.0:1619677474.915224:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.915228:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.915229:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677474.915229:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677474.915232:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677474.915232:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.915233:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:1.0:1619677474.915233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.915234:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.915235:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:17.0:1619677474.915236:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.915236:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.915236:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.915237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:17.0:1619677474.915238:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.915239:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:25.2:1619677474.915241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:17.0:1619677474.915241:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.915242:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.915243:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.915244:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.915247:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.915249:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:17.0:1619677474.915249:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677474.915251:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:17.0:1619677474.915251:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677474.915252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.915252:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.915252:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:20.0:1619677474.915253:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.915254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677474.915254:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.915255:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.915255:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677474.915256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677474.915256:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.915260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.915268:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.915270:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.915271:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.915271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.915274:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.915276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.915794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.915800:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.915802:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.915804:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.915805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.915807:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.915809:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.915810:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.915811:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.915812:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.915813:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.915817:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.915818:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.915821:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.915822:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.915823:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.915824:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.915825:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.915826:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.915827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.915831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.915838:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.915841:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.915842:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.915842:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.915844:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.915847:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.916225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.916233:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.916236:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.916238:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.916241:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.916241:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.916242:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.916244:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.916245:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.916246:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.916247:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.916251:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.916253:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.916255:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.916256:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.916256:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.916258:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.916258:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.916260:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.916261:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.916262:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.916263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.916267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.916272:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.916274:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.916274:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.916276:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.916276:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.916276:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.916277:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.916278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.916279:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.916280:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.916521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.916529:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.916532:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.916534:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.916537:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.916537:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.916538:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.916540:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.916541:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.916542:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.916543:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.916545:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.916547:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.916550:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.916551:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.916551:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.916553:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.916553:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.916554:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.916554:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.916560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.916560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:6.0:1619677474.916568:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.916569:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:22.0:1619677474.916571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.916571:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:6.0:1619677474.916571:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.916572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:6.0:1619677474.916572:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.916574:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.916575:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.916577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677474.916579:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.916580:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677474.916581:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.916583:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.916584:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.916585:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677474.916588:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.916589:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.916591:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.916592:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.916592:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.916594:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.916594:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.916595:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.916595:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677474.916600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.916606:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.916607:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.916608:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.916609:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.916610:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.916611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677474.916619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677474.916624:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.916627:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677474.916627:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.916627:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.916628:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.916629:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.916730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677474.916737:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.916740:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677474.916741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677474.916742:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.916745:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.916746:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677474.916747:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.916749:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.916750:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.916751:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677474.916754:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.916756:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677474.916758:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.916758:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677474.916759:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677474.916761:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677474.916761:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.916762:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.916762:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677474.916777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.916785:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.916787:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.916788:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.916788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.916790:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.916793:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.916939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.916946:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.916949:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.916952:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.916953:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.916955:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.916956:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.916957:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.916959:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.916960:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.916961:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.916964:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.916966:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.916968:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.916969:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.916969:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.916971:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.916971:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.916972:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.916973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.916976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.916983:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.916986:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.916986:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.916988:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.916990:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.916990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.2:1619677474.917090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.917096:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.917099:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.917100:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.917102:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.917104:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.917105:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.917106:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.917108:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.917108:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.917109:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.917112:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.917114:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.917115:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.917116:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.917117:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.917119:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.917119:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.917120:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.917121:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.917125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.917132:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.917134:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.917135:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.917135:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.917137:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.917138:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.917473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.917476:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.917479:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.917481:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.917481:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.917483:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.917485:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.917486:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.917487:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.917488:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.917489:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.917493:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.917495:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.917496:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.917497:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.917498:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.917500:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.917500:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.917501:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.917501:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.917506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.917514:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.917516:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.917517:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.917519:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.917519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.917521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.917625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.917631:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.917634:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.917635:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.917637:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.917639:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.917640:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.917641:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.917643:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.917644:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.917645:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.917648:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.917650:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.917652:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.917653:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.917654:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.917655:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.917656:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.917657:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.917657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.917662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.917670:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.917672:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.917673:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.917674:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.917675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.917677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.918466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.918474:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.918478:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.918480:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.918482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.918482:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.918483:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.918484:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.918485:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.918486:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.918488:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.918491:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.918493:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.918496:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.918497:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.918497:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.918499:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.918500:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.918501:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.918501:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677474.918506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.918514:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.918517:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.918518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.918519:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.918520:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.918523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.918679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677474.918686:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.918689:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677474.918689:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677474.918692:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.918693:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.918695:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:17.0:1619677474.918696:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.918698:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.918699:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.918700:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:17.0:1619677474.918703:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.918705:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677474.918708:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.918708:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:17.0:1619677474.918709:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677474.918711:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:17.0:1619677474.918711:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677474.918712:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677474.918713:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677474.918713:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677474.918718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.918719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677474.918724:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:21.0:1619677474.918726:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:1.0:1619677474.918726:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677474.918726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677474.918727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:21.0:1619677474.918728:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.918729:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.918731:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.918733:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.918734:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.919173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.919178:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.919181:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.919182:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.919183:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.919186:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.919187:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.919188:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.919190:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.919190:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.919192:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.919195:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.919197:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.919199:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.919200:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.919201:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.919202:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.919203:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.919204:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.919204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.919209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.919216:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.919218:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.919219:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.919220:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.919221:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.919222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.919320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.919328:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.919331:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.919332:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.919333:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.919336:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.919337:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.919339:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.919340:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.919341:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.919342:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.919345:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.919347:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.919349:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.919350:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.919351:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.919352:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.919353:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.919354:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.919354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.919359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.919367:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.919370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.919370:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.919371:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.919373:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.919375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.919477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.919485:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.919488:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.919489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.919490:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.919493:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.919494:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.919496:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.919497:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.919498:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.919499:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.919502:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.919503:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.919506:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.919507:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.919507:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.919509:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.919509:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.919510:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.919511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.919515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.919522:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.919525:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.919526:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.919526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.919528:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.919531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.919585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.919591:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.919593:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.919595:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.919596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.919597:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.919599:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.919600:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.919602:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.919603:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.919604:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.919607:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.919609:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.919611:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.919612:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.919613:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.919614:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.919615:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.919616:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.919616:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.919620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.919628:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.919630:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.919631:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.919633:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.919634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.919635:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.919858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.919864:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.919867:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.919868:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.919869:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.919872:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.919873:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.919875:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.919876:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.919877:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.919878:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.919881:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.919883:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.919885:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.919886:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.919887:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.919888:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.919889:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.919890:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.919890:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.919894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.919903:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.919905:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.919906:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.919906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.919908:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.919911:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677474.920055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.920060:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.920063:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.920063:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677474.920065:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.920067:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.920069:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.920070:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.920071:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.920072:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.920073:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.920076:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.920078:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.920081:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.920082:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.920082:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.920084:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.920085:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.920086:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.920087:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.920087:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.920090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.920092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.920098:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.920098:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.920100:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677474.920100:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.920101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677474.920102:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.920102:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.920102:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.920104:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.920106:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.920219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.920227:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.920230:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.920232:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.920234:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.920235:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.920237:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.920238:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.920239:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.920240:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.920242:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.920245:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.920247:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.920249:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.920250:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.920250:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.920252:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.920252:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.920253:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.920254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.920257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.920264:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.920267:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.920267:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.920269:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.920271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.920272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.920274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.920281:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.920284:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.920285:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.920286:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.920286:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.920287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.920413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.920421:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.920423:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.920424:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.920426:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.920429:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.920430:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.920432:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.920433:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.920434:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.920435:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.920437:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.920439:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.920442:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.920442:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.920443:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.920445:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.920445:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.920446:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.920447:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.920448:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.920451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.920454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.920459:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.920462:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.920462:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.920463:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.920463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.920464:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.920464:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.920465:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677474.920465:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.920468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.920469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.920476:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.920478:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.920479:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.920479:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.920481:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.920481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.920530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.920538:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.920541:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.920543:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.920545:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.920546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.920547:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677474.920548:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.920550:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.920551:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.920552:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.920555:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.920557:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677474.920560:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.920561:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.920561:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.920563:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677474.920564:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.920565:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.920566:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.920567:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677474.920569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.920573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677474.920576:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.920578:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.920579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.920580:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.920581:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677474.920582:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.920583:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.920584:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.920585:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677474.920585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.920693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.920699:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.920701:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.920702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.920704:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.920706:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.920707:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.920708:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.920710:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.920711:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.920712:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.920715:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.920716:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.920719:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.920719:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.920720:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.920722:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.920722:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.920724:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.920725:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.920725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.920728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.920730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.920735:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.920737:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677474.920737:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.920738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.920739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.920740:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.920741:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.920742:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.920743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.920744:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.920748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.920755:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.920758:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.920759:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.920760:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.920760:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.920762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.921128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.921136:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.921139:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.921141:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.921143:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.921144:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.921145:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.921146:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.921148:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.921149:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.921150:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.921153:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.921155:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.921157:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.921158:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.921159:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.921160:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.921161:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.921162:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.921162:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.921166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.921173:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.921175:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.921176:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.921178:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.921180:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.921181:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.922002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.922010:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.922012:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.922014:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.922017:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.922017:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.922018:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.922019:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.922020:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.922021:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.922022:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.922024:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.922025:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.922027:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.922028:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.922028:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.922030:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.922030:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.922031:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.922032:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.922033:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.922036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.922039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.922044:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.922046:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.922046:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.922048:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.922048:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.922049:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.922049:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.922049:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.922050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.922051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.922174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.922182:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.922185:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.922186:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.922189:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.922189:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.922190:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.922191:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.922193:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.922194:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.922194:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.922197:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.922199:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.922201:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.922202:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.922202:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.922204:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.922205:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.922205:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.922206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.922210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.922217:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.922219:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.922220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.922220:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.922222:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.922225:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.922821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.922824:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.922826:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.922829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.922829:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.922830:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.922832:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.922833:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.922834:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.922836:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.922837:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.922839:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.922841:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.922843:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.922844:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.922845:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.922846:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.922847:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.922848:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.922849:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.922850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.922853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.922856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.922859:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.922861:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.922861:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.922861:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.922862:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677474.922862:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.922863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.922864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.922865:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677474.922867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677474.923085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.923091:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.923095:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.923096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.923098:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.923100:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.923102:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.923103:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.923105:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.923106:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.923107:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.923110:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.923112:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.923115:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.923115:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.923116:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.923118:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.923118:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.923119:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.923120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.923124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.923133:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.923135:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.923136:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.923138:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.923138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.923140:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.923627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.923635:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.923637:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.923638:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.923640:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.923643:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.923645:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.923646:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.923647:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.923648:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.923649:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.923653:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.923655:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.923658:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.923658:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.923659:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.923661:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.923661:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.923662:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.923662:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.923667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.923676:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.923678:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.923679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.923680:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.923682:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.923684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.923686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.923693:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.923696:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.923697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.923697:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.923698:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.923698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.924199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.924207:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.924210:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.924212:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.924215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.924215:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.924216:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.924218:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.924219:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.924220:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.924221:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.924223:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.924225:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.924227:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.924228:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.924228:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.924230:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.924231:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.924232:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.924232:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677474.924232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:9.2:1619677474.924236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677474.924239:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.924242:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.924242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:24.0:1619677474.924243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.924245:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:18.0:1619677474.924245:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.924246:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.924247:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.924247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:18.0:1619677474.924249:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677474.924250:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:18.0:1619677474.924251:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.924252:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.924253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:18.0:1619677474.924253:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.924254:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.924257:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677474.924258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:18.0:1619677474.924259:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.924261:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.924262:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.924262:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.924263:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.924264:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677474.924264:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677474.924265:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.924266:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:8.0:1619677474.924266:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.924267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677474.924267:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:8.0:1619677474.924267:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:9.2:1619677474.924269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677474.924269:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.924269:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.924272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.924278:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.924279:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.924280:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677474.924280:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.924281:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.924281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.924282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.924282:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.924284:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.924284:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.924307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.924315:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.924317:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.924319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.924320:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.924323:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.924325:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.924326:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.924327:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.924328:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.924329:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.924333:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.924335:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.924338:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.924339:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.924339:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.924341:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.924341:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.924343:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.924344:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.924345:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.924347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.924351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.924355:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.924358:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.924358:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:22.0:1619677474.924359:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.924359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.924360:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677474.924361:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.924361:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.924362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.924364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.924654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.924659:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.924661:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.924662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.924664:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.924666:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.924667:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.924668:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.924669:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.924671:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.924671:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.924674:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.924676:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.924679:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.924679:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.924680:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.924682:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.924682:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.924683:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.924683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.924688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.924695:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.924697:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.924698:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.924700:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.924701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.924702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.925072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.925081:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.925085:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.925089:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.925089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.925092:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.925094:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677474.925097:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.925099:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.925100:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.925102:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677474.925107:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.925110:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677474.925113:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.925115:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677474.925115:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677474.925118:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677474.925118:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.925120:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.925121:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677474.925123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.925130:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.925133:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.925134:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.925135:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.925137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.925138:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.925164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.925171:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.925174:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.925175:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.925177:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.925180:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.925182:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.925183:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.925184:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.925185:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.925186:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.925189:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.925190:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.925193:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.925194:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.925194:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.925196:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.925196:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.925197:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.925198:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.925202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.925208:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.925211:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.925212:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.925213:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.925215:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.925217:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.925221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.925229:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.925231:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.925232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.925234:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.925236:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.925237:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.925239:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.925240:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.925241:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.925241:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:5.2:1619677474.925242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:10.0:1619677474.925244:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.925246:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.925248:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.925249:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.925249:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677474.925249:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677474.925251:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.925251:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677474.925251:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.925252:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:10.0:1619677474.925252:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.925253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:1.0:1619677474.925254:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677474.925257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677474.925257:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.925258:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.925259:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.925261:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.925261:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.925262:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.925264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677474.925265:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.925267:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677474.925267:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.925268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.925269:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.925270:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.925270:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:22.0:1619677474.925271:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.925271:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.925273:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.925273:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.925274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677474.925274:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.925275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.925279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.925286:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.925289:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.925289:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.925290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.925291:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.925299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.925368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.925377:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.925380:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.925382:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.925384:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.925385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.925386:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.925387:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.925388:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.925389:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.925390:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.925393:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.925395:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.925397:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.925398:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.925398:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.925400:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.925400:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.925401:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.925402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.925406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.925413:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.925416:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.925416:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.925418:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.925420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.925421:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.925879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.925887:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.925889:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.925891:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.925893:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.925893:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.925895:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.925896:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.925897:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.925898:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.925898:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.925901:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.925903:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.925905:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.925905:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.925906:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.925907:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.925908:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.925908:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.925909:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.925913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.925920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.925923:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.925924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.925924:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.925926:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.925928:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.926051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.926058:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.926061:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.926062:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.926064:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.926066:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.926068:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.926069:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.926070:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.926071:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.926072:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.926075:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.926077:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.926080:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.926080:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.926081:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.926083:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.926083:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.926084:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.926084:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.926089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.926097:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.926100:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.926101:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.926103:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.926103:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.926106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.926162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.926168:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.926171:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.926172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.926173:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.926176:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.926177:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.926178:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.926180:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.926181:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.926181:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.926184:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.926186:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.926189:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.926189:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.926190:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.926192:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.926192:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.926193:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.926193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.2:1619677474.926197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:25.2:1619677474.926198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677474.926205:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677474.926205:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:24.0:1619677474.926208:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:20.0:1619677474.926208:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.926209:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.926209:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.0:1619677474.926211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.926211:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.926212:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.926213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:24.0:1619677474.926215:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.926216:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677474.926217:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.926218:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.926220:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.926220:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677474.926223:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.926225:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 352 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.926226:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.926227:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.926228:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.926229:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.926230:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677474.926231:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677474.926231:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677474.926235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.926241:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.926243:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.926243:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.926244:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.926245:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.926247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.926378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.926385:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.926388:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.926389:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.926391:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.926393:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.926394:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.926396:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.926397:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.926398:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.926399:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.926402:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.926403:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.926407:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.926408:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.926408:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.926410:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.926411:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.926412:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.926412:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:25.2:1619677474.926417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.926426:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.926428:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.926429:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.926431:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.926433:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.926434:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.926452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.926460:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.926463:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.926466:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.926468:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.926468:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.926469:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.926471:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.926472:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.926473:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.926474:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.926477:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.926479:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.926481:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.926482:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.926482:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.926484:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.926485:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.926486:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.926486:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.926490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.926497:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.926499:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.926500:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.926501:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.926503:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.926504:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.926508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.2:1619677474.926511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:30.0:1619677474.926515:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.926518:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677474.926518:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:30.0:1619677474.926519:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.926519:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.926520:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.926520:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:24.0:1619677474.926521:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677474.926522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677474.926523:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.926525:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.926526:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677474.926527:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.926529:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.926530:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.926531:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677474.926534:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.926535:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677474.926537:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.926538:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677474.926538:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677474.926540:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677474.926540:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677474.926541:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677474.926542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:9.2:1619677474.926545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.926553:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.926555:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.926556:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677474.926556:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.926558:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.926560:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.926757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677474.926764:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677474.926766:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677474.926767:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677474.926769:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.926771:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.926773:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:8.0:1619677474.926774:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.926775:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.926776:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.926776:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:8.0:1619677474.926779:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.926781:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:8.0:1619677474.926783:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.926784:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:8.0:1619677474.926784:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:8.0:1619677474.926786:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:8.0:1619677474.926786:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677474.926787:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:8.0:1619677474.926788:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677474.926789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677474.926791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.2:1619677474.926794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:12.0:1619677474.926798:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677474.926800:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:24.0:1619677474.926801:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:12.0:1619677474.926801:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.926802:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.926803:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677474.926804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:13.0:1619677474.926804:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:24.0:1619677474.926805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:12.0:1619677474.926805:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:25.2:1619677474.926809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:13.0:1619677474.926815:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677474.926818:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677474.926819:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677474.926819:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677474.926820:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677474.926821:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677474.926887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.926893:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.926896:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.926897:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.926899:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.926902:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.926903:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.926905:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.926906:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.926907:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.926908:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.926912:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.926913:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.926917:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.926918:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.926918:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.926920:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.926921:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.926922:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.926922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.926926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.926934:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.926937:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.926938:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.926940:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.926941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.926944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.927084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:8.0:1619677474.927091:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677474.927094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677474.927094:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:8.0:1619677474.927097:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.927098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000400:00000200:8.0:1619677474.927099:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.927101:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677474.927102:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.927104:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.927105:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:8.0:1619677474.927105:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.927106:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677474.927107:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000400:00000200:24.0:1619677474.927109:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.927110:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.927111:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.927112:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677474.927112:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:24.0:1619677474.927113:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96502385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.927114:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.927115:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.927116:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.927119:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677474.927120:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.927120:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677474.927121:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.927122:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 344 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.927123:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677474.927124:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (23)++
00000800:00000200:24.0:1619677474.927125:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.927125:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (24)--
00000800:00000200:24.0:1619677474.927126:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.927126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677474.927127:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.927127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:25.2:1619677474.927128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.927129:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.927129:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677474.927130:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677474.927130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:31.0:1619677474.927135:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:25.2:1619677474.927135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677474.927138:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.927138:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.927140:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.927142:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677474.927142:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.927143:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677474.927145:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.927146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.927147:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.927149:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.927151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.927158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.927165:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.927168:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677474.927168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:22.0:1619677474.927169:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.927169:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.927170:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.927171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677474.927175:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677474.927178:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:24.0:1619677474.927179:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:8.0:1619677474.927180:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.927183:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.927185:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:8.0:1619677474.927186:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.927187:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.927188:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.927189:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677474.927192:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.927194:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677474.927196:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.927197:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677474.927198:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677474.927199:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677474.927200:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:8.0:1619677474.927201:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:8.0:1619677474.927202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677474.927206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.927214:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.927216:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.927217:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.927219:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.927219:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.927222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.927225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.927233:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.927236:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.927237:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.927238:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.927238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.927239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.927304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677474.927312:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677474.927315:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677474.927316:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677474.927317:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.927321:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.927322:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:24.0:1619677474.927323:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.927324:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.927325:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.927326:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:24.0:1619677474.927329:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.927331:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677474.927333:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.927334:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677474.927335:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677474.927337:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677474.927337:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677474.927339:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed960000] (22)++
00000800:00000200:24.0:1619677474.927340:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (23)--
00000800:00000200:24.0:1619677474.927340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:8.2:1619677474.927343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.927345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (21)++
00000800:00000200:20.0:1619677474.927351:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.927353:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:8.0:1619677474.927353:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (22)++
00000400:00000200:20.0:1619677474.927355:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.927356:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.927356:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed960000] (23)--
00000800:00000200:8.0:1619677474.927356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000800:00000200:21.0:1619677474.927357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.0:1619677474.927358:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:20.0:1619677474.927359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.927409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.927417:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.927420:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.927421:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.927423:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.927426:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.927427:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.927429:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.927430:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.927431:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.927432:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.927435:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.927436:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.927439:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.927440:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.927441:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.927443:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.927443:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.927444:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.927444:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.927448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.927457:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.927459:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.927460:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.927462:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.927462:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.927464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.927517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.927524:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.927528:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.927529:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.927531:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.927533:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.927535:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677474.927536:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.927537:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.927538:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.927539:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677474.927542:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.927544:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677474.927547:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.927548:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677474.927548:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677474.927550:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677474.927551:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.927551:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.927552:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.2:1619677474.927556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.927564:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.927566:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.927567:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.927569:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.927570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.927572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.927748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed960000] (20)++
00000800:00000200:24.0:1619677474.927755:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677474.927758:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.9@o2ib3
00000800:00000200:8.0:1619677474.927759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (22)--
00000400:00000200:24.0:1619677474.927761:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.9@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.927764:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.927765:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:24.0:1619677474.927767:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.927768:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.927769:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.927770:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677474.927773:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.9@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.927774:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677474.927777:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.927778:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:24.0:1619677474.927779:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677474.927781:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677474.927781:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed960000] (21)++
00000800:00000200:24.0:1619677474.927782:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed960000] (22)--
00000800:00000200:24.0:1619677474.927782:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed960000] (21)--
00000800:00000200:8.2:1619677474.927786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.927794:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.927797:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.927798:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.927799:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.927800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.927802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.927807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.927815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.927817:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.927818:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.927819:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.927819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.927820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.927823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.927831:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.927834:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.927836:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.927838:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.927838:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.927839:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.927841:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.927842:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.927843:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.927844:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.927847:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.927849:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.927851:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.927852:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.927852:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.927854:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.927854:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.927856:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.927857:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.927858:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.927860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.927864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.927867:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.927869:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.927870:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.927870:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.927872:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677474.927872:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.927874:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.927875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677474.927875:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.927876:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677474.928143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.928148:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.928150:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.928152:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.928153:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.928155:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.928156:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.928157:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.928158:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.928159:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.928160:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.928163:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.928165:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.928167:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.928168:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.928168:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.928170:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.928171:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.928172:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.928172:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.928176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.928184:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.928186:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.928187:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.928188:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.928189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.928190:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.928303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677474.928309:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.928312:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677474.928313:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677474.928315:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677474.928316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:6.0:1619677474.928317:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.928318:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677474.928319:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.928321:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:6.0:1619677474.928321:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.928321:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.928323:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:21.0:1619677474.928324:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:20.0:1619677474.928326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.928327:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:6.0:1619677474.928327:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677474.928330:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.928330:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:21.0:1619677474.928331:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.928333:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.928333:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677474.928334:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677474.928334:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:21.0:1619677474.928335:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.928335:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:21.0:1619677474.928336:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.928337:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:21.0:1619677474.928340:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.928341:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.928341:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677474.928342:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677474.928342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:21.0:1619677474.928343:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.928344:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:9.2:1619677474.928344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677474.928345:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.928346:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.928347:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.928348:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.928348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:10.0:1619677474.928351:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:25.2:1619677474.928352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.928354:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.928355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.928355:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.928357:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.928360:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.928360:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677474.928362:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.928363:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.928363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.928365:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677474.928368:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.928390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.928397:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.928400:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.928401:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.928403:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.928406:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.928407:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.928408:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.928410:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.928411:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.928412:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.928415:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.928417:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.928420:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.928421:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.928422:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.928423:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.928424:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.928425:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.928425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.928430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.928437:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.928440:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.928441:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.928441:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.928443:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.928446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.928584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.928589:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.928592:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.928592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.928594:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.928596:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.928598:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.928599:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.928600:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.928601:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.928602:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.928605:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.928607:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.928610:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.928610:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.928611:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.928613:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.928613:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.928615:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.928616:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.928616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.928619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.928621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677474.928626:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.928626:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.928628:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.928628:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677474.928629:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.928629:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.928630:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.928631:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.928632:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.928633:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.928639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677474.928646:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.928648:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.928649:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.928650:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.928650:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.928651:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.928690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.928698:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.928701:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.928704:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.928706:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.928706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.928708:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.928709:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.928710:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.928712:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.928713:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.928716:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.928718:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.928721:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.928722:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.928723:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.928724:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.928725:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.928726:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.928727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.928731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.928739:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.928741:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.928743:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.928744:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.928745:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.928748:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.928749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.928757:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.928760:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.928761:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.928761:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.928762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.928763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.928934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.928942:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.928945:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.928947:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.928949:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.928950:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.928952:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677474.928953:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.928954:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.928955:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.928956:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677474.928959:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.928961:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.928963:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.928964:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.928964:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.928966:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.928966:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.928967:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.928968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:25.2:1619677474.928971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.928978:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.928981:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.928981:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.928983:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.928985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.928986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.928991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.928998:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.929000:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.929001:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.929002:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.929002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.929003:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.929168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.929175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.929177:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.929178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.929180:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.929183:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.929184:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.929185:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.929186:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.929187:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.929188:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.929192:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.929193:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.929196:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.929197:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.929197:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.929199:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.929200:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.929200:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.929201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.929205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.929213:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.929216:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.929216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.929217:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.929219:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.929221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.929235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.929241:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.929245:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.929246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.929248:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.929250:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.929252:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.929253:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.929254:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.929256:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.929256:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.929259:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.929261:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.929264:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.929264:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.929265:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.929266:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.929267:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.929268:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.929268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.929273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.929280:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.929283:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.929283:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.929285:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.929287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.929288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.929617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.929625:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.929628:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.929630:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.929633:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.929633:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.929634:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.929635:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.929637:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.929638:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.929638:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.929642:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.929643:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.929645:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.929646:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.929647:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.929649:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.929649:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.929650:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.929651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.929655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.929662:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.929665:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.929666:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.929666:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.929668:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.929670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.929674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.929680:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.929682:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.929683:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.929683:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.929684:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.929685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677474.929926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.929932:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.929935:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.929936:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.929938:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.929940:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.929942:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.929943:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.929944:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.929945:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.929946:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.929950:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.929951:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.929954:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.929955:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.929955:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.929957:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.929957:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.929958:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.929958:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.929963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.929972:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.929974:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.929975:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.929976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.929978:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.929980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.930005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.930013:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.930016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.930016:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.930019:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.930021:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.930023:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.930024:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.930025:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.930026:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.930027:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.930031:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.930033:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.930035:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.930036:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.930037:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.930038:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.930039:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.930040:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.930040:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.930044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.930052:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.930054:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.930055:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.930055:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.930058:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.930060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.930142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.930147:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.930149:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.930150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.930152:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.930154:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.930155:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.930156:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.930158:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.930159:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.930160:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.930163:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.930165:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.930168:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.930168:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.930169:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.930171:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.930171:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.930172:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.930172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.930177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.930184:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.930187:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.930187:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.930189:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.930190:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.930192:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.930199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.930207:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.930210:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.930212:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.930214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.930215:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.930216:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.930217:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.930219:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.930220:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.930221:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.930224:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.930226:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.930228:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.930228:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.930229:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.930230:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.930231:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.930232:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.930234:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.930234:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.930236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.930240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.930244:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.930246:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.930247:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.930248:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.930249:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.930249:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.930250:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.930251:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.930252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.930253:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.930831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.930835:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.930837:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.930839:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.930840:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.930841:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.930843:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.930844:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.930846:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.930847:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.930848:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.930851:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.930853:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.930855:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.930856:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.930857:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.930859:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.930860:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.930860:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.930861:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677474.930867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.930874:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.930877:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.930878:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.930880:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677474.930882:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.930883:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.931314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.931322:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.931325:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.931327:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.931328:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.931329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.931330:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.931331:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.931332:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.931333:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.931334:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.931337:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.931338:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.931341:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.931342:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.931342:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.931343:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.931344:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.931345:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.931346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.931350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.931357:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.931360:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.931360:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.931362:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.931364:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.931364:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.931425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.931431:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.931434:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.931435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.931437:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.931439:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.931441:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.931442:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.931444:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.931445:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.931446:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.931449:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.931450:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.931452:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.931453:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.931453:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.931455:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.931455:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.931456:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.931457:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.931461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.931468:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.931470:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.931471:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.931471:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.931472:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.931474:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.931484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.931491:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.931494:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.931494:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.931495:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.931495:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.931496:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.931562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.931569:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.931572:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.931575:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.931576:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.931576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.931578:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.931579:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.931580:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.931581:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.931582:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.931586:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.931588:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.931590:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.931591:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.931591:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.931593:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.931594:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.931595:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.931595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.931599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.931606:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.931609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.931609:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.931610:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.931612:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.931613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.931674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.931680:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.931683:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.931684:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.931686:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.931688:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.931689:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.931691:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.931692:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.931693:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.931694:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.931697:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.931699:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.931701:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.931702:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.931703:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.931704:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.931705:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.931706:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.931706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.931710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.931718:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.931720:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.931721:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.931723:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.931723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.931725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.931793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.931799:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.931802:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.931803:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.931805:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.931807:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.931808:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.931809:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.931811:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.931812:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.931813:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677474.931816:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.931818:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.931821:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.931821:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.931822:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.931824:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.931825:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.931826:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.931826:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.931830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.931838:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.931841:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.931841:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.931843:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.931845:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.931846:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.931851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677474.931858:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677474.931861:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677474.931862:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.931862:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677474.931863:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.931865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.932186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.932190:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.932193:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.932195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.932195:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.932197:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.932199:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.932200:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.932201:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.932202:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.932203:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.932206:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.932208:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.932210:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.932211:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.932212:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.932213:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.932214:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.932215:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.932215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.932219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.932227:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.932229:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.932230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.932230:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.932232:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:25.2:1619677474.932234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.932237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.932238:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.932240:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.932242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.932242:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.932243:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.932243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.932465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.932472:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.932475:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.932476:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.932479:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.932481:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.932483:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677474.932484:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.932485:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.932487:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.932487:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677474.932491:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.932493:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677474.932496:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.932497:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677474.932497:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677474.932499:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677474.932499:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.932500:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.932502:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.932502:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677474.932506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.932509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.0:1619677474.932514:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.932516:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677474.932516:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:21.0:1619677474.932517:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.932518:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:21.0:1619677474.932519:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.932519:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:20.0:1619677474.932520:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677474.932520:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.0:1619677474.932522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.933248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.933255:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.933257:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.933258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.933260:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.933263:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.933264:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.933266:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.933267:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.933268:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.933269:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.933272:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.933273:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.933276:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.933277:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.933277:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.933279:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.933279:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.933281:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:21.2:1619677474.933281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:26.0:1619677474.933282:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.933282:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.933286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.933288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677474.933289:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.933292:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:10.0:1619677474.933296:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:31.0:1619677474.933298:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.933298:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.933298:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.933298:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.933299:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677474.933300:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677474.933300:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:30.0:1619677474.933301:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.933302:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677474.933302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.933302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:30.0:1619677474.933304:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.933304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677474.933305:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677474.933305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:30.0:1619677474.933307:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.933307:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677474.933309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:30.0:1619677474.933310:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.933310:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677474.933312:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677474.933313:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.933313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:30.0:1619677474.933314:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.933314:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677474.933315:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:22.0:1619677474.933315:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.933315:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677474.933316:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.933317:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.933318:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.933319:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.933320:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.933323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.933330:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.933332:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.933334:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.933334:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.933336:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.933338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.933443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.933447:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.933449:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.933451:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.933452:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.933453:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.933455:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.933456:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.933457:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.933458:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.933459:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.933462:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.933464:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.933466:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.933467:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.933468:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.933469:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.933470:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.933470:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.933471:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.933475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.933483:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.933485:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.933486:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.933488:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.933488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.933490:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.934017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.934023:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.934026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.934026:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.934028:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.934030:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.934032:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.934033:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.934034:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.934035:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.934036:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.934039:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.934041:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.934044:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.934044:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.934045:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.934047:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.934047:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.934048:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.934048:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.934053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.934060:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.934062:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.934063:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.934065:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.934066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.934067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.934356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.934364:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.934367:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.934367:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677474.934369:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.934372:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.934374:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.934375:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.934376:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.934377:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.934378:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.934382:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.934384:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.934387:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.934387:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.934388:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.934390:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.934390:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.934391:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.934391:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.934396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.934405:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.934407:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.934408:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.934408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.934411:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.934414:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.934700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.934706:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.934709:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.934710:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.934712:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.934714:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.934716:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.934717:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.934718:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.934719:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.934720:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.934724:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.934725:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.934728:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.934729:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.934729:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.934731:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.934731:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.934732:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.934733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.934737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.2:1619677474.934744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677474.934744:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.934747:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.934748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.934748:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.934750:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.934752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.934753:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.934756:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.934758:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.934760:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.934760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.934762:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.934763:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.934765:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.934766:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.934767:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.934770:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.934772:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.934775:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.934776:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.934777:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.934779:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.934779:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.934780:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.934781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.934785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.934793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.934795:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.934796:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.934798:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.934799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.934801:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.934887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.934894:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.934897:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.934898:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.934900:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.934903:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.934904:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.934906:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.934907:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.934908:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.934909:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.934912:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.934913:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.934916:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.934917:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.934917:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.934919:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.934920:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.934920:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.934921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.934925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.934932:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.934935:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.934936:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.934936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.934938:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.934941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.935099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.935106:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.935109:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.935110:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.935112:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.935114:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.935116:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.935117:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.935119:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.935119:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.935120:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.935123:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.935125:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.935127:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.935128:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.935129:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.935130:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.935131:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.935131:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.935132:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677474.935136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.935144:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.935146:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.935147:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.935148:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.935151:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.935152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.935310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.935318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.935321:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.935323:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.935326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.935326:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.935327:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.935329:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.935330:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.935331:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.935332:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.935335:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.935336:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.935338:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.935339:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.935339:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.935341:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.935342:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.935343:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.935345:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.935345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677474.935352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:9.2:1619677474.935359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.935360:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.935362:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.935363:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.935364:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677474.935365:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.935368:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.935369:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.935369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.935370:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.935373:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.935533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.935539:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.935542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.935542:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.935544:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.935546:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.935548:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.935549:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.935551:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.935552:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.935553:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.935556:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.935557:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.935560:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.935561:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.935561:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.935563:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.935564:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.935565:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.935566:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.935566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.935569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.935571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.935576:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.935577:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.935578:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.935579:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.935579:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677474.935580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.935580:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.935582:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.935583:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.935584:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.935590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677474.935597:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.935600:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.935601:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.935601:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.935602:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.935603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.935675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.935683:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.935686:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.935689:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.935690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.935691:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.935693:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.935694:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.935695:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.935697:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.935697:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.935701:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.935703:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.935705:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.935706:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.935706:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.935708:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.935708:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.935709:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.935710:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.935713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.935720:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.935723:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.935723:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.935725:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.935727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.935727:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.935731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.935738:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.935741:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.935742:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.935742:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.935743:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.935744:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.935897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.935901:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.935903:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.935905:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.935906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.935907:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.935909:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.935910:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.935911:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.935912:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.935913:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.935917:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.935918:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.935921:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.935922:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.935923:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.935924:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.935925:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.935926:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677474.935927:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677474.935928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.935930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.935933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677474.935937:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677474.935938:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.935940:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:8.0:1619677474.935940:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.935941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.935941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:8.0:1619677474.935941:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.935942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677474.935943:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.935946:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.935949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.935955:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.935958:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.935959:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.935959:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.935960:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.935960:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.936471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.936480:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.936482:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.936485:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.936487:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.936487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.936488:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.936489:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.936491:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.936492:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.936493:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.936496:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.936498:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.936500:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.936501:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.936502:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.936503:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.936504:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.936504:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.936506:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.936507:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.936509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.936513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.936518:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.936519:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.936520:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.936522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.936522:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677474.936522:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.936523:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.936524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.936525:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.936526:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.936924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.936932:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.936934:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.936936:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.936939:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.936939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.936940:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.936941:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.936942:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.936943:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.936944:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.936947:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.936948:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.936951:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.936951:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.936952:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.936953:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.936954:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.936955:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.936955:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.936959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.936966:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.936969:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.936970:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.936970:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.936971:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.936973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.937010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.937016:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.937019:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.937020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.937022:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.937024:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.937026:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.937027:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.937028:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.937029:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.937030:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.937034:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.937035:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.937038:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.937038:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.937039:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.937041:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.937041:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.937042:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.937042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.937047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.937054:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.937057:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.937058:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.937059:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.937061:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.937062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.937753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.937761:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.937765:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.937767:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.937770:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.937774:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.937776:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.937778:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.937780:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.937782:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.937783:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.937788:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.937791:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.937795:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.937796:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.937797:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.937799:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.937800:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.937802:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.937804:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.937805:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677474.937806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677474.937810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677474.937814:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.937817:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.937818:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.937819:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677474.937819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.937820:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:18.0:1619677474.937822:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.937823:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677474.937824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677474.937825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:4.2:1619677474.938070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.938080:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.938085:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.938086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.938089:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.938093:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.938095:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.938098:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.938100:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.938102:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.938103:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.938109:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.938112:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.938116:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.938117:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.938118:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.938120:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.938121:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.938122:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.938123:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.938127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.938135:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.938138:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.938139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.938140:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.938142:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.938145:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.938841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.938845:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.938848:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.938850:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.938851:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.938853:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.938854:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.938856:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.938857:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.938858:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.938859:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.938862:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.938864:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.938867:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.938868:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.938868:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.938870:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.938871:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.938871:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.938872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.938877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.938884:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.938887:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.938888:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.938888:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.938890:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.938892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.938939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.938944:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.938947:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.938949:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.938949:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.938952:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.938953:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.938954:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.938956:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.938957:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.938958:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.938961:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.938962:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.938964:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.938964:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.938965:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.938967:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.938967:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.938968:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.938969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.938973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.938981:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.2:1619677474.938982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677474.938983:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.938984:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.938985:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.938988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.938989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677474.938990:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.938993:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:8.2:1619677474.938994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:30.0:1619677474.938996:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.938998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.938998:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.938999:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.939001:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.939002:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677474.939002:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:30.0:1619677474.939003:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.939004:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.939005:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.939006:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.939006:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000400:00000200:30.0:1619677474.939007:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.939007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.939008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677474.939009:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.939011:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.939012:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.939013:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.939014:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.939015:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.939016:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.939017:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.939020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.939026:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.939029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.939029:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.939030:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.939032:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.939033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.939054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.939060:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.939063:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.939064:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.939066:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.939068:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.939069:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.939071:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.939072:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.939073:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.939074:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.939077:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.939079:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.939081:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.939081:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.939082:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.939083:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.939084:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.939085:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.939086:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.939086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.939090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.939092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.939099:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.939099:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.939100:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677474.939100:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677474.939100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.939101:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.939101:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:18.0:1619677474.939102:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.939102:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.939104:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.939288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.939295:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.939300:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.939303:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677474.939305:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.939308:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.939309:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.939311:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.939312:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.939313:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.939314:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.939317:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.939319:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.939322:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.939323:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.939323:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.939325:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.939326:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.939326:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.939327:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.939332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.939337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.939342:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.939343:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.939343:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.939345:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.939345:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.939347:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.939348:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.939350:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.939350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.939351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:22.0:1619677474.939352:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:6.0:1619677474.939352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.939353:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.939355:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.939356:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.939357:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.939360:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.939362:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.939364:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.939365:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.939366:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.939367:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.939368:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.939368:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.939369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.939373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.939381:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.939383:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.939384:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.939385:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.939386:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.939388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.939701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.939709:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.939712:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.939713:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.939715:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.939718:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.939719:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.939721:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.939722:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.939723:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.939724:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.939727:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.939729:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.939732:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.939733:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.939733:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.939735:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.939735:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.939736:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.939737:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.939738:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.939741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.939743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.939749:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.939751:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.939751:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.939752:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.939752:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.939753:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.939754:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.939755:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677474.939755:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.939757:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.939759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.939766:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.939769:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.939770:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.939770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.939771:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.939772:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.939969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.939976:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.939979:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.939979:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.939982:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.939984:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.939985:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.939986:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.939988:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.939989:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.939990:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.939993:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.939995:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.939997:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.939998:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.939999:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.940000:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.940001:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.940002:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677474.940003:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.940003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.940006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.940008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.940013:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677474.940014:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.940016:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677474.940016:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.940016:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.940017:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.940018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.940018:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.940020:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.940022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.940291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.940301:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.940305:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.940307:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.940309:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.940309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.940311:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.940312:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.940313:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.940314:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.940315:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.940318:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.940319:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.940322:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.940322:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.940323:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.940324:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.940325:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.940326:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.940326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.940331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.940338:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.940340:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.940341:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.940343:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.940344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.940345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.940653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677474.940657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:30.0:1619677474.940661:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.940663:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:17.0:1619677474.940663:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:30.0:1619677474.940665:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677474.940666:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:30.0:1619677474.940667:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.940667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:31.0:1619677474.940668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.940669:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.940670:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.940670:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.940671:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.940672:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.940672:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.940673:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677474.940674:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677474.940675:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.940677:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.940678:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.940679:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677474.940679:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:30.0:1619677474.940681:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.940684:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677474.940684:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.940685:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.940686:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.940686:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677474.940687:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.940688:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.940689:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.940689:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:17.0:1619677474.940689:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.940690:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677474.940691:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677474.940693:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677474.940693:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:9.2:1619677474.940693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677474.940694:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.940694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.940698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.0:1619677474.940700:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.940702:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.940704:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.940704:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.940705:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:8.0:1619677474.940706:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.940708:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:8.0:1619677474.940708:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677474.940709:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.940709:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.940711:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.940711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bfc00] (20)++
00000800:00000200:9.2:1619677474.940713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677474.940714:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677474.940717:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)++
00000800:00000200:24.0:1619677474.940720:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677474.940720:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.1.70@o2ib3
00000800:00000200:6.0:1619677474.940721:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (22)--
00000800:00000200:24.0:1619677474.940723:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.940723:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.940724:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000400:00000200:22.0:1619677474.940724:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.70@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.940725:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.940725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:22.0:1619677474.940726:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.940728:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:22.0:1619677474.940730:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.940731:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.940732:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.940733:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:22.0:1619677474.940737:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.940738:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:22.0:1619677474.940741:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.940742:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:22.0:1619677474.940743:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:22.0:1619677474.940745:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:22.0:1619677474.940746:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bfc00] (21)++
00000800:00000200:22.0:1619677474.940746:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bfc00] (22)--
00000800:00000200:22.0:1619677474.940747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)--
00000800:00000200:24.2:1619677474.940751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677474.940755:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677474.940758:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677474.940759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677474.940759:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.940761:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.940764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677474.940772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bfc00] (20)++
00000800:00000200:6.0:1619677474.940777:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)++
00000800:00000200:22.0:1619677474.940780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (22)--
00000800:00000200:6.0:1619677474.940780:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.70@o2ib3
00000400:00000200:6.0:1619677474.940783:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.70@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.940786:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.940787:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:6.0:1619677474.940789:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.940790:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.940791:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.940792:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677474.940795:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.940797:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677474.940800:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.940800:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677474.940801:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677474.940803:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677474.940803:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bfc00] (21)++
00000800:00000200:6.0:1619677474.940804:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bfc00] (22)--
00000800:00000200:6.0:1619677474.940804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)--
00000800:00000200:9.2:1619677474.940808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.940815:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.940817:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.940818:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.940818:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.940820:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.940823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.940823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bfc00] (20)++
00000800:00000200:22.0:1619677474.940828:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)++
00000800:00000200:22.0:1619677474.940831:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.70@o2ib3
00000800:00000200:6.0:1619677474.940832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (22)--
00000400:00000200:22.0:1619677474.940833:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.70@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.940836:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.940838:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677474.940839:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.940840:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.940841:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.940842:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677474.940844:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.940846:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.940847:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.940848:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.940848:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.940850:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.940850:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bfc00] (21)++
00000800:00000200:22.0:1619677474.940851:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bfc00] (22)--
00000800:00000200:22.0:1619677474.940851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)--
00000800:00000200:25.2:1619677474.940867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677474.940871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bfc00] (20)++
00000800:00000200:21.2:1619677474.940875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.940876:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.940877:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)++
00000800:00000200:6.0:1619677474.940877:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.940878:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.70@o2ib3
00000400:00000200:6.0:1619677474.940878:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.940880:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.940881:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.70@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677474.940881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.940882:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.940882:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.940883:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677474.940884:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:22.0:1619677474.940885:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.940886:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.940887:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.940888:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:6.0:1619677474.940890:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.940891:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.940893:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:6.0:1619677474.940893:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.940894:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.940895:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.940896:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.940897:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.940898:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.940901:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677474.940901:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.940902:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:22.0:1619677474.940903:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677474.940903:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.940905:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:6.0:1619677474.940905:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.940906:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bfc00] (22)++
00000800:00000200:6.0:1619677474.940906:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.940907:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198623bfc00] (23)++
00000800:00000200:6.0:1619677474.940907:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.940908:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bfc00] (24)--
00000800:00000200:22.0:1619677474.940909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (23)--
00000800:00000200:6.0:1619677474.940909:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.940910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.940910:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.940910:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.940911:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (22)--
00000800:00000200:9.2:1619677474.940911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677474.940911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.940912:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.2:1619677474.940914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bfc00] (21)++
00000800:00000200:25.2:1619677474.940915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677474.940917:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:22.0:1619677474.940919:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bfc00] (22)++
00000800:00000200:10.0:1619677474.940920:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.940921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:22.0:1619677474.940921:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198623bfc00] (23)--
00000400:00000200:10.0:1619677474.940921:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.940922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (22)--
00000800:00000200:22.0:1619677474.940923:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)--
00000800:00000200:20.0:1619677474.940923:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:10.0:1619677474.940923:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.940925:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.940926:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.940927:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:20.0:1619677474.940928:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.940929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.940930:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.940991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bfc00] (20)++
00000800:00000200:6.0:1619677474.940997:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)++
00000800:00000200:22.0:1619677474.941000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (22)--
00000800:00000200:6.0:1619677474.941000:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.70@o2ib3
00000400:00000200:6.0:1619677474.941002:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.70@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.941004:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.941006:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677474.941007:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.941008:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.941009:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.941010:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677474.941013:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.941014:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677474.941017:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.941017:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677474.941018:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677474.941020:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677474.941020:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bfc00] (21)++
00000800:00000200:6.0:1619677474.941021:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bfc00] (22)--
00000800:00000200:6.0:1619677474.941021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)--
00000800:00000200:9.2:1619677474.941025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.941032:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.941034:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:21.2:1619677474.941035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:10.0:1619677474.941035:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677474.941036:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.941037:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.941040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:1.0:1619677474.941040:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.941043:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.941044:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.941046:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.941049:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.941050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bfc00] (20)++
00000400:00000200:1.0:1619677474.941050:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.941052:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.941053:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.941054:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.941055:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)++
00000400:00000200:1.0:1619677474.941055:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677474.941058:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.70@o2ib3
00000800:00000200:6.0:1619677474.941058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (22)--
00000400:00000200:1.0:1619677474.941058:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.941060:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:22.0:1619677474.941061:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.70@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677474.941063:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677474.941064:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.941064:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.941065:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:22.0:1619677474.941066:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:1.0:1619677474.941066:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:22.0:1619677474.941067:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677474.941067:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.941067:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:22.0:1619677474.941068:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.941068:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:22.0:1619677474.941069:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.941070:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.2:1619677474.941073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:22.0:1619677474.941074:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.941076:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.2:1619677474.941076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677474.941079:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.941079:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677474.941080:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677474.941082:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677474.941082:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bfc00] (21)++
00000800:00000200:22.0:1619677474.941083:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bfc00] (22)--
00000800:00000200:31.0:1619677474.941084:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677474.941084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)--
00000800:00000200:21.0:1619677474.941085:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.941087:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:31.0:1619677474.941088:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677474.941088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:21.0:1619677474.941088:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.941090:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.941090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:31.0:1619677474.941091:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.941092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.941093:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:31.0:1619677474.941094:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.941094:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:31.0:1619677474.941095:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.941095:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.941096:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.941096:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.941097:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.941098:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.941098:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.941100:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.941103:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.941104:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.941106:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.941107:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.941107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677474.941108:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.941108:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.941110:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.941110:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.941111:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.941112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.941115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.941122:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.941125:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.941125:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.941127:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.941128:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.941130:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.941169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677474.941172:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677474.941175:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:21.2:1619677474.941177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bfc00] (20)++
00000400:00000200:20.0:1619677474.941177:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.941179:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.941181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677474.941181:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.941182:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.941182:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)++
00000400:00000200:20.0:1619677474.941183:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.941184:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.941185:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.941185:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.70@o2ib3
00000800:00000200:22.0:1619677474.941186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (22)--
00000400:00000200:6.0:1619677474.941189:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.70@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.941190:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.941193:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:6.0:1619677474.941194:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.941195:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.941196:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.941196:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677474.941196:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677474.941197:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677474.941198:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.941198:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677474.941199:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000400:00000200:6.0:1619677474.941199:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677474.941200:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000400:00000200:6.0:1619677474.941200:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.941201:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677474.941202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:25.2:1619677474.941204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677474.941204:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.941205:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.2:1619677474.941207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:6.0:1619677474.941209:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677474.941209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:6.0:1619677474.941210:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677474.941210:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.941212:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.941213:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677474.941213:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:6.0:1619677474.941213:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677474.941213:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bfc00] (21)++
00000400:00000200:22.0:1619677474.941214:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.941214:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677474.941214:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bfc00] (22)--
00000800:00000200:21.0:1619677474.941215:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:6.0:1619677474.941215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bfc00] (21)--
00000800:00000200:21.0:1619677474.941216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677474.941216:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:22.0:1619677474.941217:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.941217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677474.941217:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.941218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.941218:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:8.2:1619677474.941219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677474.941220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:20.0:1619677474.941221:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.941223:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.941224:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.941225:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.941226:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.941227:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.941227:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.941229:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:20.0:1619677474.941230:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677474.941230:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.941231:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.941232:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.941233:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.941234:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.941235:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.941235:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.941235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677474.941237:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.941238:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.941239:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.941240:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.941241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677474.941243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677474.941246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.941250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677474.941251:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.941252:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.941252:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.941252:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.941253:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.941253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:21.0:1619677474.941255:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677474.941255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.941257:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.941387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.941395:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.941398:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677474.941399:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677474.941401:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.941403:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.941405:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.941406:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.941407:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.941408:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.941409:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.941412:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.941414:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.941416:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.941417:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.941418:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.941419:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.941420:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.941421:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.941421:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.941426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.941434:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.941436:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.941437:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.941438:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.941440:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.941441:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.942396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.942402:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.942405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.942405:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.942407:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.942410:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.942411:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.942412:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.942414:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.942415:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.942416:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.942419:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.942421:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.942423:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.942424:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.942425:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.942426:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.942427:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.942428:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.942429:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.942429:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.942433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.942435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.942440:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.942441:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.942442:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.942443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677474.942443:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677474.942444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677474.942444:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.942445:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.942447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.942448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.942452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677474.942460:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.942462:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.942463:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.942464:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.942464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.942465:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.942787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.942795:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.942798:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.942801:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.942803:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.942803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.942804:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.942806:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.942807:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.942808:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.942809:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.942812:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.942814:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.942816:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.942817:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.942818:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.942820:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.942820:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.942822:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.942823:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.942824:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.942825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.942830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.942832:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.942835:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.942836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.942836:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.942837:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677474.942838:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.942840:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.942840:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677474.942840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.942841:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.942951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.942958:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.942961:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.942962:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.942964:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.942966:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.942967:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.942969:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.942970:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.942971:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.942972:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.942975:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.942976:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.942979:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.942980:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.942980:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.942982:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.942982:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.942983:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.942983:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.942988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.942996:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.942999:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.943000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.943000:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.943002:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.943005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.943007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.943014:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.943016:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.943017:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.943017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.943018:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.943019:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.943433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.943438:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.943441:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.943443:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.943443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.943445:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.943447:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.943448:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.943449:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.943450:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.943451:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.943454:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.943456:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.943458:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.943459:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.943460:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.943461:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.943462:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.943463:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.943463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.943468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.943476:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.943478:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.943479:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.943480:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.943481:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.943484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.944135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.944143:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.944146:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.944147:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.944148:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.944152:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.944153:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.944154:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.944156:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.944157:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.944158:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.944161:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.944162:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.944165:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.944166:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.944166:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.944168:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.944168:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677474.944169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677474.944170:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.944171:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.944172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.944174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.944177:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677474.944177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677474.944179:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.944181:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.944181:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677474.944184:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.944184:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.944184:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.944184:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:30.0:1619677474.944185:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677474.944185:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.944185:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677474.944187:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.944187:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.944187:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:30.0:1619677474.944188:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677474.944188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677474.944189:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.944189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.944190:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677474.944191:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.944194:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677474.944195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677474.944196:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.944199:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.944200:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.944200:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.944202:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677474.944202:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.944203:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.944203:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.944204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677474.944205:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.944206:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.944206:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.944207:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.944208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677474.944218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.944226:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.944228:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.944230:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.944231:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.944232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.944234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677474.944310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.944315:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.944318:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.944320:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.944321:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.944322:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.944324:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.944325:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.944326:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.944328:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.944328:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.944332:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.944334:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.944336:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.944337:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.944337:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.944339:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.944339:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.944340:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.944342:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.944342:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.944345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.944352:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.944354:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.944355:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.944357:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.944358:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.944359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.944375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.944380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.944382:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.944383:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.944384:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677474.944535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.944540:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.944542:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.944544:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.944545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.944546:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.944547:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.944549:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.944550:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.944551:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.944551:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.944554:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.944556:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.944558:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.944559:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.944560:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.944561:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.944562:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.944562:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.944563:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.2:1619677474.944567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:9.2:1619677474.944568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677474.944573:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677474.944574:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:2.0:1619677474.944576:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.944577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677474.944577:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.944578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.944578:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.944580:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.944581:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:2.0:1619677474.944582:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.944583:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:8.0:1619677474.944584:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:2.0:1619677474.944585:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.944586:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.944587:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.944588:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.944592:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.944593:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.944597:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.944598:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.944598:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.944600:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:4.2:1619677474.944601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677474.944601:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.944602:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.944602:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.944606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.944609:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.944612:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.944613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677474.944614:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:10.0:1619677474.944615:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.944616:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.944618:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.944618:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.944620:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:10.0:1619677474.944620:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677474.944621:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:10.0:1619677474.944621:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677474.944623:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:10.0:1619677474.944623:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.944624:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.944625:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.944628:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.944630:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.944633:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.944634:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.944634:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.944636:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.944637:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.944637:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.944638:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.944642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.944650:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.944653:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.944653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.944654:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.944656:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.944658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.944765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.944771:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677474.944772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:1.0:1619677474.944774:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.944775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677474.944777:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:1.0:1619677474.944777:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.944779:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.944780:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.944781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:1.0:1619677474.944781:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.944782:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.944783:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.944784:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.944785:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.944786:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.944786:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.944790:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677474.944792:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:1.0:1619677474.944792:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:6.0:1619677474.944794:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.944795:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.944795:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.944795:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:6.0:1619677474.944796:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677474.944796:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677474.944797:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677474.944798:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.944798:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.944799:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.944799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677474.944801:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677474.944803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.944803:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.944805:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.944807:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.944807:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.944810:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.944811:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.944812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.944812:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.944813:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.944813:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:22.0:1619677474.944814:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.944814:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:25.2:1619677474.944816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:22.0:1619677474.944816:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.944818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677474.944823:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.944825:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.944826:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.944828:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.944829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.944830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.944924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.944932:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.944935:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.944937:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.944939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.944940:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.944941:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.944943:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.944944:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.944945:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.944946:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.944949:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.944951:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.944954:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.944955:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.944955:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.944957:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.944958:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.944959:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.944959:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.944963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.944970:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.944972:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.944973:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.944973:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.944975:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.944977:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.944986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.944992:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.944994:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.944995:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.944995:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.944996:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.944996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.945043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.945050:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.945052:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.945054:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.945057:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.945057:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.945058:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.945059:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.945060:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.945061:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.945062:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.945065:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.945067:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.945068:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.945069:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.945070:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.945071:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.945072:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.945073:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.945074:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.945075:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677474.945081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677474.945087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.945091:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.945093:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.945093:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.945095:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.945096:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.945096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677474.945097:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.945097:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677474.945098:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.945100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.945109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.945115:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.945118:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.945119:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.945120:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.945120:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.945121:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.945906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.945914:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.945917:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.945919:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.945920:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.945921:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.945921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.945923:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.945924:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.945925:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.945926:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.945928:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.945930:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.945931:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.945932:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.945933:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.945934:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.945935:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.945935:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.945936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.945951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.945958:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.945960:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.945961:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.945962:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.945964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.945965:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.946325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.946330:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.946333:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.946334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.946336:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.946338:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.946339:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.946340:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.946342:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.946343:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.946344:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.946347:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.946349:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.946352:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.946353:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.946353:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.946355:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.946355:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.946356:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.946357:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.946361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.946370:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.946372:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.946373:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.946375:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.946375:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.946377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.946758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.946766:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.946768:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.946771:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.946772:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.946773:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.946774:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.946775:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.946776:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.946777:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.946778:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.946781:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.946783:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.946784:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.946785:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.946786:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.946787:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.946788:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.946789:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.946789:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.946803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.946810:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.946813:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.946814:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.946815:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.946817:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.946818:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.946862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.946869:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.946872:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.946874:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.946875:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.946876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.946877:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.946878:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.946879:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.946880:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.946880:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.946883:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.946885:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.946887:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.946888:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.946888:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.946890:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.946890:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.946891:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.946892:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.946896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.946903:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.946905:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.946906:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.946906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.946908:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.946910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.947255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.947262:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.947265:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.947266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.947268:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.947270:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.947272:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.947273:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.947274:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.947275:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.947276:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.947279:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.947281:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.947284:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.947284:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.947285:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.947286:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.947287:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.947287:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.947288:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.947292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.947301:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.947304:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.947305:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.947305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.947308:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.947310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.947842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.947847:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.947850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.947850:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.947853:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.947854:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.947855:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.947857:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.947858:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.947859:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.947860:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.947863:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.947865:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.947867:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.947868:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.947869:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.947870:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.947871:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.947871:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.947871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.947877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.947885:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.947887:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.947888:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.947890:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.947891:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.947893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.948439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.948447:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.948450:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.948452:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.948455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.948455:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.948456:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.948458:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.948459:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.948460:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.948461:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.948464:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.948465:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.948468:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.948469:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.948469:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.948471:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.948471:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.948473:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.948473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.948477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.948485:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.948487:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.948488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.948488:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.948490:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.948492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.948798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.948804:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.948807:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.948808:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.948810:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.948812:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.948814:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.948815:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.948816:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.948817:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.948818:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.948822:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.948823:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.948826:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.948827:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.948827:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.948829:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.948829:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.948830:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.948831:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677474.948835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.948843:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.948845:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.948846:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.948846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.948849:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.948851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.948984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.948991:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.948994:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.948995:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.948997:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.949000:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.949002:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.949004:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.949005:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.949006:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.949007:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.949010:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.949012:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.949015:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.949016:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.949016:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.949018:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.949019:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.949020:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.949020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.949025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.949032:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.949035:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.949036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.949037:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.949039:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.949042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.949121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.949126:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:4.2:1619677474.949126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:21.0:1619677474.949129:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.949131:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.949132:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:31.0:1619677474.949134:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000400:00000200:21.0:1619677474.949134:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.949135:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.949136:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.949137:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:21.0:1619677474.949138:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.949140:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.949141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:21.0:1619677474.949141:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.949142:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.949144:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.949145:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677474.949146:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.949147:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.949147:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.949149:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.949150:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.949150:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.949151:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.949151:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.949152:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.949154:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.949154:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:31.0:1619677474.949155:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.949155:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.949156:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:31.0:1619677474.949157:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.949157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.949159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.949160:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.949161:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:31.0:1619677474.949162:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (23)++
00000800:00000200:4.2:1619677474.949162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:31.0:1619677474.949164:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (24)--
00000800:00000200:31.0:1619677474.949164:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677474.949165:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:20.0:1619677474.949165:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:31.0:1619677474.949166:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:8.0:1619677474.949166:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:20.0:1619677474.949168:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:8.0:1619677474.949168:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677474.949169:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (24)++
00000800:00000200:20.0:1619677474.949169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677474.949170:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000800:00000200:20.0:1619677474.949170:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677474.949170:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.949171:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.949172:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:24.0:1619677474.949173:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.949175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.949175:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.949176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.949183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.949189:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.949191:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.949192:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.949192:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.949193:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.949193:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677474.949548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.949554:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.949557:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.949558:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.949559:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.949562:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677474.949563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:1.0:1619677474.949564:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.949565:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.949566:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.949567:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.949568:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677474.949572:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:1.0:1619677474.949572:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.949574:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677474.949576:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:1.0:1619677474.949576:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.949577:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:2.0:1619677474.949578:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677474.949578:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.949579:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.949580:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.949580:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677474.949581:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677474.949581:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677474.949584:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677474.949586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:18.0:1619677474.949587:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677474.949589:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.949591:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.949593:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677474.949594:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:18.0:1619677474.949594:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677474.949597:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.949598:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.949598:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:18.0:1619677474.949600:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677474.949601:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.949603:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677474.949604:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677474.949608:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.949610:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677474.949610:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677474.949613:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677474.949614:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.949615:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.949616:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.949619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677474.949620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.949623:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.949626:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:13.0:1619677474.949626:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:20.0:1619677474.949628:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.949629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:13.0:1619677474.949629:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:20.0:1619677474.949630:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.949630:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.949632:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:13.0:1619677474.949633:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.949633:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:20.0:1619677474.949634:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.949636:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.949636:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677474.949637:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677474.949638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:20.0:1619677474.949638:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.949640:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.949642:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.949644:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.949644:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.0:1619677474.949645:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.949645:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.949647:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:12.0:1619677474.949647:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677474.949648:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677474.949648:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:20.0:1619677474.949649:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:12.0:1619677474.949649:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:20.0:1619677474.949650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677474.949651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.949652:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677474.949654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.949662:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.949664:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.949665:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.949667:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.949667:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.949669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.949673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.949681:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.949684:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.949685:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.949685:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.949686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.949687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.949853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.949861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.949865:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.949867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.949870:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.949874:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.949877:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.949879:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.949881:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.949883:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.949884:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.949890:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.949893:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.949895:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.949897:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.949898:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.949900:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.949901:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.949902:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.949903:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.949906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.949915:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.949919:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.949920:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.949921:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.949923:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.949924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677474.949925:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677474.949933:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.949936:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.949937:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.949939:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.949942:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.949944:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.949945:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.949946:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.949947:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.949948:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.949952:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.949954:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.949957:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.949958:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.949958:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.949960:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.949960:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.949961:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.949961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.949966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.949974:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.949976:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.949977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.949978:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.949980:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.949982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.949987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.949994:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.949997:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.949998:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.949998:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.949999:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.950000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.950024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.950033:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.950036:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.950038:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.950040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.950041:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.950041:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.950042:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677474.950042:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.950043:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.950044:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.950045:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.950046:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.950047:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.950051:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.950053:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.950054:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.950055:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677474.950056:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.950056:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.950056:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677474.950058:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677474.950058:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677474.950059:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.950060:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.950061:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.950062:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.950063:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677474.950063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.950064:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.950064:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677474.950065:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.950066:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.950066:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.950067:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.950068:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.950069:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.950069:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:30.0:1619677474.950069:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.950070:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:30.0:1619677474.950070:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.950071:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.950072:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.950072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.950072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.950073:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.950075:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.950075:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.950077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.950081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.950083:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.950085:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.950086:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.950087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.950088:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.950089:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677474.950090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.950091:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.950092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.950093:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.950183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.950189:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.950192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.950192:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.950194:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.950196:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.950198:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.950199:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.950200:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.950202:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.950203:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.950206:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.950208:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.950210:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.950211:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.950212:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.950213:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.950214:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.950215:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.950216:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.950217:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.950220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.950222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.950227:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677474.950227:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.950229:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677474.950229:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677474.950230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677474.950230:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.950230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.950232:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.950232:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.950234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.950240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677474.950247:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.950250:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.950250:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.950251:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.950252:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.950253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.950792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.950800:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.950803:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.950805:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.950807:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.950808:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.950809:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.950810:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.950811:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.950812:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.950814:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.950817:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.950819:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.950820:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.950821:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.950822:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.950823:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.950824:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.950825:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.950826:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.950827:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.950829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.950832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.950837:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.950839:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.950840:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.950840:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.950841:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:31.0:1619677474.950842:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.950842:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.950843:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.950844:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.950846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.950848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.950854:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.950857:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.950858:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.950858:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677474.950859:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.950860:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.951277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.951284:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.951287:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.951289:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.951290:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.951291:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677474.951291:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.951292:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.951301:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.951302:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.951303:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.951305:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.951307:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.951309:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.951310:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.951311:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.951312:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.951313:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.951314:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.951314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.951319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.951325:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.951328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.951328:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677474.951329:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.951331:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.951333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.951714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.951722:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.951725:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.951727:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.951729:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.951730:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.951731:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.951732:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.951733:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.951734:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.951735:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.951738:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.951739:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.951742:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.951742:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.951743:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.951745:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.951745:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.951746:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.951746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677474.951746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:9.2:1619677474.951750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677474.951752:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.951755:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:24.0:1619677474.951756:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:6.0:1619677474.951756:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677474.951758:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:22.0:1619677474.951758:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.951759:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.951760:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.951760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.951762:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.951762:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.951763:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.951764:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:22.0:1619677474.951765:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.951766:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.951767:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.951770:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.951772:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.951774:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.951775:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.951776:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.951777:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.951778:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.951779:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.951779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.951783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.951791:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.951794:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.951795:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.951796:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.951798:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.951798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.952050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.952058:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.952060:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.952061:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.952064:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.952067:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.952068:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.952070:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.952071:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.952072:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.952073:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.952076:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.952078:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.952081:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.952082:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.952082:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.952084:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.952084:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.952086:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.952087:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.952087:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.952090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.952093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.952098:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.952100:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.952100:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.952101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.952101:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.952102:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.952102:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.952103:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.952104:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.952107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.952110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677474.952116:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.952119:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.952120:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.952120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.952121:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.952122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.952419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.952426:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.952428:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.952429:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.952432:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.952434:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.952435:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677474.952437:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.952438:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.952439:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.952440:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677474.952443:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.952445:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677474.952448:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.952449:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677474.952450:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677474.952451:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677474.952452:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.952452:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.952453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677474.952458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677474.952465:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677474.952468:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.952469:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.952469:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677474.952471:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.952475:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677474.952479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677474.952485:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.952487:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677474.952488:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.952489:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.952489:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677474.952490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.952724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.952732:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.952735:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.952737:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.952739:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.952739:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.952741:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.952742:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.952743:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.952744:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.952745:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.952749:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.952750:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.952754:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.952755:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.952756:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.952757:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.952758:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.952759:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.952759:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.952764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.952773:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.952775:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.952776:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.952778:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.952779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.952781:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.952788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677474.952795:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.952797:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.952798:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.952799:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.952800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.952801:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.953281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.953287:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.953290:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.953291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.953293:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.953300:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.953301:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.953303:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.953304:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.953305:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.953306:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.953309:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.953311:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.953313:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.953314:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.953315:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.953317:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.953317:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.953318:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.953318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.953323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.953331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.953333:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.953334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.953336:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.953337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.953338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.953480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.953486:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.953489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.953489:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677474.953492:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.953494:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.953496:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.953497:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.953499:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.953499:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.953501:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.953504:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.953506:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.953508:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.953509:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.953510:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.953511:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.953512:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.953513:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.953515:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.953515:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.953518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.953521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.953526:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677474.953527:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.953528:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.953529:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.953529:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677474.953530:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.953530:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.953532:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.953533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.953534:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.953570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.953573:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.953576:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677474.953578:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677474.953579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.953581:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.953582:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677474.953584:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.953585:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.953586:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.953587:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677474.953590:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.953592:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677474.953595:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.953596:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677474.953596:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677474.953598:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677474.953599:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.953600:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.953600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677474.953604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.953610:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.953614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.953614:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.953615:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.953617:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.953619:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.953678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677474.953683:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.953686:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677474.953688:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.953688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677474.953690:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.953692:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.953693:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.953695:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.953695:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.953697:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.953700:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.953701:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.953703:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.953704:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.953704:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.953706:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.953706:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.953707:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.953708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.953712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.953719:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.953721:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.953722:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.953724:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.953725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.953726:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.953839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.953846:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.953849:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.953850:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.953852:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.953854:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.953856:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.953857:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.953859:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.953860:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.953861:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.953864:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.953866:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.953869:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.953870:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.953870:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.953872:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.953873:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.953873:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.953874:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.953878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.953886:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.953889:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.953890:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.953891:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.953893:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.953896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.953940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907e3cc800] (20)++
00000800:00000200:21.2:1619677474.953945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:2.0:1619677474.953946:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cc800] (21)++
00000800:00000200:18.0:1619677474.953950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cc800] (22)--
00000800:00000200:2.0:1619677474.953950:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.38@o2ib3
00000400:00000200:2.0:1619677474.953952:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.3.38@o2ib3 : GET - for me
00000800:00000200:30.0:1619677474.953953:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:2.0:1619677474.953954:0:19932:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.3.38@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:30.0:1619677474.953956:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:2.0:1619677474.953957:0:19932:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.3.38@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000400:00000200:30.0:1619677474.953958:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.953958:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907e3cc800] (21)++
00000800:00000200:2.0:1619677474.953959:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907e3cc800] (22)++
00000800:00000200:31.0:1619677474.953960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.953961:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.953961:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907e3cc800] (23)--
00000400:00000200:30.0:1619677474.953963:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:2.0:1619677474.953963:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.3.38@o2ib3 local destination
00000400:00000200:30.0:1619677474.953966:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.953968:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677474.953968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907e3cc800] (22)++
00000400:00000200:2.0:1619677474.953968:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.3.38@o2ib3(10.51.3.38@o2ib3:10.51.3.38@o2ib3) : REPLY try# 0
00000400:00000200:30.0:1619677474.953969:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.953970:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:2.0:1619677474.953972:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.3.38@o2ib3
00000800:00000200:18.0:1619677474.953974:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cc800] (23)++
00000800:00000200:2.0:1619677474.953974:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91906ffa8b80] -> 10.51.3.38@o2ib3 (2) version: 12
00000400:00000200:30.0:1619677474.953975:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.953975:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907e3cc800] (24)--
00000800:00000200:2.0:1619677474.953975:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91907e3cc800] (23)++
00000800:00000200:18.0:1619677474.953976:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cc800] (24)--
00000800:00000200:2.0:1619677474.953976:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907e3cc800] (24)++
00000800:00000200:30.0:1619677474.953977:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677474.953977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cc800] (24)--
00000800:00000200:2.0:1619677474.953977:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91907e3cc800] (25)--
00000800:00000200:2.0:1619677474.953977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cc800] (22)--
00000800:00000200:30.0:1619677474.953980:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.953980:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.953981:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.2:1619677474.953982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907e3cc800] (21)++
00000800:00000200:30.0:1619677474.953983:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.953983:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.953984:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.953985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:18.0:1619677474.953988:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907e3cc800] (22)++
00000800:00000200:9.2:1619677474.953988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677474.953990:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907e3cc800] (23)--
00000400:00000200:18.0:1619677474.953991:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.953992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cc800] (22)--
00000400:00000200:18.0:1619677474.953993:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.3.38@o2ib3: REPLY: OK
00000800:00000200:18.0:1619677474.953995:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907e3cc800] (21)--
00000800:00000200:8.0:1619677474.953995:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.953997:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.953998:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.953998:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.954000:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.954002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.954011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.954017:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.954019:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.954020:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.954020:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.954020:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.954021:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677474.954307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.954313:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.954316:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.954317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.954319:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.954321:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.954322:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.954323:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.954325:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.954326:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.954327:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.954330:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.954332:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.954335:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.954335:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.954336:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.954338:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.954338:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.954339:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.954339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.954345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.954353:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.954356:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.954357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.954357:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.954359:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.954361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.954799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.954807:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.954810:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.954810:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.954813:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.954815:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.954817:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.954818:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.954819:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.954820:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.954821:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.954825:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.954827:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.954830:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.954830:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.954831:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.954833:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.954833:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.954834:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.954834:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.954839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.954846:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.954849:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.954849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.954850:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.954852:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.954855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.955087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.955092:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.955095:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.955097:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.955098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.955099:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.955101:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.955102:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.955103:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.955104:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.955105:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.955109:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.955111:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.955113:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.955114:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.955114:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.955116:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.955116:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.955117:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.955118:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.955122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.955130:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.955132:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.955133:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.955134:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.955135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.955136:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.955143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.955149:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.955152:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.955154:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.955155:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.955158:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.955159:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677474.955160:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.955162:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.955163:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.955164:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677474.955167:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.955168:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677474.955171:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.955171:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677474.955172:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677474.955174:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677474.955174:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.955175:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.955175:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.955180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.955188:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.955190:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.955191:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.955191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.955193:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.955196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677474.955201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677474.955207:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:4.2:1619677474.955207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:6.0:1619677474.955209:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677474.955210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677474.955212:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.955214:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677474.955215:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:6.0:1619677474.955216:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677474.955217:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677474.955218:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:6.0:1619677474.955219:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.955220:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.955221:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.955221:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:13.0:1619677474.955222:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.955223:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.955225:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677474.955226:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:12.0:1619677474.955227:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.955227:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:12.0:1619677474.955228:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.955230:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677474.955230:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:12.0:1619677474.955231:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677474.955231:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677474.955232:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677474.955234:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677474.955234:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000400:00000200:12.0:1619677474.955235:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.955235:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677474.955235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:12.0:1619677474.955237:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:12.0:1619677474.955240:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677474.955240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:12.0:1619677474.955241:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:12.0:1619677474.955241:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:12.0:1619677474.955243:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (24)--
00000800:00000200:12.0:1619677474.955244:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.955245:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677474.955246:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677474.955247:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:1.0:1619677474.955247:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:1.0:1619677474.955250:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (24)--
00000800:00000200:17.0:1619677474.955251:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (24)++
00000800:00000200:17.0:1619677474.955251:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (24)--
00000400:00000200:1.0:1619677474.955251:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.955252:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.955252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000400:00000200:17.0:1619677474.955254:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677474.955254:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.955257:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.955257:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.955258:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:13.0:1619677474.955260:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677474.955263:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677474.955263:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.955264:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:4.2:1619677474.955435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (20)++
00000800:00000200:21.0:1619677474.955441:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)++
00000800:00000200:21.0:1619677474.955443:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.72@o2ib3
00000400:00000200:21.0:1619677474.955446:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.72@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.955446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000400:00000200:21.0:1619677474.955448:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.955450:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.955451:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.955452:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.955453:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677474.955453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (21)++
00000400:00000200:21.0:1619677474.955454:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.955456:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)++
00000400:00000200:21.0:1619677474.955458:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.955458:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.72@o2ib3
00000800:00000200:21.0:1619677474.955459:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.955460:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.72@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.955462:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.955463:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.955463:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677474.955463:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.955464:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:21.0:1619677474.955465:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:20.0:1619677474.955466:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.955467:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.955468:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.955469:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:21.0:1619677474.955470:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1600] (23)++
00000800:00000200:21.0:1619677474.955471:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1600] (24)--
00000800:00000200:21.0:1619677474.955471:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (23)--
00000800:00000200:25.2:1619677474.955472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.955472:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000400:00000200:20.0:1619677474.955473:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.955474:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677474.955477:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.955478:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:20.0:1619677474.955479:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:22.0:1619677474.955480:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677474.955481:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:20.0:1619677474.955481:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1600] (21)++
00000800:00000200:22.0:1619677474.955482:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677474.955482:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1600] (22)--
00000400:00000200:22.0:1619677474.955483:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.955483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)--
00000800:00000200:6.0:1619677474.955483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.955485:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.955486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:22.0:1619677474.955487:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.955487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (20)++
00000800:00000200:21.0:1619677474.955493:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)++
00000800:00000200:10.0:1619677474.955494:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:21.0:1619677474.955495:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.72@o2ib3
00000800:00000200:10.0:1619677474.955496:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.955497:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:21.0:1619677474.955497:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.72@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.955497:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.955498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000400:00000200:10.0:1619677474.955499:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.955500:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.955501:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:21.0:1619677474.955503:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677474.955503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (21)++
00000400:00000200:21.0:1619677474.955504:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.955505:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.955506:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677474.955506:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)++
00000800:00000200:20.0:1619677474.955507:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.72@o2ib3
00000800:00000200:10.0:1619677474.955508:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:21.0:1619677474.955511:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677474.955511:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.72@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.955514:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:20.0:1619677474.955514:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.955516:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:21.0:1619677474.955517:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677474.955517:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.955518:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:21.0:1619677474.955518:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:20.0:1619677474.955519:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.955519:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.955520:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677474.955521:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1600] (23)++
00000400:00000200:20.0:1619677474.955521:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677474.955522:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1600] (24)--
00000800:00000200:21.0:1619677474.955523:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (23)--
00000400:00000200:20.0:1619677474.955523:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.955524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000800:00000200:20.0:1619677474.955525:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677474.955526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677474.955527:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.955528:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:20.0:1619677474.955529:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677474.955530:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677474.955531:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1600] (21)++
00000800:00000200:20.0:1619677474.955532:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e84c1600] (22)++
00000800:00000200:20.0:1619677474.955533:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1600] (23)--
00000800:00000200:8.0:1619677474.955533:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.0:1619677474.955534:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000800:00000200:25.2:1619677474.955536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677474.955536:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.955537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.955537:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.955539:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.955539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (21)++
00000800:00000200:8.0:1619677474.955542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677474.955543:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:21.0:1619677474.955544:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)++
00000800:00000200:21.0:1619677474.955546:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e84c1600] (23)--
00000800:00000200:6.0:1619677474.955546:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:21.0:1619677474.955547:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000400:00000200:6.0:1619677474.955547:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677474.955548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677474.955548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)--
00000400:00000200:6.0:1619677474.955549:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.955552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.955607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.955615:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.955618:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.955621:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677474.955622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:31.0:1619677474.955623:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.955623:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.955624:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677474.955624:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:31.0:1619677474.955626:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.955626:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.955627:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.955628:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.955629:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.955633:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.955635:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.955636:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.955637:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677474.955637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677474.955638:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.955638:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677474.955638:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.955639:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677474.955640:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677474.955640:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.955641:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.955642:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.955643:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677474.955643:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677474.955644:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.955645:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.955645:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.955645:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.955646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677474.955647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.955647:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.955648:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.955649:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677474.955651:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.955652:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:30.0:1619677474.955653:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.955653:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.955654:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:22.0:1619677474.955654:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96502256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677474.955655:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:25.2:1619677474.955655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:22.0:1619677474.955655:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.955656:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:22.0:1619677474.955656:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.955657:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677474.955657:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677474.955658:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677474.955659:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:22.0:1619677474.955660:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.955661:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.955661:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677474.955661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677474.955662:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677474.955663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677474.955663:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.955664:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.955665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677474.955665:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.955666:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677474.955668:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677474.955668:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.955668:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.955669:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677474.955670:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:22.0:1619677474.955670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:24.0:1619677474.955671:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.955671:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.955673:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.2:1619677474.955674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677474.955675:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677474.955681:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.955683:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.955684:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.955686:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.955686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.955688:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.955738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (20)++
00000800:00000200:20.0:1619677474.955741:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)++
00000800:00000200:20.0:1619677474.955744:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.72@o2ib3
00000800:00000200:21.0:1619677474.955746:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000400:00000200:20.0:1619677474.955746:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.72@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.955748:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.955749:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.955751:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.955752:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.955753:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.955754:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.955757:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.955759:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.955761:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.955762:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.955763:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.955764:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.955765:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1600] (21)++
00000800:00000200:20.0:1619677474.955766:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1600] (22)--
00000800:00000200:20.0:1619677474.955767:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)--
00000800:00000200:4.2:1619677474.955768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e84c1600] (20)++
00000800:00000200:25.2:1619677474.955770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.955773:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)++
00000800:00000200:21.0:1619677474.955776:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.72@o2ib3
00000800:00000200:6.0:1619677474.955777:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677474.955778:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.72@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.955779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (22)--
00000400:00000200:21.0:1619677474.955780:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.955780:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.955781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.955781:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.955782:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677474.955783:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.955783:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:21.0:1619677474.955785:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.955786:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.955787:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677474.955790:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677474.955791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.955791:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.955793:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.955794:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.955794:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677474.955794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.955795:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.955796:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.955797:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e84c1600] (21)++
00000800:00000200:21.0:1619677474.955797:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e84c1600] (22)--
00000800:00000200:21.0:1619677474.955798:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e84c1600] (21)--
00000800:00000200:6.0:1619677474.955798:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.955799:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.955799:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.955799:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.955800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677474.955802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.955810:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.955812:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.955814:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.955815:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.955816:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.955818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.956258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.956266:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.956269:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.956271:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.956273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.956274:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.956275:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.956277:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.956278:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.956279:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.956280:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.956283:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.956285:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.956288:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.956289:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.956289:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.956291:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.956291:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677474.956297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.956299:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.956301:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.956301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677474.956305:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.2:1619677474.956307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.956307:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.956308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.956308:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.956311:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.956313:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677474.956316:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.956318:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.956319:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.956320:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.956408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.956415:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.956418:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.956419:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.956421:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.956424:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.956425:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.956426:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.956428:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.956429:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.956430:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.956433:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.956435:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.956436:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.956437:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.956438:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.956439:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.956440:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.956441:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.956441:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.956446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.956452:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.956455:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.956455:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.956456:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.956457:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.956458:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.956866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.956874:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.956877:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.956878:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.956880:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.956883:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.956884:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.956886:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.956887:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.956888:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.956889:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.956892:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.956893:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.956897:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.956897:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.956898:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.956900:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.956900:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.956901:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.956901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.956906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.956914:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.956917:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.956918:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.956918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.956920:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.956923:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.957173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.957178:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.957181:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.957181:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.957183:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.957185:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.957186:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.957188:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.957189:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.957190:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.957191:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.957194:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.957196:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.957198:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.957199:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.957200:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.957202:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.957202:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.957203:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.957204:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.957205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.957208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.957210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.957215:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.957217:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.957217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.957217:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.957218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.957219:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.957220:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.957222:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.957223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.957224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.957226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.957234:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.957237:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.957238:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.957238:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.957239:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.957240:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.957297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677474.957305:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.957308:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677474.957309:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677474.957311:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.957314:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.957316:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677474.957317:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.957318:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.957319:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.957320:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677474.957323:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.957325:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677474.957329:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.957329:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677474.957330:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677474.957332:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677474.957332:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677474.957333:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.957333:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.957339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.957347:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.957350:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.957351:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.957351:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.957354:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677474.957356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.957360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.957361:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.957363:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.957364:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.957365:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.957365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.957366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.957431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.957439:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.957442:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.957444:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.957446:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.957446:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.957448:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.957449:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.957450:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.957452:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.957452:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.957456:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.957457:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.957460:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.957461:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.957461:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.957463:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.957463:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.957465:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.957466:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.957467:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.957469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.957473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677474.957476:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.957478:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.957480:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.957480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.957481:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677474.957482:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.957483:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677474.957483:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.957484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.957485:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.957489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677474.957496:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:21.2:1619677474.957498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677474.957498:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677474.957499:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677474.957501:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.957504:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.957505:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677474.957505:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677474.957507:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677474.957508:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677474.957508:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.957509:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.957510:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677474.957511:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.957512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.957513:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.957514:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.957515:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.957516:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.957516:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:31.0:1619677474.957517:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.957518:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.957519:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.957519:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.957520:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677474.957521:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:31.0:1619677474.957522:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.957522:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677474.957523:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:31.0:1619677474.957524:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.957524:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677474.957525:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:31.0:1619677474.957526:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.957527:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.957528:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.957529:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:9.2:1619677474.957529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677474.957530:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.957531:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.957531:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677474.957537:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.957540:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.957541:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.957541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.957543:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.957545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677474.957546:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.957549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677474.957552:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.957554:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.957555:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.957555:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677474.957556:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:24.0:1619677474.957557:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.957559:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:24.0:1619677474.957560:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677474.957560:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.957560:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.957561:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.957562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.957564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.957570:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.957573:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.957574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.957574:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.957575:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.957576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.957614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.957622:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.957624:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.957625:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.957627:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.957630:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.957632:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.957633:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.957634:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.957635:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.957636:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.957639:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.957641:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.957643:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.957644:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.957644:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.957646:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.957646:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.957647:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.957647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.957652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.957659:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.957661:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.957662:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.957663:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.957666:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.957667:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677474.958650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.958658:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.958660:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:21.2:1619677474.958661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.0:1619677474.958662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.958663:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.958666:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.958668:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677474.958669:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:24.0:1619677474.958669:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.958670:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.958671:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677474.958672:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:24.0:1619677474.958672:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.958675:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.958676:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.958677:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.958678:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.958678:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677474.958679:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.958680:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.958681:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677474.958681:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.958681:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:30.0:1619677474.958682:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677474.958682:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677474.958683:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.958684:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677474.958686:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.958687:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.958687:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.958688:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.958688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677474.958688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677474.958690:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.958690:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.958691:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:25.2:1619677474.958691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.958692:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.958693:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.958694:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.958694:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677474.958697:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:9.2:1619677474.958698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677474.958699:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.958701:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.958702:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.958702:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.958703:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.958704:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.958705:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:22.0:1619677474.958706:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.958706:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.0:1619677474.958707:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:6.0:1619677474.958707:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96502242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677474.958708:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.958708:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.958708:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.958710:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:6.0:1619677474.958710:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.958711:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677474.958712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677474.958714:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.958715:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.958715:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.958716:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677474.958717:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.958720:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.958721:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.958721:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.958723:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.958724:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.958724:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.958725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.958729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.958737:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.958739:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.958740:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.958742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.958742:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.958744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677474.959022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.959028:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.959031:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.959031:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677474.959033:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.959036:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.959037:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.959039:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.959040:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.959041:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.959042:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.959045:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.959047:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.959050:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.959050:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.959051:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.959053:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.959054:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.959055:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.959056:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.959056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.959059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.959061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677474.959066:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.959068:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.959069:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.959070:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.959070:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677474.959070:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.959071:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677474.959072:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.959072:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677474.959075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.959077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.959084:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.959087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.959087:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.959088:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.959088:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.959089:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.959549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.959555:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.959557:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.959560:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.959560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.959562:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.959564:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.959565:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.959566:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.959567:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.959568:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.959571:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.959573:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.959575:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.959576:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.959577:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.959578:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.959579:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.959580:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.959580:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.959585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.959594:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.959597:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.959598:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.959598:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.959600:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:4.2:1619677474.959601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677474.959602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677474.959610:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.959612:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.959613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.959615:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.959617:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.959619:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.959620:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.959622:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.959623:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.959625:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.959628:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.959630:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.959633:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.959633:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.959634:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.959636:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.959636:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.959637:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.959638:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.959639:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.959643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.959645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.959651:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.959652:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677474.959654:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677474.959654:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.959655:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677474.959655:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.959655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.959657:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677474.959657:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.959660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.959691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.959699:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.959702:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.959704:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.959707:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.959707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.959708:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.959709:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.959710:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.959711:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.959712:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.959715:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.959717:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.959719:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.959719:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.959720:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.959721:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.959722:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.959723:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.959723:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.959727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.959733:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.959736:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.959737:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.959738:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.959740:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.959741:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.960219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.960226:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.960229:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.960231:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.960233:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.960233:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.960234:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.960235:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.960236:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.960237:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.960238:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.960241:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.960242:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.960245:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.960245:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.960246:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.960247:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.960248:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.960249:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.960249:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.960253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.960260:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.960262:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.960263:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.960263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.960265:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.960267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.960273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.960279:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.960282:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.960282:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.960282:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.960283:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.960284:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.960592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.960601:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.960604:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.960606:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.960608:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.960608:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.960609:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.960610:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.960611:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.960612:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.960613:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.960616:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.960618:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.960620:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.960620:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.960621:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.960622:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.960623:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.960624:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.960624:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.960629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.960636:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.960639:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.960639:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.960641:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.960643:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.960644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.961084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.961089:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.961092:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.961093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.961095:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.961097:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.961098:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.961100:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.961101:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.961102:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.961103:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.961107:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.961108:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.961111:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.961112:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.961112:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.961114:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.961114:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.961115:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.961115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.961120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.961127:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.961130:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.961130:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.961132:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.961134:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.961134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.961226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677474.961233:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.961236:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677474.961237:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677474.961239:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.961242:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.961243:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677474.961245:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.961246:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.961247:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.961248:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677474.961251:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.961252:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677474.961255:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.961256:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.961257:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677474.961258:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.961259:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.961260:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.961260:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.961265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.961272:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.961275:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.961276:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.961276:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.961278:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.961281:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.961429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.961437:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.961439:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.961441:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.961443:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.961444:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.961444:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.961445:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.961447:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.961448:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.961448:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.961451:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.961453:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.961455:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.961456:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.961456:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.961458:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.961458:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.961459:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.961460:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.961464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.961472:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.961474:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.961475:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.961475:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.961477:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.961479:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.961491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.961499:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.961502:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.961504:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.961506:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.961506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.961507:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.961508:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.961509:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.961510:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.961511:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.961514:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.961515:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.961517:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.961518:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.961518:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.961520:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.961520:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.961522:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.961523:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.961524:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.961526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.961530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.961534:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.961535:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.961536:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.961537:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.961538:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.961539:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.961539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.961540:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.961541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.961542:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.961544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.961551:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.961553:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.961554:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.961554:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.961555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.961556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.961951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.961956:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.961959:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.961961:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.961962:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.961963:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.961965:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.961966:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.961968:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.961969:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.961970:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.961973:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.961974:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.961977:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.961977:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.961978:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.961980:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.961980:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.961981:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.961982:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.961983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.961986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.961989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.961995:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.961996:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.961997:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.961997:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677474.961998:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.961998:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.961999:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.962000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.962001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.962002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677474.962008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:8.2:1619677474.962009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.2:1619677474.962014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:17.0:1619677474.962015:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677474.962017:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.962018:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.962018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677474.962019:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.962020:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:18.0:1619677474.962020:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:20.0:1619677474.962021:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000400:00000200:17.0:1619677474.962021:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.962022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.962023:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677474.962023:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000400:00000200:17.0:1619677474.962023:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.962024:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:17.0:1619677474.962025:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.962026:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.962027:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.962028:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.962029:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.962029:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.962030:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:18.0:1619677474.962031:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677474.962032:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.962034:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.962034:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677474.962035:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.962036:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677474.962036:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.962039:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.962039:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:18.0:1619677474.962040:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.962040:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.962042:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.962042:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:18.0:1619677474.962043:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677474.962043:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.962044:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677474.962046:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.962047:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677474.962048:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.2:1619677474.962048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677474.962050:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.962050:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.962051:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.962052:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:25.2:1619677474.962056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677474.962056:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.962058:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.962059:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.962061:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.962062:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:30.0:1619677474.962063:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:21.0:1619677474.962063:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677474.962066:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.962067:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.962069:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.962070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.962071:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.962322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.962326:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.962328:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677474.962331:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.962331:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.962333:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.962334:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.962336:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.962337:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.962338:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.962339:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.962342:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.962344:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.962346:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.962347:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.962348:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.962350:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.962350:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.962351:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.962352:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.962356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.962363:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.962365:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.962366:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.962368:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.962368:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.962370:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.962378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.962386:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.962390:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.962392:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.962395:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.962399:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.962401:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.962403:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.962405:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.962407:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.962409:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.962414:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.962417:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.962421:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.962423:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.962424:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.962426:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.962427:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.962429:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.962431:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.962431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.962432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677474.962436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677474.962440:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.962443:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.962444:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.962444:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:31.0:1619677474.962446:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.962447:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677474.962448:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:31.0:1619677474.962449:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:18.0:1619677474.962450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.962451:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.962455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.962462:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.962465:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.962466:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.962467:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.962467:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.962469:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.962633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.962640:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.962643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.962643:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.962645:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.962647:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.962649:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.962650:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.962651:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.962652:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.962653:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.962657:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.962658:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.962661:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.962662:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.962663:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.962664:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.962665:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.962665:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.962666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.962670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.962677:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.962680:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.962680:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.962682:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.962683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.962684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.963369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.963377:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.963380:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.963382:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.963385:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.963385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.963386:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.963387:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.963389:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.963390:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.963391:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.963394:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.963396:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.963399:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.963400:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.963400:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.963402:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.963402:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.963404:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.963405:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.963406:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.963408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.963414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.963416:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.963418:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.963419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.963419:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.963421:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.963422:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.0:1619677474.963423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.963424:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.963424:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.963425:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677474.963442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.963448:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.963451:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.963452:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.963454:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.963456:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.963458:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.963459:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.963461:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.963461:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.963463:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.963466:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.963468:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.963470:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.963470:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.963471:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.963473:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.963473:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.963474:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.963475:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.963479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.963486:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.963488:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.963488:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.963489:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.963490:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.963492:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.963580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.963588:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.963591:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.963593:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.963595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.963596:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.963598:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.963599:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.963600:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.963601:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.963602:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.963606:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.963608:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.963610:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.963611:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.963612:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.963613:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.963614:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.963615:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677474.963617:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677474.963617:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677474.963619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.963623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:24.0:1619677474.963625:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.963627:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.963628:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.963629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.963630:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.963632:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:24.0:1619677474.963632:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.963634:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677474.963635:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677474.963636:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:5.2:1619677474.963754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.963761:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.963764:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.963764:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677474.963766:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.963769:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.963770:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.963771:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.963773:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.963774:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.963775:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.963779:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.963781:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.963783:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.963784:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.963784:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.963786:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.963786:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.963787:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.963787:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.963799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.963807:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.963809:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.963810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.963810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.963813:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.963815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.964040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677474.964048:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.964051:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677474.964052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677474.964053:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.964056:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.964058:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.964059:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.964061:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.964062:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.964063:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.964066:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.964068:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.964071:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.964072:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.964072:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.964074:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.964075:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677474.964076:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677474.964077:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677474.964078:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677474.964080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.964083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677474.964087:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.964089:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.964091:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677474.964091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.964091:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.964093:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:6.0:1619677474.964093:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.964094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.964095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677474.964096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.964143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.964148:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.964150:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.964151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.964153:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.964155:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.964156:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.964157:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.964159:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.964160:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.964161:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.964164:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.964166:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.964168:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.964169:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.964169:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.964171:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.964172:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.964173:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.964174:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.964175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.964177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.964181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.964186:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677474.964187:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.964187:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.964188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.964189:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677474.964189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677474.964190:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.964192:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.964193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.964194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.964199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.964207:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.964209:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.964210:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.964211:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.964211:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.964212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.964234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.964241:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.964244:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.964247:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.964248:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.964249:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.964250:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.964252:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.964253:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.964254:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.964255:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.964259:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.964261:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.964263:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.964264:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.964264:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.964266:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.964266:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.964267:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.964268:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.964271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.964278:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.964281:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.964281:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.964283:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.964285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.964286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.964455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677474.964464:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.964468:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677474.964470:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677474.964473:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.964477:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.964479:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677474.964481:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.964483:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.964485:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.964486:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677474.964491:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.964494:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677474.964498:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.964499:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677474.964500:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677474.964502:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.964503:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.964504:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677474.964505:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.964509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.964517:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.964520:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.964521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.964522:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.964524:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.964527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.964529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.964536:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.964538:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.964539:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.964539:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.964540:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.964541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.965123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.965131:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.965134:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.965136:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.965137:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.965138:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.965139:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.965140:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.965141:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.965142:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.965143:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.965146:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.965148:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.965150:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.965150:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.965151:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.965152:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.965153:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.965154:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.965155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.965158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.965165:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.965167:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.965168:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.965169:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.965171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.965172:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.965321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.965329:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.965332:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.965333:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.965335:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.965336:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677474.965336:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.965337:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.965338:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.965339:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.965340:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.965343:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.965345:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.965347:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.965348:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.965348:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.965350:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.965350:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.965351:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.965351:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.965356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.965364:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.965366:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.965367:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.965367:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.965369:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.965371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.965381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.965387:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.965389:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.965390:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.965390:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.965391:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.965392:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.965635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.965639:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.965641:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.965643:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.965644:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.965645:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.965647:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.965648:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.965649:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.965650:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.965651:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.965655:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.965657:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.965659:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.965660:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.965660:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.965662:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.965663:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.965664:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.965664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.965668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.965677:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.965679:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677474.965680:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.965681:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.965683:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.965685:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.965768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677474.965774:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.965777:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677474.965778:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677474.965780:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.965783:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.965784:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677474.965785:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.965787:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.965788:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.965789:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677474.965792:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.965794:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677474.965798:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.965799:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677474.965799:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677474.965801:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677474.965802:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677474.965803:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677474.965804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677474.965807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677474.965814:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677474.965817:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677474.965818:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677474.965819:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.965821:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.965824:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677474.965859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.965866:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.965869:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.965871:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.965873:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.965874:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.965875:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.965876:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.965878:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.965879:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.965879:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.965883:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.965885:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.965886:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.965887:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.965888:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.965889:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.965890:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.965891:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.965891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.965895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.965902:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.965905:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.965906:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.965907:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.965909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.965910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.966125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.966132:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.966134:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.966135:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.966136:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.966139:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.966141:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.966142:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.966143:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.966144:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.966145:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.966148:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.966150:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.966153:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.966153:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.966154:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.966155:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.966156:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.966157:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.966157:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.966162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.966169:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.966172:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.966173:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.966173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.966175:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.966178:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.966492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.966498:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.966500:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.966501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.966503:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.966505:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.966506:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.966507:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.966508:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.966509:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.966510:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.966513:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.966515:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.966517:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.966518:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.966519:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.966520:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.966521:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.966522:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.966522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.966527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.966535:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.966538:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.966538:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.966540:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.966541:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.966542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.967152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.967160:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.967162:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.967165:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.967166:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.967167:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.967168:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.967169:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.967170:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.967171:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.967172:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.967176:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.967177:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.967180:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.967181:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.967181:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.967183:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.967184:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.967185:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.967187:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.967187:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677474.967189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677474.967193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677474.967196:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.967198:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.967199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.967199:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.967200:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.967202:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:8.0:1619677474.967202:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.967203:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.967204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677474.967204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.967220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.967228:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.967230:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.967233:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.967235:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.967235:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.967236:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.967237:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.967239:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.967240:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.967240:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.967244:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.967245:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.967247:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.967248:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.967249:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.967250:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.967251:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.967252:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.967252:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.967256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.967262:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.967264:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.967265:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.967266:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.967267:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.967269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.967310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.967316:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.967319:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.967320:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.967322:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.967325:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.967327:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.967328:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.967329:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.967330:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.967331:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.967335:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.967336:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.967339:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.967340:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.967341:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.967343:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.967344:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.967345:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.967345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.967349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.967357:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.967360:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.967361:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.967362:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677474.967364:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.967365:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677474.967980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.967983:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.967986:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.967988:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.967989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.967990:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.967992:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.967993:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.967994:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.967996:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.967996:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.967999:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.968001:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.968004:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677474.968005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.0:1619677474.968005:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.968005:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.968007:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.968007:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.968008:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.968008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677474.968011:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.2:1619677474.968013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677474.968014:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.968015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.968017:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.968019:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.968020:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.968021:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677474.968021:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:22.0:1619677474.968023:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.968024:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677474.968024:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:22.0:1619677474.968025:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677474.968025:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.968027:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.968027:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:22.0:1619677474.968029:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.968031:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677474.968032:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677474.968033:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.968034:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.968035:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.968036:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.968037:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.968038:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.968038:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.968042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.968050:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.968052:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.968053:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.968054:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.968055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.968057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.968238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.968245:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.968248:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.968249:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.968250:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.968253:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.968254:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.968255:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.968256:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.968258:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.968259:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.968262:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.968264:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.968266:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.968267:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.968268:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.968269:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.968270:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.968270:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677474.968271:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.968275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.968282:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.968284:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.968285:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.968287:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.968287:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.968289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.968297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.968306:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.968308:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.968309:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.968310:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.968311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.968312:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677474.968604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.968610:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.968613:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.968613:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.968615:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.968618:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.968619:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.968620:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.968622:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.968623:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.968624:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:4.2:1619677474.968624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000400:00000200:17.0:1619677474.968627:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.968628:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.968631:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.968632:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.968632:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.968633:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:17.0:1619677474.968634:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.968634:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.968635:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:31.0:1619677474.968636:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000800:00000200:17.0:1619677474.968636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677474.968638:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.968640:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.968641:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:25.2:1619677474.968641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:31.0:1619677474.968642:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.968643:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.968645:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.968646:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.968647:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677474.968648:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:31.0:1619677474.968650:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.968651:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:31.0:1619677474.968652:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677474.968652:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.968652:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.968654:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.968655:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.968656:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:6.0:1619677474.968656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677474.968657:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.968659:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.968659:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:25.2:1619677474.968659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677474.968661:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:31.0:1619677474.968662:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677474.968663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:22.0:1619677474.968666:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:8.2:1619677474.968666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677474.968668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:22.0:1619677474.968669:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.968670:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.968670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.968671:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677474.968672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677474.968674:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.968676:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000800:00000200:21.0:1619677474.968676:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.968677:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.968678:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:30.0:1619677474.968679:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:21.0:1619677474.968679:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.968679:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.968680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:21.0:1619677474.968682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.968879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.968887:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.968889:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.968892:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.968893:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.968895:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.968896:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.968898:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.968899:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.968900:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.968901:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.968904:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.968905:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.968908:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.968908:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.968909:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.968910:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.968911:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.968912:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.968913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.968916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.968923:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.968926:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.968927:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.968927:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.968929:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.968930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.968950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677474.968953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677474.968957:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.968960:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677474.968960:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677474.968962:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.968962:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.968963:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677474.968964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.968965:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.968965:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.968966:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.968967:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.968968:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.968968:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.968969:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.968970:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677474.968970:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.968971:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.968973:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.968974:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.968974:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.968975:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.968977:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.968979:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.968979:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:24.0:1619677474.968979:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.968980:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.968981:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677474.968981:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677474.968982:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.968983:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.968983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677474.968984:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.968985:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.968985:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677474.968987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677474.968987:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.968988:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.968988:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.968989:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.968993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677474.968994:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.968996:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.968997:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.968998:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.969001:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.969001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677474.969001:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.969003:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.969004:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:25.2:1619677474.969005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000400:00000200:6.0:1619677474.969005:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.969007:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.969010:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677474.969011:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.969014:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.969015:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.969016:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.969016:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.969018:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677474.969260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.969268:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.969271:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.969272:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.969274:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.969277:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.969278:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.969279:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.969281:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.969282:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.969283:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.969286:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.969288:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.969290:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.969291:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.969292:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677474.969300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677474.969300:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.969301:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.969301:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.969302:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677474.969308:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.969311:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.969312:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.969312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.969314:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.969317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.969546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.969551:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.969553:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.969554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.969556:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.969558:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.969560:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.969560:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.969562:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.969563:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.969564:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.969567:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.969569:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.969572:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.969572:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.969573:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.969575:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.969575:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.969576:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.969576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.969581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.969588:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.969590:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.969591:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.969593:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.969594:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.969595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.969681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.969689:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.969692:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.969694:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.969696:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.969697:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.969698:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.969699:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.969700:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.969701:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.969702:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.969706:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.969708:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.969710:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.969710:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.969711:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.969712:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.969713:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.969714:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.969715:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.969718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.969725:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.969727:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.969728:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.969729:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.969731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.969732:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.970344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677474.970351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:30.0:1619677474.970352:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.970354:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.970356:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.970357:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000400:00000200:30.0:1619677474.970358:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.970359:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.970359:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.970360:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677474.970360:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677474.970360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:30.0:1619677474.970363:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.970363:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.970364:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.970365:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.970366:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.970368:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677474.970369:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.970370:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677474.970371:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677474.970372:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677474.970372:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.970373:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677474.970374:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.970375:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.970375:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.970377:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.970377:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677474.970377:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.970378:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.970379:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677474.970379:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677474.970381:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.970382:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:25.2:1619677474.970383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677474.970383:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677474.970384:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677474.970385:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677474.970386:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677474.970386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:31.0:1619677474.970390:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:9.2:1619677474.970390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677474.970393:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.970394:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.970395:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.970397:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.970398:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:18.0:1619677474.970399:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.970401:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677474.970402:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.970402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.970404:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.970407:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677474.970413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677474.970420:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.970423:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677474.970424:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.970424:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.970425:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677474.970426:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677474.970795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.970802:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.970805:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.970807:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.970809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.970809:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.970810:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.970812:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.970813:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.970814:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.970815:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:4.2:1619677474.970815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:30.0:1619677474.970819:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.970820:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.970820:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677474.970823:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.970823:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:30.0:1619677474.970824:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.970825:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:21.0:1619677474.970825:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.970825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677474.970826:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.970827:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:21.0:1619677474.970827:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.970828:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000400:00000200:21.0:1619677474.970828:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677474.970829:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.970830:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:21.0:1619677474.970830:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.970831:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.970832:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677474.970832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:21.0:1619677474.970833:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677474.970836:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.970839:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.970842:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.970843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677474.970843:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677474.970843:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.970844:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:21.0:1619677474.970845:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:8.0:1619677474.970845:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.970846:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.970847:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677474.970847:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.970847:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677474.970848:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677474.970848:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.970849:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.970851:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.970852:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677474.970852:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.970853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677474.970857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.970865:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.970867:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.970867:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.970869:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677474.970870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.970872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.970974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.970981:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.970984:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.970985:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.970987:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.970990:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.970991:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.970993:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.970994:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.970995:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.970996:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.970999:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.971001:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.971004:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.971005:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.971005:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.971007:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.971008:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.971009:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677474.971010:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677474.971010:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677474.971013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677474.971016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677474.971021:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.971024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.971024:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.971024:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677474.971025:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.971026:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.971026:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.971027:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677474.971027:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.971030:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.971443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.971451:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.971454:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.971456:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.971459:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.971459:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.971460:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677474.971461:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.971463:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.971464:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.971465:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677474.971468:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.971470:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677474.971472:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.971472:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677474.971473:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.971474:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.971475:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.971476:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.971478:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.971478:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.971481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.971485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.971489:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.971491:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677474.971492:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.971493:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677474.971493:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.971494:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.971494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.971495:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.971496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.971497:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677474.971924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.971929:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.971932:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.971933:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.971935:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.971937:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.971938:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.971940:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.971941:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.971942:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.971943:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.971946:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.971948:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.971951:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.971951:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.971952:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.971954:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.971954:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.971956:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.971957:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.971957:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.971960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.971962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.971967:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.971969:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677474.971970:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.971970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.971971:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.971972:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.971973:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.971974:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.971976:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.971976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:25.2:1619677474.971981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.971988:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.971991:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.971992:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.971992:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.971993:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.971994:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.972400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.972408:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.972411:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.972413:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.972415:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.972415:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.972417:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.972418:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.972419:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.972420:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.972421:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.972424:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.972426:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.972429:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.972429:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.972430:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.972432:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.972432:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.972433:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.972434:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.972438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.972445:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.972448:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.972449:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.972449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.972451:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.972453:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.972459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.972465:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.972468:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.972468:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.972468:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.972469:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.972470:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677474.972653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.972656:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.972658:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677474.972661:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.972661:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.972663:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.972664:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.972665:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.972666:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.972668:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.972669:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.972672:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.972673:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.972676:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.972676:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.972677:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.972679:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.972679:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.972680:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.972680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.972685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.972693:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.972696:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.972697:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.972697:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.972699:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.972702:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677474.972760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.972767:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.972770:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.972770:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.972772:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.972775:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.972776:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.972777:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.972779:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.972780:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.972781:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.972784:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.972786:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.972787:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.972788:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.972789:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.972790:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.972790:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.972792:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677474.972793:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.972794:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677474.972796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677474.972799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.972804:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.972806:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677474.972806:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.972807:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677474.972807:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.972807:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.972808:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.972808:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.972809:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677474.972810:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.972946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.972953:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.972956:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.972957:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.972959:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.972962:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.972964:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677474.972965:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.972967:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.972968:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.972969:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677474.972972:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.972973:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677474.972976:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.972977:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677474.972978:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677474.972980:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677474.972980:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.972981:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677474.972981:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677474.972985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.972994:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677474.972996:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677474.972997:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.972998:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677474.973000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:17.0:1619677474.973000:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.973002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.973006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:18.0:1619677474.973007:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.973009:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.973010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.973012:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677474.973013:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:18.0:1619677474.973014:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.973015:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677474.973016:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.973017:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677474.973017:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677474.973017:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.973017:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:18.0:1619677474.973018:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677474.973018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:18.0:1619677474.973019:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.973020:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.973023:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.973024:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.973027:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.973027:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.973028:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.973030:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.973030:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.973031:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.973032:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.973046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.973054:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.973056:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.973057:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.973058:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.973059:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.973061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677474.973069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.973076:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.973078:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677474.973079:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.973080:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.973081:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.973081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677474.973407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.973412:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.973415:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.973416:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.973417:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.973419:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.973421:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.973422:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.973423:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.973424:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.973425:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.973428:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.973430:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.973432:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.973433:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.973433:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.973435:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.973436:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.973436:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.973437:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.973442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.973450:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.973453:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.973454:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.973454:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.973456:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.973458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.973470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.973475:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.973478:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.973478:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.973480:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.973482:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.973483:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.973485:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.973486:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.973487:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.973488:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.973491:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.973493:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.973496:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.973497:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.973497:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.973499:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.973500:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.973500:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.973500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.973505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.973513:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.973515:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.973516:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.973518:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.973519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.973520:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.973629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677474.973634:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.973637:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677474.973638:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677474.973640:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.973642:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.973643:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677474.973645:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.973646:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.973647:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.973648:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677474.973651:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.973652:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677474.973654:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.973655:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677474.973656:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.973657:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677474.973658:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677474.973659:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677474.973660:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677474.973660:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:21.2:1619677474.973666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.973672:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677474.973673:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677474.973674:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.973675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677474.973676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677474.973683:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677474.973685:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677474.973685:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.973686:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677474.973687:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.973688:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677474.973830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677474.973838:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.973841:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677474.973841:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677474.973843:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.973846:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.973848:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677474.973849:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.973850:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.973851:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.973852:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.973855:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.973857:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677474.973860:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.973860:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677474.973861:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.973863:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.973863:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677474.973864:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677474.973866:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677474.973866:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677474.973868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677474.973872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677474.973872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677474.973876:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:12.0:1619677474.973876:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677474.973878:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:20.0:1619677474.973879:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:12.0:1619677474.973879:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.973880:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677474.973881:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677474.973882:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:21.0:1619677474.973882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.973882:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677474.973882:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677474.973883:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.973884:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677474.973884:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:20.0:1619677474.973885:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.973886:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.973888:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.973889:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.973890:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.973891:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.973894:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.973896:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.973898:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.973899:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.973900:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.973902:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.973902:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.973903:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.973905:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.973905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.973908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677474.973910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677474.973915:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677474.973916:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.973918:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.973919:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677474.973919:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677474.973920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677474.973920:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.973920:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677474.973922:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677474.973924:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.974047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.974055:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.974058:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.974060:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.974062:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.974063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.974064:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.974065:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.974067:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.974068:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.974069:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.974072:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.974073:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.974076:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.974077:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.974078:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.974079:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.974080:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.974081:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.974081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.974085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.974091:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.974093:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.974094:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.974095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.974097:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.974099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.974330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.974338:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.974341:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.974343:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.974345:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.974345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.974346:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.974348:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.974349:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.974350:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.974351:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.974353:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.974355:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.974357:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.974358:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.974358:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.974360:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.974361:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.974361:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.974362:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.974368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.974375:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.974377:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.974378:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.974378:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.974380:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.974382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.974567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.974574:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.974577:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.974577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.974579:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.974581:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.974583:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.974584:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.974585:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.974586:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.974587:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.974590:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.974592:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.974594:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.974595:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.974596:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.974598:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.974598:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.974599:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.974599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.974604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.974612:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.974614:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:4.2:1619677474.974614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677474.974615:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.974616:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.974618:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.974620:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677474.974621:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.974624:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.974625:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.974626:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.974629:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.974630:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.974632:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.974633:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.974634:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.974634:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.974638:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.974639:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.974642:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.974643:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.974643:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.974645:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.974645:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.974646:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.974647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.974651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.974658:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.974660:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.974661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.974662:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.974664:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.974667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.974671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.974678:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.974680:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.974681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.974682:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.974683:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.974683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.974787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.974794:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.974797:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.974799:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.974801:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.974801:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.974803:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.974804:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.974805:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.974806:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.974807:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.974810:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.974812:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.974814:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.974814:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.974815:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.974816:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.974817:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.974818:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.974819:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.974822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.974829:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.974831:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.974832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.974832:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.974834:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.974836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.974981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.974986:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.974989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.974989:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677474.974991:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.974993:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.974995:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.974996:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.974997:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.974998:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.974999:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.975002:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.975004:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.975006:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.975007:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.975008:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.975010:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.975010:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.975011:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.975011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.975016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.975023:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.975025:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.975026:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.975028:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.975029:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.975030:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.975072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677474.975079:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.975082:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677474.975083:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677474.975085:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.975088:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.975090:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677474.975091:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.975092:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.975093:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.975094:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677474.975097:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.975099:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677474.975102:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.975103:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677474.975103:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677474.975105:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677474.975106:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677474.975107:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677474.975108:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677474.975108:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677474.975111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677474.975114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:2.0:1619677474.975119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:10.0:1619677474.975121:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:2.0:1619677474.975122:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.975123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677474.975123:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:2.0:1619677474.975123:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.975124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677474.975125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:2.0:1619677474.975125:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.975128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.975257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.975264:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.975266:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.975267:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.975268:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.975272:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.975273:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.975274:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.975276:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.975277:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.975277:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677474.975279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000400:00000200:26.0:1619677474.975280:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.975281:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.975284:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.975285:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.975285:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.975285:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:26.0:1619677474.975287:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.975287:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.975288:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677474.975288:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677474.975288:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:26.0:1619677474.975289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677474.975291:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677474.975293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677474.975300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.975301:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:6.0:1619677474.975302:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:22.0:1619677474.975303:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677474.975304:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.975305:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.975305:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.975306:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.975307:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677474.975307:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677474.975311:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.975313:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677474.975313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.975314:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677474.975316:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.975317:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:22.0:1619677474.975318:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677474.975320:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:22.0:1619677474.975320:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677474.975321:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677474.975321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677474.975326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677474.975333:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677474.975336:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677474.975337:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677474.975337:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.975339:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.975342:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677474.975346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677474.975352:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.975355:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677474.975355:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.975356:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.975357:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677474.975357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.975470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.975479:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.975482:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.975484:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.975487:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.975487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.975488:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.975489:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.975490:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.975492:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.975493:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.975496:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.975497:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.975500:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.975501:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.975502:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.975503:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.975504:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.975505:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.975505:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.975509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.975515:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.975518:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.975519:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.975519:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.975521:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.975523:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.975524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.975530:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.975532:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677474.975533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.975533:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.975534:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.975535:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.976001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.976008:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.2:1619677474.976010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.976011:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.976013:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.976016:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.976016:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.976017:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677474.976017:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677474.976018:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.976019:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.976020:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.976021:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.976022:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.976025:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.976026:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.976027:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.976028:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677474.976028:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.976029:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.976031:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.976031:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.976032:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.976035:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.976035:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:31.0:1619677474.976036:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.976036:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.976037:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.976037:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.976038:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.976039:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.976039:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677474.976041:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.976041:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.976041:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.976042:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:9.2:1619677474.976043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.976044:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.976044:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.976046:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.976047:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.976048:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677474.976050:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.2:1619677474.976050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677474.976053:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.2:1619677474.976053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677474.976054:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.976056:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.976056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.976059:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677474.976059:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.976061:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677474.976061:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.976062:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.976063:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.976064:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677474.976064:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.976065:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677474.976066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.976067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.977081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.977089:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.977092:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.977094:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.977096:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.977097:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.977098:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.977100:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.977101:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.977102:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.977103:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.977106:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.977108:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.977111:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.977112:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.977112:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.977114:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.977115:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.977116:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.977116:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.977121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.977129:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.977132:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.977133:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.977134:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.977135:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.977137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.977137:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677474.977138:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:21.0:1619677474.977139:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.977139:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677474.977140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.977291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.977297:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.977300:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.977301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.977303:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.977306:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.977307:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.977308:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.977310:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.977310:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.977311:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.977314:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.977316:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.977318:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.977319:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.977320:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.977321:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.977322:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.977322:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.977323:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.977328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.977336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.977339:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.977339:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.977341:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.977342:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.977343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.977950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.977958:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.977961:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.977963:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.977965:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.977966:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.977968:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.977969:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.977970:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.977971:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.977972:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.977976:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.977977:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.977980:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.977981:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.977981:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.977983:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.977984:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.977985:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.977987:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.977987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.977989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.977994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.977995:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.977997:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.977998:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.977999:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.978001:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.978002:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.0:1619677474.978003:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.978004:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.978005:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.978006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:5.2:1619677474.978153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.978159:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.978162:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.978163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.978165:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.978166:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.978167:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.978169:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.978170:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.978171:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.978172:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.978176:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.978177:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.978180:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.978181:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.978181:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.978183:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.978183:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.978184:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.978185:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.978186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.978189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.978191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.978196:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.978197:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.978198:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677474.978199:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.978199:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677474.978200:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677474.978200:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677474.978201:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.978202:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.978205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.978207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.978214:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.978216:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.978217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.978217:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.978218:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.978219:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.978455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.978458:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.978461:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.978463:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.978464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.978465:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.978466:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.978467:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.978469:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.978470:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.978471:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677474.978474:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.978476:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677474.978478:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.978479:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677474.978479:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677474.978481:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.978481:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.978482:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.978482:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.978487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.978496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.978499:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.978500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.978500:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.978502:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.978504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677474.978808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.978814:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.978817:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.978818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.978819:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.978821:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.978823:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.978824:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.978825:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.978826:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.978827:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.978830:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.978832:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.978834:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.978835:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.978836:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.978837:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.978838:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.978839:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.978840:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.978841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.978844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.978846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.978851:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.978852:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677474.978853:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677474.978853:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.978854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.978855:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.978856:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.978858:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.978858:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.978860:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.978862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.978870:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.978872:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.978873:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.978874:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677474.978875:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.978875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677474.979434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.979442:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.979445:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.979446:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.979448:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.979451:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.979452:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.979454:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.979455:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.979456:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.979457:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.979460:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.979462:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.979466:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.979467:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.979467:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.979469:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.979469:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.979470:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.979472:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.979472:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.979476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.979478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677474.979484:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.979485:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677474.979486:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677474.979487:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677474.979487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677474.979488:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677474.979488:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.979489:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677474.979490:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.979493:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.979494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.979501:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.979504:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.979505:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.979506:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.979509:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.979511:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.979512:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.979513:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.979515:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.979515:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.979519:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.979521:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.979523:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.979524:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.979525:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.979526:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.979527:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.979528:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.979528:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.979532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.979540:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.979542:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.979543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.979543:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677474.979545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:6.0:1619677474.979545:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.979548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677474.979553:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.979555:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.979556:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.979559:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.979561:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.979562:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.979563:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.979564:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.979565:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.979566:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.979569:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.979570:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.979572:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.979573:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.979574:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.979576:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.979576:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.979577:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.979578:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.979582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677474.979589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:2.0:1619677474.979589:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.979591:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.979592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.979592:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.979593:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.979594:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.979597:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.979600:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.979602:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677474.979604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677474.979605:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.979605:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.979606:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.979607:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.979609:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.979610:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.979611:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677474.979612:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:30.0:1619677474.979614:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.979615:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.979615:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.979615:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.979616:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677474.979616:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.979617:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677474.979618:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.979619:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.979620:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.979622:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.979622:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.979623:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.979624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.979627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.979634:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.979637:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677474.979637:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.979638:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.979640:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.979642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.979647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677474.979652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.979655:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677474.979656:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.979656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.979657:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677474.979657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.979681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.979688:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.979691:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.979693:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.979695:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.979695:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.979697:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677474.979698:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.979699:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.979700:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.979701:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.979704:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.979705:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.979708:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.979708:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.979709:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.979710:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.979711:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.979712:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.979712:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.979716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.979722:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.979724:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.979725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.979725:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.979727:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.979729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677474.979813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.979816:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.979818:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.979820:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.979821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.979822:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.979824:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.979825:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.979826:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.979827:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.979828:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.979832:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.979834:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.979836:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.979836:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.979837:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.979839:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.979839:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.979840:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.979840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.979845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.979852:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.979855:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.979855:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.979857:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.979858:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.979859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677474.980048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:4.2:1619677474.980053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677474.980053:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.980056:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.980057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.980059:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677474.980060:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:2.0:1619677474.980061:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.980063:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:2.0:1619677474.980063:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:26.0:1619677474.980064:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:2.0:1619677474.980064:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.980066:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.980067:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.980068:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.980068:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.980070:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.980071:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677474.980072:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677474.980073:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.980074:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677474.980074:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677474.980075:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.980076:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677474.980078:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.980079:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.980079:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:10.0:1619677474.980080:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.980081:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677474.980082:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677474.980082:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.980083:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.980083:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677474.980085:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.980086:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677474.980087:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:25.2:1619677474.980088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677474.980088:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677474.980089:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677474.980089:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677474.980090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.980094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:13.0:1619677474.980095:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.980098:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.980099:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.980101:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.980102:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:13.0:1619677474.980103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.980104:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677474.980105:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.980106:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.980106:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:25.2:1619677474.980107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:22.0:1619677474.980109:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.980111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:13.0:1619677474.980114:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.980117:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677474.980118:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.980119:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677474.980119:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.980120:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.980315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.980321:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.980324:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.980325:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.980326:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.980328:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.980330:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.980331:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.980332:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.980333:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.980334:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.980337:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.980338:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.980341:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.980342:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.980342:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.980344:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.980345:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.980345:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.980346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.980351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.980359:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.980361:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.980362:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.980364:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.980365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.980366:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.980600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a464fe7e00] (20)++
00000800:00000200:13.0:1619677474.980608:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a464fe7e00] (21)++
00000800:00000200:13.0:1619677474.980611:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.31@o2ib3
00000400:00000200:13.0:1619677474.980613:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.3.31@o2ib3 : GET - for me
00000400:00000200:13.0:1619677474.980616:0:19942:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.3.31@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:12.0:1619677474.980617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a464fe7e00] (22)--
00000400:00000200:13.0:1619677474.980619:0:19942:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.3.31@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:13.0:1619677474.980620:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a464fe7e00] (21)++
00000800:00000200:13.0:1619677474.980621:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a464fe7e00] (22)--
00000400:00000200:13.0:1619677474.980623:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.3.31@o2ib3 local destination
00000800:00000200:13.0:1619677474.980625:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a464fe7e00] (21)--
00000800:00000200:21.2:1619677474.980807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.980815:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.980818:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.980820:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.980823:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.980823:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.980824:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.980825:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.980827:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.980828:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.980828:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.980831:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.980833:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.980835:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.980835:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.980836:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.980837:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.980838:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.980839:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.980839:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.980843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.980850:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.980852:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.980853:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.980855:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.980856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.980857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.980860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.980866:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.980869:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.980870:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.980870:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.980871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.980872:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677474.981423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677474.981431:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.981433:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677474.981435:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677474.981437:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.981438:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677474.981439:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.981440:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.981441:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.981442:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.981443:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.981446:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.981448:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.981451:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.981451:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.981452:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.981454:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.981454:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677474.981455:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677474.981455:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677474.981460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.981469:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.981472:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.981473:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.981473:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.981475:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.981479:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.981491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.981499:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.981502:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.981504:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.981505:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.981506:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.981508:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.981509:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.981510:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.981511:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.981512:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.981515:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.981516:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.981519:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.981519:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.981520:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.981522:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.981522:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.981523:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.981524:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.981527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.981534:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.981537:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.981538:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.981538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.981540:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.981542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677474.981595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677474.981602:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.981605:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677474.981605:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000400:00000200:8.0:1619677474.981607:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.981609:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.981611:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677474.981612:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.981613:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.981614:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.981615:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677474.981618:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.981620:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.981623:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.981624:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677474.981625:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677474.981626:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677474.981627:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.981627:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677474.981628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677474.981632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.981640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.981643:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677474.981644:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.981644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677474.981646:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.981649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.981824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677474.981830:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.981833:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677474.981835:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.981835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677474.981837:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.981839:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677474.981840:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.981841:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.981842:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.981843:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677474.981846:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.981848:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677474.981851:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.981852:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677474.981853:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677474.981855:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677474.981855:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677474.981856:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677474.981857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:9.2:1619677474.981861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677474.981868:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677474.981871:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677474.981871:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677474.981872:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.981874:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677474.981877:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677474.982067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677474.982074:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.982077:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677474.982078:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677474.982080:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.982082:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.982083:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677474.982085:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.982086:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.982087:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.982088:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677474.982092:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.982093:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677474.982096:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.982097:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677474.982098:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677474.982100:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677474.982100:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.982101:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.982101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677474.982106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.982114:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.982116:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.982117:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.982119:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.982120:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.982121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.982641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.982647:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.982649:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.982650:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.982652:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.982654:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.982655:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.982657:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.982658:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.982659:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.982660:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.982663:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.982665:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.982668:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.982669:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.982669:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.982671:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.982671:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.982672:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.982672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.982678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.982686:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.982688:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.982689:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.982691:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.982692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.982693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677474.982850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.982856:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.982858:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.982859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.982861:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.982864:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.982865:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.982866:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.982868:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.982869:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.982870:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.982873:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.982874:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.982877:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.982878:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.982878:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.982881:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.982881:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.982882:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.982882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.982887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.982894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.982897:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.982898:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.982898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.982900:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.982902:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.982981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.982985:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.982987:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.982990:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.982991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.982992:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.982994:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677474.982995:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.982996:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.982997:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.982998:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677474.983002:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.983004:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677474.983007:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.983008:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677474.983008:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677474.983010:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677474.983011:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.983012:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677474.983013:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677474.983014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677474.983016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677474.983019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677474.983024:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:21.0:1619677474.983024:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677474.983027:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:21.0:1619677474.983027:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:30.0:1619677474.983028:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.983028:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677474.983028:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677474.983030:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.983031:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.983033:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.983288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.983296:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.983299:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.983301:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.983303:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.983303:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.983305:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.983306:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.983308:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.983309:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.983310:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.983313:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.983315:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677474.983318:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.983319:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.983320:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.983321:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.983322:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.983324:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.983325:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.983326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.983328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.983332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.983340:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.983340:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.983342:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677474.983343:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.983343:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677474.983343:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.983344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677474.983345:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.983345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.983349:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.984155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.984161:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.984164:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.984165:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.984167:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.984169:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.984170:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.984172:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.984173:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.984174:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.984175:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.984178:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.984180:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.984183:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.984183:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.984184:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.984186:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.984186:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.984187:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.984187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.984192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.984200:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.984202:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.984203:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.984204:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.984206:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.984206:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.984318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677474.984325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677474.984326:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.984329:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.984330:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.984332:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677474.984333:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677474.984335:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677474.984336:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677474.984336:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.984337:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677474.984337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677474.984339:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.984340:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.984340:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.984341:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.984342:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.984344:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.984345:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96502140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.984345:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677474.984347:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.984347:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677474.984347:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.984348:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.984350:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677474.984351:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.984351:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.984352:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.984353:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.984353:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.984354:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.984355:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.984355:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:24.0:1619677474.984356:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.984357:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.984357:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.984359:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677474.984359:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677474.984360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677474.984361:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677474.984362:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677474.984362:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677474.984368:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:4.2:1619677474.984368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677474.984371:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677474.984371:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000400:00000200:22.0:1619677474.984372:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.984372:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677474.984373:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.984374:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.984375:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:6.0:1619677474.984375:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.984377:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677474.984378:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677474.984378:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677474.984379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.984380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677474.984380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.984382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677474.984389:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.984392:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.984393:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.984393:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.984394:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677474.984394:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.984542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.984548:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.984551:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677474.984552:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677474.984554:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677474.984556:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.984557:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677474.984558:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.984560:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.984561:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.984562:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677474.984565:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677474.984567:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677474.984570:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.984571:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677474.984572:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677474.984573:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677474.984574:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.984575:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677474.984576:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677474.984577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677474.984579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677474.984582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.984587:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677474.984587:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677474.984589:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677474.984590:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:12.0:1619677474.984590:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677474.984591:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.984591:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677474.984593:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677474.984594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677474.984595:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677474.984604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.984613:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.984615:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.984618:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.984620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.984620:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.984622:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.984623:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.984624:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.984625:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.984626:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.984630:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.984632:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.984634:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.984635:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.984636:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.984637:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.984638:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.984639:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.984640:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677474.984641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677474.984643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677474.984647:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677474.984650:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.0:1619677474.984650:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677474.984652:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:21.0:1619677474.984652:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.984652:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677474.984653:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.984654:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.984655:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.984656:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677474.984656:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677474.984657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:21.0:1619677474.984658:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.984659:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.984660:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.984661:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.984664:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.984666:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.984668:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.984669:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.984670:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.984671:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.984672:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.984673:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.984674:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.984677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.984685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.984687:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677474.984688:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.984690:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677474.984690:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.984692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677474.984698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.984706:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.984708:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.984709:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.984710:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.984711:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.984712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.984980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.984987:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.984990:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.984991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.984993:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.984996:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.984998:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.985000:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.985001:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.985002:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.985003:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.985006:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.985008:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.985011:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.985012:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.985012:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.985014:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.985014:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.985015:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.985016:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.985020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.985028:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.985031:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677474.985031:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.985032:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.985034:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.985036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.985535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.985543:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.985546:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.985548:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.985550:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.985550:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.985551:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.985553:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.985554:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.985555:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.985556:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.985559:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.985560:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.985562:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.985563:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.985564:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.985565:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.985566:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.985566:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.985567:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.985571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.985579:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.985581:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.985582:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.985584:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.985585:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.985586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.985676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.985681:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.985684:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.985685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.985687:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.985689:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.985690:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.985692:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.985693:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.985694:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.985695:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.985698:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.985700:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.985703:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.985703:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.985704:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.985706:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.985706:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.985708:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.985709:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.985709:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.985712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.985715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.985720:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.985722:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677474.985722:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.985723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.985724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.985724:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.985725:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.985727:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.985728:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.985729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.985732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.985739:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.985742:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.985743:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.985743:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.985744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.985745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677474.985863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.985865:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.985867:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.985869:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.985871:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.985871:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.985872:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.985874:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.985875:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.985876:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.985877:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.985880:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.985881:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.985884:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.985885:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.985885:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.985887:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.985887:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.985889:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677474.985890:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677474.985891:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.985893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.985896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.985899:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.985900:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677474.985901:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677474.985901:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.985902:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677474.985902:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.985903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.985904:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.985905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677474.985906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677474.986206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677474.986214:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.986216:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677474.986217:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677474.986219:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677474.986221:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.986223:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677474.986224:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.986225:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.986226:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.986227:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677474.986230:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677474.986232:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.986233:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.986234:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677474.986235:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.986236:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.986236:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677474.986237:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677474.986239:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677474.986239:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:4.2:1619677474.986245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677474.986251:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:9.2:1619677474.986253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.986254:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677474.986254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677474.986256:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:24.0:1619677474.986257:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.986258:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.986259:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.986260:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.986261:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677474.986263:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.986928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677474.986937:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.986939:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677474.986941:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.986943:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.986944:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677474.986944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677474.986945:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.986946:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.986947:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.986948:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677474.986951:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.986953:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677474.986955:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.986956:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677474.986956:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677474.986958:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677474.986958:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.986959:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.986960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.986964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.986971:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.986974:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.986975:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.986975:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.986976:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.986979:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.986990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677474.986998:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.987000:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677474.987003:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.987004:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677474.987005:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.987007:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677474.987008:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.987009:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.987010:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.987011:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677474.987014:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.987015:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677474.987018:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.987018:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677474.987019:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677474.987020:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677474.987021:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677474.987022:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.987022:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.987037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.987043:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.987045:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.987046:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.987047:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.987048:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.987050:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.987356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677474.987362:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.987365:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677474.987366:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677474.987368:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.987370:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.987372:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677474.987373:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.987374:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.987375:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.987376:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677474.987379:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.987381:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677474.987383:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.987384:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677474.987385:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677474.987386:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677474.987387:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677474.987387:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677474.987388:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677474.987393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.987401:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.987404:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.987405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.987405:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.987407:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.987409:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677474.987624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.987630:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.987633:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.987634:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.987636:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.987638:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.987639:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.987641:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.987642:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.987643:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.987644:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.987647:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.987648:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.987650:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.987651:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.987651:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.987653:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.987653:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.987654:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677474.987654:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.987659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.987666:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.987668:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.987669:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.987670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.987672:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.987673:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.987679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.987686:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.987689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.987689:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677474.987690:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.987691:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.987692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677474.987724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677474.987730:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.987733:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677474.987734:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677474.987736:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677474.987739:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.987740:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677474.987742:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.987743:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.987744:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.987745:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677474.987748:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677474.987749:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.987751:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.987752:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.987753:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.987755:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.987755:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677474.987756:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677474.987757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.987761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.987768:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.987770:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677474.987770:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677474.987771:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.987772:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.987773:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.987919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.987924:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.987927:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.987928:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.987930:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.987932:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.987933:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.987934:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.987935:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.987937:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.987938:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.987940:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.987942:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.987944:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.987945:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.987946:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.987947:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.987948:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.987949:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.987949:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.987954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.987962:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.987964:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.987965:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.987966:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.987967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.987969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.988335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.988339:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.988342:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677474.988344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.988344:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677474.988345:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.988347:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677474.988348:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.988349:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.988351:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.988352:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677474.988355:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.988356:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677474.988359:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.988359:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677474.988360:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677474.988362:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677474.988362:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.988363:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.988364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677474.988369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.988377:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.988380:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.988381:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.988381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.988383:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.988385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677474.988883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677474.988884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677474.988890:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:31.0:1619677474.988892:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.988893:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:31.0:1619677474.988895:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677474.988895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677474.988896:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.988898:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.988898:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.988900:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.988900:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677474.988900:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677474.988901:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.988902:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:21.0:1619677474.988903:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.988904:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.988905:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.988905:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.988906:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.988907:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.988908:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:21.0:1619677474.988910:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677474.988913:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.988913:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.988914:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.988915:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677474.988915:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.988916:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677474.988917:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.988918:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.0:1619677474.988918:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.988918:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:31.0:1619677474.988919:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677474.988919:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:31.0:1619677474.988920:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.0:1619677474.988920:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:31.0:1619677474.988921:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677474.988921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:31.0:1619677474.988922:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677474.988924:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.988925:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.988925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:25.2:1619677474.988926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.2:1619677474.988928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.2:1619677474.988931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.988935:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677474.988936:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.988937:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:20.0:1619677474.988937:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:30.0:1619677474.988938:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.988938:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:20.0:1619677474.988938:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.988939:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:31.0:1619677474.988940:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677474.988940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677474.988940:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.988941:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.988941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.988941:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.988942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.988943:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:30.0:1619677474.988944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677474.988944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:31.0:1619677474.988945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677474.988946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.989149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.989158:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.989161:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.989162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.989165:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.989168:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.989170:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.989172:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.989173:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.989175:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.989176:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.989179:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.989182:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.989185:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.989186:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.989187:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.989188:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.989189:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.989190:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.989190:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.989194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.989203:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.989205:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.989206:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.989207:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.989209:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.989212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.989235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.989245:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.989249:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.989253:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.989253:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.989256:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.989258:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.989260:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.989262:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.989264:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.989266:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.989271:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.989274:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.989278:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.989279:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.989280:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.989282:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.989283:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.989285:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.989287:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.989288:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.989288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677474.989296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.989299:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.989303:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677474.989304:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677474.989305:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.989305:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.989307:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.989307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.989308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:8.0:1619677474.989308:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.2:1619677474.989309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.989313:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.989315:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.989318:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.989318:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.989319:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.989321:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.989322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.989471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.989477:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.989480:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.989480:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.989482:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.989485:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.989486:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.989487:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.989488:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.989489:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.989491:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.989493:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.989495:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.989498:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.989498:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.989499:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.989501:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.989501:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.989502:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.989502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.989507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.989514:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.989517:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.989517:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.989519:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.989520:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.989521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.989775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.989784:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.989788:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.989790:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.989793:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.989796:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.989799:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.989801:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.989803:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.989805:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.989807:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.989812:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.989815:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.989819:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.989820:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.989821:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.989823:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.989824:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.989825:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.989826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.989829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.989837:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.989840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677474.989840:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677474.989841:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.989843:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.989846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.989848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.989855:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.989857:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677474.989858:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.989859:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677474.989860:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.989860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.989878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.989886:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677474.989886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:26.0:1619677474.989888:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.989889:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.989891:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677474.989893:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:26.0:1619677474.989894:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.989895:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:26.0:1619677474.989896:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:18.0:1619677474.989897:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:26.0:1619677474.989898:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.989899:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.989900:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.989901:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.989902:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677474.989902:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.989904:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677474.989905:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.989907:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677474.989907:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.989908:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.989909:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.989910:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677474.989913:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.989914:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.989915:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.989915:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677474.989915:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677474.989917:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.989918:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.989919:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:2.0:1619677474.989919:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.989920:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:2.0:1619677474.989920:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677474.989921:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677474.989921:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:25.2:1619677474.989923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677474.989923:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677474.989923:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677474.989924:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677474.989925:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:4.2:1619677474.989926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677474.989926:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677474.989930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.2:1619677474.989931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:6.0:1619677474.989931:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677474.989933:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677474.989933:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677474.989934:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677474.989935:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677474.989936:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677474.989936:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677474.989937:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000400:00000200:6.0:1619677474.989937:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677474.989938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677474.989939:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:8.0:1619677474.989939:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677474.989939:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:18.0:1619677474.989940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677474.989941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.0:1619677474.989943:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.989945:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.989945:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.989949:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.989952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677474.990065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677474.990072:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.990075:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677474.990076:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677474.990079:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.990082:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.990083:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677474.990085:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.990086:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.990088:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.990089:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677474.990093:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.990095:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677474.990098:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.990099:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677474.990099:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677474.990101:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677474.990102:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677474.990102:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677474.990103:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677474.990107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677474.990118:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677474.990121:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677474.990122:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.990125:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677474.990126:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677474.990128:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677474.990402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677474.990408:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.990411:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677474.990412:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677474.990414:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677474.990416:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.990418:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677474.990419:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677474.990420:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677474.990421:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677474.990422:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677474.990426:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677474.990428:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677474.990430:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677474.990431:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677474.990431:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677474.990433:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677474.990434:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677474.990435:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677474.990435:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677474.990440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677474.990449:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677474.990452:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677474.990453:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677474.990453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677474.990455:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677474.990458:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677474.990726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.990729:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.990732:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.990734:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.990735:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.990735:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.990737:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677474.990738:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.990739:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.990740:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.990741:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677474.990744:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.990746:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.990748:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.990749:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.990749:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.990751:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.990751:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.990752:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677474.990752:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.990757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.990764:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.990767:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.990767:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.990769:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.990770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.990771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.990808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.990816:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.990819:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.990821:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.990824:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.990824:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.990825:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.990827:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.990828:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.990829:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.990830:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.990833:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.990835:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.990837:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.990838:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.990839:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.990841:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.990841:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.990842:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.990843:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.990847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.990853:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677474.990856:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677474.990857:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.990857:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677474.990859:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.990861:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677474.990868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677474.990874:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677474.990877:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.990877:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677474.990877:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.990878:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677474.990879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.991010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.991018:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.991021:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.991023:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.991025:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.991025:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.991026:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.991027:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.991028:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.991029:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.991030:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.991033:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.991035:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.991037:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.991038:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.991038:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.991040:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.991040:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.991041:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.991042:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.991045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.991051:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.991054:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.991055:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.991055:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.991056:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.991058:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.991165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.991171:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.991173:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.991174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.991176:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.991178:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.991180:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.991181:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.991182:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.991183:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.991185:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.991188:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.991190:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.991192:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.991193:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.991194:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.991195:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.991196:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.991196:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.991197:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.991201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.991208:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.991211:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.991212:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.991213:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.991214:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677474.991215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.991620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.991625:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.991628:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.991630:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.991630:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.991632:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.991634:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677474.991635:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.991636:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.991637:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.991638:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677474.991642:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.991644:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677474.991646:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.991646:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677474.991647:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677474.991649:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677474.991649:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.991650:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677474.991651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.991655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.991663:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.991665:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.991666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.991666:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.991668:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.991671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.991900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677474.991908:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.991911:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677474.991913:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.991915:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677474.991916:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.991917:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677474.991919:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.991920:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.991921:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.991922:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677474.991926:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.991928:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677474.991931:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.991931:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677474.991932:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677474.991934:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677474.991934:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677474.991935:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677474.991936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677474.991940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.991947:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.991950:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677474.991951:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677474.991952:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.991954:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.991957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677474.991959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677474.991967:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.991969:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677474.991970:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677474.991970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.991972:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677474.991972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677474.991977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677474.991980:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.991982:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677474.991984:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.991985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677474.991986:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677474.991987:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677474.991987:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677474.991989:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.991990:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.991991:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.991992:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677474.991995:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677474.991995:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.991996:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677474.991998:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677474.991999:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.992000:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:31.0:1619677474.992001:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.992001:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677474.992002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677474.992002:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677474.992003:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677474.992003:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677474.992004:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.992004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677474.992006:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677474.992007:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.992008:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677474.992008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:31.0:1619677474.992009:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.992010:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677474.992013:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.992014:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677474.992017:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:31.0:1619677474.992018:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.992019:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677474.992019:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677474.992020:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:31.0:1619677474.992021:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:17.0:1619677474.992021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.992021:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.992022:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.992023:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.992023:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677474.992023:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.992025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.992027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.992035:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.992038:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677474.992038:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.992040:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.992041:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677474.992043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677474.992269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677474.992274:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.992277:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677474.992278:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677474.992280:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677474.992282:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677474.992283:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677474.992284:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677474.992286:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677474.992287:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677474.992288:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677474.992291:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677474.992293:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677474.992300:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677474.992301:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677474.992302:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677474.992303:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677474.992304:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.992304:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677474.992305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677474.992310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.992318:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.992321:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.992322:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.992322:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.992324:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.992327:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.992957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.992966:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.992969:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.992972:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.992973:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.992974:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.992976:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.992977:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.992978:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.992980:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.992981:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.992984:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.992986:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.992989:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.992990:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.992991:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.992992:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.992993:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.992994:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.992994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677474.992999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677474.993007:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677474.993010:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.993011:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.993013:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.993014:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.993016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677474.993019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677474.993026:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677474.993029:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677474.993030:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677474.993030:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677474.993031:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677474.993032:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677474.993049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.993056:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.993059:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.993062:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.993063:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.993065:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.993066:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.993067:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.993069:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.993070:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.993071:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.993074:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.993075:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.993077:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.993078:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.993079:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.993080:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.993081:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.993082:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.993082:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.993086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677474.993092:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.993095:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677474.993096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677474.993096:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.993098:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.993100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.993401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.993407:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.993409:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.993410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.993412:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.993414:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.993416:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.993417:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.993418:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.993419:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.993420:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.993423:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.993425:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.993427:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.993428:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.993429:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.993431:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.993431:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.993432:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677474.993433:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677474.993434:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677474.993437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677474.993439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.993444:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677474.993446:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677474.993446:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677474.993446:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677474.993447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677474.993448:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.993449:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.993450:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.993452:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.993453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677474.993457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677474.993464:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.993467:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677474.993468:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.993469:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677474.993469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.993470:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677474.993865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.993873:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.993875:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.993878:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.993880:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.993880:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.993881:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677474.993883:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.993884:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.993885:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.993886:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677474.993890:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.993891:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677474.993895:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.993896:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677474.993897:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677474.993899:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677474.993899:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.993901:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.993902:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.993903:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677474.993905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677474.993908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677474.993914:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677474.993916:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677474.993916:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677474.993917:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.993918:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.993919:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677474.993920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677474.993920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677474.993921:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677474.993921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677474.994000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677474.994008:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677474.994011:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.994011:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677474.994014:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.994016:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.994018:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.994019:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.994020:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.994021:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.994022:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.994025:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.994027:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.994030:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.994030:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.994031:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.994033:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.994033:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.994034:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.994034:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677474.994039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.994046:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.994049:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.994050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.994050:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.994052:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.994055:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677474.994360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677474.994367:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.994369:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677474.994371:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677474.994372:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677474.994375:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.994377:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677474.994378:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.994379:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.994380:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.994381:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677474.994383:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.994385:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677474.994387:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.994388:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677474.994388:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677474.994390:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677474.994391:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677474.994391:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677474.994392:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677474.994397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.994404:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.994405:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.994406:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.994407:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.994408:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.994409:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677474.994489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677474.994497:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.994499:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677474.994502:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677474.994504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677474.994505:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677474.994506:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677474.994508:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677474.994509:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677474.994510:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677474.994511:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677474.994514:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677474.994516:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677474.994519:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677474.994519:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677474.994520:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677474.994522:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677474.994522:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677474.994523:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677474.994524:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.994528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677474.994535:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677474.994538:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677474.994539:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677474.994539:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.994541:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677474.994544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677474.994671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.994678:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.994681:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.994682:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.994684:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.994687:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.994689:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.994690:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.994691:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.994692:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.994693:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.994697:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.994698:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.994701:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.994702:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.994702:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.994704:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.994705:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.994705:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677474.994705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:26.0:1619677474.994706:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.994710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677474.994713:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.994716:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.994718:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677474.994719:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:31.0:1619677474.994721:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.994721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677474.994721:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:31.0:1619677474.994722:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677474.994723:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.994723:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:31.0:1619677474.994724:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.994725:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:31.0:1619677474.994726:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.994727:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.994728:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677474.994728:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:31.0:1619677474.994731:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.994733:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.994735:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.994736:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.994736:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.994738:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.994739:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:4.2:1619677474.994739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677474.994740:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.994740:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.994744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.994749:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.994751:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.994752:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.994753:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.994754:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.994754:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.994756:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:24.0:1619677474.994757:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.994758:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.994760:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.994761:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.994762:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.994763:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677474.994764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.994765:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677474.994766:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:24.0:1619677474.994767:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.994769:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.994772:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.994773:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.994773:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.994775:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.994775:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.994776:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.994777:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.994781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.994788:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.994791:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.994792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.994792:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.994794:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.994797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.994933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677474.994938:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.994940:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.994941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.994942:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.994944:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.994946:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.994947:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.994948:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.994949:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.994950:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.994952:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.994954:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.994956:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.994957:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.994958:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.994959:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.994960:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.994961:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.994961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.994966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677474.994973:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677474.994975:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677474.994976:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.994978:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.994979:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.994980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677474.995459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677474.995463:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.995465:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677474.995468:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677474.995468:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677474.995470:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.995471:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677474.995472:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.995474:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.995475:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.995476:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677474.995479:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677474.995481:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677474.995483:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.995484:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677474.995484:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677474.995486:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677474.995486:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.995487:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677474.995488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677474.995493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.995501:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.995503:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.995504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.995504:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.995506:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677474.995508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677474.995511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677474.995517:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677474.995520:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677474.995521:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.995521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.995522:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677474.995522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677474.995611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.995619:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.995622:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.995624:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.995627:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.995627:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.995628:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.995630:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.995631:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.995632:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.995633:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.995636:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.995638:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.995640:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.995641:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.995641:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.995643:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.995643:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.995644:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.995645:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.995648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677474.995655:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.995658:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.995658:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.995660:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.995662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677474.995663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677474.995668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.995674:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.995677:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.995678:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677474.995678:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677474.995679:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.995680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677474.995811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677474.995817:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.995821:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677474.995822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677474.995824:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.995827:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.995828:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677474.995829:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.995831:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.995832:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.995833:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677474.995836:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.995838:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677474.995840:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.995841:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677474.995842:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677474.995844:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677474.995844:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677474.995845:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677474.995846:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677474.995847:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677474.995851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677474.995852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677474.995858:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677474.995860:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677474.995860:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677474.995860:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677474.995861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677474.995862:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677474.995863:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.995865:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.995865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.995868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677474.996647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677474.996653:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.996655:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677474.996656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677474.996658:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677474.996660:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.996661:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677474.996663:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677474.996664:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677474.996665:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677474.996666:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677474.996669:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677474.996671:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677474.996674:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677474.996675:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677474.996675:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677474.996677:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677474.996677:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677474.996678:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677474.996678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.996684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.996692:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.996694:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.2:1619677474.996696:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:20.0:1619677474.996697:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677474.996698:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677474.996699:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677474.996699:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677474.996700:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677474.996701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677474.996702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.996703:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677474.996705:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.996707:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677474.996708:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.996709:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.996710:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.996711:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677474.996715:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.996717:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677474.996719:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.996720:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677474.996720:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677474.996722:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677474.996723:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.996723:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.996724:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.996729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677474.996738:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.996740:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677474.996741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677474.996741:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.996744:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677474.996746:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677474.996747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677474.996754:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677474.996757:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677474.996758:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677474.996758:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.996759:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677474.996759:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677474.997018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677474.997026:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.997029:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677474.997030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677474.997032:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677474.997035:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.997036:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677474.997037:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677474.997039:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677474.997040:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677474.997041:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677474.997044:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677474.997046:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677474.997049:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677474.997049:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677474.997050:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677474.997051:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677474.997052:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677474.997053:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677474.997053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.2:1619677474.997057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.997065:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.997068:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.997068:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.997070:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677474.997072:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.997073:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.997081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.997089:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.997091:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.997094:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.997095:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677474.997095:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.997097:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.997098:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.997099:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.997100:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.997101:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.997104:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.997105:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.997108:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.997108:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.997109:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.997111:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.997111:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.997113:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677474.997114:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677474.997115:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677474.997117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.997123:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.997126:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.997127:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.997127:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.997128:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677474.997130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677474.997139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.2:1619677474.997145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677474.997145:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.997148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.997148:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677474.997148:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677474.997149:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677474.997150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677474.997152:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677474.997155:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677474.997155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677474.997156:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677474.997331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677474.997335:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.997337:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677474.997340:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677474.997341:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677474.997342:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677474.997344:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677474.997345:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677474.997346:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677474.997348:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677474.997349:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677474.997352:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677474.997353:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677474.997356:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677474.997357:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677474.997358:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677474.997359:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677474.997360:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677474.997361:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677474.997361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677474.997366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677474.997374:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677474.997377:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677474.997378:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677474.997378:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677474.997381:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.997383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677474.997449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677474.997455:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.997457:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677474.997458:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677474.997460:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677474.997462:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677474.997464:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677474.997465:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677474.997467:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677474.997468:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677474.997469:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677474.997472:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677474.997473:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677474.997476:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677474.997477:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677474.997477:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677474.997479:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677474.997480:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677474.997481:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677474.997481:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677474.997482:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677474.997486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677474.997487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677474.997492:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677474.997493:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677474.997494:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677474.997495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677474.997496:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677474.997496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677474.997497:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.997498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.997499:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.997501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677474.997505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677474.997512:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677474.997514:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677474.997515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.997515:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677474.997516:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677474.997517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677474.997895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.997903:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.997906:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.997908:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.997910:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.997911:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.997912:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.997913:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.997915:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.997916:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.997917:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.997920:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.997922:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.997924:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.997924:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.997925:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.997927:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.997927:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.997928:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.997929:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677474.997932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.2:1619677474.997932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677474.997938:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677474.997939:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:2.0:1619677474.997940:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:31.0:1619677474.997942:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:18.0:1619677474.997942:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677474.997943:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677474.997943:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677474.997944:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:2.0:1619677474.997945:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677474.997946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.997947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:2.0:1619677474.997947:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677474.997949:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677474.997950:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677474.997951:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677474.997952:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677474.997956:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677474.997958:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677474.997961:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677474.997961:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677474.997962:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677474.997964:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677474.997965:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677474.997966:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677474.997966:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677474.997970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677474.997977:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677474.997979:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677474.997980:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.997982:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.997983:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.997984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677474.998142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677474.998150:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.998153:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677474.998156:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677474.998158:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677474.998159:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677474.998160:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677474.998161:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677474.998163:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677474.998164:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677474.998165:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677474.998169:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677474.998171:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677474.998174:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677474.998174:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:13.0:1619677474.998175:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:13.0:1619677474.998177:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677474.998177:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677474.998178:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677474.998179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677474.998183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677474.998190:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677474.998193:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677474.998193:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677474.998194:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677474.998196:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677474.998199:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677474.998298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677474.998304:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.998306:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677474.998308:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677474.998309:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677474.998310:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.998312:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677474.998313:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.998315:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.998316:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.998317:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677474.998319:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677474.998321:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677474.998323:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.998324:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677474.998325:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677474.998326:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.998327:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.998328:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677474.998330:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677474.998330:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677474.998333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677474.998337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677474.998343:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677474.998344:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.998344:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677474.998345:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.998345:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677474.998346:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.998346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677474.998347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677474.998348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677474.998349:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677474.998351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677474.998359:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677474.998361:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677474.998362:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677474.998363:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677474.998363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677474.998364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677474.998419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677474.998426:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.998429:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677474.998430:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677474.998432:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.998435:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.998436:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677474.998438:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.998439:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.998440:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.998441:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677474.998445:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.998446:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677474.998449:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.998450:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677474.998450:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677474.998452:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677474.998452:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677474.998453:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677474.998453:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.2:1619677474.998458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.998466:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.998469:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677474.998470:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.998471:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.998473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677474.998474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677474.998479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677474.998486:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677474.998488:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677474.998489:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677474.998489:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677474.998490:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677474.998491:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677474.998633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.998641:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.998643:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.998645:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.998647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.998647:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.998648:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677474.998649:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.998650:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.998651:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.998652:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677474.998655:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.998656:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677474.998659:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.998659:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677474.998660:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677474.998661:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677474.998662:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.998663:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677474.998663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677474.998667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677474.998673:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677474.998676:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677474.998677:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677474.998677:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677474.998679:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677474.998681:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677474.998758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677474.998765:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.998768:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677474.998770:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677474.998772:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677474.998773:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.998774:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677474.998775:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677474.998776:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677474.998777:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677474.998778:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677474.998781:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677474.998782:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677474.998785:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677474.998785:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677474.998786:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677474.998787:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677474.998788:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677474.998789:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677474.998789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677474.998804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677474.998810:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677474.998812:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677474.998813:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677474.998814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677474.998815:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677474.998816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:24.0:1619677474.998817:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677474.998822:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.998825:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677474.998825:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677474.998827:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677474.998830:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677474.998831:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677474.998832:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677474.998834:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677474.998835:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677474.998836:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677474.998838:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677474.998840:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677474.998842:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677474.998843:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677474.998844:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677474.998845:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677474.998846:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677474.998847:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677474.998847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677474.998852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677474.998859:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677474.998862:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677474.998863:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677474.998863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.0:1619677474.998865:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677474.998865:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677474.998868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677474.998871:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.998874:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677474.998875:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677474.998877:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677474.998880:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677474.998881:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677474.998883:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677474.998884:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677474.998885:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677474.998886:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677474.998889:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677474.998891:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677474.998894:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677474.998895:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677474.998895:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677474.998897:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677474.998898:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677474.998899:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677474.998900:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677474.998900:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677474.998903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677474.998906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677474.998911:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677474.998913:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677474.998913:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.998914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.998915:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677474.998916:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677474.998916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677474.998917:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677474.998917:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.998920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677474.999155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677474.999160:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.999163:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677474.999165:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677474.999166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677474.999167:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677474.999168:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677474.999170:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677474.999171:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677474.999172:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677474.999173:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677474.999177:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677474.999178:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677474.999181:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677474.999182:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677474.999183:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677474.999184:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677474.999185:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677474.999186:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677474.999187:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677474.999188:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677474.999190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677474.999193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677474.999197:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677474.999197:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677474.999199:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:24.0:1619677474.999200:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677474.999200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.0:1619677474.999200:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677474.999201:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677474.999202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677474.999203:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677474.999206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677474.999316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677474.999325:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.999328:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677474.999330:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677474.999332:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677474.999332:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677474.999334:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677474.999335:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677474.999336:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677474.999337:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677474.999338:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677474.999342:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677474.999344:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677474.999346:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677474.999346:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677474.999347:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677474.999349:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.999349:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677474.999351:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.999352:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.999353:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677474.999355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677474.999359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677474.999364:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677474.999365:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677474.999366:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677474.999367:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677474.999367:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677474.999368:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677474.999369:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677474.999370:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677474.999371:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677474.999372:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677474.999544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677474.999554:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.999558:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677474.999559:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677474.999563:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677474.999566:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677474.999569:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677474.999571:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677474.999573:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677474.999574:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677474.999576:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677474.999581:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677474.999584:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677474.999587:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677474.999589:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677474.999590:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677474.999592:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677474.999593:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677474.999594:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677474.999595:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677474.999598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677474.999606:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677474.999609:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677474.999610:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677474.999610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677474.999613:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677474.999616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677474.999709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677474.999717:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.999720:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677474.999721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677474.999722:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677474.999725:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677474.999726:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677474.999728:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677474.999729:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677474.999730:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677474.999731:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677474.999734:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677474.999736:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677474.999738:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677474.999739:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677474.999740:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677474.999741:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677474.999742:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677474.999742:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677474.999743:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677474.999747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677474.999754:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677474.999757:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677474.999758:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677474.999758:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677474.999760:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677474.999763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677475.000030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.000038:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.000041:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.000042:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.000045:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.113@o2ib7 : GET - for me
00000400:00000200:1.0:1619677475.000047:0:19931:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.113@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1619677475.000050:0:19931:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.113@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:1.0:1619677475.000051:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.000052:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.000054:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677475.000057:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : REPLY try# 0
00000800:00000200:1.0:1619677475.000059:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.000061:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.000062:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.000062:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.000064:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.000064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.2:1619677475.000069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677475.000070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677475.000077:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677475.000077:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:30.0:1619677475.000080:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677475.000080:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.000080:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.000081:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.000083:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: REPLY: OK
00000800:00000200:31.0:1619677475.000084:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.000084:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.000085:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:30.0:1619677475.000087:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.000088:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.000090:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.000092:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.000093:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.000094:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.000098:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.000100:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.000102:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.000103:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.000104:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.000106:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.000106:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.000107:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.000108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.000112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.000118:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.000121:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.000122:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.000123:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.000125:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.000126:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.000344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.000350:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.000352:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.000353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.000355:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.000357:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.000359:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.000360:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.000361:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.000362:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.000363:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.000366:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.000367:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.000370:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.000371:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.000371:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.000373:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.000373:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.000375:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.000376:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.000376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.000380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.000382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.000387:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.000387:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.000389:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.000390:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.000390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.000391:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.000391:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.000392:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.000394:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.000395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.000400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.000407:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.000409:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.000410:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.000411:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.000412:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.000413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.000797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.000805:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.000807:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.000810:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.000811:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.000812:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.000813:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.000814:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.000816:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.000816:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.000818:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.000821:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.000822:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.000825:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.000826:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.000827:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.000828:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.000829:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.000830:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.000830:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.000834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.000839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.000841:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.000844:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.000844:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.000845:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.000846:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677475.000847:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.000849:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677475.000849:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:30.0:1619677475.000851:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.000853:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.000853:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.000855:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.000856:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.000857:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.000858:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.000859:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.000862:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.000863:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.000867:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.000867:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.000868:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.000870:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.000870:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.000871:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.000871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.000876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.000884:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.000886:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.000887:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.000889:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.000889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.000892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.001164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.001171:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.001173:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.001175:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.001176:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.001179:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.001181:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.001182:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.001183:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.001184:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.001185:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.001189:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.001190:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.001192:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.001193:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.001194:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.001195:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.001196:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.001198:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.001198:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.001199:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.001201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.001204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.001210:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.001211:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.001211:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677475.001212:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.001212:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.001212:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.001213:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.001214:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.001214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677475.001215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.001736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677475.001742:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.001744:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677475.001745:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677475.001746:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.001748:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.001750:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677475.001751:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.001752:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.001753:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.001754:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677475.001756:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.001758:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677475.001760:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.001761:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677475.001761:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.001763:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677475.001763:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.001764:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677475.001765:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677475.001766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:25.2:1619677475.001771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.001771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.001778:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:22.0:1619677475.001779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677475.001779:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677475.001780:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677475.001781:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:22.0:1619677475.001782:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.001783:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.001783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.001785:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.001788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.002034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.002039:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.002042:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.002043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.002045:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.002046:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.002048:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.002049:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.002050:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.002051:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.002052:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.002055:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.002057:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.002059:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.002060:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.002061:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.002063:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.002063:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.002064:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.002064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.002069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.002076:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.002079:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.002079:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.002081:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.002082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.002083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.002214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.002220:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.002223:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.002224:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.002225:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.002228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:1.0:1619677475.002228:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.002229:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.002230:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.002232:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.002233:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.002234:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677475.002237:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:1.0:1619677475.002237:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.002238:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677475.002240:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:1.0:1619677475.002241:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.002242:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:31.0:1619677475.002243:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.002243:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677475.002244:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:1.0:1619677475.002244:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.002245:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677475.002246:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.002246:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.002246:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:31.0:1619677475.002247:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.002248:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.002250:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.002251:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.002251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:31.0:1619677475.002252:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.002255:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.002257:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677475.002258:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:31.0:1619677475.002259:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.002260:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:22.0:1619677475.002260:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:31.0:1619677475.002261:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677475.002261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.002262:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.002263:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.002263:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.002264:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.002264:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:22.0:1619677475.002264:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.002266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677475.002268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.002275:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.002278:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.002279:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.002279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.002281:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.002283:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.002290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.002297:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.002299:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.002300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.002300:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.002301:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.002302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.002397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:4.2:1619677475.002402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:2.0:1619677475.002403:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.002406:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:21.0:1619677475.002407:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:18.0:1619677475.002407:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.002409:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.002410:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.002412:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.002412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:2.0:1619677475.002412:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.002414:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677475.002415:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.002415:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.002416:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.002417:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677475.002418:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.002419:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.002419:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.002421:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.002423:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.002424:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677475.002424:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.002427:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.002428:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.002430:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.002431:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677475.002431:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.002432:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.002433:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.002434:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677475.002434:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.002435:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.002435:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677475.002436:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677475.002436:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677475.002437:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:2.0:1619677475.002437:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.0:1619677475.002438:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.002438:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.002440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:25.2:1619677475.002444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.002447:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.002450:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.002451:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.002452:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677475.002453:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.002454:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.002455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:6.0:1619677475.002455:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.002456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.002457:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:6.0:1619677475.002458:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.002460:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.002461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.002468:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.002471:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.002472:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.002472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.002473:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.002474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.002861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.002864:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.002867:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.002869:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.002870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.002871:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.002873:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.002874:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.002876:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.002877:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.002878:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.002881:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.002883:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.002886:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.002886:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.002887:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.002889:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.002889:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.002890:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.002891:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.002896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.002904:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.002907:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.002908:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.002908:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.002910:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.002913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.003124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.003129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.003132:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.003133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.003135:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.003137:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.003138:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677475.003139:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.003141:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.003142:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.003143:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677475.003146:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.003148:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677475.003151:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.003152:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677475.003153:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677475.003154:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:1.0:1619677475.003155:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.003155:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.003156:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:25.2:1619677475.003160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.003167:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.003170:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.003171:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.003173:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.003174:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.003176:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.003186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677475.003192:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.003195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.003195:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677475.003197:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.003199:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.003201:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677475.003202:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.003203:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.003204:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.003205:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677475.003208:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.003210:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677475.003212:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.003213:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:6.0:1619677475.003213:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.003215:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677475.003215:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.003217:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677475.003218:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677475.003218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:8.2:1619677475.003221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.2:1619677475.003223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.003228:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:17.0:1619677475.003229:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:22.0:1619677475.003230:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677475.003231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.003232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:17.0:1619677475.003232:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.003233:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.003233:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.003235:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.003237:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.003264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.003272:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.003275:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.003277:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.003278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.003280:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.003281:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.003282:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.003284:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.003285:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.003286:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.003289:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.003291:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.003293:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.003294:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.003302:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.003303:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.003304:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.003305:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.003305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.003310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.003317:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.003320:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.003320:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.003322:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.003324:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.003325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.003326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.003334:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.003336:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.003337:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.003338:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.003338:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.003339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.003436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.003443:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.003446:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.003448:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.003449:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.003450:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677475.003450:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.003452:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.003453:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.003454:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.003454:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677475.003457:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.003459:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677475.003462:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.003463:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677475.003463:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677475.003465:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677475.003466:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.003467:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.003469:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.003469:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.2:1619677475.003472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.003475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:1.0:1619677475.003480:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:30.0:1619677475.003482:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:1.0:1619677475.003483:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.003484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.003484:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.003485:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.003486:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:1.0:1619677475.003486:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.003487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:1.0:1619677475.003489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.003779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.003787:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.003790:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.003791:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.003793:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.003796:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.003797:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.003799:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.003800:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.003801:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.003802:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.003805:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.003807:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.003810:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.003811:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.003811:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.003813:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.003813:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.003814:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.003814:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.003819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.003827:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.003830:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.003831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.003831:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.003833:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.003836:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.003892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.003900:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.003902:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.003903:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.003905:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.003908:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.003910:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.003911:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.003912:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.003913:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.003914:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.003918:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.003919:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.003921:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.003922:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.003923:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.003924:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.003925:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.003925:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.003926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.003930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.003937:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.003940:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.003941:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.003941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.003943:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.003946:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.004235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.004241:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677475.004244:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:21.0:1619677475.004247:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677475.004247:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.004250:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.004250:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677475.004252:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.004252:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.004253:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677475.004254:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.004255:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.004256:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.004257:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.004258:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.004258:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.004259:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677475.004260:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.004261:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96502053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.004263:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.004263:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.004264:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.004265:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:6.0:1619677475.004265:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.004267:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.004268:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.004269:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677475.004269:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.004270:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.004271:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677475.004271:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.004272:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.004273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677475.004274:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.004275:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.004275:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.2:1619677475.004276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677475.004277:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.004277:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.004278:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.004279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.004283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.004287:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.004289:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.004290:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.004291:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:20.0:1619677475.004291:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.004292:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.004293:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.004300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.004301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:21.0:1619677475.004302:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.004303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.004304:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.004415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.004423:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.004426:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.004429:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.004431:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.004432:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.004433:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677475.004435:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.004436:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.004437:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.004438:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.004442:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.004443:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677475.004447:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.004448:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:30.0:1619677475.004448:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.004450:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677475.004451:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.004452:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.004452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677475.004457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.004466:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.004468:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.004469:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.004469:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.004472:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.004475:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.004639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.004646:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.004649:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.004650:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.004652:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.004655:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.004657:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.004658:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.004659:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.004660:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.004661:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.004665:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.004666:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.004669:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.004670:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.004670:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.004672:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.004672:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.004673:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.004675:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.004675:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.004679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.004681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.004687:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.004688:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.004689:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677475.004690:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677475.004690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677475.004691:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.004691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677475.004692:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.004693:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.004696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.004878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.004885:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.004888:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.004889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.004891:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.004893:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.004895:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.004896:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.004897:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.004898:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.004899:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.004902:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.004904:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.004907:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.004908:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.004909:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.004910:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.004911:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.004912:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.004912:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677475.004917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.004924:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.004927:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.004928:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.004929:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.004931:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.004933:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.005215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.005222:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.005225:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.005228:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.005229:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.005230:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.005231:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.005233:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.005234:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.005235:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.005236:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.005239:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.005240:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.005242:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.005243:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.005244:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.005245:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.005246:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.005247:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.005249:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.005249:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.005251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.005255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.005259:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.005261:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.005262:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.005263:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.005263:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.005264:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.005264:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.005265:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.005266:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.005267:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677475.005468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677475.005474:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677475.005477:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677475.005478:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677475.005480:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677475.005482:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.005484:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:1.0:1619677475.005485:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.005486:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.005488:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.005489:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677475.005492:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677475.005493:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677475.005496:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.005497:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677475.005498:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677475.005500:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677475.005500:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677475.005501:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677475.005501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677475.005517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.005524:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.005527:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.005528:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.005528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.005531:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.005533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.005822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.005828:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.005831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.005831:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.005833:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.005835:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.005837:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.005838:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.005839:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.005841:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.005842:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.005845:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.005846:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.005849:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.005849:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.005850:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.005852:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.005852:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.005853:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.005853:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.005858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.005865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:20.0:1619677475.005865:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.005867:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.005868:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.005870:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.005871:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:21.0:1619677475.005872:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.005873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677475.005874:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.005875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.005877:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.005880:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.005881:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.005883:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.005884:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.005885:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.005886:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.005889:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.005891:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.005893:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.005894:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.005895:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.005897:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.005897:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.005898:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.005898:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.005903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.005910:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.005912:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.005913:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.005915:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.005916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.005918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.005993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.006001:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.006004:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.006006:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.006009:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.006009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.006010:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.006011:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.006012:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.006014:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.006015:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.006018:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.006020:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.006022:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.006023:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.006024:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.006025:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.006026:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.006027:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.006028:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.006031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.006038:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.006040:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.006041:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.006041:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.006043:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.006045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.006286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.006290:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.006292:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.006296:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.006297:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.006299:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.006301:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.006302:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.006303:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.006304:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.006305:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.006309:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.006310:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.006312:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.006313:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.006314:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.006316:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.006316:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.006317:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.006318:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.006322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.006330:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.006333:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.006334:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.006334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.006336:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.006338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.006578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.006586:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.006589:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.006590:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.006592:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.006595:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.006596:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.006598:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.006599:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.006600:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.006601:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.006605:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.006606:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.006609:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.006610:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.006610:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.006612:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.006612:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.006613:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.006613:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.006618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.006625:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.006627:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.006628:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.006630:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.006632:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.006633:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.006636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677475.006643:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.006645:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.006646:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.006647:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.006648:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.006649:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.006832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.006838:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.006840:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.006842:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.006843:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.006845:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.006846:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677475.006848:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.006849:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.006850:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.006851:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.006855:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.006856:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.006859:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.006860:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.006861:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.006863:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.006863:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.006864:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.006864:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677475.006869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.006876:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.006879:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.006880:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.006881:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.006883:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.006885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:5.2:1619677475.006920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.006926:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.006929:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.006930:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.006932:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.006935:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.006936:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.006937:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.006939:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.006940:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.006941:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.006943:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.006945:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.006948:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.006948:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.006949:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.006950:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.006951:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.006952:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.006952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.006957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.006964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.006967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.006968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.006968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.006970:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.006972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.007252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.007260:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.007263:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.007265:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.007267:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.007267:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.007268:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.007270:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.007271:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.007272:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.007273:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.007277:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.007279:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.007281:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.007282:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.007282:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.007284:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.007284:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.007285:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.007286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.007289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.007299:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.007302:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.007303:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.007305:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.007307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.007307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.2:1619677475.007523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.007529:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.007532:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.007533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.007535:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.007537:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.007538:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.007539:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.007541:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.007542:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.007543:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.007547:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.007548:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.007551:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.007551:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.007552:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.007554:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.007555:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.007556:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.007557:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.007557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.007561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.007562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.007567:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.007568:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.007569:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.007570:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.007571:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.007571:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.007571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.007573:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.007574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.007576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.007581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.007588:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.007591:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.007592:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.007593:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.007593:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.007595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.008091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.008098:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.008101:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.008103:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.008105:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.008105:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.008106:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.008108:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.008109:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.008110:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.008111:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.008115:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.008116:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.008119:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.008120:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.008120:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.008122:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.008123:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.008124:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.008124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.008128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.008135:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.008138:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.008139:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.008139:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.008141:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.008143:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.008185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.008192:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.008195:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.008197:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.008199:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.008199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.008200:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.008202:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.008203:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.008204:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.008205:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.008208:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.008209:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.008213:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.008213:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.008214:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.008216:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.008216:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.008217:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.008218:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677475.008221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.2:1619677475.008222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677475.008227:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677475.008230:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677475.008230:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:21.0:1619677475.008232:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:18.0:1619677475.008232:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:21.0:1619677475.008233:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.008235:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.008235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:2.0:1619677475.008236:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.008238:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:2.0:1619677475.008239:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.008242:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.008244:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.008246:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.008248:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.008249:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.008255:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.008258:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.008261:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.008263:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.008264:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.008266:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.008267:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.008268:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.008270:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.008271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677475.008272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677475.008276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677475.008280:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.008283:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.008284:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.008284:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.008285:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:17.0:1619677475.008286:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:18.0:1619677475.008288:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.008289:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677475.008290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677475.008292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:4.2:1619677475.008650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.008659:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.008661:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.008662:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.008665:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.008667:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.008669:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.008670:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.008671:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.008672:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.008673:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.008677:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.008679:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.008681:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.008682:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.008683:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.008685:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.008685:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.008687:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.008688:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.008688:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.008692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.008694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.008700:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.008701:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.008702:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677475.008703:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677475.008703:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677475.008704:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.008704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677475.008705:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677475.008707:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.008709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.009159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.009167:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.009169:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.009170:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.009172:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.009175:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.009176:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.009178:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.009179:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.009180:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.009180:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.009183:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.009185:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.009187:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.009188:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.009188:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.009190:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.009190:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.009191:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.009192:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.009196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.009203:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.009206:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.009207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.009207:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.009209:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.009212:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.009216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.009223:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.009225:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.009226:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.009226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.009227:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.009228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.009538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.009546:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.009549:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.009549:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.009551:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.009554:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.009555:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.009557:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.009558:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.009559:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.009560:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.009563:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.009564:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.009567:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.009568:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.009569:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.009570:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.009571:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.009572:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.009572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.009577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.009585:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.009588:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.009589:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.009589:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.009591:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.009593:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.009765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:21.2:1619677475.009771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.009775:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.009777:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.009777:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.009778:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.009778:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.009779:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.009779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.009780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677475.009781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.009781:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.009783:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.009784:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.009786:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.009787:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.009788:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.009789:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.009792:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.009794:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.009796:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.009797:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.009798:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.009800:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.009800:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.009801:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.009801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.009806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.009813:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.009815:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.009816:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.009818:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.009819:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.009820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.010001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.010009:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.010012:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.010015:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.010017:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.010018:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.010019:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.010021:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.010022:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.010023:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.010024:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.010027:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.010029:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.010031:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.010031:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.010032:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.010033:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.010034:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.010035:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.010036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.010039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.010046:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.010048:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.010049:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.010051:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.010052:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.010053:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.010103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.010110:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.010112:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.010114:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.010117:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.010118:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.010118:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.010119:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.010120:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.010121:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.010122:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.010125:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.010126:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.010128:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.010129:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.010129:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.010131:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.010132:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.010133:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.010135:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.010135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.010137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.010141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.010143:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.2:1619677475.010143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:24.0:1619677475.010146:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.010147:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.010147:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.010148:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000400:00000200:24.0:1619677475.010149:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.010150:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:24.0:1619677475.010151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.010152:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:21.0:1619677475.010153:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.010153:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:31.0:1619677475.010155:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:21.0:1619677475.010155:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.010156:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:21.0:1619677475.010157:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:31.0:1619677475.010158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:21.0:1619677475.010158:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.010159:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.010160:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.010161:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677475.010165:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.010166:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.0:1619677475.010170:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.010171:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677475.010171:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677475.010173:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677475.010174:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.010175:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677475.010175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677475.010179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677475.010187:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677475.010190:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677475.010191:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.010192:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.010195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677475.010196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677475.010335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.010341:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.010344:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.010345:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.010348:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.010350:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.010352:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.010353:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.010354:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.010355:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.010356:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.010359:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.010361:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.010363:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.010364:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.010364:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.010366:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.010366:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.010367:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.010368:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.010372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.010380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.010382:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.010383:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.010383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.010385:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.010386:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.010389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:9.2:1619677475.010391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.010392:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.010395:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.010397:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.010398:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677475.010399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:20.0:1619677475.010400:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.010401:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.010402:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:2.0:1619677475.010402:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000400:00000200:20.0:1619677475.010403:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.010403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.010403:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000400:00000200:20.0:1619677475.010404:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677475.010404:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:20.0:1619677475.010405:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.010406:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.010409:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.010411:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.010413:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.010414:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.010415:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.010417:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.010417:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.010418:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.010419:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.010424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.010432:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.010435:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.010436:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.010436:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.010438:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.010440:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.010442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.010449:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.010452:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.010452:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.010453:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.010454:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.010454:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.011303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.011308:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.011311:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.011312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.011314:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.011316:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.011317:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.011319:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.011320:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.011321:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.011322:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.011325:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.011327:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.011329:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.011329:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.011330:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.011332:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.011332:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.011333:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.011333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.011339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.011347:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.011349:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.011350:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.011352:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.011353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.011354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.011630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677475.011638:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677475.011640:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677475.011641:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677475.011644:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.011646:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.011648:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677475.011649:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.011651:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.011652:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.011653:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677475.011656:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.011658:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677475.011660:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.011661:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677475.011661:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677475.011663:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677475.011664:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677475.011664:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677475.011665:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:25.2:1619677475.011669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.011677:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.011679:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.011680:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.011681:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.011684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.011684:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.011691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.011697:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.011700:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.011700:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.011701:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.011702:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.011703:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.012202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.012210:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.012212:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.012214:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.012216:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.012216:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.012217:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.012218:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.012219:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.012220:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.012221:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.012224:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.012226:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.012228:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.012229:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.012229:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.012231:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.012232:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.012232:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.012233:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.012237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.012243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.012245:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.012246:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.012247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.012248:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.012250:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.012256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.012262:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.012265:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.012266:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.012266:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.012267:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.012267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.012868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.012875:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.012878:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.012880:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.012882:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.012883:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.012883:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.012885:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.012886:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.012887:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.012888:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.012891:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.012892:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.012894:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.012895:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.012895:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.012897:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.012897:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.012899:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.012900:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.012901:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.012903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.012907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.012912:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.2:1619677475.012912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:31.0:1619677475.012913:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.012914:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.012915:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.012915:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.012916:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.012917:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.012917:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.012918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:1.0:1619677475.012918:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:31.0:1619677475.012919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677475.012921:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677475.012922:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:25.2:1619677475.012924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000400:00000200:1.0:1619677475.012924:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677475.012926:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.012928:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677475.012929:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.012930:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.012931:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:1.0:1619677475.012931:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.012932:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.012934:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.012935:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.012936:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000400:00000200:1.0:1619677475.012936:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677475.012937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.012938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:1.0:1619677475.012938:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.012940:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.012941:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.012942:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.012943:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.012944:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.012945:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677475.012946:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677475.012947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:21.2:1619677475.012951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.012957:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:25.2:1619677475.012959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.012959:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:17.0:1619677475.012959:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677475.012960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:6.0:1619677475.012967:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.012970:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.012971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.012971:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.012973:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.012975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.012993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.012998:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.013001:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:9.2:1619677475.013002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677475.013002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.013004:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.013005:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.013007:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.013008:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.013009:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677475.013009:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.013010:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.013011:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677475.013012:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.013012:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.013016:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.109@o2ib7 : GET - for me
00000400:00000200:22.0:1619677475.013016:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677475.013019:0:19938:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.109@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:22.0:1619677475.013019:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:24.0:1619677475.013021:0:19938:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.109@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:22.0:1619677475.013021:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.013022:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677475.013023:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677475.013023:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677475.013024:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677475.013025:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.013025:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677475.013026:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:22.0:1619677475.013026:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.013027:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:24.0:1619677475.013030:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : REPLY try# 0
00000800:00000200:25.2:1619677475.013031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677475.013032:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.013033:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.013034:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.013035:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.013036:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.013037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677475.013039:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.013041:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:9.2:1619677475.013041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:20.0:1619677475.013042:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.013044:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.013045:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.013046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677475.013048:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.013050:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.013051:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.013051:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.013053:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: REPLY: OK
00000800:00000200:8.0:1619677475.013054:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.013467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.013476:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.013481:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.013482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.013486:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.013490:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.013493:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677475.013495:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.013497:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.013499:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.013500:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677475.013506:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.013508:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677475.013512:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.013513:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677475.013514:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677475.013516:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677475.013517:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.013518:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.013519:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.013533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.013541:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.013543:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.013544:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.013545:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.013548:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.013549:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.013911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.013919:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.013921:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.013923:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.013925:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.013926:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677475.013926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.013927:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.013928:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.013929:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.013930:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.013933:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.013935:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.013937:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.013938:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.013938:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.013940:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.013940:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.013941:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.013942:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.013945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.013948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677475.013952:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.013955:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:26.0:1619677475.013956:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677475.013956:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.013956:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.013957:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677475.013957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:24.0:1619677475.013959:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677475.013961:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.013963:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677475.013963:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000400:00000200:26.0:1619677475.013965:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.013966:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677475.013967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:26.0:1619677475.013969:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.013970:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.013971:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677475.013972:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.013973:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.013974:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677475.013976:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.013976:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.013977:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.013979:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.013979:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.013980:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677475.013981:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677475.013985:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.013988:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.013988:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677475.013992:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.013992:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.013993:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:6.0:1619677475.013994:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.013995:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677475.013996:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.013996:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:26.0:1619677475.013997:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.013997:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.013997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:26.0:1619677475.013998:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.014000:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.014001:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.014002:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.2:1619677475.014002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677475.014003:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.014006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.014010:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.014012:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677475.014013:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.014014:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677475.014014:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.014016:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.014017:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.014018:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.014018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.014018:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:22.0:1619677475.014020:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.014024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677475.014024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.014031:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.014034:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.014035:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.014035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.014036:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.014037:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.014206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.014213:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.014216:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.014219:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.014221:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.014222:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.014224:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.014225:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.014226:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.014227:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.014228:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.014232:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.014233:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.014237:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.014237:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.014238:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.014240:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.014240:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.014241:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.014242:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.014246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.014255:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.014257:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.014258:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.014260:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.014260:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.014263:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.014265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.014273:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.014276:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.014277:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.014277:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.014278:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.014279:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.014416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.014424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.014427:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.014428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.014430:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.014432:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.014433:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.014435:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96502015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.014436:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.014437:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.014438:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.014441:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.014443:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.014446:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.014447:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.014447:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.014449:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.014449:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.014450:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.014450:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.014455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.014464:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.014466:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.014467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.014468:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.014470:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.014472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.014474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.014480:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.014483:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.014484:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.014484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.014485:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.014486:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.014511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.014518:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.014521:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.014522:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.014525:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.014527:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.014529:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.014530:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.014531:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.014532:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.014533:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.014536:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.014538:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.014540:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.014541:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.014542:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.014543:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.014544:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.014545:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.014545:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.014549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.014557:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.014559:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.014560:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.014560:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.014562:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.014565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.014583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.014589:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.014592:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.014593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.014595:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.014597:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.014599:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.014600:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.014601:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.014602:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.014603:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.014606:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.014608:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.014611:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.014611:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.014612:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.014614:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.014614:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.014615:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.014615:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.014620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.014627:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.014629:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.014630:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.014630:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.014632:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.014635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.015110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.015117:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.015120:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.015121:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.015123:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.015125:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.015127:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.015128:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.015129:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.015130:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.015132:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.015135:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.015137:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.015140:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.015141:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.015141:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.015143:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.015144:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.015144:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.015145:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.015149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.015156:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.015159:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.015160:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.015162:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.015163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.015164:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.015260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.015266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.015268:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.015269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.015271:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.015273:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.015275:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.015276:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.015278:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.015278:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.015280:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.015283:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.015285:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.015288:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.015288:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.015289:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.015291:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.015291:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.015293:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:25.2:1619677475.015298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.015299:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.015299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677475.015305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.015305:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.015308:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.015308:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.015310:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:20.0:1619677475.015310:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.015312:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.015312:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.015313:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.015313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677475.015314:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.015324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.015331:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.015334:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.015335:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.015335:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.015336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.015337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.015897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.015905:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.015908:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.015911:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.015913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.015914:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.015915:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.015916:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.015917:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.015919:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.015920:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.015922:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.015924:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.015927:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.015927:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.015928:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.015930:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.015930:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.015931:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.015932:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.015946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.015954:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.015956:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.015957:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.015958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.015959:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.015961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.015964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.015969:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.015972:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.015974:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.015974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.015977:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.015978:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.015979:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.015981:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.015982:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.015983:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.015986:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.015987:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.015990:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.015990:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.015991:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.015993:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.015994:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.015995:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.015995:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.015999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.016007:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.016010:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.016011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.016011:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.016013:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.016015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.016295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677475.016302:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.016304:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677475.016305:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677475.016307:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677475.016310:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.016311:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677475.016313:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.016314:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.016315:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.016316:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.016319:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.016321:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.016324:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.016325:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677475.016325:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677475.016327:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677475.016328:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.016329:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677475.016329:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677475.016334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.016341:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.016343:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.016344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.016345:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.016347:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.016350:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.016828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.016834:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.016837:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.016838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.016840:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.016842:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.016843:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.016844:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.016846:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.016847:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.016848:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.016851:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.016853:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.016855:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.016856:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.016857:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.016858:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.016859:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.016860:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.016860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.016864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.016872:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.016875:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.016875:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.016877:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.016878:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.016879:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.017000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.017005:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.017007:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.017010:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.017010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.017012:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.017013:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677475.017015:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.017016:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.017017:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.017018:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.017021:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.017023:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.017026:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.017026:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.017027:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.017029:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.017029:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.017031:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.017032:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.017033:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677475.017035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.017038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.017042:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677475.017043:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677475.017044:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:30.0:1619677475.017045:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677475.017045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.017045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677475.017046:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.017048:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.017050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.017051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677475.017137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.017140:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.017142:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.017145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.017145:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.017147:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.017149:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.017150:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.017151:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.017152:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.017153:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.017157:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.017159:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.017161:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.017162:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.017163:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.017165:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.017165:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.017166:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.017168:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.017168:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.017171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.017173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.017179:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677475.017179:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.017181:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.017182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.017182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677475.017182:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.017183:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.017183:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.017185:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.017187:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.017522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.017530:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.017533:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.017536:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.017538:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.017538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.017539:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.017541:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.017542:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.017543:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.017544:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.017547:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.017548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.017549:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.017551:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.017552:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.017553:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.017554:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.017555:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.017555:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.017556:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.017557:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.017557:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677475.017558:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.017559:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.017561:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.017561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677475.017562:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.017564:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96502003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.017565:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.017566:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.017566:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.017567:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.017568:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.017569:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.017569:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.017570:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.017571:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.017572:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.017573:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.017574:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.017575:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.017575:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.017577:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.017578:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.017579:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.017581:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.017581:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.017583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.017589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.017590:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.017592:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.017593:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.017594:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.017595:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.017596:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677475.017597:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.017599:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.017599:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.017600:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.017602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.017608:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.017610:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.017611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.017611:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.017612:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.017613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.018542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.018548:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.018551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.018551:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.018554:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.018556:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.018557:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.018559:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.018560:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.018561:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.018562:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.018565:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.018566:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.018569:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.018569:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.018570:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.018572:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.018572:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.018573:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.018573:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.018578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.018587:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.018589:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.018590:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.018591:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.018593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.018594:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.018706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677475.018712:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.018715:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677475.018716:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.018718:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.018720:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.018721:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.018723:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96502001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.018724:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.018725:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.018726:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.018729:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.018731:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.018733:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.018734:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.018734:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.018736:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.018736:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.018737:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677475.018737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677475.018742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.018750:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.018752:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.018752:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.018753:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.018754:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.018756:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.018792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.018801:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.018803:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.018805:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.018808:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.018808:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.018809:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.018810:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96502000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.018812:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.018813:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.018814:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.018817:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.018819:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.018822:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.018822:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.018823:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.018825:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.018825:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.018826:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.018827:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.018830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.018834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:24.0:1619677475.018837:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.018839:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.018840:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.018840:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.018842:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677475.018842:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.018844:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677475.018845:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.018846:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.018848:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.018851:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.018852:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.018853:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.018854:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.018855:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.018856:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.018859:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.018861:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.018864:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.018865:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.018865:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.018867:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.018867:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.018868:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.018869:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.018870:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677475.018876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677475.018882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.018883:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.018884:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.018885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.018886:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677475.018890:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.018892:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.018893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.018894:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.018896:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.018899:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.019315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.019324:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.019328:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.019330:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.019333:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.019337:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.019340:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.019342:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.019344:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.019346:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.019347:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.019352:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.019354:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:5.2:1619677475.019355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677475.019358:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.019359:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.019360:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.019361:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677475.019363:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.019363:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.019364:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677475.019364:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:26.0:1619677475.019365:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677475.019365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.019367:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.019368:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.019369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:17.0:1619677475.019370:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.019371:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.019373:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.019373:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.019375:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.019377:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:17.0:1619677475.019378:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.019380:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.019380:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.019381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.019381:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.019383:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.019384:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.019384:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (22)++
00000400:00000200:6.0:1619677475.019384:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.019386:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.019386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677475.019387:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677475.019388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.019389:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.019390:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.019390:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677475.019395:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:5.2:1619677475.019395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677475.019398:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.019398:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:22.0:1619677475.019399:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (24)++
00000800:00000200:22.0:1619677475.019400:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (25)--
00000800:00000200:6.0:1619677475.019400:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000400:00000200:6.0:1619677475.019400:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.019401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677475.019401:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677475.019402:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.019402:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.019403:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.019403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677475.019404:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677475.019404:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677475.019553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677475.019555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.019561:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.019563:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.019563:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677475.019565:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.019566:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.019567:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.019568:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.019569:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.019570:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.019571:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.019572:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.019573:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.019574:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.019578:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.019579:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.019580:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.019581:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.019582:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.019583:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.019584:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.019586:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.019587:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.019587:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.019588:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.019589:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.019589:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.019590:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.019591:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.019592:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.019592:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.019592:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (23)++
00000800:00000200:30.0:1619677475.019593:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (24)--
00000800:00000200:31.0:1619677475.019594:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.019594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.019595:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:25.2:1619677475.019595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.019596:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.019596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.019597:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.019598:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677475.019599:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:9.2:1619677475.019600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.019601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.019602:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.019602:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.019604:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.019606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677475.019606:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:31.0:1619677475.019607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.019608:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677475.019609:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.019610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.019610:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.019611:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.019611:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677475.019611:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.019612:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677475.019613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.019716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.019722:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.019724:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.019725:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.019727:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.019730:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.019731:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.019732:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.019734:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.019735:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.019736:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.019739:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.019741:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.019743:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.019744:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.019745:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.019746:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.019747:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.019747:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.019748:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.019753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.019761:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.019763:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.019764:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.019765:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.019767:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.019768:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.019799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.019806:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.019809:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.019810:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.019812:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.019815:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.019816:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.019818:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.019819:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.019820:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.019821:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.019824:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.019826:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.019829:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.019829:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.019830:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.019832:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.019832:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.019833:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.019833:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.019838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.019846:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.019848:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.019849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.019850:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.019852:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.019854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.019867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.019873:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.019876:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.019877:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.019879:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.019882:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.019883:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.019885:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.019886:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.019887:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.019888:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.019891:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.019893:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.019895:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.019895:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.019896:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.019898:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.019898:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.019899:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.019900:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.019904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.019911:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.019913:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.019913:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.019914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.019915:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.019916:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.020193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677475.020198:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.020201:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677475.020203:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.020203:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677475.020206:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.020207:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.020209:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.020210:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.020211:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.020212:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.020215:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.020217:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.020220:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.020221:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.020221:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.020223:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.020224:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.020224:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677475.020225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677475.020229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.020237:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.020239:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.020240:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.020242:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.020243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.020244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.020252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.020260:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.020262:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.020263:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.020264:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.020264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.020266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.020444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.020450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.020453:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.020454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.020455:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.020458:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.020459:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.020460:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.020462:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.020463:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.020464:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.020467:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.020469:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.020472:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.020473:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.020474:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.020475:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.020476:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.020477:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.020477:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.020482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.020490:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.020493:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.020494:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.020495:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.020497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.020498:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.020582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.020590:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.020593:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.020595:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.020597:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.020597:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.020599:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.020600:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.020602:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.020603:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.020604:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.020607:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.020609:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.020612:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.020613:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.020614:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.020616:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.020617:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.020618:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.020618:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.020623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.020631:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.020634:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.020635:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.020636:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.020637:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.020639:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.020736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.020742:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.020744:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.020745:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.020747:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.020749:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.020751:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.020752:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.020753:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.020755:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.020756:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.020759:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.020760:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.020763:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.020764:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.020764:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.020766:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.020767:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.020767:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.020768:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.020772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.020780:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.020782:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.020783:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.020784:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.020785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.020787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.021313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.021319:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.021322:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.021323:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.021324:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.021327:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.021328:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677475.021329:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.021331:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.021332:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.021333:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677475.021336:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.021337:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677475.021339:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.021340:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:22.0:1619677475.021340:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677475.021342:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:22.0:1619677475.021342:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.021343:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.021343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.2:1619677475.021349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.021357:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.021359:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.021360:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.021361:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.021362:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.021365:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.021544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.021548:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.021551:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.021553:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.021554:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.021555:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.021556:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.021558:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.021559:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.021560:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.021561:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.021564:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.021565:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.021568:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.021569:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.021570:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.021572:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.021572:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.021573:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.021575:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.021575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.021578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.021580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677475.021584:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677475.021586:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677475.021587:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.021588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.021588:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:8.0:1619677475.021588:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.021589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.021589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677475.021590:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.021593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.021810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.021818:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.021821:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.021823:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.021825:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.021826:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.021828:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.021829:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.021830:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.021831:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.021832:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.021836:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.021838:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.021840:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.021841:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.021842:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.021843:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.021844:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.021845:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.021845:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.021849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.021855:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.021858:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.021859:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.021859:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.021861:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.021862:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.022149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.022155:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.022158:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.022159:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.022161:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.022165:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.022166:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677475.022167:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.022169:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.022170:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.022171:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677475.022174:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.022176:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677475.022178:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.022179:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677475.022180:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677475.022182:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677475.022182:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.022183:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.022183:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677475.022198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.022206:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.022208:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.022209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.022209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.022211:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.022214:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.022223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.022230:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.022233:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.022233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.022234:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.022235:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.022236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.022269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.022274:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.022276:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.022277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.022279:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.022281:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.022282:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.022283:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.022284:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.022285:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.022286:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.022289:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.022291:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.022293:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.022294:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.022298:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.022300:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.022300:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.022302:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.022303:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.022303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.022307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.022309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.022315:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.022316:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.022316:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.022317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.022318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.022319:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.022319:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.022321:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.022322:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.022323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.022330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.022337:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.022340:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.022341:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.022342:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.022342:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.022343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.022653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.022657:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.022659:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.022661:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.022662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.022663:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.022665:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.022666:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.022667:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.022668:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.022669:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.022673:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.022674:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.022676:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.022677:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.022678:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.022679:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.022680:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.022681:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.022681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.022686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.022694:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.022696:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.022697:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.022699:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.022700:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.022701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.022835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.022843:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.022846:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.022847:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.022849:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.022852:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.022854:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.022855:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.022857:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.022857:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.022859:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.022862:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.022863:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.022866:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.022867:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.022868:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.022869:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.022870:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.022870:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.022871:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.2:1619677475.022873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677475.022875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.0:1619677475.022876:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.022879:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.022881:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.022881:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.022882:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.022884:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:12.0:1619677475.022884:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:20.0:1619677475.022885:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.022887:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.022887:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.022888:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:12.0:1619677475.022888:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.022889:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.022893:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677475.022893:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.022895:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677475.022895:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.022896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.0:1619677475.022898:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.022898:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.022899:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.022901:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.022901:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.022902:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.022903:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.022907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.022915:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.022918:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.022919:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.022919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.022921:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.022923:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.023702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.023712:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.023717:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.023718:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.023721:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.023725:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.023727:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.023729:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.023732:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.023733:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.023735:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.023740:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.023743:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.023746:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.023748:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.023749:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.023751:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.023752:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.023753:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.023754:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.023758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.023766:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.023769:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.023770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.023770:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.023772:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.023775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.023872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.023880:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.023883:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.023885:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.023887:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.023888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.023889:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.023890:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.023891:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.023892:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.023893:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.023896:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.023898:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.023900:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.023901:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.023901:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.023902:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.023903:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.023904:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.023904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.023908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.023915:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.023917:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.023918:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.023919:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.023921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.023922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.023931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.2:1619677475.023933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.023938:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677475.023938:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.023941:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:30.0:1619677475.023942:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677475.023942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677475.023943:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.023943:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.023944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:22.0:1619677475.023944:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.023945:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:22.0:1619677475.023946:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.023947:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.023949:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.023950:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.023951:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.023952:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.023955:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.023958:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.023961:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.023961:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.023962:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.023964:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.023965:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.023965:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.023966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.023970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.023978:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.023980:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.023981:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.023982:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.023984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.023985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.024095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.024101:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.024104:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.024105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.024106:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.024108:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.024110:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.024111:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.024112:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.024114:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.024115:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.024118:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.024120:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.024122:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.024123:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.024124:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.024125:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.024126:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.024126:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.024127:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.024131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.024139:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.024141:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.024142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.024142:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.024145:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.024147:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.024172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.024179:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.024182:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.024183:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.024185:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.024188:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.024189:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.024191:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.024192:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.024193:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.024194:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.024197:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.024199:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.024202:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.024203:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.024204:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.024205:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.024206:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.024207:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.024207:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.024211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.024219:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.024221:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.024222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.024223:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.024225:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.024228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.024317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677475.024326:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.024331:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677475.024334:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.024334:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677475.024338:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.024340:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677475.024342:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.024345:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.024346:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.024348:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677475.024353:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.024356:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.024360:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.024361:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677475.024362:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677475.024364:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677475.024365:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.024366:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677475.024367:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677475.024370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.024379:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.024382:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.024383:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.024383:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.024385:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.024389:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.024725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.024733:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.024736:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.024738:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.024739:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.024742:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.024743:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.024745:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.024746:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.024747:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.024748:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.024752:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.024753:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.024756:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.024757:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.024757:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.024759:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.024759:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.024761:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.024762:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.024763:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.024765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.024768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.024773:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.024775:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.024776:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.024776:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.024777:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.024778:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.024779:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.024779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677475.024779:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.024781:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.024838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.024846:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.024849:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.024852:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.024855:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.024855:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.024856:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.024857:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.024859:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.024860:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.024861:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.024864:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.024865:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.024867:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.024868:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.024868:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.024869:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.024870:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.024871:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.024872:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.024875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.024884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.024887:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.024888:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.024890:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.024891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.024893:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.025044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.025052:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.025054:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.025055:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.025058:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.025061:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.025063:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677475.025064:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.025065:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.025066:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.025067:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677475.025070:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.025072:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677475.025075:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.025075:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677475.025076:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677475.025078:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677475.025078:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.025079:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.025079:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.025084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.025092:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.025095:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.025096:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.025096:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.025098:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.025101:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.025281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.025287:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.025290:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.025290:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.025293:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.025300:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.025301:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.025302:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.025304:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.025305:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.025306:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.025309:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.025310:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.025313:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.025313:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.025314:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.025316:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.025316:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.025317:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.025317:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.025322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.025330:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.025333:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.025334:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.025334:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.025336:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.025338:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.025749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.025753:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.025756:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.025758:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.025759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.025760:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.025762:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.025763:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.025764:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.025766:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.025766:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.025770:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.025772:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.025774:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.025775:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.025776:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.025778:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.025778:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.025780:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.025780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.025784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.025791:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.025794:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.025795:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.025795:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.025797:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.025800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.025806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.025812:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.025814:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.025815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.025815:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.025816:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.025817:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.026105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.026112:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.026115:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.026116:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.026117:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.026120:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.026121:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.026123:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.026124:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677475.026125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:2.0:1619677475.026125:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.026126:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.026129:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.026131:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677475.026131:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677475.026133:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:2.0:1619677475.026133:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.026134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:2.0:1619677475.026134:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.026135:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:22.0:1619677475.026136:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677475.026137:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.026137:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:22.0:1619677475.026138:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.026138:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.026139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:22.0:1619677475.026140:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.026141:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.026143:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.026143:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677475.026143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:22.0:1619677475.026145:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.026148:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.026150:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677475.026150:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677475.026152:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.026153:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677475.026153:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.026153:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.026154:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677475.026154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677475.026155:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:18.0:1619677475.026155:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.026156:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.026156:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.026157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:18.0:1619677475.026157:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:25.2:1619677475.026161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.026169:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.026171:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.026172:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.026174:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.026174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.026176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.026552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.026560:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.026563:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.026565:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.026567:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.026567:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.026569:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.026570:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.026572:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.026573:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.026574:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.026577:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.026579:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.026582:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.026583:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.026583:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.026585:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.026586:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.026587:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.026587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.026591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.026599:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.026601:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.026602:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.026602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.026604:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.026606:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.027052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.027060:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.027063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.027063:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677475.027065:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.027068:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.027070:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677475.027071:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.027072:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.027073:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.027074:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677475.027078:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.027079:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677475.027082:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.027082:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677475.027083:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:10.0:1619677475.027084:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677475.027085:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.027085:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.027086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.027090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677475.027097:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677475.027100:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677475.027100:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.027102:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.027104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677475.027105:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677475.027644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.027650:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.027653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.027653:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.027655:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.027658:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.027659:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.027660:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.027662:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.027663:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.027664:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.027667:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.027669:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.027671:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.027672:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.027672:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.027674:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.027675:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.027675:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.027676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.027681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.027689:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.027691:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.027692:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.027694:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.027695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.027696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.028326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.028334:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.028337:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.028339:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.028341:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.028342:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.028343:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.028344:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.028346:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.028347:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.028348:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.028351:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.028353:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.028355:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.028356:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.028357:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.028358:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.028358:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.028360:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.028362:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.028362:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.028364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.028368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.028373:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.028375:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.028375:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.028377:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.028377:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.028378:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.028378:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.028379:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.028380:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.028381:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.028456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.028464:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.028467:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.028467:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.028469:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.028472:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.028473:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.028474:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.028475:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.028476:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.028477:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.028480:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.028482:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.028484:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.028485:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.028486:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.028487:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.028488:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.028489:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.028490:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.028491:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.028493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.028496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.028501:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.028503:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.028504:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.028504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.028505:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.028505:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.028506:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677475.028507:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.028507:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677475.028510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.028513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.028519:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.028522:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.028523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.028523:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.028524:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.028525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.028552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.028560:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.028563:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.028565:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.028567:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.028567:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.028568:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.028570:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.028571:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.028572:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.028573:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.028576:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.028577:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.028580:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.028580:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.028581:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.028583:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.028583:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.028585:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.028586:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.028586:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.028588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.028592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.028595:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.028598:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.028599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.028599:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.028600:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.028601:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.028602:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677475.028602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.028603:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.028604:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.028636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.028643:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:4.2:1619677475.028645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:26.0:1619677475.028646:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.028647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.028649:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.028650:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:26.0:1619677475.028651:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.028653:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677475.028653:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:26.0:1619677475.028654:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.028655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:26.0:1619677475.028656:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.028656:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.028657:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.028658:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.028658:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.028660:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.028662:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.028662:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.028663:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.028664:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677475.028665:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.028665:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.028668:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.028669:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.028669:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677475.028669:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.028671:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.028671:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677475.028671:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.028672:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.028673:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.0:1619677475.028674:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.028675:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.028675:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677475.028677:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:25.2:1619677475.028678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (23)++
00000800:00000200:21.0:1619677475.028678:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.028679:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.028679:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677475.028685:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677475.028688:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677475.028688:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000400:00000200:22.0:1619677475.028689:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.028689:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677475.028690:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.028691:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.028691:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.028694:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.028695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.028695:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:25.2:1619677475.028698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.028706:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.028708:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.028709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.028709:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.028710:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.028710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.028895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.028901:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.028905:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.028905:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.028907:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.028909:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.028911:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.028912:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.028913:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.028914:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.028915:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.028918:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.028920:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.028922:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.028923:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.028924:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.028925:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.028926:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.028926:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.028927:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.028931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.028939:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.028941:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.028942:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.028942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.028944:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.028946:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.029147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.029150:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.029152:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.029154:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.029155:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.029156:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.029157:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.029159:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.029160:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.029161:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.029162:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.029165:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.029167:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.029169:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.029170:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.029171:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.029172:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.029173:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.029174:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.029175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.029180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.029188:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.029191:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.029192:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:4.2:1619677475.029192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:1.0:1619677475.029192:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.029193:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.029196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677475.029199:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.029201:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:8.2:1619677475.029201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677475.029202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.029204:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.029207:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.029209:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:10.0:1619677475.029209:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.029210:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.029211:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.029212:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.029212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.029213:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:10.0:1619677475.029213:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.029213:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.029214:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.029215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:10.0:1619677475.029217:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.029218:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.029221:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.029222:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.029222:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.029224:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.029225:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.029226:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.029227:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.029228:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.029230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.029233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677475.029238:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.029240:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:6.0:1619677475.029241:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677475.029242:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:22.0:1619677475.029242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.029242:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.029243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.029244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:6.0:1619677475.029244:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.029247:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.029286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.029291:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.029296:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.029299:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677475.029302:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.029304:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.029305:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.029306:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.029307:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.029308:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.029309:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.029313:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.029315:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.029317:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.029318:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.029318:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.029320:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.029320:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.029322:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.029323:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.029323:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.029327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.029329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.029334:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.029336:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.029336:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.029337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.029338:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.029338:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.029339:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.029341:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.029342:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.029343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.029346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.029353:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.029356:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.029357:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.029357:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.029358:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.029359:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.029518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.029526:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.029529:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.029531:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.029533:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.029533:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.029535:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.029536:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.029537:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.029538:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.029540:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.029543:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.029544:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.029546:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.029547:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.029548:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.029549:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.029550:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.029551:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.029551:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.029555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.029561:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.029564:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.029564:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.029566:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.029568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.029569:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.029621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.029629:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.029631:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.029632:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.029634:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.029637:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.029638:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.029639:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.029641:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.029642:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.029643:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.029645:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.029647:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.029650:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.029650:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.029651:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.029653:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.029653:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.029654:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.029654:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.029659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.029667:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.029669:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.029670:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.029670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.029673:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.029675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.029684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.029690:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.029694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.029694:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677475.029696:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.029698:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.029700:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.029701:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.029702:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.029703:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.029705:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.029708:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.029710:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.029712:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.029713:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.029713:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.029715:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.029715:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.029716:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.029717:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.029721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.029730:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.029732:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.029733:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.029735:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.029736:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.029737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.030193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.030201:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.030204:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.030206:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.030208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.030208:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.030209:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.030211:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.030212:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.030213:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.030214:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.030217:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.030219:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.030222:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.030222:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.030223:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.030224:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.030225:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.030226:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.030227:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.030230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.030237:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.030239:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.030240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.030240:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.030242:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.030244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.030275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.030280:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.030282:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.030285:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.030285:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.2:1619677475.030288:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:21.0:1619677475.030292:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.030297:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677475.030298:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:21.0:1619677475.030299:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.030300:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.030301:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:21.0:1619677475.030301:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.030303:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677475.030305:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.030306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.030307:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.030307:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.030309:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:21.0:1619677475.030309:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677475.030311:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.030311:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.030312:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.030312:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:31.0:1619677475.030313:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.030313:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:31.0:1619677475.030314:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.030315:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.030315:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.030316:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.030317:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677475.030318:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.030320:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.030322:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.030323:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.030324:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.030325:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.030326:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.030327:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.030328:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.030332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677475.030332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677475.030339:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.030341:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677475.030341:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:24.0:1619677475.030342:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.030343:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:8.0:1619677475.030343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.030344:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:20.0:1619677475.030344:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.030346:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.030347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677475.030347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.030348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.031026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.031033:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.031036:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.031038:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.031040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.031041:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.031042:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.031043:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.031045:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.031046:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.031047:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.031050:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.031051:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.031054:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.031054:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.031055:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.031056:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.031057:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.031058:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.031058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.031062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.031068:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.031071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.031071:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.031072:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.031074:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.031075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.031086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.031091:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.031094:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.031095:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.031095:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.031095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.031096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.031466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.031472:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.031475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.031475:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.031477:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.031480:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.031481:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.031482:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.031484:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.031485:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.031486:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.031489:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.031491:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.031494:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.031494:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.031495:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.031497:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.031497:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.031498:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.031498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.031504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.031512:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.031514:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.031515:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.031517:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.031518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.031519:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.032420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.032429:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.032432:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.032434:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.032436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.032437:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.032438:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.032439:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.032440:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.032442:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.032442:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.032445:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.032447:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.032450:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.032451:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.032451:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.032453:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.032453:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.032455:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.032457:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.032457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.032459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.032463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.032466:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.032469:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.032470:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.032470:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.032471:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.032472:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.032473:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.032473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:30.0:1619677475.032474:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.032474:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.032475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.032477:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.032480:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.032482:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.032482:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.032484:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.032485:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.032487:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.032488:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.032489:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.032490:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.032493:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.032495:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.032497:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.032498:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.032499:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.032500:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.032501:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.032502:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.032502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.032506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.032514:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.032516:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.032517:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.032518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.032519:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.032521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.032588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.032595:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.032598:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.032601:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.032602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.032603:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.032605:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.032606:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.032607:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.032608:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.032609:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.032613:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.032615:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.032617:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.032618:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.032619:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.032620:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.032621:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.032621:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.032622:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.032626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.032632:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.032634:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.032635:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.032635:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.032637:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.032639:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.032651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.032657:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.032659:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.032660:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.032660:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.032661:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.032662:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.033006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.033011:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.033014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.033014:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.033016:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.033018:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.033020:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.033021:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.033022:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.033023:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.033024:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.033027:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.033029:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.033031:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.033032:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.033033:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.033034:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.033035:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.033035:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.033036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.033040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.033048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.033050:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.033051:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.033052:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.033053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.033055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.033071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.033079:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.033084:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.033085:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.033088:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.033091:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.033094:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.033096:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.033098:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.033100:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.033101:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.033107:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.033110:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.033114:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.033115:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.033116:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.033118:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.033119:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.033121:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.033123:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.033123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.033124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677475.033128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677475.033132:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.033135:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677475.033136:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:13.0:1619677475.033136:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.033137:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:18.0:1619677475.033139:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:12.0:1619677475.033139:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.033140:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:18.0:1619677475.033141:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.033142:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:4.2:1619677475.033298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.033306:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.033309:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.033310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.033312:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.033315:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.033316:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.033317:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.033318:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.033319:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.033321:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.033324:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.033326:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.033329:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.033330:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.033330:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.033332:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.033333:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.033334:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.033334:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.033339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.033347:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.033350:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.033350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.033351:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.033353:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.033356:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.033434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.033441:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.033444:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.033445:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.033446:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.033449:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.033451:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.033452:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.033453:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.033454:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.033455:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.033459:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.033460:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.033462:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.033463:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.033463:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.033465:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.033466:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.033466:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.033468:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.033468:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.033471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.033474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.033480:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.033481:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.033481:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.033481:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.033482:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.033482:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.033483:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.033483:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.033485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677475.033485:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.033490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.033498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.033500:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.033501:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.033501:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.033502:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.033502:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.033517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.033524:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.033527:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.033528:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.033530:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.033533:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.033534:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.033536:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.033537:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.033538:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.033539:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.033541:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.033543:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.033546:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.033546:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.033547:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.033549:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.033549:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.033550:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.033550:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.033555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.033563:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.033565:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.033566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.033567:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.033569:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.033572:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.033574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.033581:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.033583:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.033584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.033585:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.033586:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.033586:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.033701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.033707:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.033710:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.033711:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.033713:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.033715:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.033716:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.033718:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.033719:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.033720:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.033721:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.033724:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.033725:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.033728:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.033729:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.033729:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.033731:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.033731:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.033732:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.033733:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.033737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.033745:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.033747:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.033748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.033748:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.033750:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.033752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.034241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.034249:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.034252:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.034254:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.034256:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.034257:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.034258:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.034259:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.034261:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.034262:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.034263:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.034266:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.034268:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.034270:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.034270:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.034271:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.034272:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.034273:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.034274:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.034275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.034278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.034285:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.034287:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.034288:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.034290:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.034291:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.034292:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.034299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.2:1619677475.034301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.034306:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.034308:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.034309:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.034310:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.034310:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.034311:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.034312:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.034313:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677475.034313:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.034315:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.034317:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.034318:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.034320:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.034321:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.034322:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.034323:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.034326:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.034327:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.034329:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.034330:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.034330:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.034332:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.034332:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.034333:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.034334:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.034338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.034345:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.034347:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.034348:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.034349:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.034350:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.034352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.034522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.034529:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.034531:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.034532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.034533:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.034536:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.034537:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.034538:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.034540:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.034540:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.034541:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.034544:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.034546:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.034549:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.034549:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.034550:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.034552:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.034552:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.034553:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.034554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.034558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.034565:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.034568:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.034569:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.034569:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.034572:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.034574:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.034729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.034734:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.034736:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.034737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.034739:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.034740:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.034742:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.034743:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.034744:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.034745:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.034746:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.034749:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.034751:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.034753:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.034754:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.034755:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.034756:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.034757:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.034757:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.034758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.034762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.034770:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.034772:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.034773:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.034775:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.034776:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.034777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.035044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677475.035051:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.035054:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677475.035056:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.035057:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677475.035059:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.035060:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677475.035061:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.035063:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.035064:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.035065:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.035068:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.035070:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677475.035073:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.035074:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:30.0:1619677475.035074:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.035076:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677475.035077:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.035078:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677475.035078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677475.035082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.035090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.035093:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.035094:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.035095:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.035097:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.035100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.035360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.035368:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.035371:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.035374:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.035376:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.035376:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.035378:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.035379:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.035380:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.035381:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.035382:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.035386:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.035387:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.035389:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.035389:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.035390:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.035391:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.035392:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.035393:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.035393:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.035398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.035404:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.035407:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.035408:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.035409:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.035411:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.035412:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.035959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.035965:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.035968:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.035970:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.035970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.035972:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.035974:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.035975:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.035977:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.035978:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.035979:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.035982:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.035984:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.035986:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.035987:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.035988:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.035990:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.035990:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.035992:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.035993:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.035993:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.035996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.035998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.036002:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677475.036003:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.036004:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:24.0:1619677475.036005:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677475.036005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:24.0:1619677475.036006:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.036006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677475.036006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.036009:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.036012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.036559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.036567:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.036570:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.036572:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.036574:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.036574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.036576:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.036577:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.036578:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.036579:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.036580:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.036583:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.036585:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.036588:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.036588:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.036589:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.036591:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.036591:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.036593:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.036594:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.036595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.036597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.036601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.036604:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.036606:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.036607:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.036607:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.036609:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.036609:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.036611:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.036612:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.036612:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.036613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.036970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.036976:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.036979:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.036980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.036982:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.036985:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.036986:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.036987:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.036989:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.036990:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.036991:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.036994:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.036996:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.036998:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.036999:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.037000:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.037001:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.037002:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.037003:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.037004:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.037005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.037007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.037010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.037015:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.037015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.037017:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.037017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.037018:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.037018:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.037018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.037020:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.037022:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.037023:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.037023:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.037024:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.037025:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.037025:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.037025:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (24)--
00000800:00000200:21.0:1619677475.037026:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.037027:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.037154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.037162:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.037165:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.037167:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.037169:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.037169:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.037171:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.037172:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.037173:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.037174:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.037176:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.037179:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.037181:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.037184:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.037184:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.037185:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.037186:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.037187:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.037188:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.037189:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.037192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.037198:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.037201:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.037202:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.037202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.037204:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.037206:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.037663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:4.2:1619677475.037663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.037669:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677475.037670:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:21.0:1619677475.037672:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:22.0:1619677475.037673:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.037673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:21.0:1619677475.037675:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.037675:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:22.0:1619677475.037676:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.037678:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.037678:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.037680:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:21.0:1619677475.037680:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.037681:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.037682:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.037684:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.037684:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.037685:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.037686:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.037687:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:21.0:1619677475.037687:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677475.037693:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.037693:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677475.037696:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.0:1619677475.037697:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.037699:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.037699:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.037700:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:21.0:1619677475.037700:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.037701:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.037702:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677475.037703:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.0:1619677475.037703:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.037704:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.037705:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:21.0:1619677475.037705:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677475.037706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:21.0:1619677475.037706:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.037706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.037710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.2:1619677475.037711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.037718:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677475.037719:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677475.037720:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.037721:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.037721:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.037722:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:6.0:1619677475.037722:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:17.0:1619677475.037723:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.037723:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.037725:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.037726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677475.037727:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.037730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.037737:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.037740:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.037741:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.037741:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.037742:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.037743:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.038034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.038043:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.038046:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.038048:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.038050:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.038051:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.038052:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.038053:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.038055:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.038056:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.038057:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.038060:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.038062:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.038065:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.038066:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.038067:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.038068:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.2:1619677475.038068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:31.0:1619677475.038069:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.038070:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.038072:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.038073:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.038074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677475.038075:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.2:1619677475.038078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677475.038078:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.038079:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677475.038080:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:2.0:1619677475.038081:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677475.038083:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:2.0:1619677475.038083:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.038084:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.038084:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.038084:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677475.038086:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.038086:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:2.0:1619677475.038086:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.038087:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.038088:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677475.038088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:2.0:1619677475.038088:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.038089:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677475.038089:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677475.038090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677475.038093:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.038095:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.038097:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.038098:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.038099:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.038101:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.038101:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.038102:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.038103:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.038106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.038114:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.038117:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.038118:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.038118:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.038120:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.038123:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.038188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.038193:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.038195:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.038196:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.038198:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.038200:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.038202:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.038203:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.038204:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.038205:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.038206:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.038209:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.038211:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.038213:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.038214:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.038215:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.038216:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.038217:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.038219:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.038220:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.038220:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.038222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.038225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:21.2:1619677475.038229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677475.038230:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677475.038231:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677475.038233:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677475.038233:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000400:00000200:20.0:1619677475.038234:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.038234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.038235:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:30.0:1619677475.038236:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677475.038236:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.038236:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.038238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677475.038239:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.038242:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.038243:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.038244:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.038244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:30.0:1619677475.038245:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.038247:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.038248:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.038249:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.038250:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.038252:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:30.0:1619677475.038254:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.038255:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677475.038256:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.038256:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.038256:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.038257:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:30.0:1619677475.038258:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.038258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677475.038259:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.038260:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.038261:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.038262:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.038263:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.038264:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.038267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.038272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677475.038273:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.038276:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.038277:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.038277:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.038279:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.038280:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677475.038280:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.038283:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.038284:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.038286:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677475.038287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:10.0:1619677475.038289:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.038290:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.038291:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.038293:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.038293:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.038298:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:10.0:1619677475.038299:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.038300:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677475.038302:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.038303:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000400:00000200:10.0:1619677475.038304:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.038304:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.038304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677475.038306:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.038309:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.038310:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.038311:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.038312:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.038313:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.038313:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.038314:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.038319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.038327:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.038330:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.038330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.038331:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.038333:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.038336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.038338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.038345:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.038348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.038348:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.038349:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.038350:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.038350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.038455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.038461:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.038464:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.038465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.038467:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.038469:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.038470:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.038472:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.038473:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.038474:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.038475:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.038478:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.038480:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.038482:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.038483:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.038484:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.038485:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:21.2:1619677475.038486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:1.0:1619677475.038486:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.038487:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.038488:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.038489:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.038491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.038494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677475.038496:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.038499:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:17.0:1619677475.038499:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:22.0:1619677475.038501:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.038501:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677475.038501:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677475.038502:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.038503:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677475.038503:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.038504:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.038504:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.038506:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.038506:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.038507:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.038509:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.038510:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.038511:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.038513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.038514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.038515:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.038516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677475.038517:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.038520:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.038520:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.038521:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.038523:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.038523:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.038523:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.038524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.038529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.038536:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.038538:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.038539:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.038541:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.038542:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.038543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.038647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.038655:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.038658:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.038660:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.038662:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.038662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.038664:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.038665:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.038666:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.038667:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.038668:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.038671:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.038673:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.038675:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.038676:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.038677:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.038678:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.038679:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.038680:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.038680:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.038684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.038690:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.038693:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.038694:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.038694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.038695:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.038697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.038701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.038707:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.038710:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.038710:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.038710:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.038711:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.038712:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.038729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.038733:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.038735:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.038737:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.038738:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.038739:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.038741:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.038742:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.038743:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.038744:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.038745:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.038748:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.038750:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.038752:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.038753:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.038753:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.038755:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.038756:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.038756:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.038757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.038761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.038769:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.038771:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.038772:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.038773:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.038774:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.038775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.038874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.038882:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.038885:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.038885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.038888:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.038891:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.038892:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.038894:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.038895:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.038896:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.038897:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.038900:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.038901:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.038904:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.038905:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.038906:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.038907:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.038908:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.038909:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.038910:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.038910:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.038913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.038916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.038921:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.038923:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.038923:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.038924:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.038925:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677475.038925:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.038926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.038927:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.038927:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.038930:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.039006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.039014:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.039016:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.039017:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.039019:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.039022:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.039023:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677475.039024:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.039026:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.039026:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.039027:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677475.039030:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.039031:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.039032:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.039033:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.039033:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.039035:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.039035:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.039036:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.039036:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.039041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.039047:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.039049:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.039049:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.039050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.039051:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.039053:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677475.039107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677475.039110:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.039113:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677475.039115:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.039116:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677475.039117:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.039119:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.039120:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.039121:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.039122:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.039123:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.039126:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.039128:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.039130:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.039131:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.039132:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.039133:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.039134:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.039134:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677475.039135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677475.039139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.039147:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.039149:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.039150:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.039150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.039152:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.039154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.039159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.039166:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.039168:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.039169:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.039169:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.039170:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.039171:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.039241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.039246:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.039248:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.039249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.039251:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.039253:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.039254:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677475.039255:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.039256:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.039257:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.039258:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677475.039262:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.039264:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677475.039266:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.039267:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:22.0:1619677475.039267:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677475.039269:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:22.0:1619677475.039270:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.039271:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677475.039272:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677475.039272:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:8.2:1619677475.039276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.2:1619677475.039277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.039283:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:1.0:1619677475.039283:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677475.039285:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677475.039285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.039286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:1.0:1619677475.039286:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677475.039287:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.039288:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.039289:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.039292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.039305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.039314:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.039317:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.039319:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.039321:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.039321:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.039323:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.039324:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.039326:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.039327:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.039328:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.039331:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.039333:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.039337:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.039337:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.039338:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.039340:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.039340:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.039341:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.039342:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.039346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.039355:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.039357:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.039358:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.039360:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.039361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.039363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.039518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.039524:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.039527:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.039527:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.039529:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.039531:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.039533:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677475.039535:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.039536:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.039537:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.039538:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.039541:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.039543:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.039546:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.039546:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677475.039547:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677475.039549:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677475.039549:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.039550:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.039550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677475.039555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.039562:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.039565:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.039566:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.039566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:4.2:1619677475.039567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:24.0:1619677475.039568:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.039572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677475.039573:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.039575:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.039575:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.039578:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.039580:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.039582:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.039583:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.039584:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.039585:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.039586:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.039590:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.039591:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.039594:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.039595:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.039596:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.039598:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.039598:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.039599:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.039599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.039604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.039612:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.039615:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.039615:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.039616:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.039618:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.039621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.039636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.039644:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.039647:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.039647:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.039649:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.039651:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.039653:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.039654:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.039656:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.039657:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.039658:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.039660:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.039662:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.039665:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.039665:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.039666:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.039668:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.039668:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.039669:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677475.039670:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.039671:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.039674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.039677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677475.039683:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677475.039684:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677475.039686:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677475.039686:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.039686:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:12.0:1619677475.039687:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.039687:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:12.0:1619677475.039690:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.039691:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.039692:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.039693:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.039694:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.039694:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.039694:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.039695:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.039934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.039939:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.039942:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.039943:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.039945:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.039948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.039950:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677475.039951:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.039953:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.039954:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.039955:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677475.039958:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.039960:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677475.039962:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.039963:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677475.039964:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677475.039965:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677475.039966:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.039967:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.039967:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677475.039972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.039980:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.039982:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.039983:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.039983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.039986:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.039988:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.040179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.040185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.040188:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.040188:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.040191:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.040193:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.040194:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.040196:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.040197:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.040198:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.040199:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.040202:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.040204:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.040206:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.040207:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.040208:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.040209:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.040210:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.040211:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.040211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.040215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.040223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677475.040223:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.040225:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.040226:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.040228:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.040230:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677475.040230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.040231:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.040234:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.040236:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.040238:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.040239:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.040241:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.040242:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.040243:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.040244:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.040245:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.040249:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.040251:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.040253:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.040254:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.040255:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.040256:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.040257:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.040258:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.040258:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.040262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.040266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.040269:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.040271:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.040272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.040272:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.040273:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677475.040274:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.040276:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677475.040276:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:30.0:1619677475.040278:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.040280:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.040280:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.040281:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.040282:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.040284:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.040285:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.040285:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.040288:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.040290:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.040291:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.040292:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.040293:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.040299:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.040300:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.040301:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.040301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.040306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.040314:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.040316:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.040317:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.040318:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.040320:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.040321:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.040448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677475.040456:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677475.040459:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677475.040459:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677475.040461:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.040464:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.040465:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677475.040467:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.040468:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.040469:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.040470:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677475.040473:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.040474:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677475.040477:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.040478:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677475.040478:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677475.040480:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677475.040481:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677475.040482:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677475.040482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:25.2:1619677475.040487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.040494:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.040496:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.040497:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.040499:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.040500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.040502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.041356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.041362:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.041364:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.041366:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.041367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.041368:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.041370:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.041371:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.041372:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.041373:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.041374:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.041377:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.041379:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.041381:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.041382:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.041382:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.041384:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.041385:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.041385:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.041386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.041391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.041399:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.041401:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.041402:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.041403:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.041404:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.041407:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.041884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.041892:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.041895:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.041897:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.041899:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.041899:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.041900:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.041901:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.041903:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.041903:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.041904:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.041907:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.041909:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.041911:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.041912:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.041913:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.041914:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.041915:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.041916:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.041916:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.041920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.041927:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.041931:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.041931:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.041932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.041933:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.041936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.042379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.042385:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.042388:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.042388:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.042390:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.042392:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.042394:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.042395:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.042396:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.042398:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.042399:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.042402:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.042403:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.042406:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.042407:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.042407:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.042409:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.042409:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.042410:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.042411:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.042416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.042424:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.042426:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.042427:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.042428:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.042430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.042431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.042437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.042444:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.042447:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.042448:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.042448:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.042449:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.042450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.042665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.042671:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.042674:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.042675:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.042677:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.042679:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.042681:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.042682:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.042683:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.042684:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.042685:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.042688:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.042690:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.042692:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.042692:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.2:1619677475.042693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.042693:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.042695:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.042696:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.042697:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.042697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677475.042699:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:9.2:1619677475.042700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.042701:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.042703:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.042704:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.042705:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:2.0:1619677475.042705:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.042707:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.042707:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.042707:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677475.042709:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:2.0:1619677475.042709:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.042710:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.042711:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.042712:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:18.0:1619677475.042713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.042714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:2.0:1619677475.042715:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.042717:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677475.042719:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.042720:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677475.042720:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677475.042722:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677475.042723:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.042723:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.042724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677475.042729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.042737:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.042740:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.042740:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.042742:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.042743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.042745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.042929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.042932:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.042934:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.042936:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.042937:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.042938:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.042940:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.042941:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.042942:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.042943:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.042945:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.042948:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.042949:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.042952:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.042952:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.042953:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.042955:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.042955:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.042956:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.042957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.042961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.042968:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.042970:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.042971:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.042972:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.042974:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.042975:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.043090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.043098:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.043101:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.043102:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.043104:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.043108:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.043109:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.043110:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.043112:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.043113:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.043114:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.043117:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.043119:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.043122:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.043123:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.043123:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.043125:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.043125:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.043126:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.043127:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.043142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.2:1619677475.043145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.043149:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.043152:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:30.0:1619677475.043153:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677475.043153:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.043153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.043155:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.043156:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677475.043157:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:30.0:1619677475.043158:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.043161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.043161:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.043162:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.043163:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.043165:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.043166:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.043166:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.043170:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.043171:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.043173:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.043174:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.043174:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.043176:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.043176:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.043178:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.043179:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.043180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.043181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.043185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.043190:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.043191:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.043192:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.043193:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.043194:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.043194:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.043195:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.043196:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.043196:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.043197:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677475.043265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.043272:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.043275:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.043275:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.043278:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.043280:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.043281:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.043283:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.043284:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.043285:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.043286:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.043290:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.043291:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.043299:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.043300:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.043301:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.043303:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.043304:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.043305:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.043305:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.043310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.043318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.043321:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.043321:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.043323:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.043325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.043326:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677475.043458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:5.2:1619677475.043464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677475.043466:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.043469:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000800:00000200:17.0:1619677475.043470:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677475.043471:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.043473:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.043473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:17.0:1619677475.043473:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:31.0:1619677475.043474:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:1.0:1619677475.043474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677475.043476:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.043477:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.043478:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.043479:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.043479:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.043480:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677475.043481:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.043482:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.043484:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.043485:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677475.043485:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.043486:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677475.043489:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:17.0:1619677475.043490:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.043492:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.043492:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:31.0:1619677475.043493:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677475.043493:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677475.043494:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677475.043495:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:17.0:1619677475.043495:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.043496:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:17.0:1619677475.043496:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677475.043497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:17.0:1619677475.043497:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.043498:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.043499:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.043500:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.043500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:9.2:1619677475.043500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:25.2:1619677475.043505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.043507:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.043510:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.043511:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.043511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677475.043512:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:26.0:1619677475.043513:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.043515:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677475.043515:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.043516:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.043516:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.043518:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.043520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.043717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677475.043721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677475.043724:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.043727:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.043728:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677475.043729:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677475.043730:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.043731:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677475.043732:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.043734:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.043734:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677475.043735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677475.043735:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.043736:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.043737:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.043737:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.043738:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.043739:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.043739:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.043740:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.043741:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.043742:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.043744:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.043746:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.043747:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.043748:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.043750:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.043751:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.043751:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.043751:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.043751:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.043752:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.043753:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.043754:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.043754:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.043754:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.043754:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677475.043755:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677475.043755:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677475.043756:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.043759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:25.2:1619677475.043760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.043767:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.043768:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677475.043770:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677475.043771:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677475.043771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677475.043772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:22.0:1619677475.043772:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.043772:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.043775:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.043775:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.043778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677475.043779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.043846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677475.043855:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.043858:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677475.043859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.043861:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.102@o2ib7 : GET - for me
00000400:00000200:2.0:1619677475.043864:0:19932:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.102@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:2.0:1619677475.043866:0:19932:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.102@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:2.0:1619677475.043867:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.043868:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.043870:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:21.2:1619677475.043871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:2.0:1619677475.043874:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : REPLY try# 0
00000800:00000200:2.0:1619677475.043876:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:2.0:1619677475.043877:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.043878:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677475.043879:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:2.0:1619677475.043879:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.043880:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (23)--
00000800:00000200:31.0:1619677475.043881:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:2.0:1619677475.043881:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:31.0:1619677475.043884:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.043885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.043886:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.043886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:31.0:1619677475.043887:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.043889:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.043890:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.043891:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.043892:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677475.043894:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:31.0:1619677475.043895:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.043896:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:31.0:1619677475.043897:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:18.0:1619677475.043897:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.043897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.043899:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: REPLY: OK
00000800:00000200:31.0:1619677475.043900:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.043901:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.043901:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677475.043901:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:31.0:1619677475.043903:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.043904:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.043905:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.043907:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.043908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.043909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.043913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.043915:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.043917:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.043918:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.043918:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.043920:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.043921:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677475.043922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.043923:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.043924:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.043925:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.043929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.2:1619677475.043932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.0:1619677475.043935:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677475.043937:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677475.043938:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.043938:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.043939:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.043940:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.043940:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677475.043940:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.043941:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.043942:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.043944:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.043946:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.043947:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.043948:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.043949:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.043950:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.043954:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.043955:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.043958:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.043958:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.043959:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.043961:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.043961:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.043962:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.043963:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.043964:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.043967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.043969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.043974:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.043974:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.043976:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.043977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.043977:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.043978:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.043978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.043980:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.043981:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.043982:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.044127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.044132:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.044135:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.044137:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.044137:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.044139:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.044141:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677475.044142:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.044143:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.044144:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.044145:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.044149:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.044151:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.044154:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.044155:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.044155:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.044157:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.044158:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.044159:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.044160:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.044161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677475.044163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.044166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677475.044170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677475.044170:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677475.044173:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677475.044173:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677475.044173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677475.044174:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.044174:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677475.044176:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.044177:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.044179:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.044183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.044190:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.044193:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.044194:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.044196:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.044196:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.044197:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.044355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.044363:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.044365:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.044367:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.044369:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.044370:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.044371:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.044372:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.044373:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.044374:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.044375:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.044378:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.044379:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.044382:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.044383:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.044384:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.044385:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.044386:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.044386:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.044387:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.044391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.044398:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.044401:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.044402:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.044402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.044404:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.044406:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.044547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.044555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.044557:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.044558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.044560:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.044562:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.044564:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.044565:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.044566:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.044567:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.044568:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.044571:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.044572:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.044575:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.044576:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.044577:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.044578:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.044579:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.044580:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.044582:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.044582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.044584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.044588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.044592:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.044595:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.044595:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.044595:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.044596:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.044597:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.044598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677475.044598:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.044599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.044600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.044603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.044610:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.044613:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.044613:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.044614:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.044615:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.044615:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.044968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.044971:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.044974:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.044976:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.044977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.044979:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.044980:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.044982:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.044983:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.044984:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.044985:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.044988:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.044990:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.044992:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.044993:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.044994:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.044996:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.044996:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.044997:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.044997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.045009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.045015:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.045018:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.045019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.045019:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.045021:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.045024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.045255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.045263:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.045266:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.045268:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.045270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.045270:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.045272:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.045273:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.045274:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.045275:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.045276:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.045280:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.045282:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.045285:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.045286:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.045286:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.045288:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.045289:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.045290:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.045290:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.045296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.045304:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.045307:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.045308:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.045309:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.045311:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.045312:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.045313:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.045314:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:20.0:1619677475.045314:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.045315:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (23)++
00000800:00000200:21.0:1619677475.045316:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.045317:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.045318:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.045614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.045620:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.045622:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.045623:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.045625:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.045627:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.045629:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.045630:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.045631:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.045632:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.045633:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.045636:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.045638:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.045640:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.045641:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.045642:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.045643:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.045644:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.045645:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.045645:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.045650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.045658:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.045660:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.045660:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.045662:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.045663:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.045665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.045803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.045811:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.045814:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.045814:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.045817:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.045820:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.045821:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.045823:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.045824:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.045825:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.045826:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.045829:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.045831:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.045833:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.045834:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.045835:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.045836:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.045837:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.045837:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.045838:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.045842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.045849:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.045852:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.045852:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.045854:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.045856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.045857:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.045885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.045893:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.045896:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.045898:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.045901:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.045901:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.045903:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.045904:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.045905:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.045906:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.045907:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.045910:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.045911:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.045914:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.045915:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.045915:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.045917:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.045917:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.045918:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.045918:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.045935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.045941:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.045944:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.045945:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.045945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.045946:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.045949:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.046589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.046596:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.046599:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.046600:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.046602:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.046604:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.046606:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.046607:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.046608:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.046610:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.046610:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.046613:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.046615:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.046618:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.046618:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.046619:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.046621:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.046621:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.046622:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.046622:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.046627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.046636:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.046639:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.046639:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.046641:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.046642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.046643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.046767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677475.046776:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677475.046778:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677475.046779:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677475.046781:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.046784:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.046786:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677475.046787:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.046788:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.046789:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.046790:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677475.046793:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.046795:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.046798:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.046799:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677475.046799:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677475.046801:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677475.046801:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677475.046802:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677475.046802:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:8.2:1619677475.046807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.046815:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.046818:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.046819:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.046819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.046821:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.046824:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.047767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.047773:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.047775:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.047776:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.047778:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.047781:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.047782:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.047784:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.047785:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.047786:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.047787:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.047790:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.047792:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.047795:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.047795:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.047796:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.047798:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.047798:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.047799:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.047799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.047805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.047813:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.047815:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.047816:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.047817:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.047819:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.047819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.047906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.047914:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.047917:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.2:1619677475.047919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:30.0:1619677475.047920:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.047923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.047923:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.047924:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677475.047924:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677475.047925:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.047925:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.047927:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.047928:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.047929:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.047932:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.047933:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.047934:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.047935:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677475.047935:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.047936:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.047938:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.047938:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.047939:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.047942:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.047944:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.047945:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.047946:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.047946:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.047947:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.047947:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.047947:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.047949:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.047949:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.047950:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.047950:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.047951:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (23)++
00000800:00000200:30.0:1619677475.047951:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.047951:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677475.047953:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (24)--
00000800:00000200:30.0:1619677475.047953:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (24)--
00000800:00000200:31.0:1619677475.047954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.047955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:9.2:1619677475.047955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.047959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.047962:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:30.0:1619677475.047964:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.047965:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:8.0:1619677475.047965:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:30.0:1619677475.047966:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.047966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.047966:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.047967:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.047968:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677475.047968:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.047969:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677475.047969:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.047970:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.047971:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.0:1619677475.047971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.047972:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.047975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.047981:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.047984:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.047985:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.047985:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.047986:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.047987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.048044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.048052:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.048054:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.048055:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.048057:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.048060:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.048061:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.048062:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.048063:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.048064:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.048065:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.048068:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.048070:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.048072:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.048073:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.048074:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.048075:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.048076:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.048077:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.048078:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.048079:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.048082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.048084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.048089:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.048091:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.048092:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.048093:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.048093:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677475.048093:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.048094:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.048095:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677475.048095:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.048098:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.048103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.048110:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.048112:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.048113:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.048114:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.048115:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.048115:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.048279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.048285:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.048288:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.048289:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.048291:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.048298:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.048300:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.048301:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.048303:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.048304:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.048305:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.048308:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.048310:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.048312:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.048313:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.048314:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.048316:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.048316:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.048317:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.048317:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.048322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.048330:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.048333:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.048334:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.048334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.048336:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.048338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.048417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.048423:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.048426:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.048427:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.048429:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.048432:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.048433:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677475.048435:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.048436:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.048438:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.048438:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677475.048442:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.048443:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677475.048446:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.048447:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677475.048447:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.048449:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677475.048450:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.048451:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677475.048452:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677475.048452:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:25.2:1619677475.048455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.2:1619677475.048457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:6.0:1619677475.048462:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677475.048463:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677475.048465:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:6.0:1619677475.048465:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.048466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677475.048466:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:6.0:1619677475.048466:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.048467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:6.0:1619677475.048468:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.048471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.048533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.048542:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.048544:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.048545:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.048547:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.048550:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.048552:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.048553:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.048554:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.048555:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.048556:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.048560:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.048562:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.048565:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.048566:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.048566:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.048568:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.048568:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.048569:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.048569:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.048574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.048581:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.048583:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.048584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.048585:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.048587:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.048589:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.048640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.048650:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.048654:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.048657:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.048660:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.048660:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.048662:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.048665:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.048667:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.048669:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.048670:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.048675:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.048678:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.048681:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.048682:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.048683:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.048684:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.048685:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.048687:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.048688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.048690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.048699:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.048702:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.048703:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.048706:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.048708:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.048709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.049038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.049047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.049051:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.049054:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.049056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.049057:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.049059:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.049061:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.049063:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.049064:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.049066:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.049071:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.049073:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.049076:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.049077:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.049078:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.049080:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.049081:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.049082:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.049083:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.049085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.049094:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.049097:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.049098:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.049101:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.049103:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.049104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.049200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.049208:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.049211:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.049212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.049214:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.049218:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.049219:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.049221:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.049223:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.049224:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.049225:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.049228:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.049230:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.049233:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.049234:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.049235:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.049237:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.049237:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.049238:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.049238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.049243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.049251:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.049254:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.049255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.049255:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.049257:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.049259:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.049330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.049335:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.049338:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.049338:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.049340:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.049342:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.049344:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.049345:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.049346:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.049347:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.049348:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.049351:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.049353:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.049355:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.049356:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.049357:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.049358:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.049359:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.049360:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.049360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.049365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.049374:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.049376:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.049377:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.049378:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.049379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.049380:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.049382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.049390:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.049392:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.049393:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.049394:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.049395:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.049395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.2:1619677475.049643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.049651:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.049654:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.049656:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.049658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.049659:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.049660:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.049661:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.049663:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.049664:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.049665:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.049668:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.049670:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.049672:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.049673:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.049673:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.049675:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.049676:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.049677:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.049677:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.049681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.049687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.049688:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.049690:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.049691:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.049691:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.049693:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.049695:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.049696:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.049697:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:9.2:1619677475.049698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:31.0:1619677475.049699:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.049701:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.049701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.049702:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.049703:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.049704:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:31.0:1619677475.049705:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.049705:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.049706:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.049707:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.049708:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.049708:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:31.0:1619677475.049709:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.049709:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.049710:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.049711:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.049713:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.049714:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.049714:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.049716:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.049716:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.049717:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.049718:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.049721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.049728:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.049730:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.049731:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.049731:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.049733:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.049735:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.050289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.050298:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.050301:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.050301:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.050304:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.050306:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.050308:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.050309:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.050310:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.050311:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.050312:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.050315:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.050317:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.050318:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.050319:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.050320:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.050321:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.050322:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.050323:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.050323:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.050328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.050337:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.050339:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.050339:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.050340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.050341:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.050342:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.050571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.050577:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.050580:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.050581:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.050582:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.050584:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.050585:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.050586:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.050587:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.050588:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.050589:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.050592:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.050593:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.050596:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.050597:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.050598:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.050599:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.050600:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.050601:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.050601:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.050605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.050613:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.050615:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.050616:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.050618:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.050620:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.050622:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.051047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.051053:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.051056:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.051057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.051059:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.051061:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.051062:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.051064:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.051065:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.051066:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.051067:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.051070:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.051071:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.051073:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.051074:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.051075:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.051077:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.051077:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.051078:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.051079:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.051080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.051083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.051085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.051090:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.051090:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.051092:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.051092:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.051093:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.051093:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.051094:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.051095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.051095:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.051097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.051201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.051210:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.051212:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.051215:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.051217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.051217:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.051219:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.051220:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.051221:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.051222:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.051223:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.051227:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.051228:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.051232:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.051233:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.051234:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.051235:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.051236:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.051238:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.051239:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.051240:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.051242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.051246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.051253:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677475.051254:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.051256:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677475.051256:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677475.051257:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.051257:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.051258:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677475.051259:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.051259:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.051262:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.051992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.052000:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.052003:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.052005:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.052008:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.052008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.052009:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.052011:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.052012:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.052013:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.052014:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.052017:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.052018:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.052021:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.052021:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.052022:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.052024:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.052024:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.052025:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.052027:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.052027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.052030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.052034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.052037:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.052040:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.052041:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.052041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.052042:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.052042:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.052044:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.052045:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677475.052045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.052046:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.052603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f76ba00] (20)++
00000800:00000200:26.0:1619677475.052613:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (21)++
00000800:00000200:26.0:1619677475.052618:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.61@o2ib3
00000800:00000200:10.0:1619677475.052618:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)++
00000800:00000200:10.0:1619677475.052621:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.61@o2ib3
00000400:00000200:26.0:1619677475.052623:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.61@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.052625:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.61@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.052628:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.052629:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.052631:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677475.052633:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.052636:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.052638:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.052639:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.052642:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677475.052644:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.052646:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.052647:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.052649:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677475.052650:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.052651:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677475.052655:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.052657:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:10.0:1619677475.052657:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.052658:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677475.052660:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677475.052661:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677475.052663:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f76ba00] (23)++
00000800:00000200:10.0:1619677475.052664:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.052665:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f76ba00] (24)--
00000800:00000200:10.0:1619677475.052665:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677475.052666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (23)--
00000800:00000200:10.0:1619677475.052666:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677475.052668:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)--
00000800:00000200:10.0:1619677475.052668:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (23)--
00000800:00000200:9.2:1619677475.052668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (23)++
00000800:00000200:10.0:1619677475.052670:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f76ba00] (21)++
00000800:00000200:10.0:1619677475.052672:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f76ba00] (22)++
00000800:00000200:10.0:1619677475.052674:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f76ba00] (23)--
00000800:00000200:10.0:1619677475.052675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)--
00000800:00000200:4.2:1619677475.052677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f76ba00] (21)++
00000800:00000200:2.0:1619677475.052677:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (23)++
00000800:00000200:2.0:1619677475.052679:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (24)--
00000800:00000200:18.0:1619677475.052680:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (23)++
00000800:00000200:18.0:1619677475.052681:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (24)--
00000400:00000200:2.0:1619677475.052681:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.052682:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.052683:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:18.0:1619677475.052684:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.052687:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)++
00000800:00000200:2.0:1619677475.052687:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.052688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.052689:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:26.0:1619677475.052691:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.61@o2ib3
00000800:00000200:10.0:1619677475.052692:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (23)++
00000800:00000200:10.0:1619677475.052694:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f76ba00] (24)--
00000800:00000200:10.0:1619677475.052695:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (23)--
00000400:00000200:26.0:1619677475.052696:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.61@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.052697:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)--
00000400:00000200:26.0:1619677475.052701:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.052703:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677475.052705:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.052707:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.052709:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.052710:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677475.052715:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.052718:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.052720:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677475.052720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f76ba00] (21)++
00000800:00000200:26.0:1619677475.052721:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677475.052722:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.052725:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.052727:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f76ba00] (22)++
00000800:00000200:26.0:1619677475.052728:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f76ba00] (23)--
00000800:00000200:26.0:1619677475.052729:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)--
00000800:00000200:10.0:1619677475.052730:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (21)++
00000800:00000200:9.2:1619677475.052732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.052733:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.61@o2ib3
00000800:00000200:26.0:1619677475.052736:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)++
00000400:00000200:10.0:1619677475.052737:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.61@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.052739:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.61@o2ib3
00000400:00000200:10.0:1619677475.052740:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.052743:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.61@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.052743:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677475.052745:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.052747:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.052749:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.052750:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677475.052753:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.052755:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677475.052756:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.052758:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96501856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.052760:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677475.052761:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.052762:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.052763:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.052767:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677475.052769:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.052769:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677475.052770:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.052773:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.052773:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677475.052774:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f76ba00] (23)++
00000800:00000200:10.0:1619677475.052775:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f76ba00] (24)--
00000800:00000200:26.0:1619677475.052776:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.052777:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (23)--
00000800:00000200:26.0:1619677475.052778:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.052778:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:10.0:1619677475.052779:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.052780:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:10.0:1619677475.052780:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.052781:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f76ba00] (22)++
00000400:00000200:10.0:1619677475.052781:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.052783:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f76ba00] (23)--
00000800:00000200:26.0:1619677475.052784:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)--
00000400:00000200:10.0:1619677475.052784:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.052785:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (21)++
00000800:00000200:10.0:1619677475.052787:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.052788:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.61@o2ib3
00000800:00000200:10.0:1619677475.052788:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677475.052790:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)--
00000400:00000200:26.0:1619677475.052792:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.61@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.052794:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.052797:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:4.2:1619677475.052797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f76ba00] (21)++
00000400:00000200:26.0:1619677475.052798:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96501855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677475.052798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:26.0:1619677475.052802:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.052803:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.052805:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.052807:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)++
00000800:00000200:17.0:1619677475.052808:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:10.0:1619677475.052810:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.61@o2ib3
00000400:00000200:26.0:1619677475.052811:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.052812:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (24)--
00000400:00000200:17.0:1619677475.052814:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.052814:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.61@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.052814:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:26.0:1619677475.052815:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677475.052815:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (24)--
00000400:00000200:10.0:1619677475.052817:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.052817:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.052818:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.052820:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677475.052821:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.052822:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:10.0:1619677475.052822:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:26.0:1619677475.052823:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.052825:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:10.0:1619677475.052825:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.052827:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.052829:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.052830:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:9.2:1619677475.052830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677475.052834:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.052835:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.052836:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f76ba00] (23)++
00000800:00000200:17.0:1619677475.052836:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:10.0:1619677475.052837:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.052837:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:26.0:1619677475.052838:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f76ba00] (24)++
00000800:00000200:8.0:1619677475.052839:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677475.052840:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f76ba00] (25)--
00000800:00000200:24.0:1619677475.052840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677475.052840:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677475.052840:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.052840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.052841:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (24)--
00000800:00000200:10.0:1619677475.052841:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:17.0:1619677475.052842:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677475.052843:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (23)++
00000800:00000200:17.0:1619677475.052843:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:8.0:1619677475.052843:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.052844:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677475.052845:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.052846:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.61@o2ib3
00000800:00000200:8.0:1619677475.052846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677475.052847:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677475.052848:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:26.0:1619677475.052850:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.61@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.052851:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:9.2:1619677475.052851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.052852:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f76ba00] (24)++
00000400:00000200:26.0:1619677475.052854:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.052854:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f76ba00] (25)--
00000800:00000200:10.0:1619677475.052855:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (24)--
00000400:00000200:26.0:1619677475.052856:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:24.0:1619677475.052857:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.052857:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (23)++
00000800:00000200:9.2:1619677475.052857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:26.0:1619677475.052859:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.052859:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.052859:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.61@o2ib3
00000800:00000200:8.0:1619677475.052860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.052861:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.052861:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:26.0:1619677475.052862:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677475.052862:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:26.0:1619677475.052863:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.052865:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:18.0:1619677475.052865:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.052867:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.052868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.052869:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.052871:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.052871:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.052874:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677475.052875:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:10.0:1619677475.052876:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.61@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.052877:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.052879:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.052880:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:10.0:1619677475.052880:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.052881:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.052883:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677475.052885:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.052887:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:9.2:1619677475.052887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:10.0:1619677475.052889:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.052890:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:26.0:1619677475.052892:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f76ba00] (24)++
00000800:00000200:26.0:1619677475.052893:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f76ba00] (25)--
00000800:00000200:26.0:1619677475.052895:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (24)--
00000800:00000200:26.0:1619677475.052896:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (23)++
00000400:00000200:10.0:1619677475.052897:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.052898:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f76ba00] (24)--
00000800:00000200:26.0:1619677475.052899:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (23)--
00000800:00000200:26.0:1619677475.052901:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.052901:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677475.052902:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.052903:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.052905:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677475.052907:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.052907:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677475.052908:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677475.052909:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.052910:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)++
00000800:00000200:10.0:1619677475.052911:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:4.2:1619677475.052911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.052912:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.61@o2ib3
00000800:00000200:10.0:1619677475.052912:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f76ba00] (23)++
00000800:00000200:10.0:1619677475.052914:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f76ba00] (24)--
00000800:00000200:10.0:1619677475.052915:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (23)--
00000400:00000200:26.0:1619677475.052916:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.61@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.052917:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.2:1619677475.052917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:26.0:1619677475.052918:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.052919:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677475.052920:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:26.0:1619677475.052921:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:10.0:1619677475.052921:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677475.052923:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.052924:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.052925:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)--
00000400:00000200:26.0:1619677475.052926:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.052926:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:1.0:1619677475.052927:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:26.0:1619677475.052928:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677475.052929:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677475.052930:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677475.052931:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.052933:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.052933:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.052934:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.052937:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:26.0:1619677475.052938:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.052938:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.052941:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:26.0:1619677475.052942:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.052943:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.052946:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.052948:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.052949:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.052952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677475.052955:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677475.052956:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.052957:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.052958:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.052960:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.052960:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.052962:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f76ba00] (21)++
00000800:00000200:26.0:1619677475.052964:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f76ba00] (22)++
00000800:00000200:10.0:1619677475.052964:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.052966:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f76ba00] (23)--
00000800:00000200:25.2:1619677475.052966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.052966:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.052967:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.052968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)--
00000800:00000200:26.0:1619677475.052969:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677475.052969:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.052970:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.052971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.052972:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:4.2:1619677475.052973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f76ba00] (21)++
00000800:00000200:13.0:1619677475.052974:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.052976:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:25.2:1619677475.052977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:13.0:1619677475.052977:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.052977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:13.0:1619677475.052979:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.052979:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)++
00000800:00000200:12.0:1619677475.052981:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.052982:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677475.052982:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f76ba00] (23)--
00000800:00000200:10.0:1619677475.052983:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)--
00000800:00000200:10.0:1619677475.052984:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (21)--
00000800:00000200:6.0:1619677475.052985:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.052988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.052988:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.052989:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.052991:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.052994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.053156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f76ba00] (20)++
00000800:00000200:26.0:1619677475.053165:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (21)++
00000800:00000200:26.0:1619677475.053169:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.61@o2ib3
00000800:00000200:10.0:1619677475.053170:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)--
00000400:00000200:26.0:1619677475.053174:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.61@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.053177:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.053180:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.053182:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.053185:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.053186:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.053187:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.053192:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.053195:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.053198:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.053199:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.053200:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.053202:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.053202:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f76ba00] (21)++
00000800:00000200:26.0:1619677475.053203:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f76ba00] (22)--
00000800:00000200:26.0:1619677475.053203:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (21)--
00000800:00000200:8.2:1619677475.053208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.053216:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.053218:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.053220:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.053221:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.053222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.053224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.053236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.053242:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.053244:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.053245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.053247:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.053249:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.053251:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.053252:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.053253:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.053254:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.053255:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.053259:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.053260:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.053263:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.053264:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.053264:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.053266:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.053266:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.053267:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.053268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.053272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.053280:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.053282:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.053283:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.053285:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.053286:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.053287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.053401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.053410:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.053412:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.053413:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.053415:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.053418:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.053420:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.053421:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.053422:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.053423:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.053424:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.053428:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.053429:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.053432:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.053433:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.053433:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.053435:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.053436:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.053436:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.053437:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.053442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.053450:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.053453:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.053454:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.053455:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.053457:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.053460:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.053462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.053469:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.053472:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.053473:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.053473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.053474:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.053475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.053494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677475.053499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f76ba00] (20)++
00000800:00000200:17.0:1619677475.053500:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.053503:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.053504:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:26.0:1619677475.053506:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f76ba00] (21)++
00000400:00000200:17.0:1619677475.053506:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.053508:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.053509:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.61@o2ib3
00000800:00000200:10.0:1619677475.053509:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (22)--
00000400:00000200:17.0:1619677475.053510:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.053511:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.053513:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.61@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.053513:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.053514:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.053515:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.053516:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.053518:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677475.053519:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.053520:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.053522:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.053522:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.053523:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.053524:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:17.0:1619677475.053524:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.053525:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.053526:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:26.0:1619677475.053527:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.61@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.053528:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.053528:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:26.0:1619677475.053529:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677475.053529:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.053529:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677475.053532:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.053533:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677475.053533:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:25.2:1619677475.053533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.053535:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677475.053536:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f76ba00] (21)++
00000800:00000200:26.0:1619677475.053536:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f76ba00] (22)--
00000800:00000200:26.0:1619677475.053537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f76ba00] (21)--
00000800:00000200:25.2:1619677475.053541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677475.053541:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.053543:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.053544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.053544:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.053546:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.053548:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:6.0:1619677475.053548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677475.053551:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.053552:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.053554:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.053556:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.053556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.053918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.053926:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.053928:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.053930:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.053932:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.053933:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.053934:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.053935:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.053936:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.053937:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.053938:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.053941:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.053943:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.053945:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.053946:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.053947:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.053949:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.053949:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.053950:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.053950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.053954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.053961:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.053963:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.053963:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.053965:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.053967:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.053968:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.054414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.054418:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.054420:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677475.054423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.054423:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.054425:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.054427:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677475.054428:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.054429:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.054430:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.054431:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677475.054434:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.054436:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677475.054439:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.054439:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677475.054440:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677475.054442:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677475.054443:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.054444:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.054444:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677475.054449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677475.054452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.054457:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.054459:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.054460:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677475.054460:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.054462:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.054462:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.054465:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.054465:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.054466:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:31.0:1619677475.054467:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.054467:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.054468:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.054470:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.054471:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.054472:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.054473:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.054476:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.054478:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.054481:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.054481:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.054482:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.054483:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.054484:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.054485:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.054486:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.054489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.054496:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.054499:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.054500:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.054500:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.054502:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.054504:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.054578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.054585:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.054588:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.054589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.054590:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.054593:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.054594:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.054595:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.054596:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.054597:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.054598:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.054601:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.054603:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.054605:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.054606:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.054607:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.054608:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.054609:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.054609:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.054610:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.054616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.054624:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.054627:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.054627:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.054628:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.054630:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.054632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.054695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.054703:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.054705:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.054708:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.054709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.054710:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.054711:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.054712:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.054714:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.054715:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.054716:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.054718:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.054720:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.054722:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.054723:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.054724:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.054725:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.054726:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.054727:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.054727:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.054731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.054737:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.054740:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.054741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.054741:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.054742:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.054745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.054814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.054820:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.054822:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.054823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.054825:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.054827:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.054828:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.054830:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.054831:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.054832:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.054833:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.054836:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.054838:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.054840:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.054841:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.054842:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.054844:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.054844:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.054845:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.054845:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.054849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.054857:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.054859:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.054860:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.054862:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.054862:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.054864:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.055094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.055101:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.055104:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.055107:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.055108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.055109:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.055110:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.055112:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.055113:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.055114:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.055115:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.055118:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.055120:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.055124:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.055125:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.055125:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.055127:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.055128:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.055128:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.055129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.055133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.055141:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.055143:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.055145:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.055146:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.055146:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.055150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.055417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.055424:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.055426:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.055428:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.055430:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.055432:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.055434:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.055435:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.055437:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.055438:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.055439:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.055442:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.055444:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.055447:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.055448:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.055448:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.055450:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.055451:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.055452:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.055453:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.055453:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.055457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677475.055459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.055465:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677475.055466:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677475.055467:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.055467:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.055468:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677475.055470:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.055471:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.055474:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.055475:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.055477:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.055477:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.055479:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.055480:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.055481:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.055482:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.056042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.056050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.056053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.056053:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.056056:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.056058:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.056059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677475.056060:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.056061:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.056062:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.056063:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.056064:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.056067:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.056068:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.056068:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677475.056070:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677475.056071:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.056072:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.056072:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.056072:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.056074:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.056074:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677475.056075:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.056075:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.056075:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.056076:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:30.0:1619677475.056077:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.056078:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.056079:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.056080:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.056080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:30.0:1619677475.056081:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.056084:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.056085:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:12.0:1619677475.056088:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:30.0:1619677475.056089:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.056089:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.056090:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:12.0:1619677475.056091:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:30.0:1619677475.056092:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.056092:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:12.0:1619677475.056092:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.056093:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.056095:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:13.0:1619677475.056095:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677475.056095:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.056096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677475.056097:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677475.056098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.056102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.056110:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.056110:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677475.056112:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.056113:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.056113:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677475.056114:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677475.056114:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.056116:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.056116:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.056118:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.056119:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.056119:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:20.0:1619677475.056119:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (24)--
00000800:00000200:20.0:1619677475.056120:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.056121:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.056280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.056287:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.056290:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.056293:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.056294:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.056300:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.056301:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.056303:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.056304:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.056305:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.056306:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.056310:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.056311:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.056314:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.056315:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.056315:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.056317:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.056317:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.056319:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.056319:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.056323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.056330:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.056332:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.056333:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.056333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.056335:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.056338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.056522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.056528:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.056531:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.056532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.056534:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.056536:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.056537:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.056539:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.056540:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.056541:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.056542:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.056545:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.056547:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.056549:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.056550:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.056551:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.056553:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.056553:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.056554:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.056555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.056559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.056566:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.056569:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.056570:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.056571:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.056572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.056573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.056578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.056586:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.056588:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.056589:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.056590:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.056591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.056591:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.056663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.056671:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.056674:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.056676:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.056678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.056679:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.056681:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:13.0:1619677475.056682:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.056684:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.056685:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.056686:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677475.056689:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.056691:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.056693:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.056694:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677475.056695:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.056696:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.056697:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.056697:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677475.056698:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677475.056702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.056708:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.056710:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.056711:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.056712:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.056714:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.056715:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.056754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.056761:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.056764:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.056765:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.056767:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.056769:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.056771:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.056772:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.056773:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.056774:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.056775:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.056778:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.056779:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.056782:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.056783:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.056784:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.056786:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.056786:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.056787:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.056787:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.056792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.056800:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.056803:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.056803:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.056805:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.056806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.056808:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.057427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.057430:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.057433:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.057435:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.057436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.057437:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.057439:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.057440:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.057441:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.057442:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.057443:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.057447:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.057449:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.057451:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.057452:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.057453:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.057454:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.057455:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.057456:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.057457:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.057458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.057461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.057463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.057468:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677475.057469:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.057470:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.057470:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.057471:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677475.057472:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.057473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.057474:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.057475:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.057478:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677475.057631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.057637:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.057640:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.057641:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.057643:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.057645:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.057647:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.057648:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.057649:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.057650:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.057651:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.057654:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.057655:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.057658:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.057658:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.057659:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.057661:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.057661:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.057662:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.057663:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.057668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.057675:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.057677:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.057678:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.057678:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:4.2:1619677475.057679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:18.0:1619677475.057680:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.057681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677475.057687:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.057689:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.057690:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677475.057691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000400:00000200:10.0:1619677475.057692:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.057695:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.057697:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.057698:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677475.057698:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677475.057699:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.057700:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.057701:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677475.057701:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.057702:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.057702:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.057703:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.057703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:10.0:1619677475.057705:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.057707:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.057710:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.057711:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.057711:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.057713:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.057714:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.057714:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.057715:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.057719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.057727:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.057730:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.057731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.057731:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.057733:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.057736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.057739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.057746:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.057748:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.057749:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.057749:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.057750:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.057751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.058028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677475.058033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:30.0:1619677475.058037:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677475.058038:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:30.0:1619677475.058039:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.058042:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.058042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677475.058042:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.058042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.058043:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:31.0:1619677475.058044:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677475.058045:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.058045:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000400:00000200:30.0:1619677475.058046:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.058048:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:17.0:1619677475.058048:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.058048:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.058049:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.058050:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677475.058051:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.058051:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.058052:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.058053:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.058054:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677475.058055:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.058057:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677475.058059:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.058059:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.058061:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677475.058062:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96501825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.058063:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:17.0:1619677475.058063:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.058064:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.058065:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677475.058065:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.058066:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.058066:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677475.058066:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.058066:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.058067:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.058068:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.058069:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677475.058069:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.058070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677475.058070:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:1.0:1619677475.058070:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.058071:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.058072:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:17.0:1619677475.058073:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.058074:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.058075:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.058075:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.058076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677475.058076:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677475.058076:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677475.058077:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.058078:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677475.058079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.058079:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.058081:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677475.058082:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677475.058082:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:25.2:1619677475.058083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.2:1619677475.058084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.058089:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:31.0:1619677475.058090:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.058092:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.058093:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.058093:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.058094:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.058094:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:31.0:1619677475.058095:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.058095:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.058095:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:17.0:1619677475.058095:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:30.0:1619677475.058096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:20.0:1619677475.058096:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.058097:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.058097:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677475.058097:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:31.0:1619677475.058098:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:20.0:1619677475.058098:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.058099:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:21.0:1619677475.058100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.058101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:25.2:1619677475.058102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.0:1619677475.058102:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.058102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677475.058103:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.058104:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.058104:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.058105:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.058105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677475.058109:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.058112:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.058112:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.058113:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.058114:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.058115:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.058123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.058130:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.058133:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.058134:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.058136:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.058139:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.058140:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.058142:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.058143:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.058144:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.058145:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.058148:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.058150:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.058153:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.058154:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.058154:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.058156:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.058156:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.058158:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.058159:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.058159:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677475.058161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.058166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677475.058170:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.058173:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677475.058173:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:20.0:1619677475.058174:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.058175:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.058175:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.058175:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.0:1619677475.058176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.058177:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.0:1619677475.058178:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.058201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.058209:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.058211:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.058213:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.058215:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.058215:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.058217:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.058218:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.058219:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.058220:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.058221:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.058224:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.058225:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.058228:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.058228:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.058229:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.058231:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.058231:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.058232:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.058232:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.058233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:9.2:1619677475.058236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677475.058240:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.058243:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:8.0:1619677475.058243:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:10.0:1619677475.058244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.058245:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.058245:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.058246:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.058246:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.058248:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.058248:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677475.058250:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677475.058250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677475.058251:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.058252:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.058253:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.058254:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.058256:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677475.058257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677475.058258:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.058261:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.058261:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.058262:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.058263:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677475.058264:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.058264:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:5.2:1619677475.058264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677475.058265:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677475.058265:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677475.058266:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677475.058266:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.058266:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677475.058267:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677475.058267:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.058268:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677475.058269:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677475.058270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.058272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677475.058272:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.058273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.058275:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.058277:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.058278:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677475.058278:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677475.058279:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.058280:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:1.0:1619677475.058280:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.058281:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.058281:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:1.0:1619677475.058281:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.058282:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677475.058282:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.058282:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.058283:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.058284:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.058284:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677475.058287:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.058289:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.058290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677475.058292:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.058292:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.058298:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.058299:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.058300:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.058301:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.058302:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.058302:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.058306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.058308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677475.058313:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677475.058314:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677475.058316:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677475.058316:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677475.058317:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.058317:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677475.058317:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.058318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677475.058319:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.058321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.058656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677475.058661:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.058664:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677475.058665:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677475.058667:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.058669:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.058670:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677475.058671:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.058672:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.058673:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.058674:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677475.058677:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.058679:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.058681:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.058682:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677475.058682:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677475.058684:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677475.058685:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.058686:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677475.058687:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677475.058687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:8.2:1619677475.058691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.058692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.058697:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677475.058699:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677475.058700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:17.0:1619677475.058700:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677475.058701:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:17.0:1619677475.058702:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.058703:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.058704:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.058706:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.058708:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.058756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677475.058762:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.058764:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677475.058765:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:21.2:1619677475.058767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:6.0:1619677475.058767:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.058770:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.058772:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677475.058773:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.058774:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.058774:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.058775:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.058776:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.058777:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677475.058779:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.058781:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.058782:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.058783:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:6.0:1619677475.058783:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:22.0:1619677475.058784:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.058785:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.058786:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.058787:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.058790:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.058790:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.058791:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677475.058792:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.058792:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677475.058794:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677475.058795:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.058795:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.058795:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.058796:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.058796:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.058796:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:6.0:1619677475.058797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.058798:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.058799:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677475.058800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677475.058800:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.058801:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.058801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.058804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.058806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677475.058808:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.058810:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.058811:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.058812:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:30.0:1619677475.058813:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.058813:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.058814:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.058814:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677475.058815:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.0:1619677475.058816:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.058816:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.058816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677475.058817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:21.0:1619677475.058818:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.058819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.058821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.058905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.058912:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.058915:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.058917:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.058919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.058920:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.058921:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.058922:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.058924:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.058925:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.058926:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.058928:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.058930:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.058932:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.058933:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.058934:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.058935:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.058936:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.058937:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.058938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.058941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.058947:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.058950:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.058950:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.058952:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.058954:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.058955:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.059459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.059467:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.059469:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.059471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.059472:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.059475:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.059476:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.059477:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.059479:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.059480:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.059481:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.059484:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.059485:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.059488:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.059489:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.059489:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.059491:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.059491:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.059492:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.059493:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.059498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.059506:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.059509:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.059509:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.059510:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.059512:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.059515:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.059660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.059663:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.059666:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.059668:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.059669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.059670:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.059672:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.059673:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.059674:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.059675:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.059676:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677475.059680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677475.059680:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.059681:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.059684:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.059684:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.059685:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.059686:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.059687:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677475.059688:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.059688:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.059688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677475.059691:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677475.059692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:30.0:1619677475.059693:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.059695:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.059696:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.059696:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.059697:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.059699:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.059700:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.059700:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:30.0:1619677475.059701:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.059702:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.059703:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.059705:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.059705:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.059705:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.059707:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677475.059708:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.059710:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.059711:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.059712:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.059713:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.059714:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.059715:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.059715:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.059719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.059726:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.059728:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.059729:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.059731:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.059732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.059733:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.060271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.060276:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.060279:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.060280:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.060282:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.060284:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.060285:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.060287:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.060288:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.060289:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.060290:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.060297:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.060299:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.060302:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.060302:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.060303:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.060305:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.060305:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.060306:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.060307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.060312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.060319:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.060321:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.060322:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.060324:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.060325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.060326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.060572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.060580:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.060583:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.060585:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.060587:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.060587:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.060589:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.060590:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.060591:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.060592:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.060593:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.060596:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.060598:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.060601:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.060601:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.060602:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.060603:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.060604:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.060606:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.060607:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.060608:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.060609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.060614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.060616:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.060619:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.060620:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.060620:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.060621:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.060622:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.060624:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677475.060624:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.060625:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.060625:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.061330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.2:1619677475.061333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677475.061336:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.061339:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.061340:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.061342:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.061342:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:22.0:1619677475.061344:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.061346:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:18.0:1619677475.061346:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677475.061347:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677475.061348:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677475.061349:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.061351:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.061353:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.061354:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:22.0:1619677475.061356:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.061357:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.061358:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.061359:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:22.0:1619677475.061360:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.061361:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:18.0:1619677475.061361:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.061362:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.061363:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.061364:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:18.0:1619677475.061364:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.061365:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.061365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:18.0:1619677475.061365:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.061367:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:25.2:1619677475.061370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677475.061373:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.061377:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.061379:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.061381:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:18.0:1619677475.061381:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.061382:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.061383:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:21.0:1619677475.061384:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.061384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677475.061384:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.061386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677475.061386:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.061387:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.061388:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.061389:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.061392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.061400:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.061403:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.061404:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.061405:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.061407:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.061409:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.061532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.061540:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.061543:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.061545:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.061547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.061547:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.061548:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.061550:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.061551:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.061552:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.061553:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.061557:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.061558:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.061561:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.061562:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.061562:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.061564:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.061565:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.061566:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.061568:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.061568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.061570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.061574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.061577:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.061580:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.061581:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.061581:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.061582:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.061583:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.061584:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.061585:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677475.061585:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.061586:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.062169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.062175:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.2:1619677475.062177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:22.0:1619677475.062178:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.062178:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.062181:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.062183:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.062183:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:6.0:1619677475.062184:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.062185:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.062186:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.062187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:6.0:1619677475.062187:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.062189:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.062190:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.062191:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677475.062192:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.062194:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.062194:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677475.062196:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.062197:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:1.0:1619677475.062197:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.062198:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.062200:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677475.062200:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:6.0:1619677475.062201:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.062201:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.062203:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.062203:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:1.0:1619677475.062203:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.062204:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.062205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:1.0:1619677475.062205:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:25.2:1619677475.062209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677475.062209:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.062210:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677475.062210:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677475.062212:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677475.062213:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.062214:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.062214:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:20.0:1619677475.062217:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:9.2:1619677475.062218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:20.0:1619677475.062219:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.062220:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.062222:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.062223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.062224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677475.062226:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.062229:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:25.2:1619677475.062229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000400:00000200:26.0:1619677475.062230:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.062230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.062233:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.062236:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.0:1619677475.062236:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.062238:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.062239:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.062240:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.062240:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.062241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677475.062243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677475.062249:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.062252:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.062253:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.062253:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.062254:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.062254:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.062296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677475.062302:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.062304:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677475.062305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677475.062307:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.062310:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.062311:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677475.062313:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.062314:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.062315:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.062316:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677475.062320:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.062321:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677475.062324:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.062325:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677475.062325:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677475.062327:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677475.062328:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.062328:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677475.062329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677475.062334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.062344:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.062348:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677475.062350:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.062351:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.062353:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:9.2:1619677475.062355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.062359:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.062360:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.062363:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677475.062364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.062364:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.062366:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.062367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.062466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.062474:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.062477:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.062477:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.062479:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.062482:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.062483:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.062485:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.062486:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.062487:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.062488:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.062491:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.062493:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.062496:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.062497:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.062497:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.062499:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.062500:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.062500:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.062500:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.062505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.062513:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.062516:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.062516:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.062517:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.062519:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.062522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.062633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.062641:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.062644:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.062646:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.062648:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.062648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.062649:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.062651:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.062652:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.062653:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.062654:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.062657:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.062659:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.062661:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.062662:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.062662:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.062664:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.062664:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.062665:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.062666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.062669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.062676:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.062679:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.062680:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.062680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.062681:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.062684:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.062832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.062840:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.062843:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.062844:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.062847:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.062847:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.062848:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.062849:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.062850:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.062851:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.062852:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.062855:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.062857:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.062859:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.062859:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.062860:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.062861:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.062862:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.062863:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.062863:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.062867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.062873:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.062876:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.062877:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.062877:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.062879:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.062880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677475.062963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.062969:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.062972:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.062973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.062975:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.062977:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.062979:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.062980:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.062982:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.062982:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.062984:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.062987:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.062989:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.062991:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.062992:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.062993:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.062994:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.062995:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.062995:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.062996:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.063001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.063008:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:21.2:1619677475.063010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.063011:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.063012:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.063012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.063013:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:22.0:1619677475.063014:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.063015:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.063017:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:6.0:1619677475.063017:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.063018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.063021:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.063022:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.063023:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.063024:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.063025:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.063026:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.063030:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.063032:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.063034:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.063035:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.063036:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.063037:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.063038:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.063038:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.063040:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.063040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.063043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.063045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.063050:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.063052:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.063052:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.063053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.063054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.063055:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.063055:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.063057:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.063058:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.063060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.063481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.063488:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.063491:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.063492:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.063494:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.063497:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.063498:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.063499:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.063501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677475.063501:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.063502:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.063503:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.063506:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.063508:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.063510:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.063511:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.063512:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.063512:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.063513:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.063514:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.063514:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.063515:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.063515:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677475.063516:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.063516:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677475.063517:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.063518:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.063519:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.063520:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.063521:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.063521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677475.063522:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.063525:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.063527:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677475.063529:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677475.063530:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.063531:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.063532:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677475.063532:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.063532:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.063533:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:6.0:1619677475.063533:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.063534:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.063535:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.063535:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677475.063535:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.063537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677475.063540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:25.2:1619677475.063541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.063548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.063549:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:22.0:1619677475.063551:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.063551:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:22.0:1619677475.063552:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677475.063552:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.063552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.063553:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.063554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:20.0:1619677475.063554:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.063555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677475.063557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.063560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.063563:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.063564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.063565:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.063566:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.063566:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.063567:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.063567:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.063568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677475.063569:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677475.063569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.063571:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677475.063573:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.063574:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.063576:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.063577:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.063578:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.063579:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.063582:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.063584:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.063586:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.063587:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.063587:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.063589:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.063590:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.063591:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.063591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.063595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.063603:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.063605:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.063605:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.063607:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.063608:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.063610:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.063792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.063798:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.063800:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.063801:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.063803:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.063806:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.063807:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677475.063808:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.063810:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.063811:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.063812:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677475.063815:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.063817:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.063819:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.063820:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.063820:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.063822:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.063823:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.063824:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677475.063825:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677475.063825:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:25.2:1619677475.063828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.063831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:6.0:1619677475.063835:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677475.063837:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:6.0:1619677475.063838:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.063839:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.063839:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:6.0:1619677475.063839:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.063840:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.063841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:6.0:1619677475.063841:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.063844:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.063885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.063890:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.063893:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.063895:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.063895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.063897:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.063899:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.063900:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.063902:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.063903:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.063904:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.063906:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.063908:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.063911:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.063911:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.063912:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.063914:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.063914:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.063915:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.063915:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.063920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.063928:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.063930:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.063931:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.063931:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.063933:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.063936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.064029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.064036:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.064039:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.064040:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.064041:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.064041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:24.0:1619677475.064044:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.064046:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677475.064046:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677475.064047:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.064049:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.064049:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.064049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677475.064051:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.064052:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.064053:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.064056:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.064058:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677475.064058:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.064060:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:24.0:1619677475.064061:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.064061:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.064063:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.064063:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677475.064063:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.064064:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.064065:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.064065:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677475.064065:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677475.064067:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.064068:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.064068:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677475.064068:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.064070:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:25.2:1619677475.064071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.064072:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.064074:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.064074:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677475.064074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.064076:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.064077:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.064078:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.064079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677475.064080:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.064081:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:25.2:1619677475.064082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.064082:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.064083:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.064083:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:6.0:1619677475.064083:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.064084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.064085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677475.064085:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.064088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.064089:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.064091:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.064092:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.064094:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.064095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.064097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.064241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.064249:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.064252:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.064254:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.064256:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.064257:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.064259:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.064260:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.064261:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.064262:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.064263:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.064267:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.064269:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.064271:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.064271:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.064272:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.064273:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.064274:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.064275:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.064276:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.064279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.064286:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.064288:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.064289:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.064290:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.064292:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.064293:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.064297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.064304:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.064306:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.064307:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.064308:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.064308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.064309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.064325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677475.064331:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.064334:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677475.064334:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677475.064336:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.064339:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.064340:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677475.064341:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.064343:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.064344:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.064345:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677475.064348:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.064350:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677475.064352:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.064353:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677475.064354:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677475.064356:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677475.064356:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.064357:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677475.064357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677475.064362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.064370:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.064372:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.064373:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.064373:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.064376:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.064378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.064531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.064538:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.064541:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.064543:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.064545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.064546:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.064547:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.064548:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.064550:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.064551:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.064552:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.064555:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.064556:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.064559:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.064560:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.064560:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.064562:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.064563:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.064564:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.064564:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.064568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.064574:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.064577:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.064578:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.064578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.064580:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.064582:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.064889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.064894:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.064897:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.064898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.064900:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.064902:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.064903:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.064905:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.064906:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.064907:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.064908:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.064911:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.064913:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.064916:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.064917:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.064917:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.064919:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.064919:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.064920:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.064921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.064925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.064933:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.064935:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.064936:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.064938:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.064939:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.064940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.065314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.065317:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.065320:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.065322:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.065323:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.065325:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.065326:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.065327:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.065328:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.065330:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.065330:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.065334:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.065335:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.065338:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.065338:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.065339:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.065341:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.065341:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.065342:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.065343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.065347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.065355:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.065357:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.065358:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.065360:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.065361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.065362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.065454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.065460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.065463:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.065464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.065466:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.065468:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.065469:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677475.065471:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.065472:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.065473:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.065474:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677475.065477:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.065479:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.065481:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.065482:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677475.065483:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.065484:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.065485:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.065486:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.065486:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677475.065491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.065499:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.065502:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.065503:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.065503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.065505:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.065507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.065577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.2:1619677475.065582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:26.0:1619677475.065585:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.065588:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:2.0:1619677475.065588:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677475.065589:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:2.0:1619677475.065591:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:26.0:1619677475.065592:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.065592:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.065594:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.065595:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.065596:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677475.065597:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.065597:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.065599:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.065599:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677475.065600:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.065601:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.065602:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677475.065602:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.065603:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.065604:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.065606:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677475.065608:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.065609:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.065611:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.065611:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.065612:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677475.065613:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677475.065613:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.065614:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677475.065614:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.065615:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.065615:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:2.0:1619677475.065615:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.065616:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:2.0:1619677475.065617:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677475.065617:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.065618:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.065619:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677475.065620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:25.2:1619677475.065623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.065626:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.065629:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.065629:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.065630:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:6.0:1619677475.065630:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:10.0:1619677475.065632:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.065633:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.065634:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.065634:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:6.0:1619677475.065634:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.065636:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.065638:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.065644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.065651:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.065654:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.065654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.065655:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.065656:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.065656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.065747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.065753:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.065755:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.065756:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.065758:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.065760:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.065761:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.065762:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.065764:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.065764:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.065766:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.065769:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.065771:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.065773:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.065774:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.065775:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.065776:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.065777:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.065777:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.065778:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.065782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.065790:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.065792:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.065793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.065794:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.065796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.065797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.065803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.065810:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.065812:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.065813:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.065814:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.065814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.065815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.066434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.066442:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.066445:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.066448:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.066450:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.066451:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.066452:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.066453:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.066455:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.066456:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.066457:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.066460:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.066462:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.066465:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.066466:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.066466:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.066468:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.066469:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.066470:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.066470:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.066474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.066482:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.066484:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.066485:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.066485:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.066487:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.066489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.066494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.066500:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.066502:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677475.066502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677475.066503:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.066503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.066504:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.066504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.066509:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.066512:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.066514:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.066516:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.066516:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.066518:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.066519:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.066521:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.066522:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.066523:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.066526:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.066527:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.066531:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.066531:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.066532:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.066534:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.066534:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.066536:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.066537:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.066538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.066540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.066546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.066552:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677475.066554:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.066554:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.066555:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.066556:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.066557:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.066557:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.066558:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.066558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.066560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.066791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.066797:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.066800:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.066800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.066802:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.066805:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.066806:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.066807:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.066809:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.066809:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.066811:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.066814:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.066816:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.066818:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.066819:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.066819:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.066821:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.066822:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.066823:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.066824:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.066824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.066827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.066830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.066834:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.066834:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.066836:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.066837:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.066837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.066838:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.066838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.066839:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.066841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.066842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.066984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.066991:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.066993:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.066994:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.066996:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.066998:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.067000:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.067001:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.067002:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.067003:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.067005:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.067008:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.067010:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.067011:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.067012:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.067013:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.067014:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.067015:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.067016:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.067017:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.067017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.067020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.067023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.067029:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.067030:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.067030:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.067031:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.067031:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.067031:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.067032:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.067033:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677475.067033:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.067034:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.067311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.067319:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.067322:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.067322:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.067324:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.067326:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.067328:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.2:1619677475.067328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:26.0:1619677475.067329:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.067330:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.067331:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.067332:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.067335:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.067335:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677475.067337:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677475.067338:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.067339:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:26.0:1619677475.067340:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.067341:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.067341:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:18.0:1619677475.067341:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677475.067343:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:18.0:1619677475.067343:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.067344:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.067345:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:18.0:1619677475.067345:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677475.067346:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.067346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677475.067346:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.067348:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.067348:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.067350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:18.0:1619677475.067350:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677475.067352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677475.067353:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.067355:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677475.067357:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:18.0:1619677475.067357:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.067358:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.067358:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677475.067359:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.067360:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:18.0:1619677475.067360:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:22.0:1619677475.067361:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.067361:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.067361:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677475.067361:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677475.067361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:18.0:1619677475.067362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677475.067362:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.067363:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677475.067363:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.067366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.2:1619677475.067366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:25.2:1619677475.067370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.067375:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677475.067377:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.067377:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677475.067378:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.067379:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677475.067380:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.067381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:1.0:1619677475.067381:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.067382:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.067383:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.067383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677475.067384:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.067387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.067395:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.067397:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.067398:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.067398:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.067399:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.067400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.067478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.067486:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.067489:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.067491:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.067493:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.067494:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.067495:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.067496:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.067498:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.067499:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.067500:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.067503:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.067505:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.067507:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.067508:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.067508:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.067510:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.067511:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.067512:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.067512:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.067516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.067522:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.067525:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.067525:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.067527:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.067529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.067529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.2:1619677475.067643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.067649:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.067651:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.067652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.067654:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.067656:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.067658:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.067659:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.067660:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.067661:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.067662:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.067665:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.067667:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.067670:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.067670:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.067671:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.067673:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.067673:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.067674:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.067674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.067679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.067686:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.067689:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.067689:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.067691:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.067692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.067693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.067723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:21.2:1619677475.067730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:10.0:1619677475.067730:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.067733:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.067734:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.067736:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.067736:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.067738:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.067739:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:10.0:1619677475.067739:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.067740:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677475.067742:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.067742:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.067744:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.067744:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.067745:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.067745:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677475.067746:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677475.067747:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.067749:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.067750:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.067752:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.067752:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.067755:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:1.0:1619677475.067756:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.067757:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.067758:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677475.067758:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677475.067758:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.067759:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.067760:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:1.0:1619677475.067760:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.067760:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677475.067761:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:1.0:1619677475.067761:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677475.067762:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:1.0:1619677475.067762:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.067763:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.067764:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.067764:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677475.067765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.2:1619677475.067768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.067771:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.067773:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.067774:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.067775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.067776:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.067776:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677475.067778:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:17.0:1619677475.067778:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.067779:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.067779:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.067780:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.067782:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.067789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.067796:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.067799:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.067800:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.067800:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.067800:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.067801:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.068047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:21.2:1619677475.068050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.068054:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:30.0:1619677475.068057:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.068057:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677475.068058:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:30.0:1619677475.068060:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677475.068060:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.068063:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.068064:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677475.068064:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.068065:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.068065:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.068066:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677475.068067:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.068068:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.068068:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.068069:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.068070:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.068071:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.068071:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.068072:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.068077:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.068077:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.068080:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677475.068080:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.068082:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.068083:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677475.068083:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.068083:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.068084:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677475.068084:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.068085:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.068085:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.068086:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677475.068086:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.068086:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:30.0:1619677475.068087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677475.068087:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677475.068088:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677475.068090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:25.2:1619677475.068095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.068097:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.068100:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.068100:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.068102:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.068102:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677475.068104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.068105:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677475.068105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.068105:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.068106:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.068108:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.068111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.068265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.068270:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.068273:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.068274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.068276:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.068278:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.068279:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.068281:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.068282:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.068283:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.068284:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.068288:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.068289:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.068292:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.068293:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:4.2:1619677475.068298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:17.0:1619677475.068300:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.068302:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.068302:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.068303:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.068304:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677475.068306:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:25.2:1619677475.068308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.068309:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.068310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.068312:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.068314:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.068315:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677475.068316:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:10.0:1619677475.068317:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.068318:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.068319:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:10.0:1619677475.068319:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.068320:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.068320:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.068320:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.068323:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.068325:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.068327:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.068328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677475.068330:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.068331:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.068332:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.068333:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.068334:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.068334:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.068335:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.068339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.068346:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.068349:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.068350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.068350:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.068352:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677475.068353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:6.0:1619677475.068354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677475.068361:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.068364:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.068366:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.068368:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.068368:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.068369:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.068371:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.068372:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.068373:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.068374:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.068377:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.068378:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.068381:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.068381:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.068382:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.068384:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.068384:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.068385:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.068387:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.068388:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.068389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.068393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.068396:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.068399:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.068400:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.068400:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.068401:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.068401:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.068403:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.068404:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677475.068404:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.068405:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.068437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.068442:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.068445:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.068446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.068448:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.068450:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.068452:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677475.068453:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.068454:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.068455:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.068456:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677475.068460:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.068462:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677475.068464:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.068465:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:22.0:1619677475.068466:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677475.068467:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:22.0:1619677475.068468:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.068468:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.068469:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.2:1619677475.068473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.068481:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.068484:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.068485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.068485:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.068487:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.068490:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.068548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.068553:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.068556:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.068557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.068558:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.068561:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.068562:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.068563:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.068564:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.068566:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.068566:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.068570:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.068571:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.068573:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.068573:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.068574:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.068575:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.068576:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.068577:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.068577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.068582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.068582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.068590:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.068590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.068592:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677475.068593:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677475.068593:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.068595:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.068595:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.068596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.068597:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.068597:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.068598:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677475.068599:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.068600:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.068601:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.068602:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.068603:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.068607:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.068609:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.068611:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.068612:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.068612:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.068614:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.068615:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.068615:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.068616:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.068620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.068626:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.068629:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.068629:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.068631:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.068633:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.068633:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.068952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.068958:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.068961:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.068962:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.068964:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.068966:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.068967:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.068969:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.068970:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.068971:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.068972:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.068975:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.068977:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.068980:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.068980:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.068981:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.068982:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.068983:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.068984:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.068984:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.068989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.068997:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.068999:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.069000:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.069002:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.069002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.069004:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.069009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:4.2:1619677475.069016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.0:1619677475.069019:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.069022:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:26.0:1619677475.069023:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677475.069023:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.069024:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.069024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.069025:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677475.069026:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.069026:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:4.2:1619677475.069027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677475.069028:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.069031:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.069032:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677475.069034:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677475.069034:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677475.069035:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.069036:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.069037:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:24.0:1619677475.069038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.069038:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.069041:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.069042:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.069043:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.069044:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.069045:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677475.069045:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677475.069046:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:8.0:1619677475.069046:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96501764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677475.069047:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:8.0:1619677475.069047:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677475.069048:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:8.0:1619677475.069048:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677475.069049:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.069049:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000400:00000200:8.0:1619677475.069049:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.069050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:8.0:1619677475.069053:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677475.069054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.0:1619677475.069054:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.069057:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.069058:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.069058:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.069060:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:8.0:1619677475.069060:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.069060:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.069061:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:10.0:1619677475.069062:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:8.0:1619677475.069062:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677475.069063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.069063:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.069065:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.069067:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:25.2:1619677475.069076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.069083:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.069086:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.069087:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.069087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.069089:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.069092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.069242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.069245:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.069247:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.069250:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.069251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.069252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.069253:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.069254:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.069256:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.069257:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.069258:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.069262:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.069263:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.069266:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.069266:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.069267:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.069269:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.069269:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.069270:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.069272:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.069272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.069275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.069278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677475.069278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677475.069282:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.069284:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677475.069285:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677475.069285:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677475.069286:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677475.069286:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.069286:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.069287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677475.069288:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.069288:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:1.0:1619677475.069288:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.069290:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.069291:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:31.0:1619677475.069292:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.069292:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.069298:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.069300:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.069301:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.069302:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.069303:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.069307:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.069308:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.069311:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.069311:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.069312:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.069314:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.069315:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.069316:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.069318:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.069318:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.069320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.069325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.069328:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.069330:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.069331:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.069331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.069332:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.069333:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.069335:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.069335:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677475.069335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.069336:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.069339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.069345:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.069347:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.069348:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.069348:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.069349:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.069349:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.069578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.069584:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.069587:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.069588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.069590:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.069592:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.069593:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.069594:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.069596:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.069597:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.069598:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.069600:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.069602:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.069604:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.069605:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.069606:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.069607:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.069608:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.069608:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.069609:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.069613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.069621:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.069623:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.069624:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.069625:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.069627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.069628:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.069634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.069642:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.069644:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.069645:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.069646:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.069647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.069647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.2:1619677475.069928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.069936:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.069939:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.069941:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.069943:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.069943:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.069945:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.069946:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.069948:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.069949:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.069950:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.069953:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.069954:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.069957:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.069958:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.069958:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.069960:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.069961:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.069962:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.069962:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.069966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.069972:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.069974:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.069975:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.069975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.069977:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.069979:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.070456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.070461:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.070464:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.070465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.070467:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.070469:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.070470:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.070472:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.070473:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.070474:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.070475:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.070478:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.070480:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.070483:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.070483:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.070484:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.070486:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.070486:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.070487:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.070488:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.070489:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.070492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.070494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.070499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.070500:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.070501:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.070502:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.070502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.070503:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.070503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.070505:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.070506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.070507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.070547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.070553:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.070555:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.070557:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.070558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.070559:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.070561:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.070562:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.070564:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.070564:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.070566:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.070568:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.070570:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.070572:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.070573:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.070574:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.070575:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.070576:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.070577:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.070577:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.070582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.070590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.070592:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.070592:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.070594:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.070595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.070596:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.070607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.070613:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.070615:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.070617:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.070618:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.070619:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.070621:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.070622:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.070624:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.070625:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.070625:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677475.070628:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.070630:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.070632:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.070632:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.070633:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.070634:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.070635:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.070635:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.070636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.070640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.070648:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.070651:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.070652:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.070653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.070654:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.070657:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.071342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.071348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.071351:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.071352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.071353:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.071355:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.071357:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.071358:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.071359:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.071360:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.071361:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.071364:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.071365:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.071368:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.071368:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.071369:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.071371:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.071371:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.071372:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.071372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.071377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.071385:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.071388:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.071388:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.071390:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.071391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.071392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.071875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.071883:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.071886:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.071888:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.071890:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.071891:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.071892:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.071893:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.071894:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.071895:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.071896:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.071899:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.071901:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.071903:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.071904:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.071905:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.071906:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.071907:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.071908:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.071909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.071912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.071919:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.071922:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.071923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.071923:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.071925:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.071927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.072199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.072208:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.072212:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.072214:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.072217:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.072220:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.072222:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.072224:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.072227:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.072228:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.072230:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.072235:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.072238:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.072242:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.072243:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.072244:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.072247:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.072248:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.072249:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.072250:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677475.072253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.072261:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.072264:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.072265:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.072267:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.072267:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.072270:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.072270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.072279:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.072283:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.072284:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.072287:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.072291:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.072301:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.072303:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.072305:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.072307:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.072308:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.072314:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.072316:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.072320:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.072321:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.072322:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.072324:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.072325:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.072326:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.072327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.072331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.072339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.072342:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.072343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.072343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.072345:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.072348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.072375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:21.2:1619677475.072381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:24.0:1619677475.072382:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.072385:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:22.0:1619677475.072386:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.0:1619677475.072386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677475.072387:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.072389:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:24.0:1619677475.072390:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.072390:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677475.072392:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677475.072392:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.072393:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.072394:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.072394:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.072395:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.2:1619677475.072395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677475.072396:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.072397:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677475.072397:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.072398:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.072400:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.072401:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.072402:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677475.072402:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.072405:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677475.072405:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677475.072405:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.072407:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.072407:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:30.0:1619677475.072408:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.072408:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677475.072408:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677475.072409:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.072409:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:31.0:1619677475.072410:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.072410:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.072410:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:22.0:1619677475.072410:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.072411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677475.072411:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:30.0:1619677475.072412:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:22.0:1619677475.072412:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.072413:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96501750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.072413:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.072414:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677475.072415:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.072415:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.072415:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.072415:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:9.2:1619677475.072415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:30.0:1619677475.072417:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:25.2:1619677475.072419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.072420:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.072421:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:30.0:1619677475.072422:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.072423:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677475.072424:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.072424:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.072424:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.072425:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.072426:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677475.072426:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677475.072426:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.072427:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.072428:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.072428:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.072429:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.072429:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.072429:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.072429:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.072431:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.072432:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.072433:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677475.072433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:9.2:1619677475.072435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.072440:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.072442:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.072442:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:24.0:1619677475.072443:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.072443:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.072444:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000400:00000200:24.0:1619677475.072445:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.072445:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.072446:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.072447:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677475.072447:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.072448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.072521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677475.072526:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.072529:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677475.072530:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677475.072532:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.072534:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.072536:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677475.072537:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.072538:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.072540:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.072541:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677475.072544:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.072546:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677475.072549:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.072550:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677475.072551:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677475.072553:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677475.072553:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.072554:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.072554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677475.072559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.072568:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.072571:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.072573:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.072573:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.072577:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.072581:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677475.072800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.072807:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.072810:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.072811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.072813:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.072815:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.072816:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.072818:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.072819:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.072820:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.072821:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.072825:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.072827:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.072830:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.072831:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.072832:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.072834:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.072834:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.072835:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.072836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.072840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.072848:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.072851:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.072851:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.072853:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.072856:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.072857:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.072955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.072963:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.072965:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.072967:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.072969:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.072969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.072971:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.072972:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.072974:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.072975:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.072975:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.072978:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.072980:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.072982:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.072983:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.072984:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.072985:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.072986:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.072987:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.072988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.072991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.072997:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.073000:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.073001:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.073001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.073003:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.073005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677475.073018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.073025:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.073027:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.073028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.073030:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.073033:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.073034:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.073035:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.073037:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.073038:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.073038:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.073041:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.073043:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.073046:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.073046:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.073047:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.073049:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.073049:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.073050:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.073050:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.073055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.073062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.073065:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.073066:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.073066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.073068:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.073070:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.073219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.073227:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677475.073228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:10.0:1619677475.073230:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.073231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677475.073233:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677475.073234:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.073236:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.073236:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.073237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677475.073237:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.073239:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.073240:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.073241:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.073242:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.073242:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.073243:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677475.073244:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.073245:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.073246:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.073247:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.073248:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.073253:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.073255:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.073256:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.073258:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.073259:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.073260:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.073260:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677475.073261:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.073261:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.073262:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.073262:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.073263:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:10.0:1619677475.073264:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.073264:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.073264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:10.0:1619677475.073265:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.073266:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.073266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.073268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677475.073272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.073276:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.073278:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.073279:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677475.073279:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.073281:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.073281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.073282:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.073283:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.073283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677475.073283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.073285:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.073286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.073289:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:25.2:1619677475.073290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.073291:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.073293:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.073294:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.073299:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.073299:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.073300:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.073301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.073302:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.073303:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.073303:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.073304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.073305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.073359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677475.073367:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.073370:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.073373:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.073375:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.073376:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.073377:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677475.073378:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.073380:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.073381:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.073382:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677475.073385:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.073387:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.073389:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.073389:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.073390:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.073391:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.073392:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.073393:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677475.073394:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:25.2:1619677475.073398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677475.073401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.073406:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.073408:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.073408:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.073410:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.073410:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.073411:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677475.073412:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.073413:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:30.0:1619677475.073414:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.073415:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.073415:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.073417:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:25.2:1619677475.073417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:30.0:1619677475.073418:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.073419:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.073420:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.073421:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.073423:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.073424:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677475.073424:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.073425:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.073425:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.073426:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.073426:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.073427:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677475.073428:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.073428:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.073429:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.073430:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.073431:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.073431:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.073432:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.073436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.073443:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.073446:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.073447:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.073448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.073449:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.073451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.073492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.073499:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.073502:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.073504:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.073506:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.073506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.073507:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.073508:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.073510:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.073510:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.073511:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.073514:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.073516:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.073518:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.073518:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.073519:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.073521:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.073521:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.073522:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.073522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.073526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.073532:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.073534:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.073535:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.073536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.073537:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.073539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.073614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.073620:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.073623:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.073624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.073626:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.073629:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.073630:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677475.073631:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.073633:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.073634:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.073635:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677475.073638:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.073639:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677475.073642:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.073643:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677475.073643:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677475.073645:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677475.073645:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.073646:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.073647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677475.073651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.073659:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.073662:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.073663:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.073663:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.073665:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.073667:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.073940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.073947:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.073950:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.073952:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.073953:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.073954:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.073956:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.073957:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.073959:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.073960:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.073961:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.073965:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.073966:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.073969:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.073970:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.073971:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.073973:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.073973:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.073975:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.073976:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.073976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.073979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.073982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.073987:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.073989:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677475.073990:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677475.073990:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.073992:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:20.0:1619677475.073992:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.073993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.073993:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.073994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.073995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.074130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.074135:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.074138:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.074139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.074141:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.074143:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.074145:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.074146:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.074147:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.074148:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.074149:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.074153:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.074154:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.074157:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.074158:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.074158:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.074160:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.074160:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.074161:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.074162:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.074163:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.074166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.074168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.074173:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.074175:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.074175:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.074176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.074176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.074178:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.074178:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.074180:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.074181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.074182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.074452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.074459:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.074462:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.074463:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.074465:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.074467:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.074469:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.074470:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.074472:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.074473:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.074474:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.074477:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.074479:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.074482:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.074483:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.074483:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.074485:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.074486:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.074486:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.074487:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.074492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.074499:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.074502:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.074503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.074503:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.074506:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.074508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.074531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.074539:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.074541:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.074543:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677475.074545:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.074546:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.074547:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677475.074549:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.074550:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.074551:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.074552:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677475.074556:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.074558:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677475.074559:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.074560:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677475.074561:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677475.074562:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677475.074563:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.074564:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677475.074566:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677475.074566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:25.2:1619677475.074568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.074572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:31.0:1619677475.074575:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.074577:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.074578:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.074579:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.074580:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:31.0:1619677475.074582:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677475.074582:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:30.0:1619677475.074583:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:12.0:1619677475.074583:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.074584:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:20.2:1619677475.074760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.074767:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.074770:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.074771:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.074773:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.074775:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.074776:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.074777:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.074779:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.074780:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.074780:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.074783:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.074785:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.074787:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.074787:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.074788:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.074790:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.074790:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.074791:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.074792:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.074796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.074803:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.074805:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.074806:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.074806:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.074807:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.074809:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.074880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.074883:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.074885:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.074888:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.074889:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.074890:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.074892:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.074893:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.074894:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.074895:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.074896:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.074900:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.074902:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.074904:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.074905:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.074906:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.074907:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.074908:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.074909:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.074909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.074914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.074922:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.074924:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.074925:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.074925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.074927:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.074929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.075216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.075221:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.075224:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.075225:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.075227:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.075229:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.075230:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.075232:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.075233:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.075234:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.075235:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.075239:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.075241:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.075243:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.075243:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.075244:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.075246:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.075246:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.075247:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.075247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.075252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.075259:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.075261:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.075262:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.075264:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.075264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.075266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.075418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.075426:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.075429:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.075431:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.075433:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.075434:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.075435:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.075436:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.075438:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.075439:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.075440:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.075443:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.075445:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.075447:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.075448:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.075449:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.075450:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.075451:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.075452:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.075454:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.075454:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.075457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.075461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.075464:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.075467:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677475.075468:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.075468:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.075468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.075469:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.075471:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677475.075471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.075472:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.075473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.075808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.075816:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.075818:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.075819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.075820:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.075823:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.075825:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.075826:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.075827:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.075828:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.075829:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.075832:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.075834:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.075837:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.075838:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.075838:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.075840:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.075840:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.075841:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.075841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.075846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.075854:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.075857:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.075858:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.075858:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.075860:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.075863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.076066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.076072:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.076075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.076075:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.076077:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.076080:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.076081:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.076082:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.076084:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.076085:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.076086:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.076089:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.076091:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.076093:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.076094:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.076095:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.076097:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.076097:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.076098:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.076098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.076103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.076110:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.076113:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.076113:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.076115:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.076116:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.076117:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.076265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.076270:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.076273:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.076275:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.076276:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.076277:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.076279:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.076280:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.076281:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.076282:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.076283:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.076287:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.076289:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.076291:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.076292:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.076292:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.076297:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.076298:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.076298:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.076299:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.076303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.076311:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.076313:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.076314:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.076315:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.076317:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.076318:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.076447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c67fb7c00] (20)++
00000800:00000200:24.0:1619677475.076455:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c67fb7c00] (21)++
00000800:00000200:24.0:1619677475.076458:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.28@o2ib3
00000800:00000200:8.0:1619677475.076459:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c67fb7c00] (22)--
00000400:00000200:24.0:1619677475.076461:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.6.28@o2ib3 : GET - for me
00000400:00000200:24.0:1619677475.076464:0:19938:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.6.28@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:24.0:1619677475.076466:0:19938:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.6.28@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:24.0:1619677475.076468:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c67fb7c00] (21)++
00000800:00000200:24.0:1619677475.076469:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c67fb7c00] (22)++
00000800:00000200:24.0:1619677475.076471:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c67fb7c00] (23)--
00000400:00000200:24.0:1619677475.076473:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.6.28@o2ib3 local destination
00000800:00000200:24.0:1619677475.076475:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919fce371f80] -> 10.51.6.28@o2ib3 (2) version: 12
00000800:00000200:24.0:1619677475.076476:0:19938:0:(o2iblnd.c:1251:kiblnd_query()) peer_ni 10.51.6.28@o2ib3 ffff919fce371f80, alive 0 secs ago
00000800:00000200:4.2:1619677475.076477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c67fb7c00] (22)++
00000400:00000200:24.0:1619677475.076480:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.6.28@o2ib3(10.51.6.28@o2ib3:10.51.6.28@o2ib3) : REPLY try# 0
00000800:00000200:24.0:1619677475.076483:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.6.28@o2ib3
00000800:00000200:24.0:1619677475.076484:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919fce371f80] -> 10.51.6.28@o2ib3 (2) version: 12
00000800:00000200:8.0:1619677475.076485:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c67fb7c00] (23)++
00000800:00000200:24.0:1619677475.076486:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919c67fb7c00] (23)++
00000800:00000200:24.0:1619677475.076486:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c67fb7c00] (24)++
00000800:00000200:8.0:1619677475.076486:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c67fb7c00] (24)--
00000800:00000200:8.0:1619677475.076486:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c67fb7c00] (24)--
00000800:00000200:24.0:1619677475.076487:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919c67fb7c00] (24)--
00000800:00000200:8.0:1619677475.076487:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c67fb7c00] (24)--
00000800:00000200:24.0:1619677475.076488:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c67fb7c00] (22)--
00000800:00000200:4.2:1619677475.076493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c67fb7c00] (21)++
00000800:00000200:8.0:1619677475.076500:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c67fb7c00] (22)++
00000800:00000200:8.0:1619677475.076503:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c67fb7c00] (23)--
00000400:00000200:8.0:1619677475.076503:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.076504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c67fb7c00] (22)--
00000400:00000200:8.0:1619677475.076505:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.6.28@o2ib3: REPLY: OK
00000800:00000200:8.0:1619677475.076506:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c67fb7c00] (21)--
00000800:00000200:21.2:1619677475.076640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.076647:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.076650:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.076652:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.076654:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.076655:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.076655:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.076657:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.076658:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.076659:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.076660:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:4.2:1619677475.076661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:30.0:1619677475.076663:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.076665:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677475.076668:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:30.0:1619677475.076669:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.076669:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.076670:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677475.076671:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:30.0:1619677475.076672:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.076672:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.076672:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:30.0:1619677475.076673:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677475.076673:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677475.076674:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:24.0:1619677475.076675:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.076677:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:8.2:1619677475.076678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:24.0:1619677475.076679:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96501727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.076681:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.076682:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.076683:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677475.076686:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.076686:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:24.0:1619677475.076688:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.076689:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.076689:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:24.0:1619677475.076690:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:21.0:1619677475.076690:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.076691:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.076692:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677475.076692:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.076692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677475.076693:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.076693:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677475.076694:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:21.0:1619677475.076695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677475.076698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677475.076699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677475.076706:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.076707:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677475.076708:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.076709:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.076710:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.076710:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.076711:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.076711:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000400:00000200:8.0:1619677475.076711:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.076712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.076713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677475.076713:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.076917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.076923:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.076926:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.076926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.076929:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.076931:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.076932:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.076933:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.076935:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.076936:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.076937:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.076940:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.076941:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.076944:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.076944:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.076945:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.076947:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.076947:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.076948:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.076948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.076953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.076960:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.076962:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.076963:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.076964:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.076965:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.076967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.076975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.076983:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.076985:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.076986:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.076987:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.076987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.076988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.077215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.077223:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.077227:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.077228:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.077230:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.077233:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.077234:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.077236:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677475.077236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:26.0:1619677475.077238:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.077239:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.077240:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.077243:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.077244:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:26.0:1619677475.077245:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677475.077247:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.077247:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:26.0:1619677475.077248:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.077249:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.077250:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:18.0:1619677475.077250:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.077252:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.077252:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677475.077252:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.077253:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677475.077253:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:26.0:1619677475.077254:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:18.0:1619677475.077255:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.077256:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.077257:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.077258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:18.0:1619677475.077258:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677475.077261:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.077263:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677475.077265:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:18.0:1619677475.077266:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.077267:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677475.077268:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677475.077268:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.077269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:18.0:1619677475.077270:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677475.077270:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:6.0:1619677475.077270:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.077271:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.077272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:6.0:1619677475.077272:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.077274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677475.077276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.077282:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:25.2:1619677475.077283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.077285:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.077286:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.077286:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.077289:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.077290:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.077291:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:22.0:1619677475.077292:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.077293:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:9.2:1619677475.077297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:22.0:1619677475.077298:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.077300:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.077300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677475.077304:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.077307:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677475.077307:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.077308:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.077309:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.077309:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.077486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.077495:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.077497:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.077500:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.077502:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.077502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.077504:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.077505:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.077506:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.077507:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.077508:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.077511:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.077513:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.077516:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.077516:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.077517:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.077519:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.077519:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.077520:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.077520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.077524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.077532:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.077534:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.077535:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.077535:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.077537:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.077539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.077543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.077548:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.077551:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.077552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.077552:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.077553:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.077554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.077558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.077566:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.077568:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.077570:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.077572:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.077573:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.077574:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.077575:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.077576:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.077577:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.077578:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.077581:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.077583:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.077585:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.077585:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.077586:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.077587:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.077588:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.077589:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.077589:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.077593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.077600:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.077602:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.077603:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.077604:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.077606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.077607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.077611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.077618:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.077621:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.077622:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.077622:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.077623:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.077624:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677475.077635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.077642:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.077645:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.077646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.077648:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.077650:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.077651:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.077653:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.077654:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.077655:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.077656:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.077659:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.077661:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.077663:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.077664:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.077665:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.077667:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.077667:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.077668:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.077669:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.077670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.077673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.077675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677475.077680:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677475.077680:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.077682:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.077682:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677475.077683:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677475.077683:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677475.077684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.077684:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.077686:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.077688:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.077806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677475.077814:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.077817:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677475.077819:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.077821:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.077821:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677475.077823:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677475.077824:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.077825:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.077826:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.077827:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677475.077831:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.077833:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.077835:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.077836:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677475.077837:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677475.077838:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677475.077839:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.077840:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677475.077841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677475.077844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.077852:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.077855:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.077856:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.077856:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.077859:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.077862:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.077870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.077877:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.077880:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.077881:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.077881:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.077882:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.077882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.077935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.077940:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.077943:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.077944:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.077946:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.077948:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.077949:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.077951:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.077952:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.077953:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.077954:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.077958:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.077960:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.077962:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.077963:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.077963:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.077965:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.077965:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.077966:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.077967:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.077968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.077971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.077973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.077978:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.077980:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.077980:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.077980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.077981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.077982:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.077983:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.077985:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.077986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.077987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.078027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.078035:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.078038:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.078039:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.078041:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.078043:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.078045:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.078046:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.078047:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.078048:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.078049:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.078053:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.078055:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.078058:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.078059:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.078059:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.078061:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.078061:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.078063:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.078064:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.078065:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.078067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.078070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.078075:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.078077:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.078077:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.078078:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.078079:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677475.078079:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.078080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.078081:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.078081:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.078084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.078087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.078094:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.078097:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.078099:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.078101:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.078102:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.078104:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.078105:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.078106:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.078107:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.078108:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.078111:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.078113:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.078114:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.078115:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.078115:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.078117:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.078118:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.078118:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.078119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.078123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.078129:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.078132:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.078132:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.078134:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.078136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.078136:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.078794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.078800:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.078803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.078803:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.078805:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.078807:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.078809:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.078810:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.078811:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.078812:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.078813:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.078817:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.078819:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.078821:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.078822:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.078823:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.078824:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.078825:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.078826:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.078826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.078831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.078839:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.078842:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.078842:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.078844:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.078846:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.078847:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.079031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.079037:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.079040:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.079040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.079043:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.079045:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.079046:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677475.079048:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.079049:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.079050:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.079051:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677475.079055:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.079056:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677475.079059:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.079060:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677475.079061:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677475.079063:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677475.079063:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.079063:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.079064:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677475.079069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.079076:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.079079:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.079080:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.079080:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.079082:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.079085:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.079153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.079161:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.079164:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.079166:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.079167:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.079169:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677475.079169:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.079170:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.079172:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.079173:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.079174:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.079176:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.079178:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.079180:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.079181:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.079182:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.079183:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.079184:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.079185:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.079187:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.079188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.079189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.079193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.079196:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.079198:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.079199:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.079199:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.079201:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.079201:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.079203:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677475.079203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.079204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.079205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.079210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.079217:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.079219:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.079220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.079220:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.079221:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.079221:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.079317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.079322:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.079325:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.079326:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.079328:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.079331:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.079332:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.079334:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.079335:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.079336:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.079337:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.079339:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.079341:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.079343:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.079344:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.079344:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.079346:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.079346:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.079347:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.079348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677475.079353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.079360:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.079362:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.079363:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.079363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.079365:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.079366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.079441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.079449:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.079452:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.079454:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.079456:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.079457:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.079458:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.079460:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.079461:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.079462:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.079463:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.079466:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.079467:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.079470:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.079470:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.079471:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.079473:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.079473:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.079474:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.079475:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.079479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.079485:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.079488:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.079488:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.079489:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.079490:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.079492:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.079644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.079650:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.079653:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.079654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.079656:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.079658:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.079659:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.079661:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.079662:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.079663:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.079664:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.079668:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.079669:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.079671:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677475.079671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677475.079672:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.079673:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.079674:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.079675:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.079676:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.079676:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:2.0:1619677475.079677:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:25.2:1619677475.079680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677475.079680:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.079681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.079683:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.079685:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.079687:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:21.0:1619677475.079688:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:2.0:1619677475.079688:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.079689:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.079690:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.079691:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.079691:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.079691:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.079693:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.079693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:2.0:1619677475.079695:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.079696:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677475.079697:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.079700:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.079701:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.079702:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.079704:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.079704:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.079706:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.079707:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.079707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.079710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677475.079712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677475.079717:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677475.079719:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677475.079720:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.079721:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.079721:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.079721:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.079722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677475.079723:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.079723:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.079726:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.079922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.079926:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.079928:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677475.079931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.079931:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.079933:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.079934:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677475.079935:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.079937:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.079938:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.079939:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.079942:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.079944:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.079946:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.079947:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.079948:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.079950:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677475.079950:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.079951:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.079953:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677475.079953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677475.079956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.079959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.079964:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:18.0:1619677475.079964:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677475.079966:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:18.0:1619677475.079966:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677475.079967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:18.0:1619677475.079967:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.079967:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677475.079969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677475.079970:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677475.079972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.080153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.080161:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.080164:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.080165:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.080167:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.080169:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.080170:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.080172:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.080173:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.080174:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.080175:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.080179:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.080180:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.080183:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.080184:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.080185:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.080186:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.080187:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.080188:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.080189:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.080190:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.080192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.080195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.080200:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.080202:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.080203:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677475.080203:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.080203:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.080206:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.080206:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677475.080206:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.080208:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677475.080208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.080468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.080476:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.080479:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.080480:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.080482:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.080484:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.080486:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.080487:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.080488:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.080490:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.080490:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.080494:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.080495:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.080497:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.080498:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.080499:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.080501:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.080501:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.080502:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.080502:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.080507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.080515:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.080517:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.080518:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.080519:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.080522:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.080523:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.080525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677475.080531:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.080533:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.080534:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.080535:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.080535:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.080536:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.080676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.080682:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.080685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.080685:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.080688:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.080690:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.080691:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.080692:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.080694:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.080695:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.080696:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.080699:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.080700:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.080702:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.080703:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.080704:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.080705:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.080706:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.080706:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.080707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.080711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.080719:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.080721:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.080722:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.080723:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.080724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.080726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.080731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.080739:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.080741:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.080742:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.080743:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.080744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.080745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.080753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.080756:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.080758:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.080760:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.080762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.080762:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.080764:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.080765:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.080766:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.080767:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.080768:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.080771:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.080773:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.080776:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.080777:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.080777:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.080779:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.080779:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.080781:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.080782:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.080782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.080786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.080787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.080793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677475.080794:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.080795:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.080796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.080796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.080796:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.080798:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.080798:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.080800:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677475.080802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.080806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.080807:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.080809:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.080810:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.080810:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.080811:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.080811:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677475.080917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:21.2:1619677475.080922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677475.080922:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677475.080925:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677475.080926:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677475.080928:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.080929:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.080930:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677475.080931:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677475.080931:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.080933:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.080934:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677475.080934:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.080935:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.080937:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.080938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.080939:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.080939:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.080941:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677475.080941:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.080943:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.080944:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.080944:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.080945:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.080945:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:1.0:1619677475.080945:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:30.0:1619677475.080946:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677475.080947:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677475.080947:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677475.080949:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000400:00000200:30.0:1619677475.080950:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.080950:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677475.080950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:30.0:1619677475.080952:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:9.2:1619677475.080953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.080955:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.080956:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.080956:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:5.2:1619677475.080957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:30.0:1619677475.080958:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.080958:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.080959:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.080960:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677475.080961:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:17.0:1619677475.080962:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677475.080964:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:8.2:1619677475.080964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677475.080964:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:17.0:1619677475.080965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:2.0:1619677475.080965:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.080966:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:17.0:1619677475.080967:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000400:00000200:2.0:1619677475.080967:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.080970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.0:1619677475.080972:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.080974:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.080975:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.080977:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.080977:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.080980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.081012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.081020:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.081023:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.081025:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.081027:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.081028:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.081030:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.081031:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.081033:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.081033:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.081035:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.081038:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.081040:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.081042:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.081042:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.081043:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.081045:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.081045:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.081046:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.081046:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.081050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.081057:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.081059:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.081060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.081060:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.081062:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.081064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.081257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.081264:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.081266:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.081267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.081269:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.081271:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.081273:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.081274:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.081275:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.081276:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.081277:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.081280:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.081281:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.081284:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.081285:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.081285:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.081287:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.081287:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.081288:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.081288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.081293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.081301:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.081303:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.081304:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.081305:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.081307:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.081309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.081312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.081320:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.081322:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.081323:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.081323:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.081324:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.081325:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.081358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (20)++
00000800:00000200:20.0:1619677475.081361:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:20.0:1619677475.081364:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000400:00000200:20.0:1619677475.081366:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.081367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:20.0:1619677475.081368:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.081369:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.2:1619677475.081372:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:20.0:1619677475.081374:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.081375:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.081377:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.081377:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677475.081378:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:20.0:1619677475.081381:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.081381:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.081382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.0:1619677475.081384:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.081384:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.081386:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.081387:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:18.0:1619677475.081387:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.081388:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677475.081389:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677475.081390:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.081390:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (21)++
00000400:00000200:18.0:1619677475.081390:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.081391:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:18.0:1619677475.081391:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.081392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)--
00000400:00000200:18.0:1619677475.081392:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.081393:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:25.2:1619677475.081396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677475.081397:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.081398:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.081400:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.081401:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.081402:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.081403:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677475.081403:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.081404:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.081405:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.0:1619677475.081406:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.081406:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.081406:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:21.0:1619677475.081408:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.081409:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:9.2:1619677475.081409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.081410:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677475.081416:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.081419:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.081419:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.081420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.081421:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.081423:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.081474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (20)++
00000800:00000200:20.0:1619677475.081476:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:20.0:1619677475.081479:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000400:00000200:20.0:1619677475.081481:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.081482:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:20.0:1619677475.081483:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.081485:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677475.081486:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.081488:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.081489:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.081490:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677475.081493:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.081495:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677475.081497:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.081498:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:20.0:1619677475.081499:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.0:1619677475.081501:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:20.0:1619677475.081501:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:20.0:1619677475.081502:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:20.0:1619677475.081503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)--
00000800:00000200:9.2:1619677475.081507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.081514:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.081517:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.081518:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.081518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.081520:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.2:1619677475.081522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (20)++
00000800:00000200:18.0:1619677475.081523:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677475.081528:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:21.0:1619677475.081531:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000400:00000200:21.0:1619677475.081533:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.081533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:21.0:1619677475.081536:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.081537:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.081539:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.081540:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.081541:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.081542:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677475.081546:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:21.0:1619677475.081549:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.081551:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677475.081552:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.081554:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.081554:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:22.0:1619677475.081555:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.0:1619677475.081555:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677475.081555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.081557:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.081557:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (21)++
00000400:00000200:22.0:1619677475.081558:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.081559:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e8d3ac00] (22)++
00000400:00000200:22.0:1619677475.081560:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.081561:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.0:1619677475.081561:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (23)--
00000800:00000200:21.0:1619677475.081561:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:22.0:1619677475.081562:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677475.081563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:22.0:1619677475.081564:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.081565:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.081566:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.2:1619677475.081566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (21)++
00000400:00000200:22.0:1619677475.081569:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.081569:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)++
00000800:00000200:8.0:1619677475.081569:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677475.081570:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.081572:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e8d3ac00] (23)--
00000800:00000200:20.0:1619677475.081572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:8.0:1619677475.081572:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.081573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677475.081573:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.081573:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.081573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)--
00000800:00000200:22.0:1619677475.081574:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677475.081574:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.081576:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.081576:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677475.081576:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.081577:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.081578:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.081579:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677475.081579:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:25.2:1619677475.081582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.081584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.081589:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.081589:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.081591:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.081591:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.081592:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.081592:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.081592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.081594:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.081595:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.081597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.081637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (20)++
00000800:00000200:21.0:1619677475.081642:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:21.0:1619677475.081645:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000400:00000200:21.0:1619677475.081647:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.081647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:21.0:1619677475.081650:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.081651:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.081652:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.081654:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.081655:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.081656:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.081659:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.081661:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.081663:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677475.081663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:21.0:1619677475.081664:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.081664:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.081665:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)++
00000800:00000200:21.0:1619677475.081666:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.081666:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (23)++
00000800:00000200:21.0:1619677475.081667:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (24)--
00000800:00000200:20.0:1619677475.081667:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000800:00000200:21.0:1619677475.081668:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (23)--
00000800:00000200:21.0:1619677475.081669:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:20.0:1619677475.081669:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.081671:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.081672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677475.081673:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677475.081674:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.081675:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.081676:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.081677:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:6.0:1619677475.081679:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677475.081680:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.081682:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677475.081682:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.081683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.081683:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.081685:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.081685:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.081686:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:20.0:1619677475.081686:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677475.081687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.081688:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:20.0:1619677475.081689:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:20.0:1619677475.081690:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:20.0:1619677475.081690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)--
00000800:00000200:9.2:1619677475.081694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.081701:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.081704:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.081705:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.081705:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.081707:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.081710:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677475.081830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (20)++
00000800:00000200:21.0:1619677475.081835:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:21.0:1619677475.081838:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000400:00000200:21.0:1619677475.081840:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.081840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:21.0:1619677475.081843:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.081845:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:21.0:1619677475.081846:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.081848:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.081849:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.081850:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:21.0:1619677475.081853:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.081855:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.081858:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.081859:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.0:1619677475.081859:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677475.081861:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.0:1619677475.081862:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:21.0:1619677475.081862:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:21.0:1619677475.081863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)--
00000800:00000200:25.2:1619677475.081867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.081874:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.081877:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.081879:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.081880:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.081882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.081883:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.081892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (20)++
00000800:00000200:20.0:1619677475.081895:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:20.0:1619677475.081898:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000800:00000200:21.0:1619677475.081900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:20.0:1619677475.081900:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.081902:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.081904:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677475.081905:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.081907:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.081907:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.081908:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677475.081911:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.081913:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677475.081916:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.081916:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.0:1619677475.081917:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677475.081919:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677475.081920:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:20.0:1619677475.081921:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e8d3ac00] (22)++
00000800:00000200:20.0:1619677475.081923:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (23)--
00000800:00000200:20.0:1619677475.081923:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:9.2:1619677475.081925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.2:1619677475.081928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:24.0:1619677475.081932:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677475.081933:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)++
00000800:00000200:24.0:1619677475.081934:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.081935:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.081935:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e8d3ac00] (23)--
00000800:00000200:8.0:1619677475.081935:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677475.081936:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:24.0:1619677475.081937:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.081937:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)--
00000800:00000200:24.0:1619677475.081940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.081984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (20)++
00000800:00000200:21.0:1619677475.081989:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:21.0:1619677475.081991:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000400:00000200:21.0:1619677475.081993:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.081994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:21.0:1619677475.081996:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677475.081996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (21)++
00000400:00000200:21.0:1619677475.081998:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:21.0:1619677475.081999:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.082000:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.082000:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)++
00000400:00000200:21.0:1619677475.082001:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.082002:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000400:00000200:21.0:1619677475.082003:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:20.0:1619677475.082006:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.082007:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677475.082008:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.082009:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:20.0:1619677475.082010:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677475.082011:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.082012:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.082013:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.082014:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677475.082017:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677475.082017:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.082018:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:21.0:1619677475.082018:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677475.082019:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.0:1619677475.082020:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677475.082021:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (23)++
00000800:00000200:21.0:1619677475.082021:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (24)--
00000800:00000200:21.0:1619677475.082022:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (23)--
00000800:00000200:20.0:1619677475.082022:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.082022:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:21.0:1619677475.082023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:20.0:1619677475.082023:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677475.082025:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677475.082025:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:9.2:1619677475.082025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677475.082026:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:20.0:1619677475.082027:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)--
00000800:00000200:25.2:1619677475.082031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677475.082032:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.2:1619677475.082033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677475.082034:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.082035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.082036:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.082038:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:21.0:1619677475.082038:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677475.082038:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.082041:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:21.0:1619677475.082041:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677475.082041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.082041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677475.082042:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.082042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:21.0:1619677475.082044:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.082045:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.082045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677475.082047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:21.0:1619677475.082047:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.082049:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:21.0:1619677475.082049:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.082050:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677475.082051:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:21.0:1619677475.082052:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.082052:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:21.0:1619677475.082053:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.082054:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677475.082055:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.082057:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.082058:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.082059:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.082060:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.082060:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:26.0:1619677475.082062:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.082062:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677475.082063:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.082063:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:26.0:1619677475.082064:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.0:1619677475.082064:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.082065:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.082066:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677475.082067:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.082067:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.082067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677475.082069:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.2:1619677475.082071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677475.082072:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.082073:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.082073:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.082075:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.082076:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.082076:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.082077:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:20.0:1619677475.082079:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:25.2:1619677475.082081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677475.082081:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.082082:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.082084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.082084:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.2:1619677475.082086:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (20)++
00000800:00000200:6.0:1619677475.082088:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:20.0:1619677475.082090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.082091:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:6.0:1619677475.082091:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.082092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677475.082092:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:20.0:1619677475.082092:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000400:00000200:6.0:1619677475.082092:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.082095:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.082095:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677475.082097:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.082098:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677475.082098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:20.0:1619677475.082100:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.082101:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.082102:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.082103:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677475.082106:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.082108:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677475.082111:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.082112:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:20.0:1619677475.082112:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677475.082114:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.0:1619677475.082115:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:20.0:1619677475.082115:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:20.0:1619677475.082116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)--
00000800:00000200:8.2:1619677475.082120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.082128:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.082131:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.082132:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.082132:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.082134:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.082136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.082327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (20)++
00000800:00000200:21.0:1619677475.082333:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:21.0:1619677475.082336:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000400:00000200:21.0:1619677475.082338:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.082339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:21.0:1619677475.082340:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.082342:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677475.082343:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.082345:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.082346:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.082347:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677475.082350:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.082352:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677475.082355:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.082356:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677475.082356:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677475.082358:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677475.082358:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:21.0:1619677475.082360:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e8d3ac00] (22)++
00000800:00000200:21.0:1619677475.082361:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (23)--
00000800:00000200:21.0:1619677475.082361:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:9.2:1619677475.082364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677475.082366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:20.0:1619677475.082369:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)++
00000800:00000200:20.0:1619677475.082371:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e8d3ac00] (23)--
00000800:00000200:20.0:1619677475.082372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:20.0:1619677475.082372:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)--
00000800:00000200:10.0:1619677475.082372:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.082375:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.082376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.082376:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.082379:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.082381:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.082394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.2:1619677475.082398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.082402:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.082404:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.082405:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677475.082407:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:31.0:1619677475.082408:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.082408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677475.082410:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677475.082410:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.082411:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.082412:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677475.082412:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.082414:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.082414:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:31.0:1619677475.082416:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.082416:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.082418:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.082419:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.082420:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.082421:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677475.082422:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.2:1619677475.082422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (20)++
00000800:00000200:20.0:1619677475.082425:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)++
00000400:00000200:31.0:1619677475.082427:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.082428:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.082428:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000800:00000200:31.0:1619677475.082429:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.082430:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:22.0:1619677475.082431:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:20.0:1619677475.082432:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.082433:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.082433:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.082434:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.082435:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677475.082435:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.082435:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677475.082435:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.082436:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.082437:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.082437:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.082437:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677475.082438:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.082438:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677475.082438:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:20.0:1619677475.082438:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96501683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.082439:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.082439:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.082439:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.082441:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.082441:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:25.2:1619677475.082442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:20.0:1619677475.082445:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.082446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.082447:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.082449:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:20.0:1619677475.082449:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.082450:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.082450:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.082452:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.082452:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.082452:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.082452:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:21.0:1619677475.082453:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.082453:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:30.0:1619677475.082454:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.082454:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.082454:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)--
00000800:00000200:31.0:1619677475.082455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677475.082455:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.082456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.082457:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:21.0:1619677475.082457:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.2:1619677475.082459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.082462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677475.082463:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.082464:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.082465:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.082466:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.082467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.082468:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.082472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e8d3ac00] (20)++
00000800:00000200:20.0:1619677475.082475:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:20.0:1619677475.082477:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.19@o2ib3
00000400:00000200:20.0:1619677475.082479:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.19@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.082480:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (22)--
00000400:00000200:20.0:1619677475.082481:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.082482:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677475.082484:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.082485:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.082486:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.082487:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677475.082490:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.082492:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.082493:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.082494:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.082495:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.082496:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.082497:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (21)++
00000800:00000200:20.0:1619677475.082497:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e8d3ac00] (22)--
00000800:00000200:20.0:1619677475.082498:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e8d3ac00] (21)--
00000800:00000200:8.2:1619677475.082502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.082510:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.082512:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.082512:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.082514:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.082515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.082516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677475.082893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.082900:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.082902:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.082903:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.082905:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.082907:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.082909:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.082910:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.082912:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.082913:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.082914:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.082917:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.082918:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.082921:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.082922:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.082922:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.082924:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.082924:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.082925:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.082926:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.082930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.082938:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.082941:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.082942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.082942:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.082944:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.082946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.082948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.082955:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.082958:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.082958:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.082959:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.082960:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.082960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.083020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.083028:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.083031:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.083033:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.083034:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.083035:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.083036:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.083038:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.083039:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.083040:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.083041:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.083044:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.083046:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.083048:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.083049:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.083050:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.083052:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.083052:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.083054:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.083055:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.083056:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.083057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.083061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.083064:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.083066:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.083067:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.083067:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.083069:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.083069:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.083071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.083072:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.083072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.083073:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.083078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.083084:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.083086:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.083087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.083087:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.083088:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.083089:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.083090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.083097:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.083100:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.083102:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.083104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.083105:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.083106:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.083107:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.083109:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.083109:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.083110:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.083113:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.083115:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.083118:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.083119:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.083120:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.083122:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.083122:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.083123:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.083123:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.083128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.083136:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.083138:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.083140:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.083141:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.083142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.083144:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.083151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.083159:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.083162:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.083163:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.083163:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.083164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.083165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.083360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.083368:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.083371:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.083372:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.083374:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.083376:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.083378:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.083379:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.083380:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.083381:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.083382:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.083386:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.083387:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.083390:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.083391:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.083391:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.083393:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.083393:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.083394:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.083395:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.083411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.083419:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.083421:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.083422:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.083423:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.083425:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.083427:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.083430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.083437:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.083440:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.083441:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.083441:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.083442:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.083443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.083466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.083471:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.083474:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.083475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.083476:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.083478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.083480:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.083481:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.083482:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.083483:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.083484:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.083487:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.083489:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.083491:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.083492:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.083493:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.083494:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.083495:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.083495:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.083496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.083500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.083508:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.083510:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.083511:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.083513:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.083514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.083515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.083519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.083526:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.083529:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.083530:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.083530:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.083531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.083532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.083778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.083782:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.083784:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.083786:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.083787:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.083788:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.083790:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.083791:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.083792:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.083793:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.083794:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.083798:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.083800:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.083802:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.083802:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.083803:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.083805:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.083805:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.083806:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.083807:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.083808:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.083810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.083813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677475.083817:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677475.083819:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677475.083820:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.083821:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.083821:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.083822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677475.083822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.083823:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.083823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677475.083826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.083841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.083850:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.083853:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.083855:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.083857:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.083858:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.083860:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.083861:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.083862:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.083863:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.083864:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.083867:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.083869:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.083873:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.083873:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.083874:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.083876:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.083876:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.083878:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.083880:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.083880:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.083882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.083886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.083894:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.083894:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677475.083896:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677475.083896:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677475.083897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.083897:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.083898:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677475.083899:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.083899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.083902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.084381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.084387:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.084390:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.084391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.084393:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.084395:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.084396:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.084397:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.084399:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.084399:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.084401:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.084404:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.084406:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.084408:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.084409:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.084410:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.084411:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.084412:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.084412:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.084413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.084418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.084426:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.084428:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.084429:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.084431:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.084432:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.084433:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.084463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.084470:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.084472:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.084474:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.084475:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.084478:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.084479:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.084480:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.084482:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.084483:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.084484:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.084488:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.084490:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.084493:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.084494:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.084494:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.084496:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.084497:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.084497:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.084498:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.084502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.084510:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.084512:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.084513:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.084515:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.084517:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.084519:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.084684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.084691:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.084695:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.084697:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.084698:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.084699:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.084701:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.084702:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.084704:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.084704:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.084706:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.084709:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.084711:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.084713:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.084714:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.084714:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.084716:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.084717:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.084718:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.084718:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.084722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.084729:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.084731:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.084732:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.084733:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.084734:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.084737:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.085037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.085044:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.085047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.085047:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.085049:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.085052:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.085054:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.085055:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.085056:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.085057:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.085058:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.085062:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.085063:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.085066:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.085067:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.085068:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.085070:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.085070:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.085071:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.085071:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.085076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.085083:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.085086:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.085087:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.085087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.085089:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.085092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.085259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.085264:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.085267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.085267:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.085269:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.085271:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.085272:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.085273:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.085275:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.085276:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.085277:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.085281:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.085282:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.085285:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.085286:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.085286:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.085288:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.085288:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.085289:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.085290:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.085290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.085294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.085297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.085303:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.085304:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.085304:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.085305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.085306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.085306:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.085307:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.085309:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.085310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.085311:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.085322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.085330:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.085333:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.085335:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.085337:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.085337:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.085339:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.085340:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.085342:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.085342:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.085344:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.085347:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.085348:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.085351:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.085352:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.085352:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.085354:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.085354:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.085356:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.085356:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.085359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.085366:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.085369:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.085370:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.085370:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.085372:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.085375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.086315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.086321:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.086324:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.086325:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.086326:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.086329:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.086330:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.086331:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.086333:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.086334:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.086335:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.086338:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.086340:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.086342:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.086343:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.086344:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.086345:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.086346:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.086347:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.086347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.086352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.086360:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.086363:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.086363:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.086365:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.086367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.086367:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.086397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.086402:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.086404:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.086406:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.086407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.086409:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.086411:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.086412:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.086413:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.086414:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.086415:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.086419:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.086420:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.086423:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.086424:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.086424:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.086426:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.086426:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.086427:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.086427:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.086432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.086440:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.086442:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.086443:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.086444:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.086445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.086446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.086454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.086461:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.086464:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.086465:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.086465:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.086466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.086467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.086559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.086567:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.086570:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.086572:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.086574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.086575:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.086576:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.086577:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.086579:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.086580:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.086581:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.086584:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.086586:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.086588:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.086589:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.086589:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.086591:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.086592:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.086592:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.086593:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.086597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.086603:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.086605:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.086606:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.086606:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.086608:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.086610:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.086870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.086877:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.086880:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.086881:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.086882:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.086885:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.086886:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.086887:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.086889:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.086889:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677475.086889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677475.086890:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.086893:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.086895:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677475.086897:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677475.086898:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.086899:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.086899:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677475.086900:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.086901:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677475.086901:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.086902:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.086902:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.086903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:10.0:1619677475.086903:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.086906:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.086907:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.2:1619677475.086907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:10.0:1619677475.086908:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.086909:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.086910:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.086911:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.086914:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.086915:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.086915:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:10.0:1619677475.086918:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.086918:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.086919:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677475.086919:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.086919:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:1.0:1619677475.086919:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.086921:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:1.0:1619677475.086921:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.086922:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.086923:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.086924:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.086924:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677475.086925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:25.2:1619677475.086927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.086930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.086935:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677475.086937:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.086938:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.086938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.086939:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677475.086939:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.086940:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.086940:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677475.086941:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.086944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.087045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.087052:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.087055:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.087057:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.087059:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.087060:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.087061:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.087062:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.087064:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.087065:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.087065:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.087069:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.087070:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.087072:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.087073:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.087073:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.087075:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.087075:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.087076:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.087077:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.087080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.087087:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.087090:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.087090:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.087092:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.087094:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.087095:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.087098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.087104:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.087107:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.087108:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.087109:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.087109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.087110:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.087205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.087210:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.087213:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.087214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.087216:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.087218:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.087219:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.087221:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.087222:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.087223:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.087224:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.087227:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.087229:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.087231:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.087232:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.087232:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.087234:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.087235:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.087236:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.087236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.087240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.087248:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.087250:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.087251:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.087252:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.087253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.087255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.087265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.087272:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.087275:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.087276:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.087276:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.087277:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.087278:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.087399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.087403:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.087405:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.087407:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.087408:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.087409:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.087411:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.087412:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.087413:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.087414:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.087415:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.087418:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.087420:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.087422:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.087423:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.087423:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.087425:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.087426:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.087426:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.087427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.087432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.087440:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.087442:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.087443:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.087443:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.087445:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.087447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.087619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.087625:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.087628:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.087628:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.087630:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.087632:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.087634:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.087635:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.087636:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.087638:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.087639:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.087642:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.087644:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.087646:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.087647:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.087648:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.087649:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.087650:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.087650:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.087651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.087655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.087663:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.087665:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.087666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.087667:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.087669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677475.087669:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.087671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677475.087677:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.087680:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.087682:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.087684:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.087685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.087685:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.087687:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.087688:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.087689:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.087690:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.087694:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.087695:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.087698:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.087699:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.087699:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.087701:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.087701:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.087702:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.087703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.087706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.087713:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.087716:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.087717:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.087718:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.087720:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.087721:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.087751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.087757:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.087759:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.087761:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.087762:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.087763:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.087765:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.087766:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.087768:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.087769:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.087770:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.087773:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.087775:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.087777:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.087778:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.087779:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.087781:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.087781:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.087782:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.087782:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.087787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.087795:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.087798:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.087799:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.087799:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.087801:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.087803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.088088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.088093:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.088096:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.088096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.088098:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.088100:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.088102:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.088103:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.088104:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.088105:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.088106:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.088109:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.088110:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.088113:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.088113:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.088114:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.088116:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.088116:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.088117:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.088118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.088122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.088129:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.088131:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.088132:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.088134:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.088134:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.088136:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.088230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.088238:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.088241:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.088242:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.088244:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.088247:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.088248:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.088250:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.088251:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.088252:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.088253:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.088256:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.088258:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.088261:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.088262:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.088262:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.088264:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.088264:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.088265:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.088265:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.088270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.088277:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.088280:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.088281:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.088281:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.088283:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.088286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.088289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.088297:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.088299:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.088300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.088301:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.088302:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.088302:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.088307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.088315:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.088318:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.088320:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.088323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.088323:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.088324:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.088325:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.088327:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.088327:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.088328:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.088331:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.088333:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.088335:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.088336:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.088336:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.088338:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.088339:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.088340:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.088340:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.088345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.088352:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.088354:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.088355:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.088355:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.088357:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.088359:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.088545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.088553:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.088556:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.088558:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.088560:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.088560:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.088561:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.088562:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.088563:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.088564:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.088565:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.088568:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.088569:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.088571:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.088572:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.088573:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.088574:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.088575:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.088576:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.088578:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.088578:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.088580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.088584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.088588:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.088590:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.088591:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.088592:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.088592:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.088593:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.088593:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.088594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.088595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.088596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.088730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677475.088738:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677475.088741:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677475.088742:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677475.088744:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.088746:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.088748:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677475.088749:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.088750:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.088751:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.088752:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677475.088755:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.088757:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 976 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.088760:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.088761:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677475.088761:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677475.088763:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677475.088763:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677475.088764:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (22)++
00000800:00000200:10.0:1619677475.088766:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:10.0:1619677475.088766:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:25.2:1619677475.088769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677475.088774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (21)++
00000800:00000200:21.0:1619677475.088777:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.088779:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.088780:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.088781:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000400:00000200:21.0:1619677475.088782:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.088783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677475.088784:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000800:00000200:26.0:1619677475.088785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:21.0:1619677475.088785:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677475.088785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:4.2:1619677475.089086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.089096:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.089099:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:13.0:1619677475.089104:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.089104:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677475.089107:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.089109:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677475.089111:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.089114:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.089116:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.089117:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677475.089122:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.089125:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:21.2:1619677475.089129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:12.0:1619677475.089129:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.089130:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677475.089131:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677475.089133:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677475.089134:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.089135:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:6.0:1619677475.089135:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:12.0:1619677475.089136:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:6.0:1619677475.089137:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.089138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:9.2:1619677475.089139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:6.0:1619677475.089140:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.089142:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.089144:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:26.0:1619677475.089145:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:6.0:1619677475.089145:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.089147:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.089147:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677475.089148:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677475.089148:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.089149:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.089149:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677475.089151:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.089153:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.089154:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677475.089155:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.089158:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.089158:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.089159:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.089161:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.089161:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.089162:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.089163:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.089164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.089166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.089168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.089173:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.089174:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.089175:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.089176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.089176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.089176:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.089177:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.089179:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.089180:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.089181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.089656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.089663:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.089665:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.089666:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.089669:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.089671:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.089673:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.089674:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.089675:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.089676:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.089677:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.089680:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.089682:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.089684:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.089685:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.089686:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.089687:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.089688:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.089689:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.089689:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.089694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.089701:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.089703:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.089704:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.089705:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.089706:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.089707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.089714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.089721:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.089723:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.089724:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.089724:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.089725:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.089726:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.089794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.089803:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.089806:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.089808:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.089810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.089811:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.089812:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.089813:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.089815:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.089816:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.089817:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.089820:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.089822:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.089826:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.089826:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.089827:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.089829:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.089830:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.089831:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.089831:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.089836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.089844:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.089847:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.089848:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.089850:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.089851:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.089853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.089862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677475.089868:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.089871:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677475.089872:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.089874:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.089876:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.089877:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677475.089879:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.089880:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.089881:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.089882:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677475.089886:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.089887:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677475.089891:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.089892:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677475.089892:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677475.089894:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677475.089895:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.089896:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677475.089897:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677475.089897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:25.2:1619677475.089900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.2:1619677475.089903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:13.0:1619677475.089907:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:1.0:1619677475.089908:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:13.0:1619677475.089910:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:1.0:1619677475.089910:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:13.0:1619677475.089911:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.089911:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.089912:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:13.0:1619677475.089913:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.089915:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677475.089917:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:4.2:1619677475.089928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.089936:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.089938:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.089939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.089941:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.089943:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.089945:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.089946:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.089947:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.089948:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.089949:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.089952:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.089954:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.089957:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.089958:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.089958:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.089960:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.089960:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.089961:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.089963:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.089963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.089966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.089969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.089974:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.089976:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.089977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.089977:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677475.089978:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.089978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677475.089978:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.089980:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677475.089980:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.089983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.090037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.090042:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.090045:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.090045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.090047:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.090049:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.090051:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.090052:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.090053:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.090054:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.090055:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.090058:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.090060:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.090063:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.090063:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.090064:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.090066:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.090066:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.090067:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.090067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.090072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.090079:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.090081:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.090082:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.090084:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.090085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.090086:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.090092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.090099:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.090102:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.090103:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.090103:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.090104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.090105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.090202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677475.090209:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.090212:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.090215:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.090216:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.090218:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.090219:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.090221:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.090222:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.090223:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.090224:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.090227:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.090229:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.090232:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.090232:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.090233:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.090235:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.090235:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.090236:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677475.090237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.090240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.090247:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.090249:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.090250:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.090250:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.090252:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.090254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.090261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.090268:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.090270:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.090271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.090271:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.090272:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.090272:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.090477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.090485:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.090488:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.090490:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.090492:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.090492:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.090493:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.090494:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.090495:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.090496:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.090497:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.090500:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.090501:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.090504:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.090505:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.090505:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.090507:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.090507:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.090509:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.090510:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.090510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.090513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.090517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.090520:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.090522:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.090523:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.090523:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.090525:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.090525:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.090527:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677475.090527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.090528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.090529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677475.090579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677475.090587:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.090589:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677475.090591:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.090594:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677475.090594:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.090595:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677475.090597:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.090598:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.090599:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.090600:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677475.090603:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.090605:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677475.090608:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.090608:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677475.090609:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677475.090610:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677475.090611:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.090612:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677475.090612:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677475.090617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.090625:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.090627:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.090628:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.090629:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.090631:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.090634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677475.090638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677475.090645:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.090648:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677475.090648:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.090649:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.090650:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.090650:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677475.090712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.090717:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.090720:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.090721:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.090722:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.090724:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.090726:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.090727:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.090728:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.090729:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.090730:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.090733:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.090735:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.090738:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.090739:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.090739:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.090741:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.090742:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.090742:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.090743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.090747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.090755:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.090757:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.090758:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.090760:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.090762:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.090763:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.090810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677475.090817:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677475.090820:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677475.090821:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677475.090823:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.090825:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.090827:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677475.090828:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.090830:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.090831:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.090832:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677475.090835:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.090837:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.090840:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.090840:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677475.090841:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677475.090843:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677475.090843:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677475.090844:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677475.090844:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:25.2:1619677475.090849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.090857:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.090859:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.090860:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.090862:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.090862:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.090864:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.090944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.090951:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.090953:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.090954:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.090956:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.090958:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.090960:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.090961:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.090963:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.090963:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.090965:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.090968:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.090970:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.090972:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.090973:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.090973:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.090975:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.090975:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.090976:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.090976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.090981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.090988:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.090990:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.090991:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.090993:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.090994:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.090995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.091427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.091435:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.091438:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.091440:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.091443:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.091443:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.091445:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.091446:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.091448:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.091449:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.091450:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.091454:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.091455:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.091459:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.091460:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.091460:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.091462:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.091463:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.091464:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.091464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.091469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.091477:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.091480:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.091480:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.091482:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.091483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.091485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.091806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.091814:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.091816:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.091817:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.091819:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677475.091820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677475.091822:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.091823:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.091824:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.091826:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.091827:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.091828:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677475.091828:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.091830:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677475.091831:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.091831:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677475.091833:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:8.0:1619677475.091833:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.091836:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677475.091836:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.091837:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.091837:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:8.0:1619677475.091837:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.091838:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96501637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677475.091839:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.091839:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677475.091839:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677475.091840:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677475.091840:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677475.091841:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677475.091841:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.091844:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.091846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677475.091846:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.091849:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.091849:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.091850:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.091851:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.091852:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.091853:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.091853:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.091854:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.091856:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:8.2:1619677475.091857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.091857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.091858:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.091860:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.091862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677475.091865:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.091868:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.091869:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.091869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.091871:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.091874:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.091980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.091985:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.091988:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.091989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.091991:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.091993:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.091994:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.2:1619677475.091995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:6.0:1619677475.091995:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.091997:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.091998:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.091999:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.092002:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:6.0:1619677475.092002:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.092004:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.092005:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:6.0:1619677475.092007:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.092008:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.092008:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.092009:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.092009:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.092010:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.092010:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677475.092011:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.092011:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.092012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:30.0:1619677475.092013:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.092014:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.092016:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.092016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.092017:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.092018:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.092021:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.092023:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.092024:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677475.092025:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.092026:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.092026:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.092026:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.092027:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.092028:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677475.092028:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.092029:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677475.092029:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.092030:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.092030:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.0:1619677475.092030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677475.092034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.092040:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.092043:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.092044:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.092044:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.092046:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.092048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.092139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.092146:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.092149:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.092151:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.092153:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.092154:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.092155:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.092156:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.092157:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.092158:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.092159:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.092163:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.092165:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.092166:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.092167:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.092168:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.092169:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.092170:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.092170:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.092171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.092175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.092181:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.092184:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.092184:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.092186:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.092188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.092189:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.092204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677475.092210:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.092212:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677475.092214:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.092215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677475.092217:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.092218:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677475.092220:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.092221:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.092222:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.092223:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677475.092226:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.092227:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.092230:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.092230:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.092231:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.092233:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.092233:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.092234:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677475.092235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677475.092239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.092246:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.092248:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.092249:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.092251:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.092252:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.092253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.092259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.092267:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.092269:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.092270:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.092271:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.092272:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.092273:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677475.092371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.092377:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.092380:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.092381:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.092382:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.092385:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.092387:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.092388:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.092389:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.092390:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.092391:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.092394:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.092395:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.092398:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677475.092399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:1.0:1619677475.092399:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.092399:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.092401:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.092402:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.092402:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.092403:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.0:1619677475.092405:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.092407:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:25.2:1619677475.092408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:21.0:1619677475.092410:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.092410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.092412:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.092413:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:22.0:1619677475.092415:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:21.0:1619677475.092415:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.092416:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.092417:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.092418:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:21.0:1619677475.092418:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677475.092419:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.092419:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.092422:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.092422:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677475.092424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677475.092425:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:25.2:1619677475.092428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:21.0:1619677475.092428:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.092428:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.092429:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.092431:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.092431:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.092432:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.092433:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677475.092435:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.092438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.092438:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.092439:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.092440:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.092440:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:9.2:1619677475.092447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.092454:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.092456:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.092457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.092458:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.092460:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.092462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.092524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.092527:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.092529:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.092531:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.092532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.092534:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.092535:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.092536:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.092538:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.092539:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.092540:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.092543:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.092545:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.092547:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.092548:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.092548:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.092550:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.092551:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.092552:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.092553:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.092554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.092556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.092559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.092564:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677475.092564:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.092566:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677475.092566:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677475.092567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677475.092567:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.092568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677475.092568:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.092569:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.092572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.092575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.092581:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.092583:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.092585:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.092586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.092588:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.092589:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.092591:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.092592:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.092593:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.092594:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677475.092597:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.092598:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.092601:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.092602:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.092602:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.092604:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.092605:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.092606:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.092606:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.092610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.092619:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.092621:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.092622:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.092623:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.092625:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.092627:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.092715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.092723:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.092726:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.092728:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.092730:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.092730:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.092732:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.092733:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.092734:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.092735:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.092736:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.092740:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.092742:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.092744:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.092745:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.092745:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.092747:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.092747:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.092749:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.092750:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.092751:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.092752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.092757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.092761:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.092763:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.092764:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.092765:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.092765:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.092766:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.092766:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.092767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.092768:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.092769:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.092770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.092777:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.092780:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.092781:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.092782:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.092782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.092783:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.092848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.092854:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.092857:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.092857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.092860:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.092862:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.092863:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.092864:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.092865:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.092866:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.092867:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.092870:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.092872:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.092874:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.092875:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.092876:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.092877:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.092878:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.092879:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.092880:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.092880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.092883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.092885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.092890:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.092890:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.092892:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.092893:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.092893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.092894:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.092894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.092895:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.092897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.092898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.092902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.092909:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.092912:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.092913:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.092913:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.092914:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.092915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.093115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.093123:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.093126:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.093126:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677475.093129:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.093131:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.093132:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.093134:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.093135:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.093136:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.093137:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.093140:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.093142:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.093145:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.093145:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.093146:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.093148:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.093148:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.093149:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.093149:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.093154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.093161:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.093164:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.093165:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.093165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.093167:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.093170:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.093721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.093726:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.093729:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.093730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.093732:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.093733:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.093735:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.093736:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.093737:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.093738:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.093739:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.093743:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.093744:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.093747:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.093747:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.093748:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.093750:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.093750:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.2:1619677475.093751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677475.093751:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.093751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.093756:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677475.093758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.093759:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.093761:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.093763:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.093763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.093764:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:21.0:1619677475.093765:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.0:1619677475.093765:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.093766:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.093766:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.093768:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.093768:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.093769:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.093770:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.093772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677475.093773:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.093773:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677475.093775:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.093777:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.093778:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.093779:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.093780:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.093781:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.093782:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.093783:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.093787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.093795:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.093798:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.093799:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.093799:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.093801:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.093803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.094030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.094038:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.094041:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.094043:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.094046:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.094046:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.094047:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.094048:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.094050:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.094051:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.094052:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.094055:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.094057:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.094060:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.094060:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.094061:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.094063:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.094064:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.094065:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.094065:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.094069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.094075:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.094078:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.094079:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.094079:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.094081:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.094083:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.094087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.094093:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.094096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.094096:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.094097:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.094097:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.094098:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.094532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.094539:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.094542:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.094544:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.094546:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.094547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.094547:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.094549:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.094550:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.094551:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.094551:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.094554:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.094556:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.094558:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.094559:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.094559:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.094560:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.094561:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.094562:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.094562:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.094567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.094574:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.094576:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.094577:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.094578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.094579:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.094581:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.094587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.094592:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.094595:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.094596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.094596:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.094597:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.094598:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.094798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.094804:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.094807:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.094808:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.094810:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.094812:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.094813:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.094815:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.094816:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.094817:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.094818:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.094821:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.094823:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.094825:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.094826:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.094826:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.094828:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.094828:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.094829:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.094829:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.094834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.094842:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.094844:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.094845:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.094847:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.094848:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.094849:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.095340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.095347:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.095350:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.095351:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.095353:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.095356:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.095357:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.095358:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.095360:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.095361:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.095362:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.095365:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.095366:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.095369:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.095370:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.095371:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.095372:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.095373:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.095373:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.095374:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.095379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.095387:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.095389:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.095390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.095391:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.095393:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.095395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.095631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.095636:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.095639:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.095641:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.095641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.095643:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.095645:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.095646:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.095647:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.095648:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.095649:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.095653:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.095654:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.095657:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.095657:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.095658:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.095660:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.095660:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.095661:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.095662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:21.2:1619677475.095664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:25.2:1619677475.095666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.095671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.095673:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.2:1619677475.095674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:22.0:1619677475.095675:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.095675:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.095676:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.095677:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.095678:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.095679:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.095679:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.095681:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.095682:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:22.0:1619677475.095682:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.095683:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.095684:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.095685:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.095688:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.095689:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677475.095689:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.095691:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.095692:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:22.0:1619677475.095692:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.095692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.095693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677475.095694:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.095694:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677475.095695:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.095695:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.095696:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.095696:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.095696:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.095696:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.095697:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:6.0:1619677475.095697:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.095698:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.095698:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.095699:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.095699:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.095699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.095700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677475.095700:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677475.095701:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.095703:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:25.2:1619677475.095704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.095705:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.095706:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.095707:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.095708:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.095709:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.095710:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.095710:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.0:1619677475.095712:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.095714:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.095715:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.095715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677475.095716:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.095717:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.095718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677475.095722:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.095725:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.095726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.095726:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.095728:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.095730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.096162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.096169:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.096172:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.096174:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.096176:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.096177:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.096178:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.096179:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.096181:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.096182:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.096183:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.096186:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.096187:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.096190:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.096191:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.096192:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.096194:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.096194:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.096195:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.096196:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.096200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.096208:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.096211:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.096212:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.096213:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.096214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.096217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.096572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.096578:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.096581:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.096582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.096584:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.096586:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.096587:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.096589:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.096590:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.096591:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.096592:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.096596:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.096597:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.096600:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.096600:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.096601:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.096603:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.096603:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.096604:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.096605:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.096605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.096609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.096611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.096616:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.096617:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.096618:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.096618:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.096619:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.096619:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.096620:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.096622:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.096623:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.096624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.096635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.096642:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.096645:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.096646:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.096646:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.096647:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.096648:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.096697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.096704:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.096707:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.096708:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.096710:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.096713:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.096715:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.096716:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.096717:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.096718:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.096719:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.096722:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.096724:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.096727:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.096728:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.096728:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.096730:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.096731:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.096731:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.096732:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.096736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.096744:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.096746:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.096747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.096748:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.096750:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.096752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.096756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.096764:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:21.2:1619677475.096765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677475.096767:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.096769:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677475.096772:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.096772:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:31.0:1619677475.096773:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:13.0:1619677475.096773:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677475.096774:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.096776:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:13.0:1619677475.096776:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.096778:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.096779:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677475.096779:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.096781:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.096782:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.096782:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.096783:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.096785:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677475.096785:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:31.0:1619677475.096786:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.096787:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.096788:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:13.0:1619677475.096788:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.096789:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:13.0:1619677475.096789:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:31.0:1619677475.096791:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.096791:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677475.096792:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.096792:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:31.0:1619677475.096793:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:13.0:1619677475.096793:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:31.0:1619677475.096794:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.096795:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.096795:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.096797:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.096797:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.096798:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.096799:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.096803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:9.2:1619677475.096808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.096809:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.096812:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.096812:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.096814:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.096816:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.096817:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:2.0:1619677475.096817:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.096821:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:4.2:1619677475.096822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:2.0:1619677475.096823:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.096824:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.096827:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.096829:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.096831:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:2.0:1619677475.096831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677475.096832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.096834:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.096837:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.096838:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.096839:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.096841:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.096842:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.096843:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.096846:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.096848:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.096850:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.096851:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.096851:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.096853:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.096853:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.096855:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.096856:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.096856:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.096859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.096862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677475.096865:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.096867:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.096868:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.096869:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:12.0:1619677475.096870:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.096871:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:13.0:1619677475.096871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.096872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677475.096872:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677475.096873:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:5.2:1619677475.097022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.097032:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.097035:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.097035:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.097037:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.097039:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.097041:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.097042:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.097044:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.097044:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.097046:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.097049:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.097051:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.097053:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.097054:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.097055:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.097056:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.097057:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.097058:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.097059:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.097059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.097062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.097065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677475.097070:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677475.097070:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677475.097072:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677475.097072:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677475.097073:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.097073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.097073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.097074:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677475.097076:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.097078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.097098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.097107:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.097111:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.097112:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.097115:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.097119:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.097121:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.097123:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.097126:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.097128:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.097130:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.097135:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.097138:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.097141:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.097142:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.097143:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.097143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:18.0:1619677475.097145:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.097146:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.097148:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.097149:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677475.097150:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.2:1619677475.097152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677475.097154:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.097154:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677475.097156:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.097159:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.097160:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677475.097161:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:10.0:1619677475.097161:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.097163:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:10.0:1619677475.097163:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.097164:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.097164:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.097164:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:10.0:1619677475.097165:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677475.097167:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:10.0:1619677475.097168:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.097170:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677475.097170:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677475.097172:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.097173:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677475.097173:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.2:1619677475.097173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677475.097175:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.097175:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.097176:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.097178:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.097178:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677475.097180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677475.097180:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.097183:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.097184:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.097184:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.097185:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.097186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.097187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.097190:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.097192:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.097193:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.097195:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.097195:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.097195:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.097196:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.097197:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.097198:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677475.097198:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.097356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.097360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:30.0:1619677475.097364:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.097366:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:30.0:1619677475.097367:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.097369:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:30.0:1619677475.097371:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.097371:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:31.0:1619677475.097372:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.097372:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677475.097373:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.097374:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:21.0:1619677475.097375:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.097376:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.097377:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.097378:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.097378:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.097380:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.097380:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.097381:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:21.0:1619677475.097381:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.097383:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.097387:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.097388:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677475.097390:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.097391:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.097392:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.097393:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.0:1619677475.097393:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.097394:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677475.097394:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.097395:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.0:1619677475.097395:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.097396:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.097397:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.097397:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:30.0:1619677475.097398:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.097398:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:30.0:1619677475.097399:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.097400:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.097400:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.097400:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.097401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:25.2:1619677475.097405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.097406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677475.097406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.097410:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:31.0:1619677475.097411:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.097412:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:20.0:1619677475.097412:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:31.0:1619677475.097413:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.097413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.097413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677475.097414:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677475.097415:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.097415:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.097415:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.097416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677475.097416:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.097417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677475.097418:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.097418:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.097419:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.097420:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677475.097420:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.097422:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.097633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.097638:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.097641:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.097641:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.097643:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.097645:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.097647:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.097648:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.097649:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.097650:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.097651:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.097655:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.097656:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.097659:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.097660:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.097661:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.097662:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.097663:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.097664:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.097664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.097668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.097675:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.097677:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.097678:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.097680:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.097681:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.097682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.097820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.097828:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.097831:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.097833:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.097835:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.097836:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.097837:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.097838:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.097840:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.097841:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.097842:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.097845:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.097846:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.097849:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.097850:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.097850:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.097852:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.097852:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.097854:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.097855:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.097856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.097858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.097862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.097865:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.097867:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.097868:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.097869:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.097870:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.097870:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.097872:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677475.097872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.097873:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.097874:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.097987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.097988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677475.097995:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.097997:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677475.097998:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677475.097999:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:31.0:1619677475.098000:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.098000:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677475.098002:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.098002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677475.098002:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.098004:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.098004:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.098005:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.098006:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.098007:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.098007:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.098008:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.098008:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.098009:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.098009:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.098010:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.098012:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.098014:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.098014:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.098017:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.098017:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.098017:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.098018:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.098019:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.098020:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.098020:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.098021:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677475.098021:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.098021:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.098022:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677475.098023:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.098023:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.098024:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.098025:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:9.2:1619677475.098025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.098026:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.098029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.098031:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:4.2:1619677475.098032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677475.098034:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.098035:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.098035:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.098037:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.098037:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.098039:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677475.098039:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677475.098040:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.098040:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677475.098041:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677475.098041:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.098042:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.098043:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.098043:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.098045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.098432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.098438:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.098440:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.098443:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.098443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.098445:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.098447:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.098448:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.098449:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.098451:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.098452:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.098455:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.098457:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.098459:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.098460:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.098461:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.098462:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.098463:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.098463:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.098464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.098469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.098477:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.098480:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.098481:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.098481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.098483:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.098485:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.098491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.098498:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.098500:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.098501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.098502:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.098503:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.098503:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.098519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.098525:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.098527:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.098528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.098530:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.098532:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.098533:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.098534:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.098536:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.098537:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.098538:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.098541:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.098543:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.098545:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.098546:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.098547:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.098548:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.098549:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.098549:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.098550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.098555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.098562:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.098565:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.098566:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.098567:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.098568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.098569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.098865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.098871:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.098874:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.098875:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.098877:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.098879:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.098881:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.098882:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.098883:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.098884:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.098885:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.098888:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.098890:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.098892:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.098892:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.098893:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.098894:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.098895:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.098896:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.098897:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.098898:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.098901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.098903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.098909:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.098910:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.098911:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.098911:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.098911:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.098912:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.098913:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.098913:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.098914:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677475.098914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677475.099077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677475.099083:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.099086:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677475.099086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677475.099088:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677475.099091:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.099092:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677475.099093:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.099095:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.099096:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.099097:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.099100:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.099102:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.099105:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.099106:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677475.099106:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677475.099108:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677475.099108:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.099109:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677475.099110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677475.099125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.099131:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.099134:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.099135:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.099135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.099137:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.099140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.099483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.099489:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.099491:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.099492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.099494:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.099496:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.099497:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.099499:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.099500:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.099501:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.099502:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.099505:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.099507:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.099510:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.099510:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.099511:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.099513:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.099513:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.099514:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.099514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.099519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.099528:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.099530:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.099531:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.099533:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.099534:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.099535:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.099804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.099812:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677475.099815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:8.0:1619677475.099815:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.099816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.099818:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.099820:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.099821:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000400:00000200:8.0:1619677475.099822:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.099823:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.099824:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677475.099825:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:8.0:1619677475.099825:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.099826:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.099828:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677475.099828:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677475.099831:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.099831:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.099832:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:8.0:1619677475.099833:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677475.099833:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.099835:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.099836:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.099837:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.099837:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:1.0:1619677475.099837:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.099838:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.099840:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:1.0:1619677475.099840:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677475.099842:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.099842:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.099843:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.099843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677475.099845:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677475.099846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.099846:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677475.099847:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.099848:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.099849:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.099850:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677475.099850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:22.0:1619677475.099854:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:9.2:1619677475.099854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677475.099856:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.099857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.099858:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.099860:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:22.0:1619677475.099860:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.099862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677475.099863:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.099864:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.099864:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.099866:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.099868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.099874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.099880:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.099883:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.099883:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.099884:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.099885:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.099885:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.100066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.100073:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.100076:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.100077:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.100079:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.100081:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.100082:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.100084:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.100085:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.100086:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.100087:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.100090:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.100092:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.100095:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.100095:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.100096:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.100097:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.100098:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.100099:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.100100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677475.100104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.100112:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.100114:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.100115:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.100115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:1.0:1619677475.100115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.100118:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.100120:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677475.100122:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.100125:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677475.100126:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677475.100127:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.100130:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.100132:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677475.100133:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.100134:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.100135:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.100136:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677475.100139:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.100141:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677475.100143:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.100144:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677475.100144:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677475.100146:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677475.100147:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.100148:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677475.100149:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677475.100150:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:25.2:1619677475.100152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.100155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:30.0:1619677475.100159:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.100162:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:8.0:1619677475.100162:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000400:00000200:30.0:1619677475.100163:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.100164:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.100164:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677475.100165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677475.100166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:31.0:1619677475.100167:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.100168:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.100171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.100178:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.100181:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.100182:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.100182:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.100183:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.100183:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.100228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.100235:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.100238:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.100240:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.100242:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.100242:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.100243:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.100245:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.100246:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.100247:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.100248:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.100251:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.100252:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.100255:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.100256:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.100256:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.100258:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.100258:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.100259:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677475.100259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.100262:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677475.100263:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.100265:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.100267:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.100267:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.100268:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.100270:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.100271:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.100272:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.100272:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.100273:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:9.2:1619677475.100274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:31.0:1619677475.100276:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.100278:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.100279:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.100280:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.100280:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:31.0:1619677475.100281:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.100282:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.100283:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.100283:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677475.100283:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:31.0:1619677475.100284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:24.0:1619677475.100284:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.100285:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.100286:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:9.2:1619677475.100288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.100290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677475.100292:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.100294:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.100299:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.100300:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.100301:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.100303:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.100471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.100475:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.100477:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.100479:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.100480:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.100481:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.100483:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.100484:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.100486:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.100486:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.100488:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.100491:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.100492:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.100495:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.100496:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.100496:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.100498:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.100499:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.100500:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.100500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.100505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.100513:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.100515:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.100516:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.100517:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.100518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.100519:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.100584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.100590:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.100593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.100593:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.100595:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.100597:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.100599:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.100600:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.100601:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.100602:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.100603:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.100607:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.100609:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.100611:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.100611:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.100612:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.100614:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.100614:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.100615:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.100616:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.100617:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.100620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.100622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.100627:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.100628:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.100629:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.100629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.100630:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.100630:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.100631:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.100632:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.100634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.100635:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.100641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.100648:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.100650:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.100651:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.100652:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.100653:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.100654:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.101028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.101035:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.101038:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.101039:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.101041:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.101043:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.101045:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.101046:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.101047:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.101048:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.101049:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.101053:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.101054:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.101057:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.101058:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.101059:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.101061:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.101061:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.101062:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.101064:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.101064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.101067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.101070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.101074:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.101077:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.101077:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.101078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.101078:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.101079:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.101080:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677475.101080:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.101081:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677475.101083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.101086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.101093:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.101095:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.101096:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.101096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.101097:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.101098:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.101292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.101301:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.101305:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.101307:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.101309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.101310:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.101311:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.101312:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.101314:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.101315:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.101316:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.101319:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.101321:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.101322:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.101323:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.101324:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.101325:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.101326:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.101327:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.101327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.101332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.101339:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.101342:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.101342:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.101344:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.101346:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.101347:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.101461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.101467:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.101470:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.101471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.101472:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.101474:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.101476:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.101477:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.101478:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.101479:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.101480:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.101483:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.101484:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.101487:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.101488:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.101488:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.101490:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.101490:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.101491:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.101491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.101496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.101504:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.101506:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.101507:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.101508:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.101509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.101511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.101593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.101601:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.101604:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.101605:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.101607:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.101610:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.101611:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.101612:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.101614:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.101615:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.101616:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.101619:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.101621:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.101624:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.101625:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.101625:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.101627:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.101628:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.101628:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.101629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.101633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.101641:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.101644:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.101644:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.101645:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.101647:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.101649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.101734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.101740:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.101743:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.101744:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.101746:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.101748:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.101749:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.101750:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.101752:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.101753:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.101754:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.101757:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.101759:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.101761:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.101762:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.101763:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.101764:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.101765:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.101766:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.101766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.101770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.101778:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.101780:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.101781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.101781:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.101783:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.101785:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.101790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.101796:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.101799:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.101800:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.101800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.101801:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.101801:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.101831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.101839:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.101842:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.101844:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.101846:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.101846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.101847:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.101849:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.101850:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.101851:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.101852:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.101856:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.101857:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.101860:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.101860:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.101861:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.101863:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.101864:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.101865:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.101866:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.101867:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.101869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.101873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.101877:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.101879:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.101880:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.101881:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.101881:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.101882:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.101882:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.101883:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.101884:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.101885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.101886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.101893:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.101896:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.101896:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.101897:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.101898:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.101898:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.102082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.102086:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.102089:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.102091:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.102092:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.102093:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.102095:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677475.102096:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.102097:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.102098:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.102099:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.102103:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.102104:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.102107:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.102107:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.102108:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.102109:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677475.102110:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.102111:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.102111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.102116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.102124:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.102127:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.102128:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.102128:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.102130:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677475.102133:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.102135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.102143:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.102145:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.102146:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.102146:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.102147:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.102148:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.102224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.102232:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.102235:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.102237:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.102239:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.102239:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.102240:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.102241:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.102243:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.102244:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.102245:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.102248:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.102249:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.102252:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.102253:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.102253:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.102255:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.102256:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.102257:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.102257:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.102261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.102268:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.102271:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.102272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.102272:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.102274:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.102276:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.102341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.102346:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.102349:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.102350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.102352:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.102354:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.102355:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.102357:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.102358:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.102359:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.102360:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.102363:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.102365:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.102368:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.102368:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.102369:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.102371:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.102371:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.102372:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.102372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.102377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.102386:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.102388:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.102389:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.102390:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.102391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.102392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.102694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.102700:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.102703:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.102704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.102706:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.102708:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.102709:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.102711:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.102712:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.102714:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.102714:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.102718:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.102720:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.102723:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.102724:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.102724:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.102726:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.102727:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.102728:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.102729:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.102729:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.102732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677475.102735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677475.102739:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:2.0:1619677475.102740:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677475.102742:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:2.0:1619677475.102742:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:30.0:1619677475.102743:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.102743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.102744:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677475.102745:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.102747:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.102749:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.103032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.103040:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.103043:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.103045:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.103047:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.103047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.103049:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.103050:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.103051:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.103052:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.103053:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.103056:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.103057:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.103061:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.103061:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.103062:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.103064:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.103065:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.103065:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.103066:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.103071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.103080:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.103082:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.103083:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.103085:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.103086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.103088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.103393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.103399:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.103402:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.103402:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.103404:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.103406:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.103408:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.103409:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.103410:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.103411:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.103412:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.103415:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.103417:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.103419:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.103420:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.103420:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.103422:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.103423:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.103424:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.103424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.103429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.103437:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.103439:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.103440:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.103442:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.103443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.103444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.103458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.103465:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.103468:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.103468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.103470:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.103473:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.103474:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.103475:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.103477:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.103478:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.103479:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.103482:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.103483:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.103486:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.103487:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.103487:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.103489:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.103490:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.103491:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.103492:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.103493:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.103496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.103498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677475.103503:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.103503:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.103505:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677475.103506:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677475.103506:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677475.103507:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.103507:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677475.103508:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.103510:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.103511:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.103726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.103734:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.103737:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.103738:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.103740:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.103743:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.103744:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.103745:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.103747:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.103748:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.103749:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.103752:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.103754:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.103756:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.103757:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.103758:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.103759:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.103760:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.103761:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.103761:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.103766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.103773:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.103776:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.103777:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.103777:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.103779:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.103782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.103818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.103826:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.103829:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.103831:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.103833:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.103834:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.103835:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.103837:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.103838:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.103839:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.103840:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.103842:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.103844:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.103846:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.103847:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.103848:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.103849:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.103850:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.103851:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.103851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.103855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.103861:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.103864:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.103865:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.103865:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.103867:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.103869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.104269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.104274:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.104277:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.104277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.104279:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.104281:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.104283:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.104284:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.104285:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.104287:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.104287:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.104291:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.104292:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.104299:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.104300:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.104300:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.104302:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.104303:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.104304:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.104305:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.104305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.104309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.104311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.104316:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.104318:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.104318:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.104319:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.104319:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.104321:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.104321:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.104323:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.104324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.104325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.104332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.104339:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.104342:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.104343:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.104343:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.104344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.104345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.104523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.104526:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.104528:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.104530:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.104531:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.104532:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.104534:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.104535:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.104536:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.104537:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.104538:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.104541:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.104543:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.104545:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.104546:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.104546:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.104548:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.104548:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.104549:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.104550:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.104551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.104554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.104557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.104561:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.104561:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.104562:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.104563:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677475.104563:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.104564:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.104564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.104565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.104566:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.104567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677475.105132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.2:1619677475.105136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:26.0:1619677475.105140:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:22.0:1619677475.105141:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677475.105143:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:22.0:1619677475.105144:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:10.0:1619677475.105144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:6.0:1619677475.105144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677475.105147:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:22.0:1619677475.105147:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677475.105148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677475.105150:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.105150:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.105152:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677475.105152:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677475.105154:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.105154:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677475.105156:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677475.105157:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.105157:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.105158:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.105158:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:26.0:1619677475.105159:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.105159:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677475.105159:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677475.105160:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677475.105163:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.105165:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.105166:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:8.0:1619677475.105166:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.105167:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677475.105168:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677475.105169:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:8.0:1619677475.105169:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96501569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.105171:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677475.105171:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677475.105172:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.105172:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:8.0:1619677475.105172:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677475.105173:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677475.105173:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677475.105174:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:8.0:1619677475.105174:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.105175:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677475.105176:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.105176:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.105177:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:8.0:1619677475.105177:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.105178:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:22.0:1619677475.105179:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.105179:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677475.105179:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.105180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.105181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677475.105181:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.105182:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.105183:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677475.105184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677475.105185:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.105185:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.105186:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.105186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677475.105188:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.105190:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:25.2:1619677475.105191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677475.105191:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.105193:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.105193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:13.0:1619677475.105194:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:21.0:1619677475.105195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:13.0:1619677475.105197:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.105198:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.105198:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:13.0:1619677475.105201:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.105201:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.105202:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:12.0:1619677475.105202:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:6.0:1619677475.105203:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.105204:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:6.0:1619677475.105205:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.105208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.105226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.105234:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.105237:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.105239:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.105241:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.105241:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.105243:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.105244:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.105245:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.105246:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.105247:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.105250:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.105252:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.105254:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.105255:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.105256:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.105257:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.105258:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.105259:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.105259:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.105263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.105269:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.105272:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.105273:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.105273:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.105274:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.105277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.105771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.105779:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.105782:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.105784:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.105786:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.105786:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.105788:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.105789:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.105790:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.105791:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.105792:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.105795:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.105796:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.105798:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.105799:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.105800:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.105801:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.105802:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.105803:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.105805:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.105805:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.105807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.105812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.105815:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.105817:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.105818:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.105818:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.105820:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.105820:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.105822:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677475.105822:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.105823:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.105824:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.105826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.105832:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.105834:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.105835:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.105835:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.105836:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.105836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.105921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.105929:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.105932:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.105934:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.105936:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.105936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.105937:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.105939:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.105940:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.105941:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.105942:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.105944:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.105946:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.105948:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.105949:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.105949:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.105951:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.105951:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.105952:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.105953:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.105956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.105963:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.105965:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.105966:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.105966:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.105968:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.105970:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.105985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.105991:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.105993:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.105994:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.105994:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.105995:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.105995:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.106176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.106181:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.106184:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.106185:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.106187:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.106189:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.106190:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.106192:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.106193:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.106194:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.106195:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.106198:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.106200:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.106202:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.106203:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.106204:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.106205:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.106205:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.106206:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.106207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.106211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.106219:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.106221:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.106222:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.106223:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.106225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.106226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.106335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.106343:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.106346:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.106349:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.106351:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.106351:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.106352:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.106354:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.106355:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.106356:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.106357:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.106360:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.106361:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.106364:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.106365:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.106365:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.106367:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.106367:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.106369:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.106371:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.106371:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.106373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.106377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.106380:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.106382:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.106383:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.106383:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.106385:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.106385:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.106387:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.106388:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.106389:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.106390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.106508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.106515:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.106518:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.106519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.106521:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.106523:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.106525:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.106526:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.106527:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.106528:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.106529:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.106532:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.106533:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.106536:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.106537:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.106537:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.106539:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.106539:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.106540:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.106542:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.106542:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.106545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.106548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.106553:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677475.106555:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.106555:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.106556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.106557:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677475.106557:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.106558:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.106558:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.106559:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.106562:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.106646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.106653:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.106655:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.106657:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.106658:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.106661:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.106663:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677475.106664:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.106665:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.106666:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.106667:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677475.106670:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.106672:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677475.106675:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.106676:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677475.106676:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677475.106678:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677475.106679:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.106679:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.106680:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677475.106684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.106692:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.106695:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.106696:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.106697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.106698:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.106702:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.106760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.106768:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.106771:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.106773:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.106775:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.106775:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.106776:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.106778:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.106779:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.106780:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.106781:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.106784:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.106786:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.106787:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.106788:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.106789:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.106790:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.106791:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.106792:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.106792:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.106796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.106803:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.106805:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.106806:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.106808:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.106809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.106810:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.106856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70a800] (20)++
00000800:00000200:21.0:1619677475.106863:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70a800] (21)++
00000800:00000200:21.0:1619677475.106865:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.14@o2ib3
00000400:00000200:21.0:1619677475.106867:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.3.14@o2ib3 : GET - for me
00000800:00000200:20.0:1619677475.106868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70a800] (22)--
00000400:00000200:21.0:1619677475.106871:0:19935:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.3.14@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:21.0:1619677475.106873:0:19935:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.3.14@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:21.0:1619677475.106874:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae70a800] (21)++
00000800:00000200:21.0:1619677475.106876:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70a800] (22)++
00000800:00000200:21.0:1619677475.106878:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae70a800] (23)--
00000400:00000200:21.0:1619677475.106879:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.3.14@o2ib3 local destination
00000400:00000200:21.0:1619677475.106883:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.3.14@o2ib3(10.51.3.14@o2ib3:10.51.3.14@o2ib3) : REPLY try# 0
00000800:00000200:4.2:1619677475.106883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70a800] (22)++
00000800:00000200:21.0:1619677475.106885:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.3.14@o2ib3
00000800:00000200:20.0:1619677475.106886:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70a800] (23)++
00000800:00000200:20.0:1619677475.106887:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70a800] (24)--
00000800:00000200:21.0:1619677475.106888:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9197c81a8780] -> 10.51.3.14@o2ib3 (2) version: 12
00000800:00000200:20.0:1619677475.106888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70a800] (23)--
00000800:00000200:21.0:1619677475.106889:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197ae70a800] (22)++
00000800:00000200:21.0:1619677475.106889:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae70a800] (22)++
00000800:00000200:20.0:1619677475.106889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70a800] (22)--
00000800:00000200:21.0:1619677475.106891:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197ae70a800] (23)--
00000800:00000200:21.0:1619677475.106891:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70a800] (22)--
00000800:00000200:4.2:1619677475.106896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae70a800] (21)++
00000800:00000200:20.0:1619677475.106900:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae70a800] (22)++
00000800:00000200:20.0:1619677475.106901:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae70a800] (23)--
00000400:00000200:20.0:1619677475.106902:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.106904:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.3.14@o2ib3: REPLY: OK
00000800:00000200:21.0:1619677475.106905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70a800] (22)--
00000800:00000200:20.0:1619677475.106906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae70a800] (21)--
00000800:00000200:5.2:1619677475.106970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.106976:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.106979:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.106980:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.106981:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.106984:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.106985:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.106986:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.106988:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.106989:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.106990:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.106993:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.106995:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.106997:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.106998:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.106998:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.107000:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.107001:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.107001:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.107002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.107006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.107014:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.107017:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.107017:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.107018:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.107020:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.107022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.107050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.107055:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.2:1619677475.107056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677475.107058:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.107059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.107060:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.107061:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677475.107062:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.107063:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.0:1619677475.107064:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677475.107065:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.107066:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.107066:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677475.107066:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.107067:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.107068:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.107068:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.107072:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.107074:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677475.107074:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:21.0:1619677475.107075:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.107076:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.107077:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.107078:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.107078:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.107078:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677475.107079:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.107080:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.107080:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.107081:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.107081:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.107082:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.107084:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.2:1619677475.107086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.107087:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.107088:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.107088:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.107090:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.107090:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.107091:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.107092:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677475.107093:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.107095:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.107096:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.107096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:21.0:1619677475.107097:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.107099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.107100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677475.107104:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.107106:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.107107:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.107108:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.107110:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.107112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.107128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.107133:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.107136:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.107138:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.107138:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.107140:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.107142:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.107143:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.107144:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.107145:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.107146:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.107149:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.107151:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.107152:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.107153:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.107154:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.107155:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.107155:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.107156:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.107157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.107161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.107169:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.107171:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.107172:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.107172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.107174:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.107177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.107369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.107378:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.107380:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.107382:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.107384:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.107385:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.107385:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.107387:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.107388:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.107389:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.107390:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.107393:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.107394:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.107397:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.107397:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.107398:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.107400:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.107400:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.107401:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.107401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.107406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.107413:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.107416:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.107417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.107417:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.107419:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.107421:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.107922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.107928:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.107931:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.107932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.107934:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.107936:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.107937:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.107938:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.107940:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.107941:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.107942:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.107945:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.107947:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.107949:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.107950:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.107950:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.107952:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.107952:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.107954:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.107955:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.107955:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.107958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.107960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.107965:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.107965:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.107967:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.107967:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.107968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.107968:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.107969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.107970:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.107971:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.107972:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.107979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.107986:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.107988:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.107989:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.107990:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.107991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.107992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.108314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.108321:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.108324:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.108325:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.108327:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.108329:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.108331:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.108332:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.108333:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.108334:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.108335:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.108338:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.108340:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.108342:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.108342:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.108343:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.108345:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.108345:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.108346:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.108347:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.108352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.108359:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.108361:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.108362:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.108362:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.108364:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.108365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.108604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.108613:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.108616:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.108617:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.108619:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.108622:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.108624:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.108625:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.108626:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.108627:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.108628:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.108632:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.108634:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.108637:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.108638:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.108638:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.108640:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.108640:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.108641:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.108641:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.108646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.108655:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.108657:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.108658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.108659:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.108661:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.108663:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.108665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:20.2:1619677475.108670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677475.108672:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.108675:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.108676:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.108676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.108677:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677475.108677:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677475.108678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677475.108680:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.108681:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.108683:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.108686:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.108687:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.108689:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.108690:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.108691:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.108692:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.108695:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.108697:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.108700:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.108701:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.108702:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.108704:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.108705:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.108706:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.108706:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.108710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.108719:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.108723:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.108724:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.108727:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.108729:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.108731:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.108731:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.108733:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.108733:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.108734:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.108735:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.108936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.108942:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.108944:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.108945:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.108947:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.108949:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.108951:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.108952:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.108953:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.108954:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.108955:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.108959:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.108961:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.108963:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.108964:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.108965:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.108966:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.108967:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.108968:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.108968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.108973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.108980:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.108982:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.108983:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.108984:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.108985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.108987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.109330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.109339:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.109342:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.109344:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.109346:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.109346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.109348:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.109349:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.109351:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.109351:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.109353:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.109356:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.109358:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.109360:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.109361:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.109362:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.109363:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.109364:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.109365:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.109365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.109370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.109377:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.109380:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.109381:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.109381:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.109383:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.109385:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.109798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.109804:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.109807:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.109808:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.109810:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.109812:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.109814:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.109815:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.109816:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.109817:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.109818:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.109822:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.109824:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.109826:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.109827:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.109827:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.109829:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.109830:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.109830:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.109831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.109835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.109843:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.109845:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.109846:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.109847:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.109849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.109850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.110053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.110061:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677475.110063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.110063:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.110064:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.110066:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.110069:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.110070:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677475.110070:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.110072:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.110073:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677475.110073:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.110075:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.110076:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.110076:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.110077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.110078:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.110079:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:8.0:1619677475.110080:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.110081:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.110082:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.110082:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.110083:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.110084:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677475.110084:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.110085:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.110086:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.110087:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.110088:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.110088:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677475.110089:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677475.110089:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677475.110090:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.110090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677475.110091:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.110092:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.110093:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.110094:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677475.110094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.110095:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.110095:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.110099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677475.110101:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.110104:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.110105:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:22.0:1619677475.110105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.110105:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.110107:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.110108:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.110109:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.110111:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.110111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.110112:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.110113:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.110386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.110392:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.110394:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.110396:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.110397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.110399:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.110401:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.110402:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.110403:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.110404:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.110405:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.110409:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.110410:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.110413:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.110413:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.110414:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.110416:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.110416:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.110417:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.110418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.110423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.110430:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.110433:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.110433:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.110435:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.110436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.110437:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.110443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.110451:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.110454:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.110455:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.110456:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.110456:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.110457:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.110582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.110587:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.110589:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.110591:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.110592:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.110593:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.110595:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.110596:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.110598:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.110598:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.110599:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677475.110603:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.110604:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.110607:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.110607:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.110608:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.110609:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.110610:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.110611:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.110611:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.110616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.110625:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.110628:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.110629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.110629:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.110631:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.110634:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.110643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.110648:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.110651:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.110652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.110654:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.110656:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.110657:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.110658:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.110660:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.110661:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.110662:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.110665:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.110666:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.110668:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.110669:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.110670:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.110671:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.110672:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.110673:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.110673:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.110678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.110685:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.110687:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.110688:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.110689:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.110690:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.110692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.110780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.110789:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.110792:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.110794:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.110796:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.110796:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.110797:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.110799:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.110800:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.110801:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.110802:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.110805:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.110807:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.110809:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.110810:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.110811:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.110812:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.110813:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.110814:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.110814:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.110818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.110824:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.110827:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.110828:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.110828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.110830:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.110832:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.111253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.111261:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.111264:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677475.111265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:31.0:1619677475.111266:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.111268:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.111269:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677475.111269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.111270:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.111270:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.111271:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.111272:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.111273:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.111274:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.111277:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.111278:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.111280:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.111280:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.111281:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.111281:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677475.111282:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.111283:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677475.111284:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.111285:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.111287:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.111288:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.111288:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.111289:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677475.111289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.111291:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.111292:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677475.111292:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.111293:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677475.111299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.111301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.111301:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.111303:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.111303:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.111304:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.111304:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.111304:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.111305:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:31.0:1619677475.111306:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.111306:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.111307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.111308:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.111308:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.111308:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.111309:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.111309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677475.111310:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.111311:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.111311:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:9.2:1619677475.111313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.111320:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.111322:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.111323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.111323:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.111325:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.111327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.111402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.111409:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.111412:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.111413:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.111415:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.111418:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.111419:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.111420:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.111421:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.111422:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.111423:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.111426:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.111428:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.111430:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.111431:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.111432:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.111434:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.111434:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.111435:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.111435:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.111440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.111448:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.111450:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.111451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.111452:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.111454:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.111456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.111502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.111507:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.111509:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.111510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.111512:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.111514:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.111516:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677475.111517:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.111518:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.111519:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.111520:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677475.111524:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.111526:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.111529:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.111529:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677475.111530:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677475.111532:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677475.111533:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.111533:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677475.111534:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677475.111535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677475.111538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.111540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:24.0:1619677475.111544:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677475.111545:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:24.0:1619677475.111547:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677475.111547:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000400:00000200:24.0:1619677475.111548:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.111548:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677475.111548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.111549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:24.0:1619677475.111551:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.111554:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.111640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.111645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.111648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.111648:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.111650:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.111653:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.111654:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.111655:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.111656:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.111657:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.111658:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.111661:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.111663:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.111665:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.111665:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.111666:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.111667:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.111668:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.111669:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.111670:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.111670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.111673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.111675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.111680:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.111682:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.111683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.111683:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.111684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.111685:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.111686:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.111687:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.111689:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.111690:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.111692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:4.2:1619677475.111693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.111700:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677475.111701:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.111703:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677475.111703:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.111704:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677475.111704:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.111705:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.111706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677475.111706:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.111707:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677475.111708:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.111710:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677475.111711:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.111713:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.111714:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.111715:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677475.111718:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.111720:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677475.111723:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.111723:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677475.111724:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677475.111726:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677475.111726:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.111728:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:5.2:1619677475.111728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677475.111729:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.111730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677475.111732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677475.111734:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677475.111735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677475.111737:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.111738:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.111739:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.111740:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677475.111742:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677475.111742:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.111743:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.111744:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.111744:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677475.111744:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677475.111745:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.111745:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677475.111745:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.111746:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.111746:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677475.111747:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677475.111747:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.111748:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677475.111749:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:17.0:1619677475.111752:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677475.111753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:17.0:1619677475.111754:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.111756:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.111757:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.111757:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.111759:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.111760:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.111760:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:2.0:1619677475.111760:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677475.111761:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677475.111763:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.111764:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.111764:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:25.2:1619677475.111765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677475.111765:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.111765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677475.111773:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.111776:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.111777:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.111777:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.111779:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.111781:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.111783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.111790:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.111792:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.111793:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.111793:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.111794:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.111795:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.111904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.111913:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.111916:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.111918:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.111920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.111920:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.111922:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.111923:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.111924:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.111925:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.111926:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.111930:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.111932:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.111934:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.111934:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.111935:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.111937:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.111937:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.111938:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.111939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.111942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.111949:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.111952:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.111952:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.111954:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.111955:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.111956:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.111963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.111969:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.111972:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.111973:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.111973:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.111974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.111975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.112076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.112079:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.112082:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.112084:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.112085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.112086:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.112088:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.112089:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.112090:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.112091:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.112092:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.112095:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.112097:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.112099:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.112100:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.112100:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.112102:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.112103:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.112104:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.112104:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.112108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.112116:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.112118:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.112119:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.112119:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.112121:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.112123:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.112519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.112525:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.112527:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.112528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.112530:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.112532:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.112533:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.112534:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.112536:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.112537:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.112538:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.112541:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.112543:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.112545:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.112546:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.112547:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.112548:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.112549:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.112550:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.112550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.112555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.112564:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.112566:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.112567:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.112568:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.112569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.112571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.112744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.112752:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.112755:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.112757:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.112759:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.112760:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677475.112760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.112762:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.112763:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.112764:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.112765:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.112768:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.112770:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.112773:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.112774:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.112774:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.112776:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.112776:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.112777:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.112778:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.112781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.112789:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.112791:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.112792:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.112793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.112794:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.112796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.112803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.112810:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.112812:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.112813:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.112813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.112814:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.112814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.112902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.112909:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.112912:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.112914:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.112916:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.112916:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.112917:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.112919:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.112920:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.112921:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.112922:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.112924:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.112926:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.112928:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.112929:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.112929:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.112931:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.112931:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.112933:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.112934:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.112935:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.112936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.112941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.2:1619677475.112942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.0:1619677475.112943:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.112945:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677475.112945:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.112946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.112946:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.112947:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677475.112949:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.112949:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.112950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.112950:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.112951:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677475.112951:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.112952:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:20.0:1619677475.112952:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.112953:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:20.0:1619677475.112954:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.112955:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677475.112955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:20.0:1619677475.112957:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.112958:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.112958:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:24.0:1619677475.112962:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677475.112962:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.112963:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677475.112964:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.112965:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.112965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.112966:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.112966:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.112967:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677475.112967:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.112967:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.112969:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.112970:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.112971:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.112971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.112975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.112984:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.112987:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.112988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.112988:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.112990:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.112992:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.113356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.113361:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.113364:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.113365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.113367:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.113369:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.113370:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.113372:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.113373:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.113374:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.113375:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.113378:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.113380:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.113382:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.113383:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.113383:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.113385:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.113385:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.113386:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.113387:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.113392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.113400:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.113402:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.113403:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.113405:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.113406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.113407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.113959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.113967:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.113969:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.113970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.113973:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.113975:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.113977:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.113978:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.113979:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.113980:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.113981:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.113985:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.113986:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.113989:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.113990:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.113990:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.113992:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.113992:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.113993:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.113994:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.113998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.114005:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.114008:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.114009:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.114009:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.114012:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.114014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.114351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.114356:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.114358:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.114359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.114361:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.114363:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.114364:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.114365:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.114366:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.114367:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.114368:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.114371:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.114372:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.114375:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.114375:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.114376:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.114378:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.114378:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.114378:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.114379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.114384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.114392:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.114395:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.114396:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.114397:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.114398:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.114400:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.114554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.114563:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.114567:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.114569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.114572:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.114575:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.114578:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.114580:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.114582:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.114584:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.114586:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.114591:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.114594:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.114598:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.114599:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.114600:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.114602:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.114603:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.114604:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.114605:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.114609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.114618:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.114622:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.114622:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.114623:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.114625:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.114627:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.114805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.114814:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.114816:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.114819:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.114821:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.114821:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.114823:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.114824:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.114825:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.114826:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.114827:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.114831:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.114833:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.2:1619677475.114834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.114835:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.114836:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.114837:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.114838:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.114839:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.114840:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.114841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677475.114842:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677475.114844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.114845:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.114846:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.114847:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.114849:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.114849:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.114849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.114851:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.114851:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.114851:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:31.0:1619677475.114852:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677475.114853:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.114854:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.114854:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.114855:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.114856:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.114857:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.114858:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.114858:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.114859:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.114859:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.114860:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.114861:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.114862:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.114862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.114866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.114874:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.114876:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.114877:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.114878:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.114880:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.114881:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.115012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.115017:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.115020:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.115021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.115023:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.115025:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.115026:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.115028:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.115029:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.115030:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.115031:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.115034:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.115035:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.115038:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.115039:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.115040:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.115041:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.115042:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.115043:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.115043:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.115047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.115055:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.115058:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.115059:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.115060:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.115061:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.115063:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.115138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.115145:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.115148:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.115149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.115151:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.115153:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.115155:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.115156:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.115157:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.115158:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.115159:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.115162:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.115164:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.115167:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.115168:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.115169:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.115170:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.115171:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.115171:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.2:1619677475.115171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:24.0:1619677475.115172:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.115176:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677475.115178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.115179:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.115181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.115182:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.115183:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.115184:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.115185:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.115185:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.115186:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.115187:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.115188:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.115189:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.115190:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.115191:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.115195:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.115195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.115196:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.115197:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.115197:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.115198:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.115199:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.115200:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.115200:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.115201:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.115201:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677475.115202:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.115204:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.115204:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.115205:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.115206:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.115207:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.115210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.115212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.115217:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.115217:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.115219:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.115219:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.115220:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.115220:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.115220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.115222:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.115223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.115224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.115230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.115237:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.115239:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.115240:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.115241:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.115242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.115243:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.115681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.115687:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.115690:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.115691:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.115693:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.115695:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.115696:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.115698:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.115699:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.115700:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.115701:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.115705:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.115706:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.115708:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.115709:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.115710:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.115711:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.115712:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.115713:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.115713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.115718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.115726:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.115728:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.115729:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.115729:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.115730:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.115732:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.115801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.115809:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.115812:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.115814:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.115816:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.115817:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.115817:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.115819:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.115820:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.115821:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.115822:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.115825:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.115827:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.115829:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.115830:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.115830:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.115832:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.115832:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.115834:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.115835:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.115836:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.115837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.115842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.115846:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.115848:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.115849:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.115850:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.115850:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.115850:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.115851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.115852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.115853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.115854:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.115952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677475.115959:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.115962:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.115964:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.115966:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.115966:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.115968:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.115969:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.115970:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.115971:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.115971:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.115974:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.115976:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.115978:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.115978:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.115979:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.115980:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.115981:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.115982:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677475.115983:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677475.115984:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677475.115986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.115989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:8.0:1619677475.115993:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.115996:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.115997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.115997:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.115998:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000400:00000200:8.0:1619677475.115999:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.116000:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677475.116000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:8.0:1619677475.116001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.116002:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:21.2:1619677475.116018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.116024:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.116027:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.116028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.116030:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.116032:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.116033:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.116034:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.116036:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.116037:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.116038:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.116041:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.116042:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.116044:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.116045:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.116046:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.116047:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.116048:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.116049:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.116049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.116053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.116061:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.116063:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.116064:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.116066:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.116066:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.116068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.116150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.116153:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.116155:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.116157:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.116158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.116160:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.116161:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.116163:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.116164:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.116165:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.116166:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.116169:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.116171:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.116173:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.116174:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.116174:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.116176:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.116177:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.116178:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.116179:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.116180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.116182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.116185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.116188:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.116189:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.116190:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677475.116190:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.116191:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677475.116191:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.116192:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.116193:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.116194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.116195:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677475.116350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.116358:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.116360:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.116361:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.116363:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.116366:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.116367:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.116369:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.116370:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.116371:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.116372:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.116375:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.116377:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.116380:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.116381:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.116381:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.116383:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.116383:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.116384:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.116384:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.116389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.2:1619677475.116397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:6.0:1619677475.116397:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.116400:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.116401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.116401:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.116403:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.116405:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.116406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677475.116408:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.116410:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.116412:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.116412:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.116414:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.116415:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.116416:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.116417:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.116418:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.116422:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.116423:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.116426:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.116427:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.116427:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.116429:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.116429:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.116430:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.116431:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.116443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.116449:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.116452:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.116453:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.116454:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.116455:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.116457:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677475.116498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.116504:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.116507:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.116508:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.116511:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.116513:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.116514:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.116516:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.116517:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.116518:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.116519:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.116522:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.116524:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.116526:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.116527:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.116528:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.116529:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.116530:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.116531:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.116532:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.116532:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.116535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.116537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677475.116542:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677475.116543:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677475.116545:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677475.116545:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677475.116546:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.116546:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.116546:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.116547:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677475.116548:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.116550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.116648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.116656:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.116659:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.116662:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.116663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.116664:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.116665:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.116667:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.116668:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.116669:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.116670:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.116674:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.116676:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.116678:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.116679:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.116679:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.116681:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.116681:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.116682:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.116683:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.116686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.116693:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.116696:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.116696:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.116698:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.116700:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.116701:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.117037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.117042:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.117045:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.117046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.117048:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.117050:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.117051:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.117052:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.117054:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.117055:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.117056:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.117059:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.117061:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.117063:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.117064:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.117064:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.117066:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.117066:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.117067:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.117068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.117072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.117080:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.117082:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.117083:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.117084:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.117085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.117087:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.117091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.117095:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.117097:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.117099:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.117100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.117101:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.117102:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.117104:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.117105:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.117106:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.117107:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.117110:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.117112:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.117115:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.117116:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.117116:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.117118:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.117119:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.117119:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.117120:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.117124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.117130:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.117133:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.117134:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.117134:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.117136:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.117139:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.117142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.117148:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.117151:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.117151:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.117151:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.117153:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.117153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.117831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.117840:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.117842:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.117844:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.117846:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.117847:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.117848:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.117849:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.117850:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.117851:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.117852:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.117855:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.117856:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.117860:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.117861:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.117861:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.117863:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.117864:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.117865:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.117865:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.117870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.117878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.117881:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.117882:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.117883:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.0:1619677475.117884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677475.117887:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.117889:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.117890:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677475.117892:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.117893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.117894:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.117897:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.117898:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.117899:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.117901:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.117902:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.117903:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.117906:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.117908:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.117910:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.117911:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.117912:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.117913:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.117914:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.117915:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.117915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.117920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.117928:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.117930:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.117931:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.117932:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.117933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.117934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.118295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.118303:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.118306:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.118307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.118309:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.112@o2ib7 : GET - for me
00000400:00000200:6.0:1619677475.118311:0:19937:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.112@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:6.0:1619677475.118314:0:19937:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.112@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:6.0:1619677475.118315:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.118316:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.118317:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677475.118321:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : REPLY try# 0
00000800:00000200:6.0:1619677475.118322:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.118324:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.118324:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.118325:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.118326:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.118327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:25.2:1619677475.118332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.118339:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.118341:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.118342:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.118343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.118344:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: REPLY: OK
00000800:00000200:22.0:1619677475.118345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.118443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.118451:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.118454:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.118456:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.118458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.118459:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.118461:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.118462:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.118464:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.118464:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.118465:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.118468:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.118470:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.118473:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.118473:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.118474:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.118476:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.118477:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.118478:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.118478:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.118482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.118489:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.118492:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.118493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.118493:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.118494:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.118497:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.118739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.118745:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.118747:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.118748:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.118750:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.118752:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.118753:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.118755:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.118756:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.118757:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.118758:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.118761:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.118763:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.118765:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.118766:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.118766:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.118768:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.118769:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.118770:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.118771:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.118771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.118774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677475.118777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.2:1619677475.118778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.118783:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.2:1619677475.118783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.118785:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.118785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677475.118786:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.118786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677475.118789:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:21.0:1619677475.118790:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677475.118790:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677475.118791:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677475.118792:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.118793:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.118793:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.118793:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.118795:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.118796:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.118796:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:30.0:1619677475.118797:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.118797:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677475.118798:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677475.118798:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677475.118799:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.118799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.118799:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (23)++
00000400:00000200:30.0:1619677475.118800:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.118800:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000400:00000200:30.0:1619677475.118801:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.118801:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.118801:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.118801:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677475.118802:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.118803:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.118803:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.118804:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.118805:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.118807:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.118809:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.118811:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.118812:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.118813:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.118815:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.118815:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.118816:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.118816:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.118817:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.118817:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.118818:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.118819:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677475.118819:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.118820:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677475.118820:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677475.118821:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677475.118821:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.118822:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.118822:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.118825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:9.2:1619677475.118825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.118828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677475.118831:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677475.118833:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.118834:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.118835:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.118835:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.118836:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.118836:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.118836:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.118837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.118837:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.118838:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677475.118839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677475.118839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677475.118839:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.118840:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677475.118842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.118915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.118921:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.118923:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.118926:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.118926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.118929:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.118930:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.118931:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.118933:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.118934:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.118935:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.118938:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.118940:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.118941:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.118942:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.118943:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.118944:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.118945:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.118946:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.118947:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.118948:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677475.118950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.118954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.118959:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.118960:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.118961:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:20.0:1619677475.118961:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.118962:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.118962:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.118963:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.118963:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.118964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677475.118965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.119040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.119045:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.119048:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.119050:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.119050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.119052:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.119054:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.119055:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.119057:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.119057:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.119058:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.119062:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.119063:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.119065:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.119066:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.119066:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.119068:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.119069:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.119069:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.119070:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.119071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.119074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.119076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.119080:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677475.119081:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.119082:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.119083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.119083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677475.119084:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.119085:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.119085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.119087:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.119090:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.119177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.119180:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.119182:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.119184:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.119185:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.119187:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.119188:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.119189:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.119191:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.119192:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.119193:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.119196:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.119198:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.119200:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.119201:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.119202:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.119203:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.119204:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.119205:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.119205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.119209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.119218:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.119221:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.119222:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.119222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.119224:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.119226:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.119769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.119774:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.119777:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.119778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.119780:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.119782:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.119783:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.119784:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.119786:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.119787:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.119787:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.119790:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.119792:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.119794:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.119795:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.119796:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.119797:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.119798:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.119798:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.119799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.119804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.119813:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.119815:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.119816:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.119817:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.119818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.119819:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.120145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.120153:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.120156:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.120158:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.120161:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.120161:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.120162:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.120163:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.120165:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.120166:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.120167:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.120170:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.120172:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.120174:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.120175:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.120175:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.120177:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.120178:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.120179:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.120181:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.120181:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.120183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.120187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.120192:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.120193:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.120194:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.120196:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.120196:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.120197:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.120197:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.120198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.120199:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.120200:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.120203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.120210:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.120213:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.120213:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.120214:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.120215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.120215:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.120391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.120398:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.120401:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.120403:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.120405:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.120406:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.120406:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.120407:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.120408:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.120409:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.120410:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.120412:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.120414:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.120416:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.120417:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.120417:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.120419:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.120420:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.120421:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.120423:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.120423:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.120426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.120430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.120433:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.120435:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.120436:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.120437:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.120438:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.120438:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.120440:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677475.120440:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.120441:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.120442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677475.120517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677475.120525:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.120527:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677475.120529:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.120531:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.120531:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677475.120533:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677475.120534:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.120535:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.120536:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.120537:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677475.120540:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.120542:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677475.120544:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.120545:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:31.0:1619677475.120546:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677475.120547:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:31.0:1619677475.120548:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.120549:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:31.0:1619677475.120550:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:31.0:1619677475.120551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:25.2:1619677475.120553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.2:1619677475.120556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:12.0:1619677475.120562:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:30.0:1619677475.120565:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:12.0:1619677475.120565:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677475.120566:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.120566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677475.120567:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:30.0:1619677475.120568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677475.120569:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000400:00000200:12.0:1619677475.120569:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.120571:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:24.0:1619677475.120573:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:13.0:1619677475.120573:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:25.2:1619677475.120576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:24.0:1619677475.120576:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.120577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.120578:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.120581:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.120583:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.120584:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677475.120584:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:24.0:1619677475.120585:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.120586:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.120587:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677475.120587:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677475.120588:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.120589:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:24.0:1619677475.120590:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.120590:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:24.0:1619677475.120592:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677475.120592:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:24.0:1619677475.120595:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.120596:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.120596:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.120598:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.120598:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.120599:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.120600:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.120604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.120612:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.120615:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.120615:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.120616:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.120618:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.120621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.120623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.120628:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.120631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.120631:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.120633:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.120635:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.120636:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.120637:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.120639:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.120640:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.120641:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.120644:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.120645:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.120648:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.120648:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.120649:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.120651:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.120651:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.120652:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.120652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.120657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.120664:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.120666:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.120667:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.120669:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.120670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.120671:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.120760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677475.120768:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677475.120771:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677475.120772:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677475.120774:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.120777:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.120778:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.120780:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.120781:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.120782:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.120783:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.2:1619677475.120786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:8.0:1619677475.120787:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.120788:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.120791:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.120792:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:30.0:1619677475.120793:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.120793:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.120794:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.120795:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:30.0:1619677475.120796:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677475.120796:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000400:00000200:30.0:1619677475.120798:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.120798:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677475.120798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:31.0:1619677475.120800:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.120800:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.120800:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:30.0:1619677475.120802:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.120803:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.120804:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677475.120804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000400:00000200:30.0:1619677475.120805:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.120806:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.120808:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.120809:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:30.0:1619677475.120810:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677475.120811:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000800:00000200:1.0:1619677475.120811:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:30.0:1619677475.120812:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.120812:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:17.0:1619677475.120812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:30.0:1619677475.120813:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677475.120813:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000400:00000200:1.0:1619677475.120813:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.120814:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677475.120814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:1.0:1619677475.120814:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.120815:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.120815:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:30.0:1619677475.120816:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.120816:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677475.120818:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:25.2:1619677475.120820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.120826:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.120829:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.120830:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.120832:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.120833:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.120834:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.121032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.121038:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.121041:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.121043:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.121044:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.121046:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.121048:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.121049:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.121050:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.121051:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.121052:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.121055:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.121057:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.121060:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.121061:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.121061:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.121063:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.121064:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.121065:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.121065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.121069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.121076:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.121079:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.121079:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.121081:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.121082:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.121084:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.121335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.121342:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.121345:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.121346:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.121348:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.121351:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.121353:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.121354:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.121355:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.121356:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.121357:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.121360:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.121362:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.121365:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.121366:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.121366:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.121368:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.121369:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.121369:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.121370:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.121375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.121383:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.121385:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.121386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.121387:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.121389:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.121391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.121453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.121459:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.121461:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677475.121464:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.0:1619677475.121464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.121467:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.121470:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.121470:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677475.121471:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.121472:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.121472:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.121473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677475.121474:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.121476:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.121476:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.121477:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677475.121478:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.121479:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677475.121481:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.121481:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.121482:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.121483:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:6.0:1619677475.121483:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.121484:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.121486:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.121487:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.121487:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677475.121488:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.121489:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.121490:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.121491:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.121491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677475.121491:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.121493:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.121494:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.121494:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.2:1619677475.121495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677475.121496:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.121496:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.121497:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.121498:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.121502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.121506:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.121507:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.121508:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.121510:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:20.0:1619677475.121510:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.121511:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.121512:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.121512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677475.121513:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.121513:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.121515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.121515:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.121516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.121516:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.121516:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.121517:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.121518:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.121657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.121665:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.121668:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.121668:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.121671:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.121673:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.121675:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.121676:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.121677:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.121679:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.121680:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.121683:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.121685:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.121687:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.121688:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.121689:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.121690:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.121691:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.121692:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.121692:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.121696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.121703:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.121706:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.121707:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.121708:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.121710:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.121711:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677475.121749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.121755:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.121758:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.121759:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.121761:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.121763:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.121764:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.121765:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.121767:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.121768:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.121769:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.121772:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.121773:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.121776:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.121776:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.121777:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.121779:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.121779:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.121780:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.121781:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.121785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.121792:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.121795:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.121796:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.121796:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.121798:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.121800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.121931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.121938:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.121941:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.121942:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.121944:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.121946:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.121948:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.121949:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.121950:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.121952:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.121953:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.121956:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.121958:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.121961:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.121962:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.121963:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.121965:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.121965:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.121966:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.121966:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.121971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.121978:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.121981:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.121981:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.121983:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.121985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.121986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.121989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.121997:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.121999:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.122000:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.122001:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.122002:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.122002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.122239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.122247:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.122250:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.122252:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.122254:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.122254:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.122255:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.122256:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.122258:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.122259:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.122260:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.122263:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.122264:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.122267:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.122267:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.122268:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.122270:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.122270:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.122271:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.122272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.122275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.122282:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.122284:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.122285:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.122285:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.122287:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.122290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.122468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.122473:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.122476:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.122477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.122478:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.122480:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.122482:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.122483:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.122484:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.122485:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.122486:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.122489:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.122491:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.122493:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.122494:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.122495:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.122496:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.122497:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.122497:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.122498:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.122499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.122503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.122505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.122510:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.122511:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.122512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.122512:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.122513:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.122514:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.122515:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.122517:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.122517:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.122519:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.122519:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.122520:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.122520:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.122521:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.122521:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.122664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.122672:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.122675:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.122678:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.122680:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.122680:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.122681:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.122683:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.122684:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.122685:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.122686:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.122689:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.122690:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.122694:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.122695:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.122696:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.122697:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.122698:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.122699:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.122699:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.122704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.122713:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.122715:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.122716:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.122718:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.122718:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.122721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.123124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.123131:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.123134:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.123134:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.123137:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.123139:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.123141:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.123142:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.123143:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.123144:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.123145:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.123148:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.123150:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.123152:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.123152:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.123153:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.123154:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.123155:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.123156:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.123156:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.123160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.123168:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.123170:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.123171:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.123172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.123173:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.123174:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.123312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.123317:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.123320:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.123321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.123323:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.123325:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.123327:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.123328:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.123329:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.123330:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.123331:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.123335:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.123336:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.123339:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.123340:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.123340:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.123342:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.123342:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.123343:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.123343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.123349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.123356:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.123358:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.123359:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.123361:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.123362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.123363:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.123416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.123421:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.123423:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.123425:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.123426:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.123428:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.123429:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.123431:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.123432:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.123433:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.123434:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.123437:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.123439:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.123442:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.123442:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.123443:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.123445:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.123446:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.123446:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.123447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.123451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.123458:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.123461:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.123462:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.123462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.123464:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.123467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.123468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.123475:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.123477:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.123478:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.123478:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.123479:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.123480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.123750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677475.123756:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.123759:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677475.123760:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677475.123762:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.123764:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.123765:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.123767:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.123768:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.123769:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.123770:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.123774:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.123775:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.123779:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.123779:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.123780:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.123782:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.123782:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.123783:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677475.123784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:25.2:1619677475.123788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.123795:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.123798:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.123799:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.123801:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.123802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.123804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.124046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.124053:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.124056:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.124058:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.124060:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.124061:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.124063:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.124064:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.124066:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.124067:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.124068:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677475.124068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:30.0:1619677475.124071:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.124073:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.124076:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.124076:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.124076:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.124077:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677475.124077:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:31.0:1619677475.124079:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.124079:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.124079:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.124080:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.124081:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.124081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677475.124082:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.124083:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.124084:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:9.2:1619677475.124084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:31.0:1619677475.124085:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.124086:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.124087:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.124090:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.124091:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:31.0:1619677475.124092:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677475.124093:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:31.0:1619677475.124094:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.124094:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:24.0:1619677475.124094:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.124094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.124095:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.124096:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:24.0:1619677475.124096:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.124097:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.124098:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.124098:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677475.124098:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:25.2:1619677475.124102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.124108:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.124111:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.124112:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.124113:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.124115:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.124116:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.124165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.124173:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.124175:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.124176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.124178:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.124181:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.124182:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.124184:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.124185:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.124186:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.124187:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.124190:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.124191:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.124193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:10.0:1619677475.124194:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.124195:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.124196:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.124198:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:10.0:1619677475.124198:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.124198:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.124199:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.124199:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677475.124201:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.124202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.124204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677475.124204:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.124206:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.124208:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.124209:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.124211:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.124212:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.124212:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.124213:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.124214:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.124214:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.124217:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.124218:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.124219:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.124220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.124221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677475.124222:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.124223:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.124224:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677475.124225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.124225:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.124226:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.124226:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.124228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677475.124229:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677475.124231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.124231:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.124232:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.124232:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.124233:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.124233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.124239:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.124241:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.124242:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.124243:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.124244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.124245:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.124980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.124990:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.124993:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.124997:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.124999:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.125001:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.125003:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677475.125006:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.125008:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.125010:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.125011:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677475.125017:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.125019:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677475.125023:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.125025:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677475.125026:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677475.125028:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677475.125029:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.125030:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.125031:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.125035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.2:1619677475.125044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:2.0:1619677475.125044:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.125046:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.125048:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.125048:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.125050:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:2.0:1619677475.125050:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.125052:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677475.125053:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:21.0:1619677475.125055:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.125055:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.125058:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.125059:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.125061:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.125062:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.125063:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.125064:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.125066:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.125068:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.125071:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.125071:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.125072:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.125074:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.125074:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.125075:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.125076:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.125081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.125090:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.125092:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.125093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.125093:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.125095:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.125098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.125099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.125107:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.125110:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.125110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.125111:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.125111:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.125112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.125118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.125126:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.125129:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.125132:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.125133:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.125134:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.125134:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.125136:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.125138:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.125139:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.125140:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.125143:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.125145:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.125147:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.125148:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.125148:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.125150:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.125150:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.125152:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.125153:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.125154:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.125155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.125160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.125165:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.125166:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.125167:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.125168:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677475.125169:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.125169:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.125170:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.125170:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.125171:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.125172:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.125245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:21.2:1619677475.125248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:26.0:1619677475.125253:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:22.0:1619677475.125254:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677475.125255:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.125256:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:22.0:1619677475.125257:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.125258:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677475.125260:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.125261:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.125262:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.125263:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.125265:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677475.125265:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.125267:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.125268:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.125269:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.125270:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.125270:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.125271:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.125272:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677475.125273:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677475.125277:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.125278:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.125279:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677475.125281:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.125282:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.125282:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.125283:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677475.125284:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.125284:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677475.125285:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677475.125285:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.125286:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.125286:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677475.125287:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677475.125287:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:22.0:1619677475.125287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677475.125288:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.125289:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.125291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677475.125295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677475.125299:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.125301:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677475.125302:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:20.0:1619677475.125302:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.125303:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.125305:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:21.0:1619677475.125305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:30.0:1619677475.125306:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.125307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677475.125308:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.125310:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.125310:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.125311:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.125312:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.125312:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.125313:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.125313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.126075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.126082:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.126085:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.126086:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.126088:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.126091:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.126092:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.126094:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.126095:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.126096:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.126097:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.126101:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.126103:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.126105:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.126106:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.2:1619677475.126107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.0:1619677475.126107:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.126108:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.126109:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.126110:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.126110:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677475.126112:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677475.126114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.126115:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.126117:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.126118:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.126119:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.126120:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.126121:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.126121:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.126123:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.126123:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.126125:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.126126:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.126127:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.126128:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.126131:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.126132:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.126133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.126134:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.126137:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.126138:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.126138:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.126140:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.126140:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.126141:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.126142:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.126143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.126146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.126148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.126153:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.126155:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.126155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.126155:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.126156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.126157:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.126158:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.126160:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.126161:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.126162:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.126168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.2:1619677475.126170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677475.126175:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.126177:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.126178:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677475.126178:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.126179:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.126180:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.126180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.126181:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.126183:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.126184:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.126185:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.126186:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.126188:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.126189:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.126190:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.126191:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.126194:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.126196:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.126198:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.126199:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.126200:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.126201:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.126202:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.126203:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.126204:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.126207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.126213:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.126216:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.126217:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.126217:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.126219:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.126221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.126226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.126233:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.126236:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.126237:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.126238:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.126241:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.126243:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:21.2:1619677475.126244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:10.0:1619677475.126244:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.126245:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.126246:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.126247:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.126250:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.126250:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:10.0:1619677475.126251:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.126253:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.126254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:10.0:1619677475.126254:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.126255:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.126256:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:1.0:1619677475.126256:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.126257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677475.126258:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.126258:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677475.126258:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.126259:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000400:00000200:1.0:1619677475.126260:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677475.126261:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.126261:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:1.0:1619677475.126261:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.126262:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.126263:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.126264:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677475.126265:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677475.126267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677475.126268:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677475.126268:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.126270:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677475.126271:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.126271:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.126273:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.126273:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.126273:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:26.0:1619677475.126274:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:25.2:1619677475.126274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.126274:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:31.0:1619677475.126275:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677475.126275:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:31.0:1619677475.126276:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96501464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677475.126276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:1.0:1619677475.126276:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.126276:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:26.0:1619677475.126277:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677475.126277:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:31.0:1619677475.126278:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.126278:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:31.0:1619677475.126279:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.126280:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.2:1619677475.126282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677475.126282:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:31.0:1619677475.126283:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.126284:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:31.0:1619677475.126285:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.126285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.126285:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.126288:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.126288:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.126289:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.126290:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677475.126290:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:6.0:1619677475.126290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677475.126291:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.126292:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.126292:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:25.2:1619677475.126298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:20.0:1619677475.126298:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.126299:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:31.0:1619677475.126300:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.126300:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677475.126300:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.126302:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:25.2:1619677475.126302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:20.0:1619677475.126302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677475.126303:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.126304:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.126306:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.126306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.126307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677475.126310:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.126313:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.126314:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.126314:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.126315:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.126316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.126342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677475.126348:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.126350:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677475.126353:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.126353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677475.126355:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.126356:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.126358:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.126359:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.126360:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.126361:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.126364:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.126366:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.126368:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.126369:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.126369:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.126371:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.126372:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.126373:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677475.126373:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677475.126377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.126385:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.126388:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.126388:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.126390:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.126391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.126392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677475.126443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.126449:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.126452:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.126453:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.126454:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.126457:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.126458:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.126459:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.126461:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.126462:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.126462:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.126465:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.126467:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.126470:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.126470:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.126471:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.126473:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.126473:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.126474:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.126474:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.126490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.126497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.126500:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.126501:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.126501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.126503:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.126505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.126625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.126632:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.126635:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.126636:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.126638:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.126641:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.126642:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.126643:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.126645:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.126646:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.126647:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.126650:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.126651:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.126654:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.126655:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.126656:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.126658:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.126659:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.126660:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.126661:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.126662:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.126664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677475.126667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677475.126671:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677475.126673:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677475.126674:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.126675:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.126675:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.126675:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.126676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677475.126677:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.126678:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.126680:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.126758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.126764:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.126766:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.126769:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.126769:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.126771:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.126773:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.126774:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.126775:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.126776:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.126777:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.126780:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.126782:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.126784:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.126785:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.126785:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.126787:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.126788:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.126789:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.126789:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.126793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.126800:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.126803:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.126804:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.126804:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.126806:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.126808:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.126993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.126997:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.127000:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.127001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.127003:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.127005:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.127006:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.127007:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.127008:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.127010:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.127011:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.127013:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.127015:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.127018:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.127019:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.127019:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.127021:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.127022:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.127022:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.127023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.127027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.127034:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.127037:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.127037:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.127039:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.127040:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.127041:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.127209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.127214:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.127216:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.127219:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.127219:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.127221:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.127222:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.127224:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.127225:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.127226:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.127227:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.127230:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.127231:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.127234:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.127235:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.127235:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.127237:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.127237:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.127239:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.127240:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.127241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.127243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.127246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.127252:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.127253:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.127254:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.127254:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677475.127255:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.127255:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.127256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.127257:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.127258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677475.127259:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677475.127315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.127322:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.127325:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.127326:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.127328:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.127331:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.127332:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.127333:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.127335:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.127336:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.127337:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.127340:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.127342:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.127344:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.127345:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.127346:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.127347:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.127348:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.127348:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.127349:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.127354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.127362:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.127364:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.127365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.127366:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.127368:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.127371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.127915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.127923:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.127926:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.127928:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.127930:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.127930:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.127931:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.127933:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.127934:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.127935:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.127936:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.127938:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.127940:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.127942:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.127943:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.127943:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.127945:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.127946:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.127947:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.127948:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.127949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.127951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.127955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.127958:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.127961:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.127962:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.127962:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.127963:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.127963:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.127965:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677475.127965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.127966:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.127967:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.127995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.128000:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.128003:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.128003:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.128005:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.128008:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.128009:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.128010:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.128011:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.128013:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.128014:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.128017:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.128019:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.128021:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.128022:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.128022:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.128024:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.128024:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.128025:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.128025:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.128030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.128038:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.128040:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.128041:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.128043:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.128044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.128045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.128281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.128286:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.128289:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.128291:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.128291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.128292:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.128298:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.128300:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.128301:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.128302:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.128303:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.128306:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.128308:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.128310:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.128311:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.128312:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.128313:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.128314:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.128315:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.128315:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.128320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.128328:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.128330:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.128331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.128332:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.128334:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:25.2:1619677475.128335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.128338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.128339:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.128341:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.128342:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.128342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.128343:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.128344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.128396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.128404:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.128407:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.128409:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.128411:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.128411:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.128412:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.128414:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.128415:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.128416:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.128417:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.128420:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.128422:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.128424:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.128424:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.128425:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.128426:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.128427:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.128428:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.128428:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.128432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.128439:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.128441:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.128442:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.128444:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.128445:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.128446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.128885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.128890:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.128893:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.128894:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.128896:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.128898:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.128899:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.128900:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.128902:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.128903:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.128904:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.128907:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.128909:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.128912:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.128912:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.128913:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.128915:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.128915:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.128916:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.128916:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.128921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.128929:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.128931:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.128932:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.128933:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.128935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.128936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.129043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.129051:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.129054:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.129055:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.129056:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.129059:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.129061:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.129062:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.129063:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.129064:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.129065:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.129069:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.129071:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.129074:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.129075:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.129075:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.129077:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.129077:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.129078:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.129079:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.129083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.129090:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.129093:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.129094:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.129094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.129096:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.129099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.129123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.129131:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.129134:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.129135:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.129137:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.129137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.129139:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.129140:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.129141:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.129142:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.129143:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.129147:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.129148:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.129150:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.129151:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.129151:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.129153:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.129153:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.129154:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.129155:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.129158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.129165:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.129167:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.129168:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.129170:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.129171:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.129172:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.129785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.129791:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.129794:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.129794:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.129796:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.129798:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.129799:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.129801:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.129802:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.129803:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.129804:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.129807:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.129809:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.129811:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.129812:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.129812:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.129814:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.129814:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.129816:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.129817:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.129817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.129821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.129823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.129828:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.129830:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.129830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.129831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.129831:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.129833:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.129834:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.129835:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.129836:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.129837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.129843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.129850:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.129853:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.129854:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.129854:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.129855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.129856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.129890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.129898:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.129901:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.129903:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.129905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.129905:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.129906:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.129907:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.129909:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.129910:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.129911:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.129914:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.129916:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.129918:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.129919:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.129920:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.129921:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.129922:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.129923:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.129924:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.129927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.129934:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.129937:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.129938:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.129938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.129940:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.129942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.129946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.129952:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.129955:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.129955:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.129956:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.129956:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.129957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.129995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.130003:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.130005:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.130007:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.130009:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.130009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.130011:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.130012:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.130013:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.130014:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.130015:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.130018:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.130019:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.130021:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.130022:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.130022:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.130024:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.130024:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.130026:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.130027:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.130028:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.130029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.130033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.130038:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.130039:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.130040:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.130041:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.130042:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.130043:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.130043:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677475.130043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677475.130044:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.130045:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.130046:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.130048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677475.130051:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.130054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.130054:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:30.0:1619677475.130055:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677475.130056:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677475.130057:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.130058:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.130058:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.130059:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:10.0:1619677475.130059:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.130060:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:10.0:1619677475.130060:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.130061:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.130063:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.130063:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.130064:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.130067:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.130069:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.130071:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.130072:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.130073:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.130074:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.130075:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.130075:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.130076:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.130080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.130089:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.130093:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.130094:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.130096:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.130099:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.130100:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.130832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.130838:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.130840:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.130841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.130843:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.130846:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.130847:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.130848:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.130850:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.130851:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.130852:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.130855:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.130857:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.130860:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.130860:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.130861:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.130863:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.130863:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.130864:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.130864:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.130869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.130877:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.130880:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.130880:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.130882:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.130883:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.130885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.130957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.130965:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.130968:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.130969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.130970:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.130973:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.130975:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.130976:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.130977:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.130978:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.130979:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.130983:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.130984:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.130987:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677475.130987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677475.130988:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.130989:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.130990:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.130991:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.130992:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:21.0:1619677475.130992:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:24.0:1619677475.130993:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.130994:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677475.130994:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:25.2:1619677475.130996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677475.130996:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.130997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.130998:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.130999:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:4.2:1619677475.131000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677475.131001:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.131002:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.131003:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.131004:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677475.131004:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677475.131006:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.131007:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.131007:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.131007:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.131007:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.131008:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677475.131009:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677475.131010:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.131010:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677475.131011:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.131011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677475.131012:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.131012:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677475.131013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677475.131014:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.131015:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.131016:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.131016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.131022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.131032:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:4.2:1619677475.131033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:1.0:1619677475.131036:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:20.0:1619677475.131037:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:1.0:1619677475.131038:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.131039:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.131040:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.131042:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677475.131042:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.131043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.131045:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.131046:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:20.0:1619677475.131047:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.131049:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.131050:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.131051:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.131052:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.131055:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.131057:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.131059:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.131059:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.131060:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.131062:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.131062:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.131063:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.131064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.131067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.131075:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.131078:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.131078:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.131080:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.131081:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.131082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.131088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.131096:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.131097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677475.131098:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.131099:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.131100:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.131100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.131101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677475.131105:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.131108:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.131110:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.131111:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.131112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.131112:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.131114:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.131115:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.131116:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.131117:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.131120:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.131121:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.131123:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.131124:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.131125:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.131126:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.131127:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.131128:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.131128:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.131132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.131138:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.131141:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.131141:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.131143:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.131145:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.131146:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.131225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.131233:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.131236:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.131237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.131238:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.131241:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.131242:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.131243:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.131245:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.131246:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.131247:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.131250:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.131251:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.131254:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.131255:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.131255:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.131257:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.131258:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.131258:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.131259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.131263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.131270:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.131273:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.131274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.131274:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.131276:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.131279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.131342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.131350:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.131354:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.131356:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.131359:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.131362:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.131364:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.131366:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.131369:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.131370:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.131372:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.131378:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.131381:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.131384:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.131385:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.131386:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.131388:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.131389:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.131390:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.131391:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.131395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.131402:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.131405:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.131406:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.131406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.131408:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.131411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.131589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.131597:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.131601:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.131601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.131604:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.131607:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.131609:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.131610:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.131611:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.131612:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.131614:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.131617:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.131619:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.131621:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.131622:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.131622:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.131624:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.131624:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.131625:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.131626:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.131630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.131638:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.131641:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.131642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.131642:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.131644:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.131647:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.131653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.131660:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.131663:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.131664:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.131664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.131665:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.131666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.131731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.131737:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.131739:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.131740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.131742:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.131744:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.131745:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.131747:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.131748:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.131749:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.131750:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.131754:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.131755:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.131758:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.131759:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.131760:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.131761:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.131762:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.131762:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.131763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.131768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.131775:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.131777:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.131778:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.131780:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.131781:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.131782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.131876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.131884:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.131887:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.131889:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.131891:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.131892:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.131893:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.131894:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.131896:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.131897:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.131898:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.131901:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.131902:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.131905:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.131905:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.131906:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.131908:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.131908:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.131909:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.131910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.131913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.131920:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.131923:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.131924:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.131924:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.131926:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.131928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.132081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.132086:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.132089:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.132091:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.132091:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.132093:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.132094:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.132096:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.132097:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.132098:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.132099:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.132102:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.132104:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.132106:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.132107:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.132108:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.132109:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.132110:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.132111:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.132112:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.132112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.132115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.132117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.132121:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677475.132122:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.132123:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:22.0:1619677475.132124:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677475.132124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.132125:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:22.0:1619677475.132126:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.132126:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.132128:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.132130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.132334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.132340:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.132342:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.132344:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.132345:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.132348:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.132349:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.132351:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.132352:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.132353:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.132354:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.132357:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.132359:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.132361:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.132362:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.132363:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.132364:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.132365:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.132366:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.132367:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.132367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.132371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.132374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.132380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.132381:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.132381:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.132381:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.132382:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.132382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.132383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.132384:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.132384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677475.132385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.132614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.132620:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.132623:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.132623:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.132625:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.132627:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.132629:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.132630:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.132631:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.132632:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.132633:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.132637:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.132639:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.132641:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.132642:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.132642:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.132644:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.132644:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.132645:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.132645:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.132650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.132657:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.132660:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.132660:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.132662:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.132663:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.132664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.132674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677475.132683:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.132686:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677475.132689:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.132691:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.132691:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677475.132693:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.132694:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.132695:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.132696:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.132697:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.132700:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.132702:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.132705:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.132705:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.132706:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.132708:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.132708:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.132710:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:31.0:1619677475.132711:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677475.132712:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:9.2:1619677475.132713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.132718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:8.0:1619677475.132720:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.132723:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.132724:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.132724:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.132726:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000400:00000200:8.0:1619677475.132727:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.132728:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:30.0:1619677475.132729:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:8.0:1619677475.132729:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.132730:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:20.2:1619677475.133109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.133114:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.133116:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.133119:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.133119:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.133121:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.133123:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.133124:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.133125:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.133126:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.133127:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.133131:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.133132:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.133135:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.133136:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.133136:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.133138:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.133138:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.133139:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.133140:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.133144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.133151:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.133154:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.133154:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.133156:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.133157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.133158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.133494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.133500:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.133503:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.133504:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.133506:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.133508:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.133509:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.133511:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.133512:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.133513:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.133514:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.133517:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.133519:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.133521:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.133522:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.133523:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.133524:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.133525:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.133526:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.133526:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.133531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.133540:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.133542:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.133543:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.133545:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.133545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.133547:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.133662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.133668:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.133671:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.133672:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.133674:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.133676:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.133677:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.133678:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.133680:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.133681:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.133682:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.133684:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.133686:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.133688:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.133689:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.133690:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.133692:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.133692:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.133693:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.133694:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.133695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.133698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.133700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.133705:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.133706:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.133706:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.133707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.133708:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.133708:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.133709:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.133711:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.133712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.133713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.133716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.133723:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.133726:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.133727:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.133728:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.133728:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.133729:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.133910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677475.133912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.133918:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.133920:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677475.133921:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.133922:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.133922:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.133924:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.133924:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.133926:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.133927:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.133928:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.133929:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.133930:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.133932:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.133933:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.133934:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.133935:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.133935:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.133936:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.133937:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.133938:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.133941:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.133941:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.133942:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.133942:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.133943:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.133944:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.133945:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.133946:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.133946:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.133946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.133947:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.133947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677475.133948:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.133949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.133950:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.133950:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677475.133950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.133951:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.133951:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.133956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677475.133957:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.133959:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.133960:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.133962:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.133962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.133964:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:24.0:1619677475.133965:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677475.133966:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.133967:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.133969:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.133970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.133972:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.134353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.134361:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.134364:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.134365:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.134367:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.134370:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.134372:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.134373:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.134374:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.134375:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.134376:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.134380:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.134381:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.134384:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.134385:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.134385:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.134387:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.134388:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.134388:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.134388:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.134394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.134402:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.134405:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.134405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.134406:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.134408:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.134411:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.134413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.134420:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.134422:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.134424:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.134424:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.134425:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.134425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.134552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.134558:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.134560:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.134561:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.134562:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.134564:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.134566:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.134567:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.134568:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.134569:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.134570:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.134573:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.134575:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.134578:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.134579:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.134579:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.134581:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.134581:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.134582:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.134582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.134587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.134595:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.134597:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.134598:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.134600:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.134601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.134602:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.135054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.135062:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.135065:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.135067:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.135070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.135070:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.135071:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.135073:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.135074:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.135075:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.135076:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.135079:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.135081:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.135083:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.135083:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.135084:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.135085:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.135086:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.135087:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.135088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.135091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.135098:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.135101:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.135101:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.135103:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.135105:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.135105:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.135150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.135155:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.135157:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.135159:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.135160:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.135162:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.135163:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.135165:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.135166:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.135167:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.135168:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.135171:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.135173:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.135176:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.135177:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.135177:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.135179:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.135179:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.135180:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.135181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.135185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.135192:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.135195:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.135195:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.135196:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.135198:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.135201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.135500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.135506:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.135509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.135509:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.135511:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.135514:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.135515:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.135516:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.135517:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.135518:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.135519:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.135522:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.135524:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.135526:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.135527:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.135527:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.135529:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.135529:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.135530:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.135530:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.135536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.135544:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.135546:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.135547:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.135548:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.135550:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.135551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.135670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.135678:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.135681:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.135684:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.135685:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.135686:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.135687:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.135688:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.135689:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.135690:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.135691:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.135694:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.135696:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.135699:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.135700:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.135700:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.135702:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.135703:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.135704:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.135706:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.135706:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.135708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.135712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.135715:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.135717:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.135718:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.135718:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.135720:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.135720:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.135722:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677475.135722:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.135723:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.135724:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.135725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.135731:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.135733:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.135734:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.135734:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.135735:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.135736:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.135755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.135762:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.135765:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.135767:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.135769:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.135769:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.135770:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.135771:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.135773:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.135774:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.135774:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.135778:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.135780:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.135781:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.135782:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.135782:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.135784:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.135785:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.135785:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.135787:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.135787:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.135790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.135793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.135798:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.135799:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.135800:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.135801:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677475.135802:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.135802:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.135803:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.135803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.135804:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.135805:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.135976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.135983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.135986:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.135987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.135988:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.135992:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.135993:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.135994:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.135996:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.135996:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.135997:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.136000:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.136002:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.136005:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.136006:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.136006:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.136008:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.136009:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.136009:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.136010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.136014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.136021:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.136024:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.136025:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.136025:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.136027:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.136030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.136172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.136180:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.136182:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.136183:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.136185:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.136188:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.136189:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.136190:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.136192:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.136193:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.136194:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.136197:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.136198:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.136201:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.136202:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.136203:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.136204:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.136205:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.136206:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.136206:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.136211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.136218:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.136220:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.136221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.136222:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.136224:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.136226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.136231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.136238:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.136241:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.136241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.136242:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.136243:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.136243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.136526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.136532:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.136535:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.136536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.136538:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.136540:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.136541:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.136543:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.136544:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.136545:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.136546:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.136549:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.136551:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.136554:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.136555:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.136555:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.136557:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.136557:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.136559:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.136560:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.136560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.136563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.136566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677475.136567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.136572:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677475.136573:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677475.136574:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.136574:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.136575:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.136575:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.136575:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677475.136576:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:1.0:1619677475.136576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677475.136577:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.136579:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.136580:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.136581:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.136581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677475.136581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.136582:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.136584:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.136585:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.136586:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.136587:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.136590:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.136592:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.136594:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.136595:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.136595:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.136597:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.136598:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.136598:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.136598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.136603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.136611:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.136613:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.136614:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.136616:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.136616:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.136618:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.136778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.136781:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.136783:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.136786:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.136786:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.136788:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.136790:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.136791:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.136792:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.136793:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.136794:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.136798:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.136799:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.136802:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.136802:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.136803:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.136805:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.136806:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.136806:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.136807:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.136812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.136821:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.136823:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.136824:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.136824:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.136827:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.136829:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.136894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.136902:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.136905:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.136907:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.136909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.136909:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.136911:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.136912:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.136913:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.136914:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.136915:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.136919:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.136921:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.136923:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.136924:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.136925:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.136926:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.136927:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.136928:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.136928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.136932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.136939:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.136941:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.136942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.136942:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.136945:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.136947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.137463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.137469:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.137472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.137472:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.137475:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.137477:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.137478:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.137479:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.137481:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.137482:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.137483:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.137486:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.137488:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.137490:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.137491:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.137492:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.137493:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.137494:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.137495:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.137496:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.137496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.137500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.137502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.137507:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.137509:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.137509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.137509:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.137510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.137511:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.137512:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.137514:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.137515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.137516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.137518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.137525:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.137527:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.137528:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.137529:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.137530:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.137530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.2:1619677475.137541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.137547:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.137550:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.137551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.137552:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.137555:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.137556:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677475.137558:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.137559:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.137560:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.137561:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677475.137564:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.137566:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.137568:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.137569:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.137570:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.137571:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.137572:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.137572:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.137573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677475.137577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.137585:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.137587:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.137588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.137588:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.137590:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.137593:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.137672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.137681:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.137684:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.137686:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.137688:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.137688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.137690:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.137691:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.137692:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.137693:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.137694:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.137698:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.137700:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.137702:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.137703:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.137704:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.137705:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.137706:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.137707:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.137708:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.137711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.137718:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.137721:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.137722:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.137722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.137724:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.137726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.138168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.138175:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.138178:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.138179:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.138182:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.138182:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.138183:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.138184:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.138185:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.138186:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.138187:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.138190:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.138191:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.138193:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.138194:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.138195:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.138196:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.138197:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.138197:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.138198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.138202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.138208:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.138210:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.138212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.138212:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.138213:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.138215:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.138218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:9.2:1619677475.138222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.138226:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:24.0:1619677475.138228:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.138228:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.138229:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:24.0:1619677475.138231:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.138232:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:10.0:1619677475.138232:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677475.138232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.138233:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.138233:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:10.0:1619677475.138234:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.138235:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.138236:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.138238:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.138239:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.138239:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.138242:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.138244:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.138246:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.138247:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.138247:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.138249:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.138249:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.138250:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.138252:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.138252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.138255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.138258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677475.138262:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.138264:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:26.0:1619677475.138265:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:12.0:1619677475.138265:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.138266:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.138267:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:12.0:1619677475.138267:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677475.138268:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677475.138268:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.138269:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.138274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677475.138280:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.138283:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.138284:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.138284:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.138285:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.138285:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.138377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.138382:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.138385:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.138386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.138388:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.138390:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.138391:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.138393:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.138394:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.138395:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.138396:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.138400:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.138401:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.138403:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.138404:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.138405:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.138407:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.138407:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.138407:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.138408:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.138413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.138421:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.138423:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.138424:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.138426:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.138427:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.138428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.138448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.138453:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.138455:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.138457:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.138458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.138460:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.138462:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.138463:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.138464:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.138465:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.138466:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.138469:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.138471:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.138473:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.138474:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.138475:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.138476:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.138476:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.138477:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.138479:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.138479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.138482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.138485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.138492:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.138493:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.138493:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.138493:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677475.138494:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.138494:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.138495:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.138496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.138497:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677475.138497:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.139072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.139078:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.139081:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.139082:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.139084:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.139085:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.139087:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.139088:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.139089:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.139090:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.139091:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.139095:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.139097:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.139100:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.139101:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.139102:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.139103:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.139104:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.139105:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.139105:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.139109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.139117:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.139119:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.139120:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.139122:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.139123:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.139126:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.139129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.139135:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.139138:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.139139:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.139140:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.139141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.139142:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.139148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.139154:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.139157:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.139158:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.139160:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.139162:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.139164:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.139165:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.139166:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.139167:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.139168:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.139172:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.139173:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.139176:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.139176:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.139177:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.139179:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.139179:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.139180:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.139181:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.139182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.139185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.139187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677475.139192:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.139192:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.139194:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677475.139195:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677475.139195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677475.139196:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.139196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677475.139197:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.139199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.139201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.139201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677475.139204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:10.0:1619677475.139209:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.139211:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.139212:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677475.139212:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.139214:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677475.139214:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.139216:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.139216:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.139217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:10.0:1619677475.139217:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.139218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:10.0:1619677475.139218:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.139220:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.139221:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.139222:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.139223:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.139226:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.139228:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.139231:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.139232:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.139232:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.139234:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.139234:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.139235:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.139237:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.139237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.139240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.139243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.139247:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.139250:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.139250:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.139251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.139251:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.139252:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.139253:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.139253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.139253:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.139256:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.139276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.139284:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.139286:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.139288:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.139290:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.139290:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.139292:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.139298:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.139299:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.139300:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.139301:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.139304:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.139305:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.139308:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.139308:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.139309:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.139311:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.139311:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.139312:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.139313:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.139317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.139324:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.139326:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.139327:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.139328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.139329:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.139331:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.139434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:4.2:1619677475.139435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:22.0:1619677475.139440:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.139442:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:26.0:1619677475.139443:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677475.139443:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.139445:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.139446:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.139446:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:22.0:1619677475.139447:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.139448:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677475.139449:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.139450:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.139451:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.139451:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.139452:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.139453:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677475.139454:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.139455:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.139457:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.139458:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.139459:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.139460:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677475.139460:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.139462:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.139463:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:26.0:1619677475.139464:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.139464:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677475.139466:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677475.139466:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.139466:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.139467:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.139467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677475.139469:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.139470:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.139470:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:25.2:1619677475.139471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677475.139472:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.139472:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.139473:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.139475:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.139475:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.139478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.0:1619677475.139480:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.2:1619677475.139483:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.139483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:12.0:1619677475.139485:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:20.0:1619677475.139486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.139487:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.139488:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.139488:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.139488:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.139490:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.139490:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:30.0:1619677475.139491:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:12.0:1619677475.139491:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.139492:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:30.0:1619677475.139493:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.139493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:13.0:1619677475.139493:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.139493:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:12.0:1619677475.139494:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:30.0:1619677475.139495:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.139495:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677475.139498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.139498:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.139499:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.139500:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.139502:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.139503:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.139504:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.139507:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.139508:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.139510:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.139511:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.139511:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.139513:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.139513:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.139514:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.139515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.139518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.139525:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.139528:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.139528:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.139530:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.139532:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.139532:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.139535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.139542:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.139544:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.139545:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.139546:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.139546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.139547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.140350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.140356:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.140359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.140359:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.140362:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.140364:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.140365:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.140366:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.140367:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.140369:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.140369:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.140373:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.140375:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.140377:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.140378:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.140379:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.140380:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.140381:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.140381:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.140382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.140387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.140395:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.140397:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.140398:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.140400:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.140401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.140402:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.140739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.140747:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.140750:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.140752:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.140753:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.140754:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677475.140754:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.140756:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.140757:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.140758:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.140759:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.140762:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.140763:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.140765:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.140766:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.140766:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.140768:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.140768:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.140770:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.140772:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.140772:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.140773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.140778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.140783:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.140784:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.140785:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.140787:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.140787:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.140788:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.140788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.140789:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.140790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.140791:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.141121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.141129:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.141132:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.141133:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.141134:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.141137:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.141139:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677475.141140:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.141141:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.141142:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.141143:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677475.141146:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.141148:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677475.141150:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.141151:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677475.141152:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677475.141153:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677475.141154:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.141154:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.141155:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677475.141159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.141168:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.141171:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.141171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.141172:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.141174:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.141178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.141190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.141197:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.141200:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.141201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.141201:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.141202:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.141202:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.141220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.141225:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.141228:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.141229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.141231:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.141233:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.141235:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.141236:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.141237:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.141238:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.141239:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.141242:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.141243:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.141245:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.141246:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.141247:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.141248:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:5.2:1619677475.141248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677475.141249:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.141250:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.141251:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.141252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.141254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677475.141254:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677475.141257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677475.141257:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.141258:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.141260:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.141262:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.141263:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677475.141263:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.141264:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:17.0:1619677475.141264:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.141264:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.141265:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:17.0:1619677475.141266:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.141266:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:17.0:1619677475.141267:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677475.141268:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:17.0:1619677475.141268:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677475.141270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.141271:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (22)--
00000400:00000200:17.0:1619677475.141271:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.141272:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.141274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677475.141274:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.141274:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677475.141275:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677475.141276:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.141277:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677475.141277:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677475.141278:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.0:1619677475.141278:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677475.141278:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.141279:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:31.0:1619677475.141280:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.141280:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677475.141280:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.141280:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.141281:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677475.141282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677475.141282:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.141282:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.141282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677475.141283:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.141283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.141284:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:25.2:1619677475.141285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:31.0:1619677475.141286:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.141287:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.141288:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.141289:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.141292:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.141293:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:31.0:1619677475.141299:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677475.141299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.141301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (22)--
00000800:00000200:31.0:1619677475.141302:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.141302:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:22.0:1619677475.141302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.141303:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:22.0:1619677475.141304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.141305:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.141305:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.141306:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677475.141306:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677475.141307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.141311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.141319:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.141321:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.141322:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.141323:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.141325:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.141327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.141394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677475.141400:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.141403:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677475.141404:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677475.141406:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.141408:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.141409:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677475.141410:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.141412:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.141413:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.141414:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677475.141417:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.141419:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677475.141422:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.141423:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677475.141423:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677475.141425:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677475.141425:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.141426:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677475.141426:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677475.141431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.141439:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.141442:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.141443:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.141443:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.141445:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.141448:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.141474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.141481:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:20.2:1619677475.141483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677475.141484:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.141485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.141487:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.141489:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.141489:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:26.0:1619677475.141491:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.141492:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677475.141492:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.141493:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:26.0:1619677475.141494:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.141495:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.141495:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.141496:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677475.141498:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.141499:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677475.141501:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677475.141501:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.141502:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677475.141504:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:2.0:1619677475.141504:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.141505:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:26.0:1619677475.141508:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.141508:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:2.0:1619677475.141508:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.141509:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677475.141510:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677475.141511:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.141511:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.141512:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677475.141512:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.141513:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:2.0:1619677475.141513:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.141514:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.141516:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.141516:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:25.2:1619677475.141517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677475.141517:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.141518:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.141521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677475.141524:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.141527:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.141528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.141528:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.141529:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:6.0:1619677475.141530:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.141531:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.141532:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.141533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677475.141533:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.141535:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.141537:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677475.141714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677475.141720:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677475.141723:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677475.141724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677475.141726:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.141728:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.141730:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:1.0:1619677475.141731:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.141732:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.141733:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.141734:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:1.0:1619677475.141738:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.141740:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:1.0:1619677475.141743:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.141744:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677475.141744:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677475.141746:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:1.0:1619677475.141747:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677475.141747:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677475.141748:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:25.2:1619677475.141752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.141760:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.141763:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.141764:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.141765:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.141767:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.141769:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.141989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.141993:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.141995:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677475.141998:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.141998:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.142000:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.142001:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677475.142003:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.142004:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.142005:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.142006:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.142009:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.142011:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.142013:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.142014:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.142014:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.142016:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677475.142017:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.142018:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.142018:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.142022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.142030:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.142032:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.142033:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.142034:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.142036:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.142038:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.142054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.142062:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.142065:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.142067:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.142068:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.142069:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.142071:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.142072:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.142074:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.142075:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.142076:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.142079:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.142081:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.142085:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.142085:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.142086:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.142088:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.142088:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.142089:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.142090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.142094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.142102:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.142105:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.142105:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.142107:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.142107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.142111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.142187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.142195:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.142198:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.142199:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.142201:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.142204:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.142205:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677475.142207:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.142208:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.142209:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.142210:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677475.142213:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.142214:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677475.142217:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.142218:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677475.142219:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677475.142220:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677475.142221:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.142222:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.142222:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677475.142226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.142234:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.142237:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.142238:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.142238:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.142241:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.142243:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.142249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.142256:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.142259:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.142260:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.142260:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.142261:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.142261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.142338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.142344:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.142347:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.142348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.142350:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.142352:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.142354:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.142355:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.142356:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.142357:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.142358:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.142362:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.142363:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.142366:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.142366:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.142367:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.142369:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.142369:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.142370:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.142370:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.142375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.142384:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.142386:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.142387:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.142388:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.142389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.142390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.142450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.142458:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.142461:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.142461:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677475.142464:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.142467:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.142468:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677475.142469:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.142471:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.142472:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.142473:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677475.142477:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.142479:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677475.142481:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.142482:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677475.142482:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677475.142484:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677475.142484:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.142485:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.142485:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.142490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.142497:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.142499:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.142500:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.142502:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.142505:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.142506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.142741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.142749:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.142752:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.142754:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.142755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.142756:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.142757:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.142759:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.142760:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.142761:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.142762:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.142764:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.142766:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.142768:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.142768:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.142769:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.142770:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.142771:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.142772:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.142772:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.142776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.142783:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.142785:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.142786:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.142787:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.142789:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.142790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.143036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.143041:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.143044:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.143046:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.143047:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.143048:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.143050:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.143051:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.143052:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.143053:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.143054:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.143058:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.143059:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.143062:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.143063:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.143063:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.143065:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.143066:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.143067:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.143067:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.143072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.143081:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.143084:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.143085:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.143085:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.143087:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.143090:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.143229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.143237:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.2:1619677475.143239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.143240:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.143242:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.143243:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.143244:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.143245:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:6.0:1619677475.143245:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677475.143246:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.143247:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:30.0:1619677475.143248:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.143248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:30.0:1619677475.143250:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.143251:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677475.143251:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.143254:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.143255:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.143255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677475.143255:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:30.0:1619677475.143256:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:6.0:1619677475.143257:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.143258:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.143259:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.143260:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.143260:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.143261:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.143261:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.143263:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.143263:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.143264:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:6.0:1619677475.143264:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.143265:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677475.143265:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.143267:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.143267:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677475.143267:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.143268:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677475.143268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.143269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677475.143269:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.143270:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.143270:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.143271:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.143272:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:6.0:1619677475.143272:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677475.143273:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96501380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.143273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.143273:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677475.143274:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.143274:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.143274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:31.0:1619677475.143275:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.143275:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:31.0:1619677475.143276:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.143277:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:25.2:1619677475.143278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:24.0:1619677475.143278:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.143279:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:31.0:1619677475.143280:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.143280:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.143281:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.143281:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:24.0:1619677475.143281:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.143282:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.143282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677475.143283:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.143285:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.143285:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677475.143286:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.143286:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:9.2:1619677475.143287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.143288:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.143288:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.143288:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677475.143289:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677475.143289:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.143290:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677475.143290:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.143291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.143292:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677475.143293:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:9.2:1619677475.143297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.143300:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.143302:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.143302:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.143303:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.143303:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (21)++
00000400:00000200:24.0:1619677475.143304:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.143304:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.143304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677475.143305:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.143306:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.143308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.144112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.144118:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.144121:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.144122:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.144124:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.144126:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.144128:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.144129:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.144130:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.144131:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.144132:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.144135:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.144137:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.144139:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.144140:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.144141:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.144142:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.144143:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.144143:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.144144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.144149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.144157:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.144159:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.144160:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.144162:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.144163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.144164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.144279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.144282:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.144284:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.144287:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.144287:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.144289:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.144290:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.144292:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.144296:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.144297:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.144298:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.144302:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.144303:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677475.144304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677475.144306:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.144307:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.144307:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.144309:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.144310:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.144310:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.144311:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677475.144313:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.144316:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677475.144316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:30.0:1619677475.144318:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.144320:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.144321:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.144321:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.144323:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.144324:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.144325:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.144325:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:30.0:1619677475.144326:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677475.144327:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:30.0:1619677475.144328:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.144328:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.144329:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677475.144331:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677475.144331:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.144333:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.144333:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677475.144334:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.144334:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.2:1619677475.144335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:30.0:1619677475.144336:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.144336:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.144338:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.144339:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.144340:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.144342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677475.144343:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.144346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677475.144346:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.144346:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.144347:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.144348:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.144348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677475.144350:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.144351:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.144352:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.144354:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.144354:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.144354:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.144355:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.144356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.144357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.144358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.144361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.144368:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.144371:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.144372:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.144373:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.144373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.144374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.144449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.144456:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.144459:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.144459:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.144462:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.144464:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.144465:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.144467:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.144468:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.144469:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.144470:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.144473:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.144474:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.144477:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.144478:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.144478:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.144480:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.144481:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.144482:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.144482:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.144486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.144487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677475.144494:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677475.144495:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677475.144496:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.144497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.144497:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.144498:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:1.0:1619677475.144499:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:10.0:1619677475.144500:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.144501:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.144502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:26.0:1619677475.144504:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.144505:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.144506:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.144507:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.144508:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.144509:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.144512:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.144514:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.144517:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.144518:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.144518:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.144520:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.144520:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.144521:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.144521:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.144526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.144534:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.144537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.144537:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.144538:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.144540:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.144542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.144546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.144553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.144556:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.144557:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.144557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.144558:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.144558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.144650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677475.144656:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.144659:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677475.144661:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.144661:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677475.144663:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.144664:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.144666:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.144667:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.144668:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.144669:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.144672:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.144674:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.144676:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.144677:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.144677:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.144679:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.144680:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.144681:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677475.144682:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677475.144682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:25.2:1619677475.144685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.2:1619677475.144687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.144690:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677475.144692:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:6.0:1619677475.144692:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.144693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677475.144694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677475.144694:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.144695:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.144696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.144697:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.144699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.145156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.145161:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.145164:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.145164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.145166:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.145168:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.145169:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.145170:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.145171:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.145172:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.145173:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.145176:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.145178:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.145180:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.145181:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.145181:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.145183:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.145183:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.145184:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.145185:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.145186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.145189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.145190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.145195:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.145196:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.145197:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.145198:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.145198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.145199:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.145199:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.145201:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.145202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.145203:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.145213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.145220:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.145223:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.145224:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.145224:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.145225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.145226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.145270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.145278:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.145280:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.145283:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.145284:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.145284:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.145286:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.145287:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.145288:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.145290:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.145290:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.145298:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.145300:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.145303:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.145303:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.145304:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.145306:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.145307:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.145308:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.145308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.145312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.145320:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.145322:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.145323:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.145324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.145325:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.145327:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677475.145974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.145980:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.145983:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.145984:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.145986:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.145988:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.145989:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.145991:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.145992:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.145993:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.145994:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.145997:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.145998:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.146001:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.146001:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.146002:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.146004:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.146005:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.146005:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.146006:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.146010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.146017:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.146020:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.146021:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.146021:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.146023:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.146025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.146029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:25.2:1619677475.146031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.146037:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.146038:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.146040:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677475.146041:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.146042:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.146043:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000400:00000200:6.0:1619677475.146043:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.146044:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.146044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.146045:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.146045:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677475.146046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:6.0:1619677475.146046:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.146048:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.146049:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.146050:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.146052:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.146054:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.146056:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.146057:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.146058:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.146060:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.146060:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.146061:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.146061:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.146066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.146074:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.146076:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.146077:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.146078:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.146079:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.146081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.146261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.146269:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.146271:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.146272:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.146274:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.146277:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.146278:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.146279:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.146281:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.146282:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.146283:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.146286:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.146288:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.146291:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.146291:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.146292:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.146300:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:25.2:1619677475.146301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.146301:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.146302:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.146303:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.146304:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677475.146309:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:4.2:1619677475.146310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.146312:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.146312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.146313:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.146315:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.146317:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.146317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677475.146319:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.146320:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.146321:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:4.2:1619677475.146430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.146438:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.146440:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.146442:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.146443:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.146446:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.146447:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.146448:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.146450:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.146451:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.146451:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.146454:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.146456:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.146459:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.146460:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.146460:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.146462:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.146462:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.146464:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.146465:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.146465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.146468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.146471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.146475:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.146478:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.146478:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.146479:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.146479:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.146481:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.146481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677475.146481:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.146482:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677475.146484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.146864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.146871:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.2:1619677475.146873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:26.0:1619677475.146874:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.146875:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.146876:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.146879:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.146879:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677475.146881:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677475.146882:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.146882:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.146882:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:26.0:1619677475.146884:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.146885:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.146886:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.146887:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677475.146888:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.146889:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.146891:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.146892:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.146893:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.146894:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677475.146895:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677475.146895:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:26.0:1619677475.146898:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.146899:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:22.0:1619677475.146899:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.146900:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677475.146901:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677475.146902:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.146902:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.146903:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.146904:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677475.146904:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.146904:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.146905:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.146907:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.146907:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.146908:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.146908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.2:1619677475.146908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:25.2:1619677475.146913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677475.146916:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.146919:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:20.0:1619677475.146920:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:17.0:1619677475.146920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.146920:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.146922:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:1.0:1619677475.146922:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:20.0:1619677475.146923:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.146925:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.146925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677475.146926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.146927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.146933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.146941:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.146944:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.146946:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.146947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:30.0:1619677475.146949:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.146950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.146950:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:31.0:1619677475.146951:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677475.146951:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.146953:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.146953:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.146954:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.146955:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.146959:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.146960:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.146961:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.146962:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677475.146962:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.146964:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.146965:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.146966:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.146967:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.146970:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.146970:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.146971:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.146971:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.146972:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.146973:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.146973:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.146974:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.146974:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.146975:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.146975:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.146975:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.146976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.146977:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.146977:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.146978:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.146978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.146979:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.146983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.146987:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.146989:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.146990:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:8.0:1619677475.146990:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.146991:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.146991:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.146992:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677475.146993:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.146995:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.146996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.146996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677475.146997:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.147666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.147674:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.147677:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.147679:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.147681:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.147682:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.147682:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.147683:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.147684:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.147685:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.147686:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.147689:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.147690:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.147692:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.147693:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.147693:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.147695:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.147695:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.147696:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.147697:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.147701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.147708:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.147711:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.147711:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.147713:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.147715:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.147715:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.147741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.147747:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.147750:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.147751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.147753:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.147755:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.147757:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.147758:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.147759:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.147760:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.147761:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.147765:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.147766:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.147768:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.147769:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.147770:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.147771:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.147772:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.147773:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.147773:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.147777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.147785:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.147787:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.147788:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.147788:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.147790:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.147791:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.147991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.147997:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.147999:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.148000:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.148002:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.148004:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.148006:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.148007:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.148008:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.148009:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.148010:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.148013:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.148015:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.148017:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.148018:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.148018:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.148020:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.148020:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.148021:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.148022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.148026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.148034:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.148036:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.148037:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.148039:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.148040:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.148041:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.148404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.148413:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.148415:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.148417:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.148419:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.148420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.148421:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.148422:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.148424:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.148425:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.148426:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.148429:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.148431:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.148433:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.148434:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.148435:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.148436:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.148437:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.148438:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.148438:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.148443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.148450:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.148452:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.148453:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.148453:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.148455:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.148457:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.148859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.148864:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.148866:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.148868:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.148869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.148870:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.148872:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.148873:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.148874:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.148875:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677475.148876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:20.0:1619677475.148877:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.148880:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.148882:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.148882:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677475.148884:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:20.0:1619677475.148884:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.148885:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677475.148885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.148886:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:22.0:1619677475.148887:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.148888:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.148888:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677475.148889:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.148889:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.148889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677475.148891:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.148892:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.148893:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.148894:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677475.148894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:22.0:1619677475.148895:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.148898:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.148899:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.148901:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.148902:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677475.148902:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677475.148903:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.148904:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677475.148904:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:22.0:1619677475.148905:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:17.0:1619677475.148905:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.148906:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:1.0:1619677475.148906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677475.148907:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:17.0:1619677475.148907:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.148908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:17.0:1619677475.148910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:25.2:1619677475.148911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.148913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.148918:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.2:1619677475.148920:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:6.0:1619677475.148921:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.148921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.148922:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.148924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.148926:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677475.148927:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:21.0:1619677475.148927:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.148929:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.148929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.148930:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.0:1619677475.148931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677475.148932:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.148932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:31.0:1619677475.148934:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.148935:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.148936:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.148937:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.148939:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.148939:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.148940:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.148941:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.148942:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.148943:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.148943:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000400:00000200:30.0:1619677475.148944:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.148944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.148945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.148946:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.148948:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.148949:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.148950:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.148951:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.148952:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.148953:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.148954:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.148957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.148963:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.148966:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.148967:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.148967:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.148969:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.148971:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.149194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.149201:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.149204:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.149206:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.149208:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.149208:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.149209:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.149211:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.149212:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.149213:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.149214:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.149216:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.149218:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.149220:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.149221:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.149221:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.149223:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.149223:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.149225:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.149227:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.149227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.149229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.149233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.149235:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.149237:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.149238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.149238:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.149240:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.149240:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.149242:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.149243:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.149243:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.149244:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.149253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.149259:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.149261:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.149262:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.149263:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.149263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.149264:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.149316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.149324:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.149327:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.149328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.149330:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.149333:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.149334:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.149335:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.149336:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.149337:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.149338:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.149341:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.149343:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.149345:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.149346:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.149347:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.149348:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.149349:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.149349:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.149350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.149355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.149363:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.149366:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.149366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.149367:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.149369:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.149371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.149603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.149610:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.149613:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.149614:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.149616:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.149618:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.149619:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.149620:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.149622:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.149623:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.149624:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.149627:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.149629:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.149632:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.149633:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.149634:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.149635:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.149636:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.149637:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.149638:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.149639:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.149641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677475.149644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.149650:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677475.149651:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677475.149652:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.149652:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.149653:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:13.0:1619677475.149655:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.149656:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.149659:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.149660:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.149663:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.149732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.149740:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677475.149742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.149743:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.149745:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.149747:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.149748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.149748:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.149749:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.149750:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.149750:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.149751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:30.0:1619677475.149753:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.149754:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.149754:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.149755:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677475.149756:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.149758:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.149760:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.149761:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.149762:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.149763:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.149764:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677475.149764:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.149766:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.149767:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.149768:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:6.0:1619677475.149768:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.149769:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.149770:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677475.149770:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.149771:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.149771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677475.149773:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.149773:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.149774:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677475.149775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677475.149776:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.149776:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.149777:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.149778:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677475.149782:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:25.2:1619677475.149782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.149785:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.149786:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.149787:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.149788:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.149789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.0:1619677475.149789:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.149792:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.149792:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.149794:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.149795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.149797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.150078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.150083:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.150086:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.150088:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.150088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.150091:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.150092:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.150094:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.150095:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.150096:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.150097:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.150101:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.150103:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.150105:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.150106:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.150106:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.150108:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.150109:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.150110:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.150111:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.150112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.150115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.150116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.150119:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.150121:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.150122:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.150123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677475.150124:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.150127:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.150128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.150128:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.150130:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.150132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.150523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.150531:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.150534:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.150536:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.150538:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.150539:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.150540:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.150541:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.150543:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.150543:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.150545:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.150548:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.150550:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.150552:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.150553:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.150554:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.150555:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.150556:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.150557:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.150558:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.150559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.150562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.150565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.150569:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.150571:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.150572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.150572:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.150574:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.150574:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.150576:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677475.150576:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.150577:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.150578:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.150579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.150585:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.150588:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.150588:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.150589:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.150589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.150590:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.150749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.150755:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.150757:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.150758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.150760:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.150762:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.150764:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.150765:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.150766:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.150767:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.150768:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.150771:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.150773:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:4.2:1619677475.150774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:22.0:1619677475.150775:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.150776:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.150777:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.150777:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:22.0:1619677475.150778:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.150779:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.150779:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.150779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.150780:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.150782:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.150783:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677475.150784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677475.150784:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.150786:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.150787:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.150788:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.150789:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.150790:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.150791:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.150792:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.150793:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.150793:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.150795:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.150796:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.150798:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.150798:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.150799:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.150800:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.150800:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.150802:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.150803:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.150803:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.150804:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.150808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.150816:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.150818:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.150819:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.150821:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.150822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.150823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.151177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.151185:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.151188:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.151190:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.151192:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.151192:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.151194:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.151195:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.151196:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.151197:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.151198:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.151202:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.151203:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.151205:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677475.151205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:31.0:1619677475.151206:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.151207:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.151208:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.151209:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.151210:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.151210:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677475.151211:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:25.2:1619677475.151214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677475.151214:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.151215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.151217:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.151219:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.151220:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:17.0:1619677475.151220:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.151222:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.151223:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:17.0:1619677475.151223:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.151224:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.151224:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.151225:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677475.151226:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.151227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677475.151227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677475.151228:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:17.0:1619677475.151229:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.151231:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:4.2:1619677475.151232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677475.151234:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.151234:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677475.151235:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.151235:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.151237:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.151237:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677475.151237:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.151238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677475.151238:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.151239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:8.0:1619677475.151239:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.151240:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677475.151242:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.151243:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:25.2:1619677475.151243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.151244:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677475.151244:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.151245:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96501345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.151246:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.151247:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.151248:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.151249:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.151249:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.151250:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:26.0:1619677475.151251:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.151252:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96501344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.151252:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.151253:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.151253:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:26.0:1619677475.151254:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.151254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:8.0:1619677475.151254:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:6.0:1619677475.151254:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.151255:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677475.151256:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.151257:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.151258:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.151259:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677475.151260:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.151260:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.151261:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677475.151261:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.151262:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.151262:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677475.151263:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677475.151265:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.151266:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.151266:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:25.2:1619677475.151266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.151268:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.151268:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.151269:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.151270:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677475.151273:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:25.2:1619677475.151275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.151277:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.151278:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.151279:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.151280:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.151281:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.151282:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.151283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:6.0:1619677475.151283:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.151284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.151284:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.151285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.151289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677475.151290:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.151292:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.151293:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.151299:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.151300:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.151301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.151629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.151634:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.151637:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.151638:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.151640:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.151642:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.151643:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.151644:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.151646:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.151647:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.151648:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.151651:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.151653:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.151655:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.151656:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.151657:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.151658:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.151659:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.151659:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.151660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.151665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.151673:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.151675:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.151676:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.151678:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.151679:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.151680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.152476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.152484:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.152487:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.152489:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.152492:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.152492:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.152493:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.152495:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.152496:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.152497:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.152498:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.152501:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.152502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.152503:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.152505:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.152506:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.152507:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677475.152508:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.152509:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.152509:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.152510:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.152511:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.152511:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.152512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.152514:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677475.152515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677475.152516:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.152517:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.152518:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.152520:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.152521:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.152522:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677475.152523:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:6.0:1619677475.152525:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.152526:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:6.0:1619677475.152526:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:24.0:1619677475.152527:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.152528:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677475.152529:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677475.152530:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.152530:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.152531:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677475.152532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677475.152532:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.152533:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.152533:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.152534:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.152535:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.152538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.152540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.152545:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.152547:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.152547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.152547:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.152548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.152550:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.152551:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.152552:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.152554:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.152554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:25.2:1619677475.152559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.152566:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.152569:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.152570:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.152570:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.152571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.152572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.152641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.152647:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.152650:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.152652:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.152654:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.152656:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.152657:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677475.152659:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.152660:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.152661:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.152662:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677475.152665:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.152667:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677475.152670:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.152671:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677475.152671:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677475.152673:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677475.152673:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.152674:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.152675:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677475.152690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.152699:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.152702:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.152703:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.152703:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.152705:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.152708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.153150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677475.153155:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.153158:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677475.153159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677475.153161:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.153163:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.153165:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677475.153166:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.153167:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.153168:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.153169:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677475.153173:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.153175:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677475.153177:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.153178:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:6.0:1619677475.153179:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:6.0:1619677475.153181:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:6.0:1619677475.153181:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.153182:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.153183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:21.2:1619677475.153184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677475.153187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:30.0:1619677475.153191:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.153194:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.153196:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.153196:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:31.0:1619677475.153198:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.153198:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.153198:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.153199:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.153200:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.153201:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.153201:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.153202:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.153203:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677475.153203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:30.0:1619677475.153204:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:12.0:1619677475.153204:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:30.0:1619677475.153207:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.153209:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.153211:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.153212:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.153213:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.153214:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.153215:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.153216:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.153217:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.153220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.153226:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.153229:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.153230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.153230:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.153232:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.153234:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.153584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.153589:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.153592:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.153593:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.153595:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.153598:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.153599:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.153601:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.153602:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.153603:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.153604:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.153607:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.153609:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.153612:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.153612:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.153613:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.153615:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.153615:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.153616:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.153616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.153622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.153630:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.153632:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.153633:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.153634:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.153636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.153637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.154257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.154265:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.154269:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.154270:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.154272:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.154275:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.154276:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.154278:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.154279:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.154280:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.154281:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.154284:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.154286:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.154288:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.154289:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.154290:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.154291:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.154292:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.154293:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.154297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.154297:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677475.154305:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.154307:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.154308:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.154310:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.154312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.154313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.154316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677475.154323:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.154325:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.154326:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.154327:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.154327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.154328:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.154465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.154473:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.154475:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.154478:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.154480:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.154480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.154481:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.154482:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.154484:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.154485:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.154486:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.154489:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.154490:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.154492:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.154493:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.154494:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.154495:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.154496:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.154497:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.154497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.154501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.154509:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.154511:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.154512:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.154512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.154514:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677475.154514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:24.0:1619677475.154516:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.154519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677475.154520:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.154523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.154523:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.154525:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.154527:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677475.154528:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.154530:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677475.154530:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.154532:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.154533:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.154534:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.154534:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.154535:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.154536:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:30.0:1619677475.154537:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.154538:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.154539:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.154540:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.154542:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.154542:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:30.0:1619677475.154543:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.154544:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677475.154545:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.154546:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.154546:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.154548:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.154548:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.154548:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.154549:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677475.154550:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677475.154550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677475.154552:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.154553:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.154553:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.154554:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:25.2:1619677475.154554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677475.154555:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.154557:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.154558:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.154559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.154560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677475.154562:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.2:1619677475.154565:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.154568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677475.154569:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.0:1619677475.154570:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677475.154570:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.154571:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.154572:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.154572:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.154574:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.154574:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.154575:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677475.154575:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.154576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.154577:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.154578:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.154579:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.154579:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.154580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.154580:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.154581:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.154581:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.154661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677475.154661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677475.154664:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.154667:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677475.154669:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677475.154669:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.154670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677475.154671:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:20.0:1619677475.154672:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.154672:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:20.0:1619677475.154674:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677475.154676:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.154676:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.154677:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.154678:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.154679:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.154679:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677475.154680:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.154682:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.154683:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.154684:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.154685:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.154686:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.154686:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.154688:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677475.154690:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.154690:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.154690:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.154692:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.154692:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.154692:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.154693:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.154694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677475.154695:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.154696:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.154697:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.154698:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.2:1619677475.154698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.154699:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.154699:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.154700:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.154704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.154707:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.154709:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.154710:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.154711:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677475.154711:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.154713:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.154714:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.2:1619677475.154714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:22.0:1619677475.154715:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.154715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.154718:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.154718:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.154719:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677475.154720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677475.154721:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.154722:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.154723:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.154723:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.154724:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.154823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677475.154830:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.154833:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677475.154834:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677475.154835:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.154838:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.154840:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677475.154841:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.154842:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.154843:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.154844:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677475.154848:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.154850:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677475.154852:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.154853:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677475.154853:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677475.154855:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677475.154855:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.154856:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677475.154856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677475.154860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.154867:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.154869:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.154870:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677475.154871:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.154873:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.154875:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677475.154883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677475.154889:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.154891:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.154892:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.154892:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.154893:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.154893:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.155292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.155302:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.155305:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.155307:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.155309:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.155309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.155310:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.155311:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.155312:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.155313:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.155314:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.155317:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.155319:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.155321:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.155321:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.155322:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.155324:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.155324:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.155325:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.155326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.155330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.155338:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.155340:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.155341:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.155342:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.155344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.155345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.155431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.155437:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.155440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.155440:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.155442:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.155444:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.155446:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.155447:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.155448:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.155449:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.155450:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.155454:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.155455:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.155458:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.155458:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.155459:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.155461:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.155461:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.155462:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.155462:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.155467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.155475:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.155477:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.155478:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.155480:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.155480:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.155482:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.155868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.155871:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.155874:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.2:1619677475.155876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677475.155876:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.155878:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.155880:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.0:1619677475.155881:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.155881:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.155882:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.155883:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677475.155883:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.155884:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677475.155886:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677475.155888:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.155889:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.155890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677475.155890:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677475.155891:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.155892:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677475.155893:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.155894:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.155894:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.155894:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:30.0:1619677475.155895:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.155896:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.155896:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:30.0:1619677475.155897:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677475.155897:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.155897:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.155898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677475.155900:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.155902:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.2:1619677475.155903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677475.155904:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.155905:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.155905:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.155906:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.155907:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.155908:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.155909:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677475.155911:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:25.2:1619677475.155912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677475.155913:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.155914:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.155915:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.155916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.155918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677475.155919:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.155922:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.155923:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.155925:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.155926:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.155926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677475.156006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.156014:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.156016:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:5.2:1619677475.156016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677475.156017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.156019:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.156021:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.156022:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:10.0:1619677475.156023:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.156024:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.156025:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.156026:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677475.156026:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.156026:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.156028:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.156029:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.156031:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.156031:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.156032:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677475.156034:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.156034:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.156035:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.156036:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677475.156037:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677475.156037:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.156038:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.156039:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.156041:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.156041:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677475.156041:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.156042:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.156042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:1.0:1619677475.156043:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.156046:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677475.156047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.156047:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.156047:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677475.156050:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (24)--
00000800:00000200:1.0:1619677475.156050:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.156052:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.156053:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.156053:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677475.156054:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:22.0:1619677475.156056:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000400:00000200:22.0:1619677475.156058:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.156058:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (23)++
00000800:00000200:5.2:1619677475.156058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677475.156059:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (24)--
00000400:00000200:6.0:1619677475.156059:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.156060:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.156061:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.156063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.156063:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.156064:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.156064:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677475.156066:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.156066:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677475.156067:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677475.156067:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677475.156067:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.156068:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.156068:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677475.156069:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.156143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.156149:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.156152:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.156153:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.156155:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.156157:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.156158:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.156159:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.156161:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.156162:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.156163:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.156166:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.156168:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.156170:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.156171:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.156171:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.156173:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.156173:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.156174:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.156175:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.156179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.156186:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.156188:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.156188:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.156189:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.156190:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.156192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.156497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677475.156498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.156504:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677475.156505:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.156507:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.156507:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:24.0:1619677475.156509:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.156509:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677475.156510:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.156512:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.156512:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.156514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:6.0:1619677475.156514:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:24.0:1619677475.156515:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.156515:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.156517:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677475.156517:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.156518:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.156519:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.156519:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:24.0:1619677475.156520:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.156521:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.156522:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:24.0:1619677475.156523:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677475.156524:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.156525:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:6.0:1619677475.156526:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:30.0:1619677475.156528:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.156528:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.156529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.156529:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.156531:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:24.0:1619677475.156531:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.156532:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.156533:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96501320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.156533:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:30.0:1619677475.156534:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.156534:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.156535:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.156535:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.156535:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:30.0:1619677475.156536:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.156536:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.156536:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.156536:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677475.156538:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.156538:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:30.0:1619677475.156539:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.156539:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.156539:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677475.156541:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.156541:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.156541:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.156542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677475.156542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677475.156543:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.156544:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.2:1619677475.156544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677475.156545:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.156546:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:25.2:1619677475.156546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.156547:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.156548:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.156548:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.0:1619677475.156550:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.156552:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.156552:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:9.2:1619677475.156552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677475.156553:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677475.156553:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.156554:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.156555:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.156555:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.156556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.156556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.156556:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.156557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.156557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677475.156558:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:6.0:1619677475.156558:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.156560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000400:00000200:6.0:1619677475.156560:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.156561:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.156562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.156562:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.156563:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:8.0:1619677475.156564:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.156567:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677475.156568:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.156570:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.156571:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.156572:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.156573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.156574:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.156589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.156597:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.156600:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.156602:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.156603:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.156604:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.156605:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.156606:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.156608:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.156609:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.156610:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.156613:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.156615:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.156617:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.156617:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.156618:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.156619:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.156620:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.156621:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.156621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.156625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.156632:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.156634:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.156635:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.156637:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.156638:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.156640:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.156674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.156680:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.156683:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.156684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.156686:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.156689:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.156690:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.156691:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.156693:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.156694:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.156695:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.156698:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.156699:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.156702:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.156703:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.156704:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.156706:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.156706:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.156707:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.156708:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.156712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.156718:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.156721:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.156722:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.156724:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.156725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.156727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.157385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.157391:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.157394:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.157394:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.157396:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.157398:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.157400:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.157401:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.157402:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.157403:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.157404:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.157407:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.157409:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.157411:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.157412:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.157412:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.157414:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.157414:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.157415:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.157416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.157421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.157429:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.157431:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.157432:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.157434:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.157435:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.157436:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.157487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.157494:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.157496:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.157497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.157499:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.157501:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.157503:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677475.157504:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.157506:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.157507:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.157508:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677475.157511:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.157513:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677475.157516:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.157517:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677475.157518:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677475.157519:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677475.157520:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.157521:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677475.157522:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677475.157522:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677475.157525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.2:1619677475.157528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:26.0:1619677475.157533:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:17.0:1619677475.157534:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:26.0:1619677475.157535:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:17.0:1619677475.157536:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:10.0:1619677475.157536:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.157537:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.157537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.157538:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:26.0:1619677475.157539:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.157542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677475.157894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677475.157902:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.157905:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.157907:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.157909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.157909:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.157910:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677475.157911:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.157913:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.157914:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.157915:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677475.157918:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.157920:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677475.157923:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.157924:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677475.157924:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677475.157926:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677475.157927:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.157927:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677475.157928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.157932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.157941:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.157943:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677475.157944:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.157945:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.157947:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.157950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.158037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.158044:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.158047:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.158050:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.158051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.158052:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.158054:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.158055:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.158056:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.158058:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.158058:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.158062:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.158064:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.158066:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.158066:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.158067:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.158069:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.158069:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.158071:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.158072:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.158073:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.158074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.158078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.158081:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.158084:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.158085:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.158085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.158086:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.158087:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.158089:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.158089:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677475.158089:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.158090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.158094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.158100:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.158102:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.158103:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.158103:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.158104:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.158105:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.158258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.158264:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.158266:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.158267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.158269:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.158272:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.158273:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.158274:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.158276:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.158277:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.158278:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.158281:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.158282:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.158284:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.158285:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.158286:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.158287:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.158288:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.158288:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.158289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.158293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.158302:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.158304:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.158305:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.158307:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.158308:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.158309:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.158569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.158575:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.158577:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.158578:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.158580:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.158582:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.158584:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677475.158585:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.158586:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.158587:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.158588:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677475.158591:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.158592:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677475.158594:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.158595:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:22.0:1619677475.158595:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677475.158597:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:22.0:1619677475.158597:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.158598:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.158598:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.2:1619677475.158604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.158612:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.158615:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.158616:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.158616:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.158618:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.158621:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.159108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.159113:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.159116:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.159118:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.159118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.159120:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.159121:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.159122:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.159124:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.159125:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677475.159127:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:21.0:1619677475.159130:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.159133:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677475.159135:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677475.159135:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.159137:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.159137:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:30.0:1619677475.159138:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.0:1619677475.159138:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:30.0:1619677475.159140:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.159140:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.159140:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677475.159142:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.159142:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677475.159143:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677475.159143:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.159143:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677475.159144:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677475.159145:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.159146:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677475.159146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:30.0:1619677475.159147:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.159149:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:4.2:1619677475.159149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677475.159152:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.159153:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677475.159154:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:30.0:1619677475.159156:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.159156:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.159156:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:30.0:1619677475.159157:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:21.0:1619677475.159157:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.159157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.159157:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.159158:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:20.0:1619677475.159158:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.159159:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:21.0:1619677475.159159:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:30.0:1619677475.159160:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677475.159160:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677475.159161:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:20.0:1619677475.159161:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677475.159162:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.159166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.159175:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.159177:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.159178:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.159179:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.159181:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:21.2:1619677475.159183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:9.2:1619677475.159184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.159188:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.159189:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.159191:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677475.159192:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677475.159192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.159192:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.159193:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.159194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:31.0:1619677475.159195:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.159197:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.159199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.159200:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.159201:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.159202:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.159204:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.159205:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.159205:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.159209:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.159210:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.159212:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.159212:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.159213:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.159214:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.159215:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.159217:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.159218:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.159219:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.159220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.159224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.159229:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.159230:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.159231:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.159233:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.159233:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.159234:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.159234:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.159235:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.159236:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.159237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.159240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.159247:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.159250:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.159250:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.159251:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.159251:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.159252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.159315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.159321:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.159324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.159324:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.159326:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.159329:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.159330:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.159331:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.159333:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.159334:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.159335:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.159338:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.159340:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.159342:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.159343:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.159343:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.159345:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.159345:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.159346:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.159346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.159352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.159360:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.159362:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.159363:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.159364:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.159366:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.159367:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.159532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.159540:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.159542:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.159543:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.159545:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.159548:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.159549:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.159550:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.159551:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.159552:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.159554:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.159557:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.159559:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.159562:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.159562:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.159563:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.159565:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.159565:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.159566:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.159568:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.159568:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.159571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.159573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.159578:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.159581:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.159581:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.159582:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.159582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:4.2:1619677475.159582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:26.0:1619677475.159583:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.159583:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.159584:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.159584:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.159587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677475.159590:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.159592:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677475.159593:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677475.159595:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677475.159598:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.159599:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.159600:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.159601:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.159602:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.159603:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.159606:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677475.159608:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.159610:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.159611:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.159612:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.159613:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.159614:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.159615:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677475.159616:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677475.159617:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:25.2:1619677475.159619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.159622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:6.0:1619677475.159626:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.159629:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.159629:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:6.0:1619677475.159629:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.159630:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.159631:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677475.159632:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:6.0:1619677475.159632:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.159633:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:6.0:1619677475.159635:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.159636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.159643:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.159645:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.159646:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.159646:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.159647:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.159648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677475.159916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.159924:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.159927:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.159929:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.107@o2ib7 : GET - for me
00000800:00000200:21.0:1619677475.159930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.159931:0:19934:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.107@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:20.0:1619677475.159933:0:19934:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.107@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:20.0:1619677475.159935:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.159935:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.159937:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.159941:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : REPLY try# 0
00000800:00000200:20.0:1619677475.159943:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.159945:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.159946:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.159946:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.159948:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.159948:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677475.159953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.159961:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.159963:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.159964:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.159966:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: REPLY: OK
00000800:00000200:20.0:1619677475.159967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.159968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.160197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.160206:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.160208:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.160210:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.160213:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.160213:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.160214:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.160215:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.160217:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.160218:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.160219:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.160223:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.160224:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.160226:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.160227:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.160228:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.160230:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.160230:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.160231:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.160232:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.160236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.160239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.0:1619677475.160242:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.160244:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:6.0:1619677475.160244:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677475.160245:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.160245:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677475.160247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:6.0:1619677475.160247:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.160248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:8.0:1619677475.160248:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.160250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677475.160250:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.160251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677475.160252:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:6.0:1619677475.160252:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.160253:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677475.160255:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.160255:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.160256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677475.160256:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.160257:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.160257:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.160258:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677475.160259:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.160260:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.160262:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.160263:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.160264:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.160267:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.160269:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:6.0:1619677475.160271:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.160273:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.160275:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.160275:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.160276:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677475.160276:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.160277:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.160278:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:6.0:1619677475.160278:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.160279:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.160279:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677475.160279:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.160280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677475.160280:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.160281:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.160282:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.160283:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.2:1619677475.160284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:25.2:1619677475.160286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.160290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.160295:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:22.0:1619677475.160297:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.160299:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.160299:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:22.0:1619677475.160300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.160300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:20.0:1619677475.160300:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.160301:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.160302:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.160302:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.160302:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.160304:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.160304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.160305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.160306:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.160307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.160308:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.160309:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.160309:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.160310:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.160310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.160507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.160515:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.160517:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.160520:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.160522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.160522:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.160523:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.160525:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.160526:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.160527:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.160528:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.160531:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.160533:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.160535:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.160536:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.160537:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.160538:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.160539:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.160540:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.160540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.160544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.160551:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.160554:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.160555:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.160556:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.160558:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.160559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.160821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.160829:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.160831:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.160832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.160834:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.160836:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.160838:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.160839:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.160840:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.160841:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.160842:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.160845:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.160847:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.160850:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.160851:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.160851:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.160853:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.160853:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.160854:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.160854:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.160859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.160867:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.160869:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.160870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.160870:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.160873:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.160875:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.161013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b9ba00] (20)++
00000800:00000200:22.0:1619677475.161019:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)++
00000800:00000200:22.0:1619677475.161021:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000800:00000200:6.0:1619677475.161022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000400:00000200:22.0:1619677475.161024:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.161026:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.161027:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677475.161029:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.161030:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.161031:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.161032:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677475.161035:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.161036:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677475.161039:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677475.161039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b9ba00] (21)++
00000800:00000200:22.0:1619677475.161040:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:22.0:1619677475.161041:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677475.161043:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677475.161044:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (22)++
00000800:00000200:22.0:1619677475.161045:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (23)--
00000800:00000200:6.0:1619677475.161045:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (23)++
00000800:00000200:22.0:1619677475.161046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (23)--
00000800:00000200:22.0:1619677475.161047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000800:00000200:6.0:1619677475.161047:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000800:00000200:9.2:1619677475.161049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677475.161049:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.161050:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.161052:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:6.0:1619677475.161053:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.161054:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.161055:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.161055:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:24.0:1619677475.161056:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.161059:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:6.0:1619677475.161059:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677475.161060:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.161060:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677475.161060:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:24.0:1619677475.161062:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.161063:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.161064:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:24.0:1619677475.161065:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677475.161065:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:6.0:1619677475.161067:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:6.0:1619677475.161067:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (21)++
00000800:00000200:6.0:1619677475.161068:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (22)--
00000800:00000200:6.0:1619677475.161069:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)--
00000800:00000200:25.2:1619677475.161073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:4.2:1619677475.161076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b9ba00] (20)++
00000800:00000200:13.0:1619677475.161080:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:22.0:1619677475.161081:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)++
00000800:00000200:13.0:1619677475.161083:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:22.0:1619677475.161084:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000400:00000200:13.0:1619677475.161084:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.161085:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000400:00000200:22.0:1619677475.161087:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677475.161087:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.161089:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677475.161090:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:22.0:1619677475.161092:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.161094:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677475.161095:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.161096:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.161097:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.161098:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677475.161100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b9ba00] (21)++
00000400:00000200:22.0:1619677475.161101:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.161104:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677475.161106:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)++
00000800:00000200:22.0:1619677475.161107:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.161107:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000800:00000200:22.0:1619677475.161108:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677475.161109:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:6.0:1619677475.161110:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.161111:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:6.0:1619677475.161111:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.161112:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (23)++
00000800:00000200:22.0:1619677475.161113:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (24)--
00000400:00000200:6.0:1619677475.161113:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:22.0:1619677475.161114:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (23)--
00000400:00000200:6.0:1619677475.161114:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.161115:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000400:00000200:6.0:1619677475.161116:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.161117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:6.0:1619677475.161117:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.161118:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677475.161120:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.161122:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.161124:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:6.0:1619677475.161124:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.161124:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677475.161125:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677475.161126:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677475.161127:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:6.0:1619677475.161127:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (21)++
00000400:00000200:30.0:1619677475.161128:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.161128:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919861b9ba00] (22)++
00000800:00000200:6.0:1619677475.161129:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (23)--
00000400:00000200:30.0:1619677475.161130:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.161130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000800:00000200:31.0:1619677475.161131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:9.2:1619677475.161132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.161133:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.161133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b9ba00] (21)++
00000800:00000200:22.0:1619677475.161138:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)++
00000800:00000200:22.0:1619677475.161141:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000800:00000200:18.0:1619677475.161141:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:6.0:1619677475.161141:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (23)++
00000400:00000200:22.0:1619677475.161143:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.161143:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919861b9ba00] (24)--
00000800:00000200:6.0:1619677475.161143:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (23)--
00000800:00000200:18.0:1619677475.161144:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:6.0:1619677475.161144:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)++
00000800:00000200:21.2:1619677475.161145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:18.0:1619677475.161145:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.161145:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000800:00000200:2.0:1619677475.161145:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:22.0:1619677475.161146:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.161148:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677475.161149:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.161149:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.161150:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.161152:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.161153:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.161154:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:22.0:1619677475.161156:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.161158:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677475.161159:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:6.0:1619677475.161160:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.161161:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.161161:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677475.161162:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.161162:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677475.161163:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.161164:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.161164:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.161165:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.161166:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:22.0:1619677475.161167:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (23)++
00000800:00000200:22.0:1619677475.161168:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (24)--
00000800:00000200:22.0:1619677475.161169:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (23)--
00000800:00000200:22.0:1619677475.161170:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)++
00000400:00000200:6.0:1619677475.161170:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.161171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.161171:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000800:00000200:6.0:1619677475.161172:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677475.161174:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.161175:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.161176:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.161176:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:6.0:1619677475.161177:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:22.0:1619677475.161178:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677475.161179:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96501294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.161179:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:22.0:1619677475.161180:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.161181:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.161182:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:6.0:1619677475.161183:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (23)++
00000800:00000200:6.0:1619677475.161184:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (24)--
00000800:00000200:6.0:1619677475.161184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (23)--
00000800:00000200:25.2:1619677475.161185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:22.0:1619677475.161185:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.161185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.161187:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677475.161187:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.161189:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.161190:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.161191:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:6.0:1619677475.161191:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.161192:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:12.0:1619677475.161193:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:6.0:1619677475.161193:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677475.161194:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:6.0:1619677475.161194:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96501293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.161195:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677475.161196:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:6.0:1619677475.161196:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.161197:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (22)++
00000400:00000200:6.0:1619677475.161197:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.161198:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (23)--
00000400:00000200:12.0:1619677475.161198:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.161199:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000800:00000200:22.0:1619677475.161199:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)++
00000800:00000200:13.0:1619677475.161200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:9.2:1619677475.161200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677475.161201:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000400:00000200:12.0:1619677475.161201:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.161202:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.161204:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.161204:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677475.161206:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:12.0:1619677475.161206:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677475.161209:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:22.0:1619677475.161209:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.161209:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677475.161209:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.161209:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:26.0:1619677475.161210:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.161210:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:22.0:1619677475.161211:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677475.161212:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96501292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.161212:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677475.161213:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.161214:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.161216:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.161216:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677475.161217:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (22)++
00000800:00000200:25.2:1619677475.161218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.161218:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:26.0:1619677475.161219:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677475.161219:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.161221:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.161221:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.161222:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.161222:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.161223:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:6.0:1619677475.161224:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:5.2:1619677475.161224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677475.161225:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.161226:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:22.0:1619677475.161226:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.161227:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:9.2:1619677475.161227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677475.161227:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.161228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677475.161228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000800:00000200:22.0:1619677475.161229:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:22.0:1619677475.161229:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (21)++
00000800:00000200:22.0:1619677475.161230:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919861b9ba00] (22)++
00000800:00000200:21.0:1619677475.161230:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.161230:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.161230:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677475.161230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677475.161232:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (23)--
00000800:00000200:22.0:1619677475.161232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000400:00000200:21.0:1619677475.161232:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.161233:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677475.161233:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:10.0:1619677475.161233:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677475.161234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677475.161234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.161235:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.161236:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.161236:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:8.2:1619677475.161236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.161237:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:4.2:1619677475.161237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b9ba00] (21)++
00000400:00000200:17.0:1619677475.161238:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.161238:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:17.0:1619677475.161239:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:10.0:1619677475.161239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:17.0:1619677475.161241:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.161242:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.161243:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.161243:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)++
00000800:00000200:20.0:1619677475.161244:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:17.0:1619677475.161244:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.161245:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919861b9ba00] (23)--
00000800:00000200:20.0:1619677475.161246:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:6.0:1619677475.161246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000800:00000200:22.0:1619677475.161247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)--
00000400:00000200:20.0:1619677475.161247:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.161247:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.161249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677475.161249:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.161249:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677475.161251:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.161252:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677475.161252:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.161252:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.161253:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.161255:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:31.0:1619677475.161256:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677475.161256:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.161256:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.161257:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.161258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.161259:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.161261:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.161262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:31.0:1619677475.161263:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.161263:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.161265:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.161266:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.161266:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000400:00000200:31.0:1619677475.161268:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.161268:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.161269:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.161269:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:6.0:1619677475.161269:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:31.0:1619677475.161270:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677475.161270:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.161271:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.161272:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677475.161272:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:31.0:1619677475.161273:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.161273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.161273:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.161275:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:6.0:1619677475.161275:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.161277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677475.161278:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.161279:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.161279:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.161281:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:25.2:1619677475.161281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677475.161282:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.161283:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.161283:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.161287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677475.161288:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.161292:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.161292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.161293:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:8.0:1619677475.161293:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.161299:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:22.0:1619677475.161299:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.161300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677475.161300:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.161301:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.161303:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.161306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.161390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677475.161395:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.161398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.161398:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677475.161400:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.161402:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.161404:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677475.161405:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.161406:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.161407:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.161408:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677475.161412:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.161413:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677475.161416:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.161417:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677475.161418:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677475.161420:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677475.161420:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.161421:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:4.2:1619677475.161421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b9ba00] (20)++
00000800:00000200:6.0:1619677475.161423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:6.0:1619677475.161424:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)++
00000800:00000200:6.0:1619677475.161426:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000800:00000200:25.2:1619677475.161427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677475.161427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000400:00000200:6.0:1619677475.161428:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.161429:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.161430:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677475.161431:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.161432:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.161433:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.161434:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:6.0:1619677475.161434:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677475.161436:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.161437:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.161438:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.161438:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:30.0:1619677475.161440:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.161441:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.161441:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.161442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677475.161442:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:30.0:1619677475.161443:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:6.0:1619677475.161444:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677475.161445:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (21)++
00000800:00000200:6.0:1619677475.161446:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (22)--
00000800:00000200:6.0:1619677475.161446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)--
00000800:00000200:25.2:1619677475.161450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.161457:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.161460:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.161461:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.161462:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.161463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.161465:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.161472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.161479:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.161481:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.161482:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.161483:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.161484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.161485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.161489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.161496:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.161499:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.161500:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.161502:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.161505:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.161506:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:4.2:1619677475.161507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b9ba00] (20)++
00000400:00000200:24.0:1619677475.161508:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.161509:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.161510:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.161511:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.161512:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)++
00000400:00000200:24.0:1619677475.161515:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.161515:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000800:00000200:6.0:1619677475.161515:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000800:00000200:24.0:1619677475.161517:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677475.161518:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677475.161519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677475.161520:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.161520:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.161521:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677475.161521:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.161523:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.161523:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677475.161523:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677475.161524:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.161524:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.161524:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.161525:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.161525:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677475.161526:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.161527:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677475.161528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.161528:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.161531:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.161531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677475.161532:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.161534:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.161535:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.161536:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.161537:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96501285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.161537:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677475.161538:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.161539:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.161539:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.161540:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677475.161540:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.161540:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.161541:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000400:00000200:21.0:1619677475.161541:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677475.161542:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677475.161542:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.161544:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (21)++
00000400:00000200:21.0:1619677475.161544:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.161545:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (22)--
00000800:00000200:6.0:1619677475.161545:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.161546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)--
00000800:00000200:21.0:1619677475.161546:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677475.161546:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.161547:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.161548:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.161549:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.161550:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677475.161550:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:6.0:1619677475.161550:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.161551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677475.161551:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.161551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b9ba00] (20)++
00000800:00000200:22.0:1619677475.161552:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.161552:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.161553:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677475.161553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677475.161554:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)++
00000800:00000200:21.0:1619677475.161554:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.161555:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:22.0:1619677475.161556:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000800:00000200:21.0:1619677475.161556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677475.161556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000400:00000200:22.0:1619677475.161558:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677475.161558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:22.0:1619677475.161559:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.161561:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.2:1619677475.161561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:22.0:1619677475.161562:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.161563:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.161564:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.161565:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:21.0:1619677475.161568:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.161568:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:22.0:1619677475.161569:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.161569:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.161569:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677475.161570:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.161570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677475.161571:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:21.0:1619677475.161571:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.161572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.161573:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677475.161574:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.161574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677475.161575:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677475.161575:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677475.161577:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677475.161578:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (21)++
00000800:00000200:22.0:1619677475.161578:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (22)--
00000800:00000200:22.0:1619677475.161579:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)--
00000800:00000200:9.2:1619677475.161583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.161589:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.161592:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.161593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.161593:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.161595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b9ba00] (20)++
00000400:00000200:8.0:1619677475.161596:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.161598:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677475.161600:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)++
00000800:00000200:6.0:1619677475.161603:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000800:00000200:22.0:1619677475.161604:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000400:00000200:6.0:1619677475.161606:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.161609:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.161610:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677475.161611:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.161612:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.161613:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.161614:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677475.161617:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.161618:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677475.161620:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.161621:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677475.161621:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677475.161623:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677475.161623:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (21)++
00000800:00000200:6.0:1619677475.161625:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919861b9ba00] (22)++
00000800:00000200:6.0:1619677475.161626:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (23)--
00000800:00000200:6.0:1619677475.161626:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000800:00000200:9.2:1619677475.161629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.161631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919861b9ba00] (21)++
00000800:00000200:22.0:1619677475.161636:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)++
00000800:00000200:2.0:1619677475.161637:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677475.161638:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919861b9ba00] (23)--
00000800:00000200:22.0:1619677475.161639:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000800:00000200:22.0:1619677475.161639:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)++
00000800:00000200:2.0:1619677475.161640:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.161641:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.161641:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.161642:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.49@o2ib3
00000800:00000200:6.0:1619677475.161643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (22)--
00000400:00000200:2.0:1619677475.161643:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.161644:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.49@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677475.161646:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:22.0:1619677475.161647:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.161649:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677475.161650:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.161651:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.161652:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.161653:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:9.2:1619677475.161653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000400:00000200:22.0:1619677475.161656:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.161657:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677475.161660:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.161661:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677475.161661:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.161661:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677475.161663:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677475.161663:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919861b9ba00] (21)++
00000800:00000200:18.0:1619677475.161663:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677475.161664:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919861b9ba00] (22)--
00000800:00000200:2.0:1619677475.161664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677475.161665:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919861b9ba00] (21)--
00000800:00000200:18.0:1619677475.161665:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.161666:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.161666:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:25.2:1619677475.161669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.161676:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.161679:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.161680:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.161682:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.161683:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.161685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.161777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677475.161784:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.161787:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677475.161789:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.161791:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677475.161791:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.161793:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.161794:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.161796:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.161797:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.161798:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.161801:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.161802:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.161805:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.161806:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.161807:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.161809:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.161809:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.161810:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677475.161811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677475.161815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.161823:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.161825:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.161826:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.161828:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.161829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.161831:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.161842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.161850:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.161853:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.161855:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.161856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.161858:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.161859:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.161860:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.161862:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.161863:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.161863:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.161866:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.161868:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.161870:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.161871:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.161872:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.161874:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.161874:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.161875:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.161875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.161879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.161885:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.161888:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.161889:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.161889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.161891:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.161893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.161932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.161939:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.161942:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.161944:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.161946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.161946:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.161947:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.161948:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.161950:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.161951:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.161952:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.161955:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.161956:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.161959:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.161960:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.161961:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.161963:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.161963:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.161964:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.161964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.161969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.161977:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.161979:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.161980:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.161982:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.161983:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.161985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.162186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.162194:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.162197:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.162198:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.162200:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.162203:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.162204:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.162206:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.162207:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.162208:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.162209:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.162212:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.162214:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.162217:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.162218:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.162218:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.162220:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.162221:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.162222:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.162222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.162227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.162234:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.162237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.162237:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.162238:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.162240:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.162243:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.162266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.2:1619677475.162267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.162272:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677475.162272:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677475.162275:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.162275:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677475.162276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.162276:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677475.162279:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.162280:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:22.0:1619677475.162283:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.162283:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.162285:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677475.162285:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677475.162287:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.162287:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.162289:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.162289:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.162291:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.162291:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.162292:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677475.162293:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677475.162301:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677475.162303:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677475.162304:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.162306:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.162306:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677475.162307:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.162308:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677475.162308:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.162309:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:22.0:1619677475.162310:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.162310:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677475.162310:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677475.162311:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.162311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:2.0:1619677475.162312:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.162312:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.162314:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.162314:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.162316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.2:1619677475.162319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677475.162324:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.162327:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:1.0:1619677475.162327:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:20.0:1619677475.162328:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.162329:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.162329:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677475.162330:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.162331:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677475.162331:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.0:1619677475.162332:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677475.162332:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.162335:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.162638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677475.162643:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.162646:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677475.162647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677475.162649:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.162652:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.162653:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677475.162655:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.162656:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.162657:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.162658:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677475.162662:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.162663:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.162666:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.162666:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677475.162667:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677475.162669:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677475.162670:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.162670:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677475.162671:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677475.162675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.162683:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.162686:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.162687:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.162687:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.162689:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.162691:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.163207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.163213:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.2:1619677475.163214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.163216:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677475.163218:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.163218:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.163219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.163220:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:22.0:1619677475.163221:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.163222:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.163222:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.163224:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.163225:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.163226:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.163227:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.163229:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.163230:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.163231:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:22.0:1619677475.163232:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.163233:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.163234:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.163235:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.163235:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677475.163238:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.163239:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.163240:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:6.0:1619677475.163240:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.163242:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.163242:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.163242:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677475.163243:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.163244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677475.163244:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.163245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.163245:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677475.163246:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:25.2:1619677475.163248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.163248:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677475.163249:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.163250:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:6.0:1619677475.163251:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677475.163251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:25.2:1619677475.163254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677475.163256:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:21.0:1619677475.163259:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.163261:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677475.163262:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:22.0:1619677475.163262:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000400:00000200:21.0:1619677475.163262:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.163263:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.163264:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:20.0:1619677475.163264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.163265:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:22.0:1619677475.163265:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:31.0:1619677475.163266:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.163266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:21.0:1619677475.163266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.163268:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.163269:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.163271:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.163364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677475.163372:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.163375:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677475.163376:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677475.163379:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677475.163382:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.163383:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.163385:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.163386:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.163387:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.163388:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.163392:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677475.163394:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.163397:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.163398:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.163399:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.163401:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.163401:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.163402:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677475.163403:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677475.163407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.163415:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.163418:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.163419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.163419:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.163422:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.163424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.163426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.163433:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.163436:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.163436:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.163437:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.163438:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.163438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.163810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.163818:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.163821:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.163824:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.163826:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.163827:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.163828:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.163829:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.163831:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.163832:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.163832:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.163835:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.163837:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.163840:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.163841:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.163841:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.163843:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.163844:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.163845:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.163847:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.163848:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.163849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.163853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.163857:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.163860:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677475.163861:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677475.163861:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.163862:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.163863:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677475.163863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.163864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.163865:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.163865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.163913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.163921:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.163924:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.163926:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.163927:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.163929:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.163930:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.163932:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.163933:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.163934:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.163935:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.163938:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.163940:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.163942:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.163942:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.163943:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.163945:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.163945:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.163946:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.163948:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.163948:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.163950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.163954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.163957:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.163959:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.163960:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.163960:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.163962:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.163962:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.163964:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677475.163964:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.163965:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.163966:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.163971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.163977:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.163979:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.163980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.163980:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.163981:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.163981:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.164050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.164056:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.164059:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.164060:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.164062:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.164065:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.164066:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.164067:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.164069:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.164070:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.164071:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.164074:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.164075:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.164077:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.164078:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.164079:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.164080:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.164081:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.164082:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.164083:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.164083:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.164086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.164089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.164095:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.164096:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.164097:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677475.164097:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.164098:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.164098:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.164099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.164099:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.164100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677475.164101:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677475.164103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.164108:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.164111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.164111:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.164113:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.164116:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.164117:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.164118:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.164119:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.164120:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.164121:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.164124:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.164126:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.164128:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.164129:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.164130:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.164131:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.164132:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.164132:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.164133:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.164134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.164138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.164139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.164144:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.164145:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.164146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.164146:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.164147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.164149:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.164149:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.164151:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.164152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.164153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.164156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.164163:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.164166:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.164167:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.164167:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.164168:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.164169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.164312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.164318:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.164321:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.164322:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.164324:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.164326:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.164328:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.164329:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.164331:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.164332:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.164333:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.164336:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.164338:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.164340:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.164341:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.164342:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.164344:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.164344:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.164345:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.164345:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.164350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.164359:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.164361:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.164362:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.164363:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.164364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.164366:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.164371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.164379:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.164381:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.164382:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.164383:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.164384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.164385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.164849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.164856:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.164859:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.164860:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.164862:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.164865:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.164866:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.164868:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.164869:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.164870:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.164871:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.164874:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.164876:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.164879:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.164879:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.164880:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.164882:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.164882:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.164883:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.164883:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.164888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.164896:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.164899:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.164900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.164900:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.164902:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.164905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.165085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677475.165090:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.165093:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677475.165094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677475.165096:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.165098:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.165099:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677475.165100:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.165102:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.165103:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.165104:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677475.165107:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.165109:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677475.165111:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.165112:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677475.165113:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677475.165114:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677475.165115:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.165116:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677475.165117:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677475.165118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677475.165121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.2:1619677475.165123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.165128:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:2.0:1619677475.165128:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:6.0:1619677475.165129:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677475.165130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677475.165131:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:2.0:1619677475.165131:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.165132:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.165132:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.165134:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.165137:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.165165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.165171:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.165173:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.165174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.165175:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.165178:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.165180:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.165181:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.165182:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.165183:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.165184:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.165186:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.165188:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.165190:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.165190:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.165191:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.165192:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.165193:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.165193:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.165194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.165198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.165206:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.165208:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.165209:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.165211:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.165212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.165213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.165443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.165451:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.165454:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.165457:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.165459:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.165459:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.165461:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.165462:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.165463:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.165464:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.165465:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.165469:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.165471:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.165473:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.165473:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.165474:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.165475:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.165476:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.165477:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.165478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.165482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.165489:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.165491:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.165492:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.165494:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.165495:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.165496:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.165690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.165697:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.165700:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.165701:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.165702:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.165705:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.165706:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.165707:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.165709:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.165710:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.165710:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.165714:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.165715:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.165718:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.165719:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.165719:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.165721:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.165721:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.165722:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.165724:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.165724:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.165727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.165730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.165735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.165737:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.165737:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.165738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.165739:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677475.165739:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.165740:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.165741:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.165741:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.165743:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.165778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.165785:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.165788:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.165789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.165791:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.165793:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.165795:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.165796:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.165797:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.165798:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.165799:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.165802:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.165803:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.165806:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.165807:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.165807:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.165809:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.165810:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.165811:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.165812:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.165813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.165815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.165818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.165822:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.165825:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677475.165825:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:22.0:1619677475.165826:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.165826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.165827:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677475.165828:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.165828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.165829:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.165831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.165867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.165874:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.165876:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.165877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.165879:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.165881:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.165883:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.165884:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.165886:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.165887:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.165888:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.165891:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.165893:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.165895:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.165896:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.165897:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.165898:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.165899:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.165899:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.165900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.165904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.165912:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.165914:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.165915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.165915:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.165917:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.165919:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.165978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.165985:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.165988:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.165990:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.165992:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.165992:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.165994:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.165995:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.165996:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.165997:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.165998:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.166002:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.166004:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.166007:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.166008:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.166008:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.166010:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.166011:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.166012:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.166012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.166016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.166023:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.166026:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.166027:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.166027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.166029:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.166032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.166051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.166056:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.166059:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.166060:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.166062:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.166063:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.166065:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.166066:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.166067:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.166068:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.166069:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.166073:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.166074:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.166077:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.166078:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.166079:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.166080:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.166081:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.166081:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.166082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.166086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.166094:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.166096:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.166097:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.166099:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.166100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.166101:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.166275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.166278:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.166281:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.166283:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.166284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.166285:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.166287:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.166288:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.166289:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.166290:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.166291:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.166298:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.166300:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.166302:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.166303:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.166304:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.166305:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.166306:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.166307:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.166307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.166312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.166313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677475.166320:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:30.0:1619677475.166321:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677475.166323:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:30.0:1619677475.166324:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:17.0:1619677475.166324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.166324:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.166326:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.166327:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.166329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677475.166329:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:30.0:1619677475.166330:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.166332:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.166333:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.166334:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.166335:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.166336:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677475.166337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000400:00000200:30.0:1619677475.166340:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.166342:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.166343:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.166344:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677475.166344:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:30.0:1619677475.166345:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.166346:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.166347:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.166347:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:30.0:1619677475.166348:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.166348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677475.166348:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677475.166349:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.166352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:8.0:1619677475.166353:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.166354:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677475.166355:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96501255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.166357:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.166357:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.166358:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.166359:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:8.0:1619677475.166361:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.166362:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.166363:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.166363:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:31.0:1619677475.166364:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.166365:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.166366:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677475.166366:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.166367:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.0:1619677475.166367:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:25.2:1619677475.166368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677475.166368:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677475.166369:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677475.166369:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677475.166370:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:25.2:1619677475.166374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.166376:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.166379:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.166379:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.166380:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.166380:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.166381:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.166382:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.166383:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.166383:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:31.0:1619677475.166384:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.166385:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.166388:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.166502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.166509:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.166512:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.166513:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.166515:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.166517:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.166519:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.166520:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.166521:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.166522:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.166523:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.166525:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.166527:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.166530:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.166530:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.166531:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.166532:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.166533:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.166534:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.166534:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.166538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.166548:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.166551:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.166552:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.166554:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.166556:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.166557:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.166869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677475.166876:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.166879:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677475.166880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677475.166881:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677475.166884:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.166885:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677475.166886:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.166888:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.166889:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.166889:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677475.166892:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677475.166894:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.166895:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.166896:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.166896:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.166898:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.166898:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.166899:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677475.166899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677475.166914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.166920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.166922:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.166923:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.166924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.166924:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.166926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.166927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.166934:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.166936:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.166937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.166938:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.166940:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.166942:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677475.166943:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.166944:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.166945:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.166945:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677475.166948:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.166950:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677475.166953:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.166953:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677475.166954:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677475.166956:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677475.166956:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.166957:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.166957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677475.166962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.166971:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.166973:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.166974:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.166974:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.166977:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.166980:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.166996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.167002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.167005:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.167005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.167007:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.167010:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.167011:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.167013:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.167014:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.167015:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.167016:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.167019:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.167021:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.167024:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.167024:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.167025:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.167027:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.167027:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.167028:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.167028:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.167033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.167040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.167043:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.167044:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.167045:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.167046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.167047:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.167542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.167545:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.167548:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.167550:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.167551:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.167552:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.167553:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.167555:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.167556:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.167557:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.167558:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.167561:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.167563:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.167566:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.167566:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.167567:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.167569:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.167569:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.167570:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.167570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.167576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.167584:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.167587:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.167588:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.167588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.167590:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.167592:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677475.167612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.167618:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.167621:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.167622:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.167623:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.167626:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.167627:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:4.2:1619677475.167628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:2.0:1619677475.167628:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.167630:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.167631:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.167632:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.0:1619677475.167633:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677475.167635:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.167636:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677475.167638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677475.167638:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:21.0:1619677475.167639:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677475.167641:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.167641:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.167642:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:21.0:1619677475.167643:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:2.0:1619677475.167643:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:21.0:1619677475.167644:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.167645:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677475.167645:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:21.0:1619677475.167646:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677475.167646:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:21.0:1619677475.167647:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:2.0:1619677475.167647:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.167648:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:21.0:1619677475.167650:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:25.2:1619677475.167652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.0:1619677475.167652:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.167654:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.167655:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.167655:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.167657:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.167657:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.167658:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.167659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677475.167660:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.167662:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.167663:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.167663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:12.0:1619677475.167665:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.167667:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.167667:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.0:1619677475.167670:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.167673:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.167673:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.167675:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.167676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.167677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.167995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.168003:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.168005:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.168008:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.168010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.168010:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.168011:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.168013:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.168014:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.168015:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.168016:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.168019:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.168020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.168021:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.168024:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.168025:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.168026:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.168027:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.168027:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.168028:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.168029:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677475.168029:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.168029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677475.168031:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.168031:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.168033:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.168033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:31.0:1619677475.168034:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.168036:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.168037:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.168038:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.168038:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.168040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:31.0:1619677475.168042:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.168043:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.168043:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.168044:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.168046:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.168046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.168047:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.168047:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.168048:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.168048:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:31.0:1619677475.168050:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.168050:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.168051:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.168052:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.168054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.2:1619677475.168055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:25.2:1619677475.168056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.168063:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677475.168063:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.168065:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.168065:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:20.0:1619677475.168066:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.168067:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677475.168067:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.168067:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000400:00000200:22.0:1619677475.168068:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.168068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:6.0:1619677475.168068:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.168069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:22.0:1619677475.168070:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.168071:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.168073:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.168074:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.168076:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.168077:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.168078:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.168079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.168080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.168081:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677475.168082:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.168083:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.168084:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.168084:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.168085:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.168085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.168087:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.168087:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.168088:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.168088:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.168090:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.168091:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.168092:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.168093:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.168094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.168096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.168099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.168103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.168104:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.168105:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.168106:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677475.168106:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.168107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.168108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.168108:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.168109:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.168110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.168115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.168122:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.168124:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.168125:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.168126:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.168126:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.168127:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.168972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.168978:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.168980:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.168981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.168983:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.168985:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.168986:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.168987:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.168988:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.168989:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.168990:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.168993:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.168994:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.168996:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.168997:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.168997:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.168999:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.169000:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.169001:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.169001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.169006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.169014:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.169016:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.169017:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.169019:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.169020:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.169021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.169726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.169734:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.169738:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.169740:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.169742:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.169744:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.169745:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.169746:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.169748:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.169749:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.169750:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.169754:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.169755:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.169759:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.169760:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.169760:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.169762:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.169763:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.169764:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.169766:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.169766:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.169769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.169772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.169777:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.169779:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.169780:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.169781:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.169782:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.169782:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.169783:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.169784:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.169785:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.169785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.169833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.169839:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.169842:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.169843:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.169845:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.169847:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.169848:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.169849:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.169851:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.169852:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.169853:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.169856:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.169858:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.169861:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.169861:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.169862:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.169864:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.169864:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.169865:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.169865:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.169870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.169878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.169881:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.169881:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.169883:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.169884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.169885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.170031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.170039:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.170042:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.170044:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.170045:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.170047:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.170049:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.170050:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.170051:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.170052:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677475.170052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000400:00000200:31.0:1619677475.170053:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.170057:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.170059:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.170059:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:31.0:1619677475.170062:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.170062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.170062:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:31.0:1619677475.170063:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.170063:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.170065:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:1.0:1619677475.170065:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.170066:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.170067:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.170067:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:1.0:1619677475.170067:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.170068:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677475.170069:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.170071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677475.170071:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.170072:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.170073:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677475.170076:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.170078:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677475.170078:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.170081:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677475.170081:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.170081:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677475.170082:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.170082:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.170082:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000400:00000200:6.0:1619677475.170084:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.170084:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677475.170084:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.170085:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.170086:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677475.170087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:25.2:1619677475.170090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.170097:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.170100:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.170101:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.170101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.170102:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.170104:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.170206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:4.2:1619677475.170212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:18.0:1619677475.170213:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.170216:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.170216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.170219:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.170222:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.170222:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:18.0:1619677475.170223:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.170224:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.170226:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.170227:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.170228:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677475.170228:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.170228:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.170233:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.170233:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.170235:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.170237:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677475.170237:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.170238:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.170238:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.170240:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:10.0:1619677475.170240:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677475.170241:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.170241:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.170242:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677475.170242:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.170244:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.170246:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677475.170246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677475.170247:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.170253:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.170253:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.170255:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:10.0:1619677475.170256:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:2.0:1619677475.170256:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.170257:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.170258:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.170260:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.170260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677475.170261:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.170262:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.170264:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.170265:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.170266:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.170267:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.170270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.170278:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.170280:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.170281:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.170282:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.170284:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.170287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.170464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.170473:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.170476:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.170478:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.170480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.170481:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.170482:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.170483:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.170485:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.170486:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.170487:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.170491:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.170492:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.170494:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.170495:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.170495:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.170497:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.170497:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.170498:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.170499:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.170513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.170521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.170523:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.170524:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.170525:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.170527:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.170528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.170614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.170622:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.170624:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.170625:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.170627:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.170630:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.170631:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.170632:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.170634:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.170634:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.170635:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.170638:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.170640:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.170643:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.170643:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.170644:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.170645:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.170646:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.170646:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.170647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.170651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.170659:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.170662:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.170662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.170663:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.170665:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.170667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.170885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.170890:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.170892:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.170893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.170895:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.170897:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.170898:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.170899:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.170901:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.170901:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.170903:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.170906:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.170908:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.170910:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.170911:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.170911:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.170913:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.170914:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.170914:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.170915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.170919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.170926:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.170928:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.170929:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.170931:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.170933:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.170933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.171057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.171063:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.171066:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.171067:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.171069:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.171071:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.171073:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.171074:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.171075:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.171076:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.171077:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.171081:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.171082:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.171085:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.171085:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.171086:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.171088:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.171088:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.171089:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.171089:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.171094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.171101:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.171103:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.171104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.171105:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.171107:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.171109:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.171112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.171119:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.171122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.171122:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.171123:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.171124:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.171125:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.171211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.171217:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.171220:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.171221:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.171223:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.171225:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.171226:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677475.171227:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.171228:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.171229:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.171230:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677475.171233:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.171234:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.171235:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.171236:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.171236:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.2:1619677475.171238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677475.171238:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.171239:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.171240:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.171240:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:30.0:1619677475.171245:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677475.171245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677475.171248:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.171250:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.171252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.171252:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.171252:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:30.0:1619677475.171253:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677475.171254:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:30.0:1619677475.171255:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.171255:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.171256:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:30.0:1619677475.171257:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.171257:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.171258:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.171259:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677475.171259:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:30.0:1619677475.171263:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.171264:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.2:1619677475.171265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:30.0:1619677475.171266:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.171267:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.171268:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.171269:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.171270:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.171271:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677475.171271:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677475.171272:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677475.171274:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:25.2:1619677475.171275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677475.171275:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.171276:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.171277:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.171277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677475.171282:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.171284:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.171285:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.171287:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.171288:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.171289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.171509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.171517:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.171520:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.171522:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.171524:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.171524:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.171526:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.171527:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.171528:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.171529:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.171530:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.171534:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.171535:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.171538:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.171539:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.171540:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.171541:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.171542:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.171544:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.171545:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.171546:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.171548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.171552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.171556:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.171559:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.171559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.171560:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.171560:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.171562:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677475.171562:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.171563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.171564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.171565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.171566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.171573:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.171575:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.171576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.171576:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.171578:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.171578:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.171832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.171836:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.171838:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.171839:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.171841:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.171843:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.171845:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.171845:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.171847:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.171848:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.171849:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.171852:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.171853:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.171856:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.171857:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.171857:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.171859:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.171859:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.171860:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.171861:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.171862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.171865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.171867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.171872:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.171874:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.171874:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.171874:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.171875:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.171876:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.171877:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.171879:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.171880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.171881:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.171883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.171891:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.171893:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.171894:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.171895:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.171895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.171896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.171952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.171960:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.171963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.171963:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.171966:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.171968:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.171970:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.171971:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.171973:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.171974:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.171975:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.171978:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.171979:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.171982:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.171983:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.171984:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.171985:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.171986:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.171986:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.171987:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.171991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.171998:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.172001:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.172002:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.172002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.172004:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.172007:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.172181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.172183:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.172186:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.172188:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.172189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.172190:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.172192:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.172193:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.172194:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.172195:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.172196:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.172200:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.172202:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.172204:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.172205:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.172205:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.172207:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.172208:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.172208:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.172209:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.172214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.172222:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.172224:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.172225:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.172225:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.172227:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.172229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.172235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.172242:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.172245:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.172246:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.172246:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.172247:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.172248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.172539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.172548:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.172551:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.172553:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.172555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.172556:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.172558:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.172559:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.172560:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.172561:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.172562:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.172566:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.172567:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.172571:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.172572:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.172572:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.172574:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.172575:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.172576:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.172576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677475.172581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.172590:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.172592:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.172593:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.172595:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.172595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.172598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.172691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.172696:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.172699:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.172700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.172702:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.172704:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.172705:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.172707:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.172708:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.172709:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.172710:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.172713:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.172715:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.172718:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.172718:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.172719:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.172721:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.172721:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.172722:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.172722:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.172727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.172735:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.172737:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.172738:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.172739:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.172740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.172742:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.173432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.173438:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.173440:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.173442:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.173443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.173445:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.173446:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.173448:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.173449:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.173450:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.173451:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.173454:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.173456:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.173458:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.173459:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.173460:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.173461:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.173462:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.173463:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.173464:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.173465:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.173468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.173470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.173473:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.173475:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.173475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.173476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.173476:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.173479:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.173480:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.173480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.173482:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.173484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.173584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.173591:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.173594:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.173595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.173597:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.173600:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.173601:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.173603:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.173604:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.173605:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.173606:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.173609:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.173611:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.173614:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.173615:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.173616:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.173617:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.173618:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.173618:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.173619:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.173623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.173631:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.173634:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.173636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.173636:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.173638:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.173641:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.173731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.173737:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.173740:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.173741:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.173743:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.173745:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.173746:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.173748:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.173750:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.173751:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.173752:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.173755:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.173757:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.173759:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.173760:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.173760:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.173762:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.173762:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.173763:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.173764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.173769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.173777:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.173779:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.173780:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.173781:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.173782:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.173783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.173809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.173817:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.173820:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.173823:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.173825:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.173826:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.173827:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.173829:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.173830:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.173831:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.173832:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.173835:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.173837:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.173840:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.173841:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.173842:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.173843:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.173844:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.173845:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.173845:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.173849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.173857:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.173859:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.173860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.173861:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.173863:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.173866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.174235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.174242:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.174245:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.174245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.174248:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.174250:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.174251:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.174253:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.174254:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.174255:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.174256:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.174260:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.174261:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.174264:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.174265:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.174266:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.174268:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.174268:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.174269:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.174270:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.174271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.174274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677475.174276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677475.174280:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:21.2:1619677475.174282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677475.174282:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677475.174283:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:2.0:1619677475.174283:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.174284:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677475.174285:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.174285:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677475.174287:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.174287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.174289:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.174290:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677475.174290:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677475.174291:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.174292:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.174295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:30.0:1619677475.174299:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.174301:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:30.0:1619677475.174301:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677475.174302:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677475.174302:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.174303:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.174304:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.174304:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.174305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.174305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:30.0:1619677475.174305:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.174306:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.174309:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.174310:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.174312:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.174313:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.174314:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.174315:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.174316:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.174316:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.174317:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.174321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.174328:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.174330:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.174331:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.174332:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.174334:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.174335:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.174636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.174646:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.174650:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.174652:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.174655:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:21.2:1619677475.174656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:10.0:1619677475.174658:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.174660:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:22.0:1619677475.174662:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677475.174662:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.174665:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:10.0:1619677475.174665:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.174665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677475.174667:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.174668:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.174669:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677475.174671:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.174672:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.174674:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.174676:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.174676:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677475.174677:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.174679:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.174680:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:22.0:1619677475.174683:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.174683:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.174685:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.174685:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.174685:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677475.174687:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.174688:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677475.174688:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677475.174689:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677475.174689:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.174690:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:22.0:1619677475.174691:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677475.174691:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:22.0:1619677475.174692:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677475.174693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677475.174693:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.174693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.174697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:13.0:1619677475.174701:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.174704:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.174704:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.174705:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:13.0:1619677475.174706:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.174707:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.174708:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.174708:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677475.174708:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:21.0:1619677475.174710:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.174710:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:25.2:1619677475.174712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:21.0:1619677475.174712:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:12.0:1619677475.174719:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.174722:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.174723:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.174724:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.174724:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.174726:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.175002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.175010:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.175012:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.175015:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.175017:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.175017:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.175018:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.175019:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.175020:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.175022:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.175022:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.175026:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.175028:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.175030:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.175031:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.175031:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.175033:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.175033:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.175035:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.175036:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.175037:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.175038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.175042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.175047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.175049:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.175049:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.175051:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.175051:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.175052:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.175052:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.175053:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.175054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.175055:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.175057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.175064:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.175066:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.175067:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.175068:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.175068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.175069:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.175082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.175091:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.175095:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.175097:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.175099:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.175103:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.175105:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.175107:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.175110:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.175111:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.175112:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.175118:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.175120:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.175124:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.175125:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.175126:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.175128:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.175129:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.175130:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.175131:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.175134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.175142:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.175145:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.175145:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.175146:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.175148:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.175150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.175450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.175458:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.175460:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.175462:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.175464:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.175465:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.175466:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.175467:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.175468:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.175469:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.175470:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.175474:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.175475:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.175478:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.175479:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.175479:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.175481:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.175482:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.175483:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.175483:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.175488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.175496:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.175499:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.175500:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.175500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.175502:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.175505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.175508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:21.2:1619677475.175514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.175517:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.175519:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.175519:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.175520:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.175521:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.175521:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.175522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.175523:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.175523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.175524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.175526:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.175527:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.175528:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.175530:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.175531:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.175532:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.175535:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.175537:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.175539:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677475.175539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677475.175540:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.175541:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.175542:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.175543:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.175543:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.175544:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.175545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677475.175547:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677475.175548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677475.175549:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.175550:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.2:1619677475.175550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677475.175553:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.175555:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:10.0:1619677475.175555:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.175555:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:10.0:1619677475.175557:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:6.0:1619677475.175557:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.175557:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.175558:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:10.0:1619677475.175558:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.175558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.175559:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.175559:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.175560:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.175561:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.175562:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677475.175562:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:4.2:1619677475.175563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:10.0:1619677475.175565:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.175567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677475.175567:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:25.2:1619677475.175569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677475.175569:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.175570:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:20.0:1619677475.175571:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:10.0:1619677475.175571:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677475.175572:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.175573:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:10.0:1619677475.175573:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.175573:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677475.175574:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.175574:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.175574:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.0:1619677475.175575:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000400:00000200:20.0:1619677475.175575:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.175576:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.175576:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.175577:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.175578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.175578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:20.0:1619677475.175578:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:25.2:1619677475.175579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:20.0:1619677475.175580:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.175581:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.175582:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.175583:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677475.175586:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:20.0:1619677475.175586:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677475.175588:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:20.0:1619677475.175588:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.175589:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.175589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.175590:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.175590:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.175591:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:22.0:1619677475.175592:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.175592:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.175593:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.175593:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:22.0:1619677475.175594:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677475.175594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.175599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.175606:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.175609:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.175610:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.175611:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.175612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.175613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.175696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.175704:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.175707:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.175709:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.175711:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.175712:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.175714:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.175715:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.175717:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.175718:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.175719:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.175722:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.175724:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.175727:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.175728:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.175728:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.175730:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.175731:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.175732:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.175732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.175736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.175744:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.175746:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.175747:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.175749:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.175750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.175752:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677475.175800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.175807:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.175809:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.175810:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.175812:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.175814:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.175815:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.175817:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.175818:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.175819:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.175820:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.175824:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.175825:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.175828:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.175829:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.175829:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.175831:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.175832:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.175833:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.175834:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.175835:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.175837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.175840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677475.175844:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677475.175845:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677475.175847:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.175848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.175848:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.175848:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.175848:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.175849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677475.175850:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.175852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.175971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.175978:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.175981:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.175982:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.175984:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.175987:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.175988:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677475.175989:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.175991:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.175992:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.175993:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:18.0:1619677475.175995:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.175997:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:18.0:1619677475.176000:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.176001:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:18.0:1619677475.176001:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677475.176003:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677475.176004:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.176005:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677475.176006:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677475.176006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.2:1619677475.176009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677475.176011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:21.0:1619677475.176017:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:2.0:1619677475.176017:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:21.0:1619677475.176019:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:2.0:1619677475.176019:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:21.0:1619677475.176020:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.176020:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.176021:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:21.0:1619677475.176022:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.176023:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.176025:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.176028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.176035:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.176038:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.176039:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.176040:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.176041:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.176042:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.176399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.176407:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.176409:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.176410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.176413:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.176415:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.176417:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.176418:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.176420:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.176421:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.176422:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.176425:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.176426:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.176429:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.176430:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.176430:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.176432:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.176433:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.176434:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.176435:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.176436:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.176439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.176442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.176447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.176449:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.176449:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.176450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.176451:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677475.176451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.176452:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.176453:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.176453:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.176455:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.176582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.176587:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.176589:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.176590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.176592:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.176594:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.176595:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.176596:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.176597:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.176598:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.176599:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.176603:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.176605:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.176607:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.176608:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.176608:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.176610:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.176610:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.176611:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.176611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.176616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.176624:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.176626:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.176627:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.176629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.176629:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.176631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.176676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.176681:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.176683:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.176686:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.176686:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.176688:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.176690:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.176691:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.176693:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.176694:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.176695:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.176698:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.176700:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.176703:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.176704:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.176704:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.176706:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.176707:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.176707:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.176708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.176712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.176719:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.176722:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.176722:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.176723:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.176725:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.176727:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.177488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.177494:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.177496:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.177497:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.177499:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.177502:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.177503:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.177504:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.177506:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.177507:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.177508:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.177511:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.177512:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.177515:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.177515:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.177516:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.177517:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.177518:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.2:1619677475.177518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:22.0:1619677475.177519:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.177519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.177524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.177527:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.177530:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.177532:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.177533:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677475.177534:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.177535:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.177535:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:31.0:1619677475.177536:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.177536:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.177537:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.177538:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.177538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677475.177539:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.177540:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.177540:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.177541:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.177544:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.177546:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.177549:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.177550:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.177550:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.177552:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.177553:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.177554:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.177556:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.177557:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.177558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.177562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.177566:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.177568:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.177569:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.177570:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677475.177570:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.177572:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.177573:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.177574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.177575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.177575:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.177695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.177701:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.177703:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.177705:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.177706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.177708:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.177709:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.177711:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.177712:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.177713:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.177714:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.177717:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.177719:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.177721:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.177722:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.177723:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.177724:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.177725:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.177726:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.177726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.177732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.177741:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.177743:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.177744:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.177744:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.177746:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.177749:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.177869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.177876:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.177879:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.177882:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.177884:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.177885:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.177887:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.177888:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.177889:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.177890:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.177891:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.177894:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.177896:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.177899:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.177900:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.177901:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.177902:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.177903:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.177904:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.177905:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.177909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.177916:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.177919:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.177920:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.177922:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.177923:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.177925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.178020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.178027:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.178030:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.178030:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.178033:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.178035:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.178037:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.178038:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.178039:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.178040:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.178041:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.178044:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.178046:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.178048:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.178048:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.178049:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.178051:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.178051:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.178052:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.178053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.178057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.178064:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.178066:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.178067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.178067:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.178068:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.178069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.178076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.178084:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.178086:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.178087:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.178087:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.178087:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.178088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.178388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.178394:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.178396:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.178397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.178399:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.178401:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.178402:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.178403:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.178405:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.178406:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.178407:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.178409:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.178411:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.178413:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.178414:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.178415:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.178416:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.178417:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.178417:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.178418:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.178423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.178432:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.178434:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.178435:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.178436:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.178437:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.178438:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.178806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677475.178812:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.178815:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677475.178816:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677475.178818:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.178820:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.178822:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677475.178823:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.178825:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.178826:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.178827:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677475.178830:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.178832:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677475.178834:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.178835:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677475.178836:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.178838:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677475.178838:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.178839:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677475.178840:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:25.2:1619677475.178844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.178851:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.178853:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.178854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.178855:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.178857:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.178859:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.179000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.179003:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.179005:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.179007:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.179008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.179010:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.179011:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.179012:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.179014:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.179015:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.179016:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.179019:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.179021:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.179023:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.179024:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.179025:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.179026:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.179027:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.179027:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.179028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.179033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.179041:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.179043:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.179044:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.179045:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.179046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.179047:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.179329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.179337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.179340:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.179342:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.179345:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.179345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.179347:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.179348:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.179349:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.179350:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.179352:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.179355:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.179357:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.179360:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.179361:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.179361:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.179363:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.179364:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.179365:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.179365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.179370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.179378:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.179381:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.179382:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.179382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.179385:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.179387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.179391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.179392:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.179394:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.179395:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.179395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.179396:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.179397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.179426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.179431:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.179433:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.179434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.179436:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.179438:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.179439:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.179440:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.179442:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.179442:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.179443:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.179447:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.179448:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.179450:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.179451:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.179452:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.179453:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.179454:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.179455:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.179456:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.179457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.179459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.179462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.179467:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.179467:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.179469:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.179469:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.179470:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.179470:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.179471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.179472:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.179473:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.179474:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.179477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.179485:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.179487:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.179488:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.179488:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.179489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.179490:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.179607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.179615:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.179618:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.179620:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.179622:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.179623:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.179625:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.179626:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.179627:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.179628:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.179629:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.179632:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.179634:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.179636:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.179637:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.179638:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.179639:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.179640:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.179640:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.179641:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.179645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.179652:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.179654:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.179655:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.179656:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.179657:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.179660:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.179665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.179671:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.179674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.179674:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.179674:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.179675:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.179676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.179894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.179901:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.179903:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.179904:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.179905:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.179908:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.179910:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.179911:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.179912:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.179913:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.179913:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.179916:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.179918:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.179921:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.179921:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.179922:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.179924:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.179924:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.179924:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.179925:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.179930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.179938:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.179941:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.179942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.179942:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.179944:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.179947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.180060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.180068:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.180070:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.180073:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.180074:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.180075:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.180076:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.180077:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.180079:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.180080:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.180080:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.180083:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.180085:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.180087:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.180087:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.180088:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.180089:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.180090:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.180091:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.180093:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.180093:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.180095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.180099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.180104:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.180105:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.180106:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.180107:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.180108:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.180108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.180108:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.180109:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.180111:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.180112:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.180352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.180358:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.180361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.180361:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.180364:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.180366:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.180367:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.180368:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.180369:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.180371:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.180371:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.180375:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.180376:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.180379:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.180380:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.180380:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.180382:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.180382:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.180383:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.180383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.180389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.180397:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.180399:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.180400:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.180402:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.180403:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.180405:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.180406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.180414:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.180417:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.180417:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.180420:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.180423:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.180424:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.180425:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.180427:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.180428:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.180429:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.180432:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.180433:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.180436:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.180437:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.180438:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.180439:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.180440:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.180440:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.180441:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.180445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.180447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677475.180453:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.180455:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677475.180455:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.180456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.180457:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.180457:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.180458:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677475.180459:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.180460:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.180462:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:10.0:1619677475.180464:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.180465:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.180467:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.180468:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.180469:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.180470:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.180473:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.180475:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.180477:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.180478:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.180478:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.180480:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.180480:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.180481:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.180483:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.180483:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.180486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.180489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.180493:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.180495:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677475.180496:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.180496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.180497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.180498:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.180499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.180499:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.180499:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.180502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.180544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.180552:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.180555:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.180557:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.180559:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.180560:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677475.180560:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.180562:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.180563:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.180564:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.180565:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.180568:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.180570:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.180572:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.180573:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.180573:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.180575:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.180575:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.180576:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.180577:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.180580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.180588:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.180590:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.180591:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.180592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.180593:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.180595:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.180720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.180727:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.180730:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.180731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.180733:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.180736:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.180737:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.180738:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.180739:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.180741:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.180742:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.180745:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.180746:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.180748:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.180749:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.180750:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.180751:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.180752:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.180753:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.180753:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.180757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.180766:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.180768:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.180769:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.180769:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.180772:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.180774:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.180981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.180987:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.180989:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.180991:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.180992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.180994:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.180995:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.180996:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.180998:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.180999:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.181000:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.181002:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.181004:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.181007:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.181008:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.181008:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.181010:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.181011:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.181012:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.181012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.181016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:5.2:1619677475.181021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:24.0:1619677475.181022:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.181025:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.181026:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.181026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.181026:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677475.181028:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.181029:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.181030:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677475.181032:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:1.0:1619677475.181032:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.181034:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.181036:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.181037:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.181038:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.181039:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.181040:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.181044:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.181046:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.181048:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.181049:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.181049:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.181051:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.181052:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.181052:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.181053:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.181057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.181065:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.181067:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.181068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.181069:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.181071:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.181073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.181234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.181239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.181241:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.181242:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.181244:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.181246:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.181247:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.181248:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.181249:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.181250:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.181251:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.181255:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.181257:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.181259:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.181260:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.181260:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.181262:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.181262:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.181263:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.181263:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.181268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.181275:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.181278:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.181278:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.181280:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.181281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.181282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.181309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:21.2:1619677475.181312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:17.0:1619677475.181315:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.181318:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.181318:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:30.0:1619677475.181321:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:17.0:1619677475.181321:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.181323:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.181324:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:17.0:1619677475.181325:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.181326:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.181327:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.181328:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:17.0:1619677475.181328:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.181329:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.181330:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.181330:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677475.181332:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.181333:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.181334:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.181335:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.181336:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.181337:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.181337:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.181339:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.181340:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:30.0:1619677475.181341:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.181341:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677475.181343:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.181343:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.181344:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.181344:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.181345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:30.0:1619677475.181346:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.181347:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.181348:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.181349:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.181350:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.2:1619677475.181350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677475.181351:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.181352:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.181356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.181358:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.181361:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.181362:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.181363:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.181364:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.181364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.181366:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677475.181367:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.181368:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.181368:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.181370:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.2:1619677475.181370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:22.0:1619677475.181373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.181378:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.181381:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.181382:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.181383:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.181383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.181384:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.181739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.181746:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.181749:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.181750:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.181751:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.181755:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.181756:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.181757:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.181759:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.181760:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.181761:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.181764:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.181765:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.181768:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.181769:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.181769:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.181771:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.181772:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.181773:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.181773:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.181777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.181785:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.181788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.181788:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.181789:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.181791:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.181794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.181880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.181886:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:21.2:1619677475.181887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677475.181889:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.181890:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.181892:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.181894:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677475.181894:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.181895:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:31.0:1619677475.181897:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677475.181897:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.181898:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.181899:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.181900:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.181900:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:30.0:1619677475.181901:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.181902:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.181903:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.181904:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:1.0:1619677475.181905:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:31.0:1619677475.181906:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.181907:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.181908:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.181909:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.181909:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:31.0:1619677475.181910:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677475.181910:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677475.181912:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677475.181912:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.181913:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:31.0:1619677475.181914:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.181914:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:31.0:1619677475.181916:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.181918:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677475.181918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677475.181919:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.181920:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.181921:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.181922:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.181923:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.181923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677475.181925:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:9.2:1619677475.181927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677475.181928:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.181929:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.181929:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.181931:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.181933:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:2.0:1619677475.181934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:24.0:1619677475.181935:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.181936:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.181936:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.181939:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.181941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:25.2:1619677475.182172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.182179:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.182182:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.182183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.182184:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.182184:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.182185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.182288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.182292:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.182298:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.182300:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.182302:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.182305:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.182306:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.182307:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.182308:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.182309:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.182310:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.182314:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.182316:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.182318:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.182319:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.182320:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.182321:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.182322:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.182322:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.182323:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.182328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.182336:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.182338:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.182339:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.182341:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.182342:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.182343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.182973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.182980:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.182984:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.182985:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.182988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.182988:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.182990:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.182991:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.182992:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.182993:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.182994:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.182997:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.182999:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.183002:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.183003:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.183003:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.183005:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.183006:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.183007:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.183007:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.183011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.183019:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.183022:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.183022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.183023:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.183025:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.183028:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.183030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.183037:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.183040:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.183040:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.183041:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.183042:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.183042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.183183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.183188:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.183191:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.183191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.183193:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.183196:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.183197:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.183198:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.183199:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.183200:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.183201:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.183205:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.183206:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.183209:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.183209:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.183210:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.183212:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.183212:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.183214:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.183215:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.183215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.183218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.183220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.183226:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.183228:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.183228:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.183228:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.183229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.183230:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.183231:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.183232:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.183233:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.183234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.183238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.183245:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.183248:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.183249:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.183249:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.183250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.183251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.183548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.183551:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.183553:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.183555:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.183556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.183558:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.183559:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.183561:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.183562:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.183563:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.183564:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.183567:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.183569:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.183571:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.183572:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.183573:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.183575:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.183575:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.183576:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.183576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.183581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.183589:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.183592:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.183592:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.183594:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.183595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.183596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.184091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:4.2:1619677475.184096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:22.0:1619677475.184097:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.184100:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.184100:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.184102:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.184103:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000400:00000200:22.0:1619677475.184104:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.184106:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.0:1619677475.184106:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677475.184107:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:22.0:1619677475.184107:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.184109:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.184109:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.184110:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.184111:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677475.184112:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.184113:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677475.184115:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96501174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.184116:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.184117:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.184118:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677475.184118:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.184120:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677475.184120:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.184121:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.184122:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677475.184123:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.184124:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.184124:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.184124:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.184125:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677475.184125:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.184126:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.184127:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.184128:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.184129:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.184129:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677475.184130:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677475.184131:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:25.2:1619677475.184132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:9.2:1619677475.184135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677475.184140:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:24.0:1619677475.184141:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.0:1619677475.184142:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:24.0:1619677475.184143:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:20.0:1619677475.184143:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.184144:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.184145:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.184145:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677475.184146:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:24.0:1619677475.184147:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.184148:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677475.184149:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.184478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.184484:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.184486:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.184488:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.184489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.184491:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.184492:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.184494:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.184495:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.184496:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.184497:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.184500:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.184502:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.184504:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.184505:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.184506:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.184507:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.184508:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.184509:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.184511:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.184511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.184514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.184521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677475.184522:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.184524:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677475.184524:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.184525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.184525:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.184526:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.184527:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:6.0:1619677475.184527:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.184528:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677475.184530:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.184554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.184562:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.184565:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.184568:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.184570:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.184571:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.184572:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.184573:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.184575:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.184576:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.184577:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.184580:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.184582:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.184584:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.184585:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.184585:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.184587:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.184587:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.184588:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.184589:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.184592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.184599:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.184602:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.184603:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.184604:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.184606:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.184607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.184775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.184777:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.184780:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.184782:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.184783:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.184784:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.184786:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.184788:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.184789:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.184790:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.184791:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.184794:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.184796:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.184798:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.184799:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.184799:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.184801:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.184801:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.184802:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.184803:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.184804:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.184808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.184811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.184814:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.184815:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.184816:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677475.184816:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.184817:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677475.184817:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.184818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.184818:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.184819:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.184820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.2:1619677475.184962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.184971:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.184975:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.184977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.184980:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.184983:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.184986:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677475.184988:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.184990:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.184992:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.184993:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677475.184999:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.185002:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677475.185006:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.185007:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677475.185008:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677475.185011:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677475.185012:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.185013:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.185014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677475.185017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.185024:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.185027:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.185027:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.185028:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.185031:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.185034:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.185041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.185047:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.185049:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.185050:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.185050:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.185051:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.185052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.185094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.185102:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.185104:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.185107:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.185108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.185109:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.185110:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.185112:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.185113:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.185114:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.185115:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.185118:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.185120:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.185124:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.185124:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.185125:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.185126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.185127:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.185127:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.185129:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.185130:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.185131:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677475.185131:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677475.185133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.185134:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677475.185136:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.185136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.185137:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.185139:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.185139:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.185140:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.185141:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.185141:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.185142:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.185143:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.185144:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.185144:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.185145:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.185146:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.185147:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.185147:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.185148:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.185149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.185150:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.185150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677475.185152:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.185155:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.185156:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.185156:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.185158:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.185158:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.185159:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.185159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.185164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.185173:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.0:1619677475.185176:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.185178:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.185179:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.185181:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:21.0:1619677475.185181:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.185182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.185183:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.185183:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677475.185186:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.185188:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.185189:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.185190:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.185191:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.185193:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.185194:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.185195:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.185198:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.185200:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.185202:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.185203:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.185203:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.185205:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.185205:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.185206:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.185208:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.185208:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.185210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.2:1619677475.185212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:21.2:1619677475.185214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.185218:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:2.0:1619677475.185219:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677475.185220:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.185221:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.185222:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.185222:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.185223:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.185224:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.185224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677475.185224:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:30.0:1619677475.185225:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:18.0:1619677475.185226:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677475.185227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:2.0:1619677475.185228:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.185232:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.185234:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.185237:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.185239:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.185241:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.185242:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.185248:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.185251:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.185253:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.185254:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.185255:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.185258:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.185259:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.185260:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.185261:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.185264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.185273:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.185276:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.185277:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.185279:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.185281:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.185283:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.185338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.185347:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.185350:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.185350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.185352:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.185355:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.185357:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.185358:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.185359:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.185360:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.185362:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.185364:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.185366:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.185369:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.185370:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.185370:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.185372:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.185372:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.185373:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.185373:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.185380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.185387:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.185390:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.185391:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.185391:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.185393:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.185396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.185598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677475.185600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:8.0:1619677475.185606:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677475.185607:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:8.0:1619677475.185608:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.185609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677475.185609:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.185610:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:8.0:1619677475.185612:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.185613:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.185614:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.185615:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.185616:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:17.0:1619677475.185617:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677475.185617:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.185619:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.185619:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.185620:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.185621:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.185621:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.185623:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677475.185623:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677475.185628:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.185629:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.185631:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677475.185632:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.185634:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.185635:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.185635:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677475.185635:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.185636:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.185637:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677475.185637:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.185638:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.185639:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.185639:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.0:1619677475.185639:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.185639:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.185640:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.185641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.185643:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677475.185647:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677475.185651:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.185653:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.185654:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.185655:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:20.0:1619677475.185656:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.185657:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.185658:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677475.185658:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.185659:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.185659:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.185661:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.185664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.185665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.185673:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.185675:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.185676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.185677:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.185678:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.185678:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.186011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.186016:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.186019:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.186020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.186022:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.186024:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.186025:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.186026:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.186028:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.186029:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.186030:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.186033:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.186035:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.2:1619677475.186036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.186038:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.186038:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.186039:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.186041:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.186041:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.186042:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.186042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677475.186044:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.186047:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677475.186047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.186049:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.186051:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.186052:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.186053:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.186054:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.186055:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:30.0:1619677475.186056:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.186057:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.186057:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.186057:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.186058:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:21.0:1619677475.186060:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.186060:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:30.0:1619677475.186062:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.186062:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.186064:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.186067:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.186068:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.186068:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.186070:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.186070:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.186071:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.186072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.186075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.186082:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.186084:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.186085:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.186085:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.186087:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.186090:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.186395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.186401:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.186404:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.186405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.186407:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.186409:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.186410:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.186412:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.186413:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.186414:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.186415:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.186419:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.186420:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.186424:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.186425:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.186425:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.186427:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.186428:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.186429:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.186429:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.186434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.186444:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.186447:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.186449:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.186451:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.186452:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.186454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.186882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.186890:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677475.186892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.0:1619677475.186892:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.186893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.186895:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.186898:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.186898:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677475.186899:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.186900:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.186901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.186901:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:8.0:1619677475.186902:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.186903:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.186905:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677475.186905:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.186907:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.186909:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.186910:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.186911:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.186912:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.186914:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:6.0:1619677475.186914:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.186915:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677475.186918:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.186919:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.186919:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:6.0:1619677475.186919:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.186921:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677475.186921:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677475.186922:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.186923:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.186924:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677475.186924:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.186924:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.2:1619677475.186925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.0:1619677475.186925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677475.186925:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.2:1619677475.186927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.186927:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.186927:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.186928:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.186929:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.186930:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677475.186930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677475.186933:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:25.2:1619677475.186933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.186935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.186936:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:1.0:1619677475.186936:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.186937:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:31.0:1619677475.186938:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.186939:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677475.186939:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.186940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677475.186940:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.186940:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:17.0:1619677475.186940:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.186940:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.186941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.186941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677475.186942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677475.186942:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:1.0:1619677475.186942:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.186943:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677475.186943:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677475.186943:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.186944:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96501157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.186944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.186945:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.186945:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.186946:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.186946:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:1.0:1619677475.186946:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:31.0:1619677475.186947:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.186948:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.186948:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.186952:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.186954:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677475.186956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:31.0:1619677475.186957:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.186958:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.186958:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.186960:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.186961:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.186962:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.186962:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677475.186963:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677475.186966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.186966:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.186967:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.186967:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.186968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.186969:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677475.186973:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.186976:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.186977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.186978:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.186980:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.186982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.186984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.186991:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.186994:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.186994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.186995:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.186996:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.186997:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.187110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.187118:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.187121:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.187122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.187124:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.187127:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.187128:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.187129:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.187131:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.187132:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.187132:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.187136:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.187137:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.187140:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.187141:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.187141:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.187143:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.187143:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.187144:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.187145:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.187146:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.187149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.187152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677475.187155:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.187158:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.187158:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.187159:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677475.187160:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:12.0:1619677475.187160:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.187161:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.187162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677475.187162:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.187163:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.187277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.187283:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.187285:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.187287:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.187288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.187290:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.187291:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.187293:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.187299:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.187300:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.187301:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.187304:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.187306:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.187309:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.187310:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.187310:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.187312:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.187313:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.187313:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.187314:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.187319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.187326:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.187328:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.187329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.187329:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.187332:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:9.2:1619677475.187334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.187337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.187338:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.187340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.187340:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.187340:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.187342:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.187342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.187343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.187346:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.187349:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.187351:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.187352:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.187353:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.187355:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.187356:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.187358:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.187358:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.187360:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.187362:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.187364:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.187365:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.187366:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.187367:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.187369:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.187369:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.187370:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.187370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.187375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.187383:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.187385:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.187386:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.187387:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.187388:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.187389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.187395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.187403:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.187405:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.187406:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.187407:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.187407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.187409:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.187593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.187601:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.187604:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.187607:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.187608:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.187610:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.187611:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.187612:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.187613:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.187615:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.187615:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.187619:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.187620:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.187623:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.187624:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.187625:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.187626:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.187627:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.187628:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.187628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.187632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.187638:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.187640:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.187641:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.187641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.187643:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.187645:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.187962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.187967:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.187970:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.187971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.187973:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.187975:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.187976:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.187977:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.187979:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.187980:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.187981:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.187984:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.187986:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.187988:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.187989:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.187989:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.187991:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.187992:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.187992:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.187993:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.187997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.188005:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.188007:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.188008:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.188009:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.188010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.188011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.188143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.188149:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.188152:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.188153:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.188155:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.188156:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.188158:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:4.2:1619677475.188158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000400:00000200:17.0:1619677475.188159:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.188160:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.188162:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.188163:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.188166:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.188166:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:17.0:1619677475.188167:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.188169:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677475.188170:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:17.0:1619677475.188170:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.188171:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.188171:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677475.188172:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.188173:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.188173:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.188174:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:10.0:1619677475.188174:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.188175:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677475.188176:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.188177:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.188178:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.188179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:10.0:1619677475.188180:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.188180:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.188184:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.188185:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.188186:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.188189:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677475.188189:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.188190:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.188190:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.188190:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677475.188190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.188192:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.188192:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.188193:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:22.0:1619677475.188194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677475.188194:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677475.188194:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677475.188198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.188207:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.188209:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.188210:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.188212:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.188213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.188215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.188446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.188454:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.188457:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.188458:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.188460:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.188462:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.188464:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.188465:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.188466:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.188468:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.188468:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.188472:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.188473:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.188476:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.188477:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.188478:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.188479:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.188480:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.188481:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.188482:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.188482:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.188486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.188488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.188493:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.188496:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.188496:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.188497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.188497:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.188498:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.188498:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.188500:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.188500:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.188502:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.188546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.188552:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.188554:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.188556:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.188557:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.188559:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.188560:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.188562:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.188563:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.188564:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.188565:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.188568:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.188570:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.188572:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.188573:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.188574:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.188575:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.188576:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.188576:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.188577:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.188581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.188588:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.188590:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.188591:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.188591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.188594:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.188596:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.188839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.188844:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.188846:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.188847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.188849:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.188851:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.188852:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.188853:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.188854:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.188855:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.188856:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.188859:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.188860:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.2:1619677475.188862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677475.188863:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.188863:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.188864:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.188866:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.188866:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.188867:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.188867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677475.188870:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677475.188872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.188873:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.188875:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.188878:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.188878:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.188879:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677475.188879:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:31.0:1619677475.188881:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.188881:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:31.0:1619677475.188882:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.188882:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.188884:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.188884:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.188885:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.188885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.188886:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.188889:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.188891:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.188892:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.188893:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.188894:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.188895:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.188896:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.188897:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.188898:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.188901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.188907:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.188910:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.188911:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.188912:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.188914:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.188916:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.188921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.188928:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.188930:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.188931:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.188932:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.188932:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.188933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.189197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.189205:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.189207:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.189209:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.189211:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.189212:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.189213:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.189214:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.189215:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.189216:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.189217:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.189220:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.189222:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.189225:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.189226:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.189226:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.189228:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.189229:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.189229:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.189230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.189234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.189241:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.189244:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.189245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.189246:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.189247:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.189250:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.189402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.189408:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.189411:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.189413:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.189413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.189415:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.189417:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.189418:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.189419:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.189420:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.189421:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.189424:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.189426:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.189428:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.189429:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.189430:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.189431:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.189432:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.189433:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.189434:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.189438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.189446:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.189448:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.189449:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.189450:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.189451:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.189452:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.189606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.189613:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.189616:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.189619:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.189621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.189622:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.189623:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.189624:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.189626:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.189627:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.189628:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.189631:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.189633:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.189635:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.189636:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.189636:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.189638:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.189638:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.189640:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.189641:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.189642:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.189644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.189647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.189652:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.189654:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.189654:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.189656:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.189656:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.189657:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.189657:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.189658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.189659:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.189660:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.189728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.189734:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.189737:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.189737:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.189739:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.189741:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.189743:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.189744:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.189745:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.189746:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.189748:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.189750:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.189752:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.189754:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.189755:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.189756:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.189757:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.189758:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.189758:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.189759:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.189763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.189771:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.189773:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.189774:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.189775:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.189776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.189777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.190081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.190088:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.190091:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.190092:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.190093:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.190096:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.190097:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.190098:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.190100:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.190101:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.190102:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.190105:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.190107:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.190109:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.190110:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.190111:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.190112:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.190113:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.190114:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.190115:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.190116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.190118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.190121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677475.190126:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677475.190126:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.190128:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677475.190129:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677475.190129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677475.190130:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.190130:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677475.190131:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.190133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.190134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.190645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.190650:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.190653:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.190655:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.190656:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.190657:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.190659:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.190660:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.190661:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.190662:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.190663:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.190667:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.190668:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.190671:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.190672:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.190672:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.190674:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.190674:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.190675:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.190675:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.190680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.190688:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.190690:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.190691:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.190692:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.190693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.190694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.190722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.190730:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.190732:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.190733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.190735:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.190738:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.190740:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.190741:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.190742:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.190743:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.190744:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.190747:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.190748:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.190751:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.190752:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.190752:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.190754:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.190755:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.190755:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.190756:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.190761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.190769:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.190771:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.190772:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.190773:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.190775:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.190777:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.190805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.190810:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.190813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.190813:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.190815:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.190817:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.190818:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.190819:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.190821:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.190821:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.190823:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.190826:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.190828:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.190830:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.190831:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.190831:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.190833:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.190833:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.190834:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.190835:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.190836:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.190839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.190842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:4.2:1619677475.190845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677475.190848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677475.190850:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.190852:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677475.190853:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677475.190853:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.190854:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.190855:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:8.0:1619677475.190855:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:30.0:1619677475.190856:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.190856:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677475.190856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677475.190857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677475.190857:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.190857:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.190857:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.190858:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.190859:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.190859:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.190859:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:30.0:1619677475.190861:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677475.190861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.190861:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:30.0:1619677475.190862:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.190862:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677475.190862:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.190863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.190863:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000400:00000200:8.0:1619677475.190863:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677475.190864:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.190864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677475.190865:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.190865:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.190866:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.190867:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.190868:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.190870:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.190870:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677475.190873:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.190874:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.190876:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.190877:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.190877:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677475.190878:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.190879:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.190880:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.190880:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:30.0:1619677475.190881:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.190881:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677475.190882:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.190882:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.0:1619677475.190882:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.190883:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.190884:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.190884:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.190886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.2:1619677475.190889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.190893:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.190897:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677475.190897:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:22.0:1619677475.190898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.190898:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.190900:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.190900:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:25.2:1619677475.190901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677475.190901:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.190902:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.190903:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.190905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.190906:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.190907:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677475.190908:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.190909:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.190910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.190910:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.190911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.191512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.191520:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.191523:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.191524:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.191526:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.191528:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.191530:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.191531:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.191533:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.191533:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.191535:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.191538:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.191540:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.191541:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.191542:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.191543:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.191544:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.191545:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.191546:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.191547:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.191547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.191551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.191554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.191560:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.191561:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.191562:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.191562:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.191562:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.191563:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.191564:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.191564:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.191565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677475.191566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.191674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.191680:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.191683:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.191683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.191685:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.191688:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.191689:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.191690:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.191692:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.191693:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.191694:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.191697:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.191699:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.191701:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.191702:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.191703:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.191704:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.191705:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.191706:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.191706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.191711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.191718:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.191721:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.191721:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.191723:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.191724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.191725:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.191828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.191835:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.191838:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:4.2:1619677475.191838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677475.191839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.191841:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.191842:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.191844:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.191845:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.191846:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677475.191846:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.191847:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.191849:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:18.0:1619677475.191849:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677475.191850:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677475.191853:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:24.0:1619677475.191854:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.191856:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:24.0:1619677475.191857:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.191858:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677475.191859:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677475.191860:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96501132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.191860:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.191860:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:24.0:1619677475.191861:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.191862:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677475.191862:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:24.0:1619677475.191863:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677475.191863:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.191864:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.191864:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:24.0:1619677475.191867:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.191868:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.191871:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.191872:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:25.2:1619677475.191873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677475.191873:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.191874:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.191875:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.191876:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.191877:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.191880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.191881:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.191883:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.191884:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.191886:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.191888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677475.191888:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677475.191889:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:6.0:1619677475.191890:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.191891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.191892:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.191894:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.191897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.192533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.192541:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.192544:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.192546:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.192548:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.192549:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.192550:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.192551:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.192552:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.192553:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.192554:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.192558:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.192560:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.192563:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.192564:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.192564:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.192566:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.192566:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.192567:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.192568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.192573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.192581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.0:1619677475.192581:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.192584:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.192585:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.192586:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.192587:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.192587:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:22.0:1619677475.192590:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:20.0:1619677475.192590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677475.192592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.192593:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.192596:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.192597:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.192599:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.192600:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.192601:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.192603:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.192607:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.192609:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:4.2:1619677475.192609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677475.192612:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.192612:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.192613:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.192615:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.192615:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.192616:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.192616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677475.192616:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.192619:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.192620:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.192621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677475.192622:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.192625:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.192627:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.192628:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96501129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.192629:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677475.192629:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.192630:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.192631:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:8.0:1619677475.192631:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:21.0:1619677475.192632:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.192634:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.192634:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.192636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677475.192636:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.192638:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.192641:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.192641:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.192642:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.192644:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.192644:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.192645:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.192646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.192650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.192659:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.192661:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.192662:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.192663:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.192665:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.192668:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.192825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.192833:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.192836:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.192838:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.192840:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.192841:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.192843:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.192844:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.192845:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.192846:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.192847:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.192850:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.192852:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.192855:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.192855:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.192856:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.192858:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.192858:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.192860:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.192861:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.192862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.192864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.192868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.192871:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.192874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.192874:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.192875:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677475.192875:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.192877:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.192878:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.192879:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.192880:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.192880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.193245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.193253:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.193256:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.193257:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.193259:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.193262:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.193264:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.193265:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.193267:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.193268:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.193268:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.193272:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.193273:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.193276:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.193277:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.193277:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.193279:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.193279:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.193280:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.193281:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.193285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.193292:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.193297:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.193299:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.193300:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.193302:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.193305:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.193418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.193424:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.193427:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.193428:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.193429:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.193431:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.193433:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.193434:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.193435:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.193436:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.193438:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.193441:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.193443:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.193445:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.193446:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.193447:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.193448:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.193449:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.193449:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.193450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.193455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.193463:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.193465:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.193466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.193466:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.193468:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.193470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.193475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.193483:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.193485:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.193486:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.193486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.193487:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.193488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.193491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.193499:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.193502:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.193505:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.193507:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.193508:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.193509:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.193511:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.193513:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.193514:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.193515:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.193519:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.193521:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.193524:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.193525:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.193525:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.193527:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.193528:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.193529:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.193530:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.193533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.193541:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.193543:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.193545:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.193545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.193547:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.193550:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.193652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.193657:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.193660:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.193661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.193663:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.193665:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.193667:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.193668:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.193669:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.193670:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.193671:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.193675:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.193677:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.193679:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.193680:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.193680:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.193682:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.193682:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.193683:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.193683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.193688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.193696:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.193698:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.193699:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.193701:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.193702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.193703:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.193963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.193971:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.193974:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.193977:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.193978:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.193979:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.193981:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.193982:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.193983:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.193985:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.193985:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.193989:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.193991:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.193994:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.193995:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.193996:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.193997:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.193998:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.193999:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.194000:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.194004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.194011:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.194013:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.194014:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.194016:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.194017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.194019:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.194021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.194029:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.194031:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.194032:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.194033:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.194033:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.194034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.194321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677475.194329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.194333:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.194335:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.194337:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:21.2:1619677475.194337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:30.0:1619677475.194338:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.194339:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.194341:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.194342:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.194343:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677475.194343:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:30.0:1619677475.194344:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.194346:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:30.0:1619677475.194347:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.194347:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:30.0:1619677475.194349:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.194349:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.194351:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.194352:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.194352:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.194353:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:17.0:1619677475.194353:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.194354:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.194355:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.194355:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.194356:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:17.0:1619677475.194356:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.194356:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.194357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000400:00000200:17.0:1619677475.194358:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.194361:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677475.194361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677475.194363:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.194365:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.194366:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.194367:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.194368:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.194368:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:8.0:1619677475.194368:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:17.0:1619677475.194369:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.194370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.0:1619677475.194371:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.194372:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.194372:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.194374:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:8.0:1619677475.194379:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.194381:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677475.194382:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.194385:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.194385:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.194387:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.194388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.194389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.194394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.194401:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.194404:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.194405:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.194406:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.194406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.194407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.194429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (20)++
00000800:00000200:31.0:1619677475.194438:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677475.194441:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.18@o2ib3
00000400:00000200:31.0:1619677475.194444:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.4.18@o2ib3 : GET - for me
00000800:00000200:30.0:1619677475.194445:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677475.194447:0:19945:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.4.18@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:31.0:1619677475.194449:0:19945:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.4.18@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:31.0:1619677475.194451:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677475.194452:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a864385200] (22)--
00000400:00000200:31.0:1619677475.194454:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.4.18@o2ib3 local destination
00000400:00000200:31.0:1619677475.194459:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.4.18@o2ib3(10.51.4.18@o2ib3:10.51.4.18@o2ib3) : REPLY try# 0
00000800:00000200:31.0:1619677475.194461:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.4.18@o2ib3
00000800:00000200:31.0:1619677475.194464:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a867b82d00] -> 10.51.4.18@o2ib3 (2) version: 12
00000800:00000200:31.0:1619677475.194465:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a864385200] (21)++
00000800:00000200:31.0:1619677475.194465:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a864385200] (22)++
00000800:00000200:31.0:1619677475.194467:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a864385200] (23)--
00000800:00000200:31.0:1619677475.194468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:20.2:1619677475.194473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a864385200] (21)++
00000800:00000200:30.0:1619677475.194482:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a864385200] (22)++
00000800:00000200:30.0:1619677475.194485:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a864385200] (23)--
00000800:00000200:21.2:1619677475.194485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:30.0:1619677475.194486:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.194487:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.4.18@o2ib3: REPLY: OK
00000800:00000200:30.0:1619677475.194490:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (22)--
00000800:00000200:31.0:1619677475.194491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a864385200] (21)--
00000800:00000200:30.0:1619677475.194491:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.194494:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.194497:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.194499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.194499:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.194501:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.194502:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.194504:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.194505:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.194506:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.194510:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.194512:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.194516:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.194516:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.194517:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.194519:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.194519:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.194520:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.194521:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.194525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.194533:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.194536:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.194536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.194537:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.194539:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.194542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.194547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.194552:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.194555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.194555:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.194557:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.194559:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.194560:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.194561:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.194562:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.194563:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.194564:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.194567:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.194569:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.194571:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.194572:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.194573:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.194574:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.194575:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.194576:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.194577:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.194577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.194580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.194583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.194588:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.194590:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.194590:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.194590:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.194591:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.194592:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.194593:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.194594:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.194595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.194596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.194598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.194605:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.194608:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.194609:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.194609:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.194610:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.194611:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.194863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.194871:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.194874:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.194876:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.194877:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.194879:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.194881:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.194882:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.194883:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.194884:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.194885:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.194889:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.194891:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.194893:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.194894:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.194895:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.194896:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.194897:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.194898:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.194900:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.194901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.194902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.194906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.194908:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.194911:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.194912:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.194912:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.194914:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.194914:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.194916:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.194917:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.194917:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.194918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.194920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.194926:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.194928:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.194929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.194929:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.194930:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.194931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.195012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.195015:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.195018:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.195020:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.195020:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.195021:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.195023:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.195024:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.195026:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.195027:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.195028:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.195031:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.195032:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.195035:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.195035:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.195036:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.195038:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.195038:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.195039:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.195040:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.195044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.195051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.195053:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.195054:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.195055:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.195056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.195057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.195338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677475.195346:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.195349:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.195351:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.195353:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.195354:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.195355:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677475.195356:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.195357:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.195358:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.195359:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.195363:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.195364:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.195367:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.195368:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677475.195369:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.195371:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677475.195371:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.195372:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677475.195373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.195378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.195386:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.195389:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.195390:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.195390:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.195392:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.195396:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.195425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.195431:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.195433:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.195434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.195436:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.195438:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.195439:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.195441:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.195442:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.195443:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.195444:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.195448:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.195449:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.195451:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.195452:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.195453:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.195455:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.195455:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.195456:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.195456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.195461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.195468:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.195470:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.195471:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.195473:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.195473:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.195475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.195556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.195564:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.195567:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.195567:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.195570:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.195572:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.195574:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.195575:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.195577:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.195578:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.195579:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.195582:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.195584:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.195587:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.195588:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.195589:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.195590:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.195591:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.195591:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.195592:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.195596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.195604:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.195607:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.195608:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.195608:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.195611:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.195614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.195617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.195619:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.195621:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.195622:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.195622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.195623:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.195624:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.196221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.196226:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.196228:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.196231:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.196231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.196233:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.196234:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.196236:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.196237:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.196238:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.196239:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.196242:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.196244:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.196247:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.196248:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.196248:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.196250:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.196251:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.196252:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.196253:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.196254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.196256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.196260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.196266:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.196267:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.196267:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677475.196268:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.196268:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.196268:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.196269:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.196270:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.196270:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.196271:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677475.196479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.196485:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.196488:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.196488:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.196490:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.196493:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.196494:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.196495:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.196496:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.196497:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.196498:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.196502:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.196503:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.196506:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.196507:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.196507:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.196509:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.196509:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.196510:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.196510:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.196516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.196524:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.2:1619677475.196527:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677475.196530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.196531:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677475.196532:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.196534:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.196535:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677475.196536:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.196539:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.196541:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.196543:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.196544:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.196546:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.196547:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.196548:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.196549:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.196550:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.196553:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.196555:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.196558:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.196559:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.196560:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.196561:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.196562:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.196563:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.196563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.196567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.196575:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.196577:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.196578:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.196579:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.196581:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.196583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.196868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.196876:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.196879:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.196880:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.196882:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.196885:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.196886:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677475.196887:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.196889:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.196890:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.196891:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677475.196894:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.196895:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677475.196897:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.196898:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677475.196898:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.196899:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.196900:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.196901:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.196902:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.196903:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677475.196905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:4.2:1619677475.196908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.196914:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.196915:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.196916:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.196917:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.196918:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.196918:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.196919:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.196919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.196920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677475.196920:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677475.196960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.196967:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.196970:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.196971:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.196973:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.196976:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.196977:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.196978:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.196979:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.196980:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.196981:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.196985:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.196986:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.196989:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.196990:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.196990:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.196992:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.196993:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.196994:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.196996:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.196996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.196999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.197002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.197007:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677475.197009:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.197010:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.197011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.197011:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677475.197011:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.197012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677475.197012:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677475.197014:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.197017:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.197023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.197031:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.197033:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.197034:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.197034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.197035:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.197036:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.197438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.197444:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.197447:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.197448:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.197449:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.197452:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.197453:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.197454:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.197456:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.197456:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.197458:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.197461:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.197462:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.197465:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.197465:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.197466:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.197468:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.197468:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.197469:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.197469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.197475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.197483:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.197485:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.197486:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.197488:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.197489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.197490:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.197627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.197636:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.197639:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.197641:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.197643:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.197644:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.197645:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.197647:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.197648:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.197649:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.197650:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.197654:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.197656:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.197658:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.197659:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.197659:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.197661:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.197661:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.197662:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.197663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.197666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.197673:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.197676:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.197677:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.197678:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.197680:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.197681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.197729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.197737:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.197740:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.197741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.197742:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.197745:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.197746:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.197748:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.197749:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.197750:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.197750:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.197753:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.197755:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.197758:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.197758:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.197759:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.197760:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.197761:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.197762:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.197762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.197767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.197775:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.197778:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.197779:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.197779:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.197781:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.197784:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.197856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.197862:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.197865:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.197865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.197867:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.197869:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.197871:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677475.197872:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.197873:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.197874:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.197875:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677475.197878:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.197880:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677475.197883:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.197884:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677475.197885:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677475.197886:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677475.197887:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.197887:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.197888:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677475.197892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.197900:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.197903:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.197903:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.197904:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.197906:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.197909:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677475.198028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677475.198034:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.198036:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677475.198038:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.198039:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677475.198040:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.198041:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677475.198043:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.198044:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.198045:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.198046:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677475.198050:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.198051:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677475.198054:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.198055:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:4.2:1619677475.198055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.0:1619677475.198056:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677475.198057:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677475.198058:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.198059:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677475.198060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:26.0:1619677475.198063:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:9.2:1619677475.198064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.198066:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:26.0:1619677475.198068:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.198068:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.198070:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:26.0:1619677475.198071:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.198071:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.198072:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.198073:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.198074:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.198074:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:26.0:1619677475.198076:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.198077:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.198078:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.198081:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.198082:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.198082:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677475.198084:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.198087:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.198088:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.198088:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.198090:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.198091:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.198092:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.198093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.198096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.198104:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.198106:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.198107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.198108:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.198110:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.198112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.198131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.198137:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.198140:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.198141:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.198142:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.198145:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.198146:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.198148:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.198149:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.198150:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.198151:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.198155:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.198156:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.198159:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.198159:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.198160:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.198161:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.198162:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.198164:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.198164:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.198165:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.198169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.198170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.198175:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677475.198176:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677475.198177:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.198177:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.198178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677475.198178:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.198180:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.198180:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.198182:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.198184:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.198200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.198207:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.198210:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.198212:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.198214:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.198214:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.198216:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.198217:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.198218:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.198219:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.198220:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.198224:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.198226:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.198228:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.198229:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.198229:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.198231:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.198231:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677475.198231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.198232:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.198233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.198235:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.198236:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:25.2:1619677475.198236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677475.198238:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.198239:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.198240:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:30.0:1619677475.198240:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.198241:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.198242:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.198242:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.198243:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.198244:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.198244:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.198245:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.198246:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.198248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.198248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:30.0:1619677475.198249:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.198251:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.198254:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.198255:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.198255:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.198257:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.198258:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.198258:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.198259:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.198263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.198270:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.198273:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.198274:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.198274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.198276:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.198279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.198314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.198319:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.198321:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.198322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.198324:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.198326:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.198327:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.198328:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.198330:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.198331:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.198332:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.198336:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.198338:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.198340:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.198341:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.198341:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.198343:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.198343:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.198344:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.198345:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.198346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.198349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.198351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.198356:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.198357:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.198358:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.198358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.198359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.198359:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.198360:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.198361:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.198363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.198364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.198372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.198380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.198382:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.198383:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.198384:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.198385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.198385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.198622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.198629:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.198631:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.198632:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.198634:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.198637:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.198638:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.198639:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.198641:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.198642:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.198643:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.198646:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.198648:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.198650:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.198651:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.198652:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.198653:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.198654:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.198655:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.198656:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.198657:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677475.198660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677475.198662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.198667:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677475.198668:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677475.198670:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.198670:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677475.198670:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677475.198671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677475.198671:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.198672:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.198673:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.198676:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.198766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677475.198772:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.198775:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677475.198775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.198777:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.198779:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.198780:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677475.198782:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.198783:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.198784:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.198785:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677475.198788:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.198790:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677475.198794:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.198794:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677475.198795:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677475.198797:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677475.198797:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.198798:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677475.198799:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677475.198800:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:9.2:1619677475.198803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.2:1619677475.198805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:10.0:1619677475.198810:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:1.0:1619677475.198810:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677475.198812:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677475.198812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:10.0:1619677475.198813:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.198814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677475.198814:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.198814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:10.0:1619677475.198816:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.198819:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677475.198987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.198994:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.198997:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.198997:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.198999:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.199002:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.199004:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677475.199005:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.199006:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.199007:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.199008:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677475.199012:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.199013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:18.0:1619677475.199013:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677475.199016:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.199016:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677475.199017:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677475.199019:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677475.199019:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677475.199019:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:18.0:1619677475.199020:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.199021:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677475.199022:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677475.199023:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:25.2:1619677475.199025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:17.0:1619677475.199025:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.199028:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.199029:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.199030:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.199032:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.199032:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.199033:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.199033:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.199035:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.199036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.199036:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.199037:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.199038:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.199039:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.199040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677475.199041:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.199042:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.199042:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.199044:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.199045:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.199045:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677475.199046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:25.2:1619677475.199050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.199058:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.199060:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.199061:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.199061:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.199063:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.199065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.199276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.199281:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.199284:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.199285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.199287:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.199289:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.199290:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.199292:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.199298:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.199299:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.199300:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.199303:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.199304:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.199306:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.199307:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.199308:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.199309:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.199310:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.199310:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.199311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.199316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.199321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.0:1619677475.199325:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.199327:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:26.0:1619677475.199328:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:21.0:1619677475.199328:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.199330:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.199330:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:26.0:1619677475.199331:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:21.0:1619677475.199332:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677475.199332:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.199334:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.199337:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.199338:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.199339:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.199341:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.199342:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.199343:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.199346:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.199347:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:21.2:1619677475.199349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:26.0:1619677475.199350:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.199350:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.199351:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.199352:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.199353:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.199354:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.199355:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:6.0:1619677475.199355:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.199357:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.199358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.199359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:6.0:1619677475.199360:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.199362:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.199363:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.199365:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677475.199366:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:6.0:1619677475.199366:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.199367:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.199368:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:12.0:1619677475.199369:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.199370:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.199372:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:6.0:1619677475.199372:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.199374:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.199375:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:6.0:1619677475.199375:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.199377:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.199378:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.199379:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.199380:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.199381:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.199381:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.199382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.199386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.199393:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.199396:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.199397:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.199398:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.199399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.199401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.199530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.199538:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.199541:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.199544:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.199546:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.199546:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.199548:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.199549:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.199551:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.199552:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.199553:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.199556:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.199558:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.199562:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.199563:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.199564:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.199565:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.199566:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.199567:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.199568:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.199571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.199579:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.199581:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.199582:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.199584:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.199584:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.199587:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.199920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.199924:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.199927:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.199930:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.199930:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.199932:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.199933:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.199935:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.199936:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.199937:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.199938:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.199941:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.199943:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.199945:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.199946:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.199947:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.199948:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.199949:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.199950:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.199950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.199954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.199962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.199964:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.199965:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.199965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.199967:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.199969:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.199971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.199978:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.199981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.199981:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.199982:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.199983:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.199983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.200037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.200043:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.200045:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.200047:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.200048:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.200049:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.200051:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.200052:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.200053:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.200054:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.200055:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.200058:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.200060:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.200061:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.200062:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.200062:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.200064:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.200065:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.200066:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.200068:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.200068:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677475.200070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.200074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.200078:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.200080:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.200080:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.200080:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.200081:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.200082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.200082:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.200083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.200084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677475.200085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.200093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.200100:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.200103:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.200104:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.200104:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.200105:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.200106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.200252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.200257:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.200260:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.200261:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.200263:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.200265:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.200267:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.200268:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.200269:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.200270:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.200271:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.200275:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.200277:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.200279:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.200280:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.200281:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.200282:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.200283:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.200284:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.200284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.200288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.200297:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.200299:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.200300:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.200302:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.200303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.200304:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.200473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.200482:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.200485:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.200487:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.200489:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.200490:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.200492:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.200493:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.200494:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.200496:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.200497:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.200500:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.200501:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.200504:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.200505:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.200505:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.200507:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.200508:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.200509:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.200511:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.200511:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.200514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.200517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.200521:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.200524:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:31.0:1619677475.200525:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.200525:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.200526:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.200527:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:24.0:1619677475.200527:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.200528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.200529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677475.200530:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.200556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.200562:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.200565:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.200565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.200568:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.200570:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.200571:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677475.200573:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.200574:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.200575:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.200576:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.200579:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.200581:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.200584:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.200584:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.2:1619677475.200585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:17.0:1619677475.200585:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677475.200587:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677475.200587:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.200588:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.200588:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:31.0:1619677475.200593:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:9.2:1619677475.200593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.200596:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.200598:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.200600:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677475.200600:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:31.0:1619677475.200601:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.200602:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677475.200603:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:31.0:1619677475.200604:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.200604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.200604:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.200605:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.200606:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.200606:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.200607:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.200610:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.200610:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.200612:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.200616:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.200617:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.200617:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.200620:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.200620:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.200621:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.200623:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.200623:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.200626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.200629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.200633:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.200636:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.200637:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.200637:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.200637:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.200640:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.200640:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677475.200640:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.200641:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.200643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.200855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.200861:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:4.2:1619677475.200862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677475.200864:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.200865:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.200867:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.200870:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:2.0:1619677475.200870:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.200871:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.200872:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.200873:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.200874:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:2.0:1619677475.200874:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.200875:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.200876:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677475.200877:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.200880:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.200880:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.200881:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:2.0:1619677475.200882:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:10.0:1619677475.200883:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677475.200884:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677475.200885:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677475.200885:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:10.0:1619677475.200886:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677475.200886:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:10.0:1619677475.200887:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:2.0:1619677475.200888:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.200889:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.200889:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:10.0:1619677475.200890:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.200890:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:10.0:1619677475.200892:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:9.2:1619677475.200894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677475.200895:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.200896:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.200896:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.200898:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.200899:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.200900:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:18.0:1619677475.200901:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:10.0:1619677475.200901:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.200901:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.200904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677475.200904:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.200905:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.200905:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.200907:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:4.2:1619677475.200907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677475.200909:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677475.200911:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.200914:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.200914:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.200915:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.200915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.200916:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.200917:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677475.200917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000400:00000200:22.0:1619677475.200918:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.200918:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677475.200920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677475.200924:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.200927:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.200928:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.200928:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.200929:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.200929:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.201142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.201148:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.201150:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.201151:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.201153:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.201154:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.201156:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.201157:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.201159:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.201159:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.201160:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.201164:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.201166:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.201168:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.201169:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.201169:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.201171:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.201171:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.201172:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.201172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.201177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.201185:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.201188:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.201188:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.201190:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.201191:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.201192:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.201194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.201197:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.201199:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.201201:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.201202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.201204:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.201205:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.201206:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.201208:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.201209:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.201210:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.201213:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.201215:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.201217:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.201218:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.201219:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.201221:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.201221:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.201222:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.201222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.201227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.201234:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.201236:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.201237:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.201239:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.201240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.201241:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.201300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.201307:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.201310:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.201311:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.201312:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.201315:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.201317:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.201318:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.201320:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.201321:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.201322:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.201325:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.201327:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.201330:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.201331:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.201332:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.201333:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.201334:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.201334:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.201335:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.201340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.201348:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.201351:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.201352:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.201352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.201354:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.201357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.201776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.201785:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.201788:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.201790:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.201792:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.201793:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.201794:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.201795:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.201796:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.201797:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.201798:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.201801:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.201803:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.201805:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.201805:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.201806:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.201807:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.201808:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.201809:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.201809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.201813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.201819:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.201822:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.201823:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.201824:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.201826:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.201827:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.201835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.201841:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.201844:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.201845:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.201846:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.201846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.201847:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.202164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.202169:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.202172:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.202173:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.202175:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.202177:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.202178:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.202180:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.202181:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.202182:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.202183:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.202186:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.202187:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.202190:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.202191:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.202191:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.202193:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.202193:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.202194:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.202195:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.202196:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.202199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.202201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.202205:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.202207:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.202207:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.202208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.202209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.202209:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.202210:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.202211:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.202213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.202214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.202218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.202225:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.202227:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.202228:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.202229:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.202229:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.202230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.202313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.202321:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.202324:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.202326:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.202329:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.202329:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.202330:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.202331:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.202333:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.202334:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.202335:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.202338:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.202340:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.202343:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.202344:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.202344:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.202346:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.202347:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.202348:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.202348:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.202353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.202361:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.202364:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.202365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.202365:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.202367:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.202370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.202374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.202380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.202383:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.202384:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.202384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.202385:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.202386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.202431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.202436:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.202438:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.202441:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.202441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.202443:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.202444:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.202446:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.202447:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.202448:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.202449:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.202452:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.202454:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.202457:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.202458:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.202458:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.202460:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.202461:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.202461:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.202462:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.202467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.202476:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.202478:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.202479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.202479:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.202481:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.202484:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.202541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.202548:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677475.202550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.202551:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.202552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.202554:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.202556:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.202558:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677475.202558:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.202559:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.202560:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.202561:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677475.202562:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.202563:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.202564:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.202565:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.202566:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.202566:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.202568:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.0:1619677475.202568:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.202569:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.202570:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.202571:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.202571:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.202572:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677475.202572:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.202573:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.202575:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.202575:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677475.202576:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.202576:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677475.202577:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677475.202577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677475.202579:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.202580:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:25.2:1619677475.202580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.202581:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.202582:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.202583:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.202583:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.202584:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.202588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677475.202588:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.202590:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.202591:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.202592:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.202594:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:6.0:1619677475.202594:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.202597:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:6.0:1619677475.202597:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677475.202598:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.202600:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.202601:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.202602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.202894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.202901:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.202903:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.202904:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.202906:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.202909:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.202910:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.202912:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.202913:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.202914:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.202915:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.202918:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.202919:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.202922:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.202923:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.202924:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.202925:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.202926:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.202926:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:5.2:1619677475.202926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677475.202927:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.202931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.202932:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.202935:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.202936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.202938:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.202939:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:17.0:1619677475.202940:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.202941:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:17.0:1619677475.202941:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.202942:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.202942:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.202943:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.202943:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.202945:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.202945:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.202946:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677475.202950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:17.0:1619677475.202950:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.202952:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.202953:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.202954:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.202954:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.202955:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677475.202956:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (23)++
00000800:00000200:22.0:1619677475.202957:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.202958:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (24)--
00000800:00000200:6.0:1619677475.202958:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (24)--
00000800:00000200:22.0:1619677475.202959:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (23)++
00000800:00000200:17.0:1619677475.202959:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.202959:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677475.202960:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677475.202960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677475.202961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:25.2:1619677475.202964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.202971:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.202973:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.202974:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.202974:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.202975:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.202978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.202996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.203001:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.203003:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.203004:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.203006:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.203007:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.203009:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.203010:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.203011:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.203012:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.203013:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.203017:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.203019:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.203021:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.203022:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.203022:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.203024:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.203024:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.203025:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.203025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.203030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.203038:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.203040:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.203041:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.203042:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.203044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.203044:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.203476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.203481:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.203483:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.203485:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.203486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.203487:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.203489:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.203490:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.203492:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.203493:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.203494:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.203497:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.203499:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.203502:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.203503:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.203503:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.203505:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.203505:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.203506:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.203508:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.203508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.203511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.203519:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.203522:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.203522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.203523:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.203525:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:4.2:1619677475.203526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677475.203527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677475.203530:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.203532:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.203533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.203534:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:21.2:1619677475.203803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.203811:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.203814:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.203816:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.203818:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.203819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.203820:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677475.203821:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.203822:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.203823:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.203824:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.203828:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.203829:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.203833:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.203833:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677475.203834:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.203836:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677475.203836:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.203837:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.203838:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.203842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677475.203849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:18.0:1619677475.203851:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.203853:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.203854:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.203854:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677475.203854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.203856:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.203857:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.203858:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.203859:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.203860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:6.0:1619677475.203862:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.203863:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.203864:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.203866:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.203867:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.203868:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.203871:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.203873:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.203875:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677475.203876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:6.0:1619677475.203876:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.203876:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.203878:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.203878:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.203879:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.203880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:1.0:1619677475.203882:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:25.2:1619677475.203884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677475.203885:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677475.203886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677475.203887:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.203890:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.203891:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:1.0:1619677475.203891:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677475.203893:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:1.0:1619677475.203893:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.203894:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.203894:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.203895:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.203896:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:1.0:1619677475.203896:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.203897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.203898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:1.0:1619677475.203900:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.203902:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677475.203905:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.203906:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677475.203906:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.203908:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.203908:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.203910:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677475.203911:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677475.203912:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:9.2:1619677475.203914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.203917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:8.0:1619677475.203921:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:17.0:1619677475.203922:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:8.0:1619677475.203923:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.203924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677475.203924:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000400:00000200:8.0:1619677475.203924:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.203925:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677475.203926:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000400:00000200:8.0:1619677475.203926:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.203929:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.204256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.204263:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.204267:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.204269:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.204270:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.204272:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.204273:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.204274:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.204276:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.204277:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.204278:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.204281:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.204282:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.204285:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.204286:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.204286:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.204288:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.204289:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.204290:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.204290:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.204293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.204300:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.204303:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.204304:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.204304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.204306:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.204308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.204850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.204858:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.204861:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.204863:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.204865:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.204865:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.204866:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.204868:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.204869:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.204870:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.204871:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.204874:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.204875:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.204878:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.204879:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.204880:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.204881:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.204882:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.204883:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.204883:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.204888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.204896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.204899:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.204900:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.204900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.204902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677475.204902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.204907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677475.204908:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.204909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.204910:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677475.204912:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.204914:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.204915:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.204916:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.204918:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.204919:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.204920:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.204923:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.204925:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.204927:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.204928:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.204928:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.204930:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.204930:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.204931:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.204931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.204936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.204944:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.204946:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.204947:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.204949:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.204950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.204951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.205562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.205568:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.205571:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.205573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.205574:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.205576:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.205578:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.205579:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.205581:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.205582:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.205583:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.205587:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.205588:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.205592:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.205593:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.205594:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.205596:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.205596:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.205597:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.205597:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.205603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.205612:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.205616:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.205617:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.205620:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.205622:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.205624:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.205624:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.205625:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.205625:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.205627:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.205627:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.205715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.205724:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.205726:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.205727:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.205729:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.205732:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.205733:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.205735:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.205736:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.205737:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.205738:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.205741:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.205743:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.205746:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.205747:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.205748:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.205749:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.205750:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.205750:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.205751:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.205755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.205763:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.205766:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.205767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.205767:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.205769:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.205772:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.205773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.205779:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.205781:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.205782:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.205784:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.205786:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.205787:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.205788:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.205790:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.205791:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.205792:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.205795:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.205795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:22.0:1619677475.205797:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.205799:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.205800:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.205801:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677475.205801:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:22.0:1619677475.205802:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.205803:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.205804:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:1.0:1619677475.205804:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:22.0:1619677475.205805:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.205805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:17.0:1619677475.205805:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.205807:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.205808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.205810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:1.0:1619677475.205810:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.205811:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677475.205812:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.205814:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.205815:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:1.0:1619677475.205815:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.205816:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:1.0:1619677475.205816:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:6.0:1619677475.205817:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.205818:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.205818:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.205819:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.205819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:1.0:1619677475.205819:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677475.205820:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.205821:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.0:1619677475.205822:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.205822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677475.205824:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.205825:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677475.205825:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677475.205827:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677475.205827:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:25.2:1619677475.205828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:1.0:1619677475.205828:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.205829:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677475.205833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677475.205835:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.205838:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.205839:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.205839:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.205840:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.205841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677475.205841:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.205843:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.205844:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.205844:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.205847:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.205850:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.206080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.206089:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.206092:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.206094:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.206096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.206097:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.206099:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.206100:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.206101:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.206102:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.206103:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.206107:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.206108:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.206111:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.206112:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.206113:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.206115:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.206115:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.206117:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.206118:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.206119:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677475.206126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:25.2:1619677475.206131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.206134:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.206136:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.206137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.206138:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.206139:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.206141:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.206142:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.206143:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.206145:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.206148:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.206153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.206160:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.206163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.206163:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.206164:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.206165:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.206165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.206219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.206227:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.206230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.206230:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.206232:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.206235:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.206236:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677475.206238:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.206239:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.206240:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.206241:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677475.206244:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.206245:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677475.206248:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.206249:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677475.206249:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677475.206251:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677475.206252:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.206253:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.206254:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.206255:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677475.206257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.206260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677475.206265:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677475.206268:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:2.0:1619677475.206268:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.206269:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.206269:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.206270:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.206271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.206272:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:2.0:1619677475.206272:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.206275:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.206378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.206385:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.206387:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.206388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.206390:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.206392:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.206394:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.206395:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.206396:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.206397:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.206398:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.206401:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.206403:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.206405:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.206406:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.206407:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.206409:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.206409:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.206410:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.206410:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.206415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.206423:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.206426:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.206426:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.206428:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.206429:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.206431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.206600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.206607:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.206610:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.206613:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.206615:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.206616:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.206617:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.206619:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.206620:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.206621:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.206622:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.206625:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.206627:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.206629:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.206630:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.206631:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.206632:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.206633:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.206634:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.206636:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.206636:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.206638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.206642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.206646:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.206648:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.206649:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.206650:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.206650:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.206651:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.206652:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.206652:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.206653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.206653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.206654:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677475.206658:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.206661:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.206662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.206664:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.206666:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.206667:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.206669:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.206670:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.206671:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.206672:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.206675:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.206676:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.206679:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.206679:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.206680:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.206682:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.206682:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.206683:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.206683:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.206688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.206695:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.206697:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.206698:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.206700:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.206701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.206702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677475.206813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.206819:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.206822:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.206823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.206823:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.206824:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.206825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.206996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.207001:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.207004:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.207006:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.207006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.207009:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.207010:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.207011:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.207013:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.207014:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.207015:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.207018:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.207019:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.207022:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.207023:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.207023:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.207026:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:21.0:1619677475.207028:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.207029:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.207030:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.207030:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.207031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.207036:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.207037:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000400:00000200:1.0:1619677475.207040:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677475.207041:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.207042:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.207043:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.207043:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.207045:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677475.207045:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677475.207046:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.207048:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.207049:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.207050:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.207052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.207053:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677475.207053:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.207054:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.207055:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.207055:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.207056:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.207057:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.207058:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.207058:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.207059:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677475.207059:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.207060:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.207062:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.207063:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.207063:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677475.207064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:25.2:1619677475.207068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.207076:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.207078:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.207079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.207079:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.207081:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.207084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.207459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.207467:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.207470:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.207472:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.207474:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.207475:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.207476:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.207477:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.207479:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.207480:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.207481:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.207484:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.207486:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.207489:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.207490:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.207491:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.207492:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.207493:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.207494:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.207494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.207499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.207506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.0:1619677475.207510:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677475.207513:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677475.207513:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.207514:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.207515:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.207517:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:21.0:1619677475.207517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.207517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677475.207518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:26.0:1619677475.207519:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.207522:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.207524:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.207525:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.207526:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.207527:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.207528:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.207531:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.207532:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.207534:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.207535:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.207536:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.207537:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.207537:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.207538:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.207539:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.207543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.207551:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.207554:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.207554:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.207556:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.207558:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.207559:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.207658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.207664:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.207667:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.207667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.207669:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.207672:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.207673:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.207674:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.207676:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.207676:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.207677:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.207680:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.207682:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.207684:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.207685:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.207686:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.207687:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.207688:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.207689:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.207689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.207693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.207701:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.207703:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.207704:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.207706:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.207706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.207708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.207710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.207717:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.207720:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.207722:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.207723:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.207726:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.207727:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677475.207729:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.207730:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.207731:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.207732:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677475.207735:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.207737:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677475.207740:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.207740:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677475.207741:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677475.207743:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677475.207743:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.207744:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.207744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677475.207749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.207758:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.207761:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.207762:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.207762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.207764:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.207768:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.207778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.207785:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.207788:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.207789:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.207791:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.207793:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.207795:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.207796:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.207797:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.207798:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.207799:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.207802:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.207804:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.207807:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.207808:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.207808:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.207810:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.207810:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.207812:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.207813:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.207813:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.207816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.207819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.207824:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677475.207826:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.207826:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.207827:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.207828:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677475.207828:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.207829:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.207830:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.207830:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.207833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.207885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.207892:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.207895:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.207896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.207898:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.207901:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.207902:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677475.207904:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.207905:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.207906:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677475.207906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:10.0:1619677475.207907:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.207910:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:10.0:1619677475.207910:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.207911:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.207913:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:10.0:1619677475.207914:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677475.207915:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677475.207915:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.207916:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:10.0:1619677475.207916:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677475.207917:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:20.0:1619677475.207918:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.207918:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.207919:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.207919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:20.0:1619677475.207920:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.207921:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.207922:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.207923:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.207924:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.2:1619677475.207924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:20.0:1619677475.207927:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.207929:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.207932:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.207932:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677475.207932:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677475.207933:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.207935:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677475.207935:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:20.0:1619677475.207936:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.207936:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000400:00000200:17.0:1619677475.207936:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.207936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.207937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:17.0:1619677475.207938:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.207941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677475.207941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.207948:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.207951:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.207951:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.207952:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.207954:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.207957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.208020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.208026:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.208028:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.208030:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.208031:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.208033:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.208034:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.208036:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.208037:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.208038:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.208039:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677475.208043:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.208044:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.208047:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.208047:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.2:1619677475.208049:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.0:1619677475.208053:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.208054:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.208055:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.208056:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.208056:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677475.208057:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.208060:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:9.2:1619677475.208060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:31.0:1619677475.208062:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.208064:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.208065:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.208066:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677475.208067:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:31.0:1619677475.208068:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.208069:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.208070:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677475.208070:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:31.0:1619677475.208071:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677475.208071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.208071:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.208074:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000400:00000200:31.0:1619677475.208076:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.208078:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677475.208078:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677475.208080:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.208081:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:9.2:1619677475.208081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677475.208082:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.208083:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.208084:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.208085:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.208086:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677475.208088:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:25.2:1619677475.208089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677475.208091:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.208092:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.208092:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.208093:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.208094:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677475.208096:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.208098:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.208099:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.208101:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.208102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.208103:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677475.208112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.208118:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.208121:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.208122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.208123:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.208126:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.208127:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.208129:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.208130:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.208131:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.208132:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.208135:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.208137:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.208140:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.208140:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.208141:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.208143:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.208143:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.208144:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.208144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.208149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.208156:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.208159:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.208160:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.208160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.208162:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.208164:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.208216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.208219:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.208221:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.208224:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.208225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.208226:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.208228:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.208229:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.208231:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.208232:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.208233:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.208236:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.208238:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.208240:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.208241:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.208242:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.208244:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.208244:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.208246:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.208247:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.208247:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.208250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.208252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.208258:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677475.208258:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.208261:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677475.208261:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677475.208262:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.208262:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677475.208262:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.208262:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.208264:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.208267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.208466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.208474:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.208477:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.208479:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.208482:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.208482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.208483:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.208485:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.208486:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.208487:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.208488:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.208491:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.208492:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.208496:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.208497:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.208497:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.208499:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.208500:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.208501:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.208501:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.208506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.208513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.208516:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.208518:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.208519:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:6.0:1619677475.208520:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.208521:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.2:1619677475.208522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000400:00000200:6.0:1619677475.208522:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.208523:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.208525:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.208526:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.208526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.208527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.208528:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:1.0:1619677475.208528:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:22.0:1619677475.208529:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.208531:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.208531:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:22.0:1619677475.208532:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677475.208532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:17.0:1619677475.208532:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:22.0:1619677475.208534:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677475.208535:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.208537:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.208538:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.208538:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:1.0:1619677475.208539:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677475.208540:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:1.0:1619677475.208540:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677475.208541:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.208542:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:1.0:1619677475.208542:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.208543:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.208544:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:2.0:1619677475.208544:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.208544:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.208545:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:1.0:1619677475.208545:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.208546:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:2.0:1619677475.208548:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.208548:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.208550:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.208550:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:2.0:1619677475.208550:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677475.208550:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.208551:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:2.0:1619677475.208551:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96501038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.208552:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.208553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:2.0:1619677475.208553:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.208553:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.208554:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:2.0:1619677475.208555:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677475.208555:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.208556:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.208557:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:2.0:1619677475.208558:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.208559:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:2.0:1619677475.208560:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677475.208560:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677475.208560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:21.0:1619677475.208561:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:25.2:1619677475.208562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.208563:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.208564:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.208564:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.208565:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.208565:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.2:1619677475.208566:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:20.0:1619677475.208566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677475.208567:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.208568:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.208568:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:21.0:1619677475.208569:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.208569:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677475.208569:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:21.0:1619677475.208571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677475.208572:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:6.0:1619677475.208572:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:25.2:1619677475.208573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:6.0:1619677475.208573:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.208574:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.208574:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677475.208575:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:6.0:1619677475.208575:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.208576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677475.208577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:13.0:1619677475.208581:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.208583:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.208584:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.208586:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.208588:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.208589:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.208609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.208618:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.208621:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.208621:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.208623:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.208626:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.208628:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.208629:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.208630:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.208631:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.208632:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.208636:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.208638:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.208641:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.208642:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.208643:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.208644:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.208645:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.208646:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.208646:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.208651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.208659:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.208661:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.208662:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.208664:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.208665:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.208667:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.208835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677475.208840:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.208842:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677475.208845:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.208845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677475.208847:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.208849:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:21.0:1619677475.208850:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.208852:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.208853:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.208854:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677475.208858:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.208859:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.0:1619677475.208862:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.208863:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677475.208864:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677475.208866:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677475.208867:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.208868:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677475.208868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677475.208872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.208879:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.208881:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677475.208882:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.208884:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.208885:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677475.208887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677475.209137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.209143:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.209146:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.209146:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.209149:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.209152:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.209153:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.209154:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.209156:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.209157:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.209158:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.209161:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.209163:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.209165:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.209165:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.209166:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.209168:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.209169:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.209170:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.209170:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.209174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.209181:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.209183:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.209183:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.209184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.209185:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.209187:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.209396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.209401:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.209404:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.209405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.209407:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.209409:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.209411:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.209412:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.209413:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.209414:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.209415:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.209418:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.209420:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.209422:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.209423:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.209424:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.209425:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.209426:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.209427:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.209428:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.209428:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.209432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.209434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.209439:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.209440:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.209441:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.209442:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.209442:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.209442:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.209443:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.209444:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.209446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.209447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.209452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.209459:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.209461:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.209462:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.209463:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.209463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.209465:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.209551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.209559:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.209562:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.209563:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.209564:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.209567:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.209568:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.209570:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.209571:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.209572:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.209573:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.209576:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.209578:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.209581:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.209582:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.209583:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.209584:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.209585:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.209586:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.209586:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.209600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.209608:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.209611:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.209612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.209612:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.209615:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.209618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.209783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677475.209788:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.209791:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677475.209792:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677475.209793:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677475.209795:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.209797:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677475.209797:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.209799:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.209800:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.209801:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.209804:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.209805:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.209808:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.209809:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677475.209809:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677475.209811:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677475.209812:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.209812:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677475.209812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677475.209817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.209824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.209826:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.209827:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.209827:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.209830:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.209833:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.210149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.210156:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.210159:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.210160:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.210162:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.210164:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.210165:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.210167:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.210168:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.210169:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.210170:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.210173:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.210175:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.210178:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.210179:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.210180:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.210182:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.210182:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.210183:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.210184:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.210188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.210196:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.210198:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.210199:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.210201:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.210203:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677475.210203:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:4.2:1619677475.210293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.210301:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.210304:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.210305:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.210307:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.210310:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.210311:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.210312:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.210313:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677475.210314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:10.0:1619677475.210315:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.210316:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.210319:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.210321:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.210322:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:10.0:1619677475.210324:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.210325:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:10.0:1619677475.210325:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.210326:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:30.0:1619677475.210327:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.210327:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.210328:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.210329:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.210329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677475.210330:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.210330:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.210332:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.210333:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.210334:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677475.210334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:30.0:1619677475.210335:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.210336:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.210340:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.210341:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.210343:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677475.210344:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.210345:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.210345:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677475.210346:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677475.210346:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.210347:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.210348:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.210348:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.210349:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677475.210349:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.210350:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.0:1619677475.210350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.210351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:25.2:1619677475.210354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.210359:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677475.210362:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677475.210362:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.210362:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.210363:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.210364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:22.0:1619677475.210365:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677475.210365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:22.0:1619677475.210366:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.210366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.210368:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.210371:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.210372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.210379:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.210381:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.210382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.210383:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.210384:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.210384:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.210480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.210485:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.210487:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.210488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.210489:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.210492:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.210493:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.210494:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.210495:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.210496:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.210497:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.210500:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.210502:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.210504:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.210505:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.210506:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.210507:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.210508:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.210508:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.210509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.210513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.210521:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.210523:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.210524:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.210525:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.210526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.210528:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.210683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:2.0:1619677475.210689:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:21.2:1619677475.210691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677475.210692:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.14.18@o2ib3
00000800:00000200:18.0:1619677475.210693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:2.0:1619677475.210695:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677475.210698:0:19932:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 16456
00000800:00000200:31.0:1619677475.210699:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:2.0:1619677475.210699:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677475.210700:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)++
00000800:00000200:2.0:1619677475.210701:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (23)--
00000800:00000200:2.0:1619677475.210701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000800:00000200:31.0:1619677475.210702:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.210705:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.210706:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.2:1619677475.210706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (21)++
00000400:00000200:31.0:1619677475.210707:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.210709:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.210710:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.210711:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677475.210712:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)++
00000400:00000200:31.0:1619677475.210713:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.210713:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677475.210714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (23)--
00000800:00000200:18.0:1619677475.210715:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:31.0:1619677475.210717:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.210719:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.210721:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.210722:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.210722:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.2:1619677475.210722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (21)++
00000800:00000200:31.0:1619677475.210724:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.210724:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.210725:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.210726:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677475.210727:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)++
00000800:00000200:25.2:1619677475.210730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677475.210730:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.14.18@o2ib3
00000800:00000200:18.0:1619677475.210731:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (23)--
00000800:00000200:2.0:1619677475.210731:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907cbe3400] (23)--
00000400:00000200:2.0:1619677475.210732:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.210734:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:2.0:1619677475.210735:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.210736:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:2.0:1619677475.210737:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.210737:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.210738:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:30.0:1619677475.210739:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.210740:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.210742:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.210743:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:2.0:1619677475.210743:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.210745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:2.0:1619677475.210745:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 16456 bytes in 256 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:2.0:1619677475.210747:0:19932:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 16456
00000800:00000200:25.2:1619677475.210748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677475.210750:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.210751:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:2.0:1619677475.210752:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677475.210754:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677475.210754:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:2.0:1619677475.210755:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677475.210756:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:2.0:1619677475.210756:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:31.0:1619677475.210758:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.210759:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.210759:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.210760:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:25.2:1619677475.210760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677475.210761:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:12.0:1619677475.210767:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.210769:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677475.210770:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:25.2:1619677475.210778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.210785:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.210788:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[2] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677475.210789:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)++
00000800:00000200:13.0:1619677475.210792:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (24)--
00000800:00000200:12.0:1619677475.210792:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (24)--
00000800:00000200:12.0:1619677475.210793:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:25.2:1619677475.210802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677475.210809:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677475.210812:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677475.210813:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.210815:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.210816:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677475.210817:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677475.210822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677475.210830:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.210833:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677475.210835:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.210837:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677475.210837:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.210838:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677475.210839:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.210841:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.210842:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.210843:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677475.210846:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.210848:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677475.210851:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.210852:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:30.0:1619677475.210852:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:30.0:1619677475.210854:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:30.0:1619677475.210854:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.210855:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677475.210856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:4.2:1619677475.210858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:9.2:1619677475.210861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.210867:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677475.210868:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.210869:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677475.210870:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.210871:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.210871:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.210873:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:26.0:1619677475.210874:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.210876:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677475.210877:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.210879:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.210879:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.210881:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:10.0:1619677475.210882:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.210883:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677475.210884:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:26.0:1619677475.210885:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.210885:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677475.210886:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.210887:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677475.210887:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677475.210887:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.210888:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:26.0:1619677475.210891:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.210891:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677475.210892:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677475.210893:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677475.210894:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677475.210894:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677475.210895:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677475.210900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.210903:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677475.210905:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677475.210906:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.210906:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677475.210908:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.210910:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677475.211367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.211373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.211376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.211376:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.211379:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.211381:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.211382:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.211383:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.211385:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.211386:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.211387:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.211390:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.211392:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.211394:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.211395:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.211396:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.211397:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.211398:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.211399:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.211399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.211404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.211412:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.211414:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.211415:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.211417:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.211418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.211419:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.211486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.211492:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.211495:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.211496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.211498:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.211501:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.211502:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677475.211503:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.211505:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.211506:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.211507:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677475.211510:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.211511:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.211513:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.211514:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.211515:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.211516:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.211516:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.211517:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.211518:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677475.211523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.211531:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.211533:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.211534:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.211535:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.211535:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.211537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.211542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.211549:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.211551:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.211552:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.211553:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.211553:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.211554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.211590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.211598:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.211601:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.211602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.211604:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.211606:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.211608:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.211609:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.211610:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.211611:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.211612:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.211616:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.211617:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.211620:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.211621:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.211622:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.211623:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.211624:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.211625:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.211625:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.211643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.211650:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.211653:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.211654:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.211654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.211657:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.211659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.212171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.212177:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.212180:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.212180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.212182:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.212184:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.212185:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.212187:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.212188:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.212189:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.212190:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.212193:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.212194:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.212197:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.212197:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.212198:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.212200:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.212200:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.212202:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.212203:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.212203:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.212206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.212209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.212214:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677475.212216:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.212216:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.212217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.212217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677475.212218:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.212219:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.212221:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.212222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.212223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.212242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.212247:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.212250:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.212251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.212253:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.212255:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.212256:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.212257:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.212259:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.212259:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.212261:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.212264:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.212266:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.212268:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.212269:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.212270:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.212271:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.212272:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.212272:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.212273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.212277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.212284:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.212286:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.212287:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.212289:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.212290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.212291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.212357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.212365:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.212368:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.212370:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.212373:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.212373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.212374:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.212376:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.212377:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.212378:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.212379:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.212383:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.212385:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.212387:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.212388:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.212388:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.212390:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.212391:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.212392:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.212394:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.212394:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.212396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.212400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.212403:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.212407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.212407:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:30.0:1619677475.212408:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.212408:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.212410:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:8.0:1619677475.212410:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.212411:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.212412:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677475.212412:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.212626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677475.212631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:24.0:1619677475.212633:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.212635:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:21.0:1619677475.212636:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677475.212636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.212638:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.212639:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:24.0:1619677475.212640:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.212641:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677475.212642:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677475.212642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:24.0:1619677475.212643:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.212644:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.212644:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.212645:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.212646:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.212647:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.212648:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.212650:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.212651:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.212652:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.212653:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.212655:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.212657:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677475.212658:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.212659:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.212659:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.212659:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.212661:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.212661:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.212662:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677475.212662:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.212662:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.212663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677475.212663:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677475.212665:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677475.212665:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.212666:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.212667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677475.212667:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677475.212674:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677475.212677:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:22.0:1619677475.212678:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.212678:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:6.0:1619677475.212678:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677475.212679:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.212681:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.212681:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.212685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.212685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.212686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.212701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.212708:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.212711:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.212713:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.212715:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.212716:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.212718:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.212719:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.212720:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.212721:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.212722:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.212726:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.212727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677475.212728:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.212731:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.212732:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.212732:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.212734:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.212734:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677475.212735:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.212736:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.212736:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:26.0:1619677475.212736:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.212737:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.212739:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.212740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677475.212742:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.212744:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.212745:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.212746:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.212747:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.212747:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:26.0:1619677475.212748:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.212749:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.212750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677475.212751:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.212751:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.212753:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:6.0:1619677475.212753:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.212756:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.212756:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677475.212757:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.212757:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.212759:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:25.2:1619677475.212759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.212760:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.212760:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.212761:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.212765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:20.2:1619677475.212769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677475.212770:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.212772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677475.212773:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.212774:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.212774:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677475.212774:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677475.212775:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.212775:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:20.0:1619677475.212776:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.212776:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.212777:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677475.212777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677475.212777:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.212778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:20.0:1619677475.212778:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.212780:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677475.212780:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677475.212781:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96501013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.212783:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.212783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:20.0:1619677475.212784:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.212785:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.212789:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.212790:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.212792:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.212793:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.212794:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.212795:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.212796:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.212797:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.212798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.212802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.212810:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.212813:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.212814:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.212814:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.212816:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.212817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.212819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677475.212824:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.212827:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.212827:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.212828:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.212829:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.212829:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.213253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:21.2:1619677475.213258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677475.213258:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.213260:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.213261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.213263:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.213265:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.213266:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677475.213266:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.213268:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.213269:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677475.213269:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.213271:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.213272:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.213273:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.213273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.213276:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.213276:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.213278:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.213279:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.213280:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96501011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.213281:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.213282:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.213282:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.213282:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.213283:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.213284:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677475.213284:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.213284:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.213285:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.213286:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:31.0:1619677475.213287:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.213287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677475.213289:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:25.2:1619677475.213290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.213292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.213293:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.213293:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:22.0:1619677475.213300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:31.0:1619677475.213301:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.213301:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.213302:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:31.0:1619677475.213303:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.213303:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.213303:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.213303:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677475.213304:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677475.213304:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.213304:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.213305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677475.213306:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.213307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.213308:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677475.213309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:25.2:1619677475.213317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.213319:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.213321:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.213322:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.213324:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677475.213324:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.213326:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.213327:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.213327:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.213327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.213328:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677475.213329:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.213330:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.213486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.213493:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.213496:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.213497:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.213498:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.213501:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.213502:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.213503:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.213505:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.213506:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.213507:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.213510:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.213512:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.213515:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.213515:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.213516:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.213518:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.213518:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.213519:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.213519:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.213524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.213531:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.213534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.213534:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.213535:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.213537:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.213539:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.213943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.213949:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.213951:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.213953:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.213954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.213956:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.213957:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.213958:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.213959:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.213960:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.213961:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.213964:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.213966:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.213968:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.213969:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.213970:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.213971:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.213972:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.213972:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.213973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.213977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.213984:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.213986:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.213987:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.213989:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.213990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.213991:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.213998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.214005:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.214008:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.214009:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.214010:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.214010:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.214011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.214113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.214118:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.214121:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.214121:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.214123:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.214125:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.214127:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.214128:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.214129:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.214130:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.214131:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.214134:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.214136:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.214138:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.214139:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.214140:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.214142:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.214142:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.214143:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.214143:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.214148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.214155:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.214157:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.214158:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.214160:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.214161:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.214162:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.214565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.214573:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.214576:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.214578:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.214580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.214581:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.214582:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.214584:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.214585:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.214586:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.214587:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.214590:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.214592:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.214596:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.214596:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.214597:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.214599:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.214599:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.214601:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.214602:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.214603:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.214605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.214609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.214613:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.214616:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.214617:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.214617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.214618:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.214620:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677475.214620:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.214621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.214622:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.214622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.214859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677475.214862:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677475.214865:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677475.214867:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.214868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:20.0:1619677475.214869:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.214870:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677475.214871:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.214873:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.214874:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.214875:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677475.214878:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.214879:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.214882:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.214882:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.214883:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.214885:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.214885:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677475.214887:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:20.0:1619677475.214888:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677475.214888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677475.214891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677475.214894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677475.214897:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.214898:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.214899:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:20.0:1619677475.214899:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.214900:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000400:00000200:20.0:1619677475.214900:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.214901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:20.0:1619677475.214902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.214903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677475.214904:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:21.2:1619677475.214996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.215002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.215005:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.215006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.215007:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.215010:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.215011:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.215012:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.215014:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.215015:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.215016:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.215018:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.215020:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.215022:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.215023:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.215023:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.215025:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.215025:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.215026:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.215027:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.215031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.215044:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.215046:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.215047:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.215049:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.215050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.215051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.215201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.215208:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.215211:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.215212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.215214:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.215217:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.215219:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677475.215220:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96501004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.215221:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.215222:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.215223:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677475.215227:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.215229:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677475.215231:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.215232:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677475.215233:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677475.215235:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677475.215235:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.215237:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.215238:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.215238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677475.215241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.215244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677475.215249:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:24.0:1619677475.215251:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677475.215252:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:24.0:1619677475.215253:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:18.0:1619677475.215253:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.215253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677475.215254:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.215255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:18.0:1619677475.215255:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.215259:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.215804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.215812:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.215815:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.215817:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.215820:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.215820:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.215821:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.215823:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.215824:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.215825:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.215826:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.215829:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.215830:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.215834:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.215834:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.215835:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.215837:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.215837:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.215838:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.215839:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.215844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.215853:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.215855:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.215856:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.215858:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.215859:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.215861:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.216033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.216039:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.216042:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.216043:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.216045:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.216047:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.216048:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.216049:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.216051:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.216052:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.216053:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.216056:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.216058:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.216060:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.216061:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.216062:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.216063:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.216064:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.216064:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.216065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.216070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.216078:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.216080:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.216081:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.216083:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.216083:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.216085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.216098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.216106:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.216109:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.216110:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.216112:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.216115:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.216116:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.216118:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96501001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.216119:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.216120:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.216121:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.216124:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.216126:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.216129:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.216130:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.216130:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.216132:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.216133:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.216133:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.216134:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.216138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.216145:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.216147:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.216148:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.216150:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.216152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.216153:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.216156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677475.216163:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.216165:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.216166:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.216167:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.216167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.216168:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.216317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.216321:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.216323:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.216325:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.216326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.216327:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.216329:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.216330:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96501000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.216331:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.216332:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.216333:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.216337:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.216338:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.216341:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.216342:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.216342:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.216344:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.216344:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.216345:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.216346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.216350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.216358:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.216361:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.216362:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.216362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.216364:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.216366:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.216540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.216546:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.216549:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.216550:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.216551:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.216554:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.216555:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.216557:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.216558:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.216559:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.216560:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.216564:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.216565:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.216568:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.216569:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.216569:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.216571:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.216572:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.216573:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.216573:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.216578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.216586:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.216589:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.216590:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.216590:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.216592:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.216595:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.216872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.216880:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.216883:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.216884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.216885:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.216888:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.216889:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.216890:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.216892:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.216893:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.216894:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.216896:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.216898:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.216901:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.216902:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.216903:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.216904:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.216905:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.216906:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.216906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.216910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.216918:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.216921:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.216922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.216922:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.216924:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.216927:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.216931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.216936:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.216938:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.216939:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.216940:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.216943:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.216944:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.216945:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.216946:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.216947:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.216948:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.216951:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.216952:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.216955:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.216955:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.216956:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.216958:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.216958:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.216959:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.216960:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.216961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.216964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.216966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.216971:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.216973:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.216973:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.216973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.216974:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.216975:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.216976:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.216978:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.216978:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.216980:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.216980:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.216981:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.216982:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (23)++
00000800:00000200:20.0:1619677475.216982:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.216983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (24)--
00000800:00000200:20.0:1619677475.216984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.216985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.217293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.217302:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.217306:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.217308:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.217310:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.217311:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.217313:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.217314:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.217315:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.217316:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.217318:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.217321:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.217323:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.217325:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.217326:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.217326:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.217328:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.217328:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.217329:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.217330:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.217334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.217341:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.217344:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.217344:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.217346:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.217348:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.217348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677475.217436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.217444:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.217447:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.217448:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.217449:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.217452:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.217454:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.217455:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.217456:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.217457:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.217458:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.217461:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.217462:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.217465:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.217466:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.217466:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.217468:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.217469:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.217469:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.217470:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.217474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.217482:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.217485:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.217486:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.217486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.217488:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.217491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.217494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.217501:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.217504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.217504:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.217505:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.217506:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.217506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.217573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.217580:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.217583:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.217584:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.217586:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.217588:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.217589:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.217591:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.217592:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.217593:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.217594:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.217597:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.217599:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.217602:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.217602:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.217603:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.217605:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.217605:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.217606:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.217606:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.217611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.217618:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.217621:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.217622:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.217622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.217624:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.217627:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.217712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.217720:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.217723:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.217725:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.217726:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.217727:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.217729:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.217730:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.217732:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.217733:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.217734:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.217737:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.217738:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.217741:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.217742:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.217743:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.217744:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.217745:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.217746:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.217746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.217750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.217757:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.217760:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.217761:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.217761:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.217763:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.217766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.217911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.217916:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.217918:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.217919:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.217921:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.217923:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.217924:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.217925:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.217927:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.217928:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.217929:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.217932:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.217933:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.217935:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.217936:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.217937:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.217939:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.217939:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.217940:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.217940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.217945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.217952:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.217954:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.217955:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.217957:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.217958:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.217959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.217987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.217993:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.217995:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.217998:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.217998:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.218000:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.218001:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.218003:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.218004:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.218005:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.218006:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.218009:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.218011:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.218013:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.218014:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.218015:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.218016:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.218017:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.218018:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.218018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.218025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.218032:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.218034:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.218035:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.218036:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.218037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.218038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.218125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.218130:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.218133:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.218135:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.218135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.218137:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.218139:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.218140:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.218141:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.218142:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.218143:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677475.218146:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.218148:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.218150:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.218151:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.218151:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.218153:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.218153:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.218154:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.218155:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.218159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.218167:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.218170:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.218171:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.218171:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.218173:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677475.218176:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.218271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.218276:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.218279:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.218280:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.218282:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.218284:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.218285:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677475.218286:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.218288:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.218289:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.218290:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677475.218299:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.218300:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677475.218303:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.218304:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677475.218304:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.218306:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677475.218307:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.218308:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677475.218309:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677475.218310:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:25.2:1619677475.218313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.2:1619677475.218315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:6.0:1619677475.218321:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677475.218322:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:6.0:1619677475.218323:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.218324:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.218325:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677475.218325:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677475.218325:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.218326:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:6.0:1619677475.218326:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.218329:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.218440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.218446:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.218449:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.218450:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.218452:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.218454:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.218455:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.218457:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.218458:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.218459:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.218460:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.218463:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.218464:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.218465:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.218466:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.218466:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.218468:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.218468:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.218469:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.218470:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.218474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.218481:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.218483:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.218484:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.218484:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.218486:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.218487:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.218563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.218572:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.218575:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.218577:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.218579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.218580:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.218582:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.218583:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.218584:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.218585:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.218586:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.218589:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.218591:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.218593:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.218594:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.218594:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.218596:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.218597:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.218598:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.218600:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.218600:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.218601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.218606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.218608:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.218611:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.218612:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.218612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.218614:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.218614:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.218616:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677475.218616:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.218617:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.218618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.218619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.218625:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.218627:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.218628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.218628:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.218629:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.218630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677475.218665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.218671:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.218674:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.218675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.218677:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.218679:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.218681:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.218682:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.218683:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.218684:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.218685:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.218688:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.218690:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.218692:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.218693:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.218694:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.218695:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.218696:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.218697:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.218698:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.218699:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.218702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.218704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677475.218709:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677475.218710:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677475.218712:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677475.218712:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677475.218713:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.218713:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677475.218713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.218714:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677475.218715:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.218717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.218852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.218855:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.218858:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.218860:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.218861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.218862:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.218864:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.218865:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.218867:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.218868:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.218869:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.218872:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.218874:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.218877:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.218878:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.218878:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.218880:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.218881:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.218882:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.218883:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.218884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.218886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.218890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677475.218892:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.218895:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.218896:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.218896:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677475.218896:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.218898:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.218898:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.218899:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.218900:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677475.218901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.218910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.218918:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.218921:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.218922:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.218922:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.218923:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.218924:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.218959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.218965:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.218968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.218968:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.218970:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.218972:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.218973:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.218975:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.218976:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.218977:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.218978:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.218981:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.218983:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.218986:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.218986:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.218987:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.218988:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.218989:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.218990:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.218990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.218994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.219002:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.219004:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.219005:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.219007:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.219008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.219010:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.219565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.219573:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.219576:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.219579:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.219581:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.219582:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.219583:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.219585:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.219586:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.219587:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.219588:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.219591:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.219593:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.219596:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.219597:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.219598:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.219599:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.219600:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.219601:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.219601:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.219606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.219609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.219611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.2:1619677475.219613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:22.0:1619677475.219614:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.219614:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:31.0:1619677475.219617:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.219617:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677475.219617:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:22.0:1619677475.219618:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.219618:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:20.0:1619677475.219619:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:1.0:1619677475.219619:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:31.0:1619677475.219620:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.219620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:22.0:1619677475.219621:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677475.219622:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.219622:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.219623:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:30.0:1619677475.219624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677475.219624:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:20.0:1619677475.219624:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.219625:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.219626:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.219627:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.219628:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.219628:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.219629:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.219630:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.219630:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.219632:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677475.219633:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677475.219633:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:1.0:1619677475.219634:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.219635:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.219635:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:1.0:1619677475.219636:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.219637:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.219638:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.219638:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677475.219638:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.219639:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:1.0:1619677475.219639:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677475.219640:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677475.219640:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.219642:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.219642:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.219643:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.219644:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:1.0:1619677475.219644:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.219645:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.219647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.219647:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677475.219648:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677475.219649:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.219650:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677475.219651:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.219651:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677475.219652:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677475.219652:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677475.219653:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677475.219653:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:31.0:1619677475.219654:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677475.219654:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.219654:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:31.0:1619677475.219655:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677475.219655:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677475.219656:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.219657:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677475.219657:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.219658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.219658:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.219659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:9.2:1619677475.219660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:6.0:1619677475.219660:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.219662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.219665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677475.219667:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:8.0:1619677475.219667:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.219670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:17.0:1619677475.219670:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:8.0:1619677475.219670:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:17.0:1619677475.219671:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.219671:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.219671:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677475.219672:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:8.0:1619677475.219674:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.219675:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677475.219675:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.219676:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.219676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677475.219676:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.219677:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.219677:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677475.219678:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677475.219680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.219688:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.219691:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.219692:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.219692:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.219693:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.219693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.219837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.219842:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.219845:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.219846:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.219848:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.219850:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.219851:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.219853:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.219854:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.219855:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.219856:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.219860:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.219861:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.219864:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.219865:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.219865:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.219867:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.219867:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.219868:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.219868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.219873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.219880:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.219883:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.219883:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.219885:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.219886:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.219887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.219939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.219945:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.219949:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.219949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.219951:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.219953:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.219954:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.219956:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.219957:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.219958:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.219959:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.219962:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.219964:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.219966:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.219967:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.219968:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.219969:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.219969:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.219970:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.219971:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.219975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.219983:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.219985:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.219986:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.219988:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.219989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.219990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.220233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.220239:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.220241:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.220243:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.220244:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.220246:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.220247:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.220248:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.220250:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.220251:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.220252:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.220255:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.220257:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.220259:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.220260:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.220261:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.220262:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.220263:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.220264:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.220265:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.220266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.220268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.220270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.220273:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.220275:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.220276:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677475.220276:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677475.220277:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677475.220279:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.220280:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.220280:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.220281:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.220284:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.220497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.220505:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.220508:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.220509:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.220511:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.220514:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.220516:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677475.220517:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.220518:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.220519:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.220520:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677475.220523:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.220524:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.220526:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.220527:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.220527:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.220529:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.220530:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.220531:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.220532:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.220533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677475.220535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:4.2:1619677475.220539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.220545:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.220546:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.220546:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.220548:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.220549:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:10.0:1619677475.220549:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.220550:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.220550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.220551:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677475.220551:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.220689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.220695:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.220698:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.220699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.220701:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.220703:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.220704:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.220705:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.220706:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.220708:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.220709:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.220712:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.220713:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.220716:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.220716:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.220717:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.220719:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.220719:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.220720:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.220722:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.220722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.220725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.220727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.220732:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.220732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.220734:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.220735:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.220735:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.220735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.220736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.220737:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.220738:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.220739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.220746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.220753:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.220756:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.220756:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.220757:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.220758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.220759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.221126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.221135:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.221138:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.221140:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.221142:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.221142:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.221144:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.221145:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.221146:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.221147:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.221148:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.221152:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.221154:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.221156:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.221157:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.221157:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.221159:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.221160:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.221161:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.221161:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.221165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.221171:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.221174:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.221175:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.221176:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.221178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.221178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.2:1619677475.221677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.221684:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.221686:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.221687:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.221689:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.221692:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.221693:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.221694:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.221696:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.221697:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.221698:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.221701:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.221703:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.221705:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.221706:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.221707:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.221708:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.221709:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.221710:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.221710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.221715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.221723:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.221726:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.221726:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.221728:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.221729:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.221730:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.221766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.221774:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.221777:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.221779:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.221781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.221781:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.221783:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.221784:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.221785:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.221786:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.221787:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.221791:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.221792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fe8666e00] (20)++
00000800:00000200:30.0:1619677475.221793:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.221796:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.221797:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.221797:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.221799:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.221800:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.221801:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.221801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677475.221801:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fe8666e00] (21)++
00000800:00000200:24.0:1619677475.221804:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.72@o2ib3
00000800:00000200:25.2:1619677475.221805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.221805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666e00] (22)--
00000400:00000200:24.0:1619677475.221807:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.72@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.221810:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.221812:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:6.0:1619677475.221812:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:24.0:1619677475.221813:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.221814:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.221815:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.221815:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:24.0:1619677475.221816:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:22.0:1619677475.221816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.221816:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.221819:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677475.221820:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.221822:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:6.0:1619677475.221822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677475.221824:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.221825:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677475.221825:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.221827:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677475.221827:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fe8666e00] (21)++
00000800:00000200:24.0:1619677475.221829:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fe8666e00] (22)--
00000800:00000200:24.0:1619677475.221829:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fe8666e00] (21)--
00000800:00000200:24.2:1619677475.221832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677475.221836:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677475.221837:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:8.0:1619677475.221838:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.221839:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677475.221840:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.221841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677475.221998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.222005:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.222007:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.222008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.222009:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.222012:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.222013:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.222014:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.222015:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.222016:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.222017:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.222020:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.222022:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.222024:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.222025:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.222025:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.222027:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.222027:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.222028:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.222028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677475.222032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677475.222033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.222039:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.222040:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677475.222042:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677475.222043:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:31.0:1619677475.222044:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.222044:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.222044:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.222046:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.222047:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.222047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.222048:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677475.222049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677475.222050:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.222051:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.222052:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.222053:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.222056:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.222057:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.222059:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.222060:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.222060:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.222062:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.222062:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.222063:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.222064:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.222068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.222074:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.222077:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.222077:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.222079:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.222081:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.222081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677475.222259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.222267:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.222270:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.222271:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.222272:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.222275:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.222276:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.222277:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.222278:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.222279:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.222280:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.222283:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.222285:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.222288:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.222288:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.222289:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.222291:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.222291:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.222292:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:25.2:1619677475.222297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.222298:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.222299:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:4.2:1619677475.222299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.222305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.222307:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.222307:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.222308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.222309:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677475.222309:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.222310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.222311:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.222311:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.222313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.222411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.222419:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.222421:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.222422:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.222424:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.222426:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.222428:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.222429:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.222430:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.222431:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.222432:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.222435:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.222436:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.222439:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.222439:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.222440:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.222442:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.222442:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.222443:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.222443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.222448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.222455:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.222458:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.222458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.222459:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.222461:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.222463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.222466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.222473:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.222476:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.222477:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.222477:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.222478:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.222479:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.222509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.222514:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.222516:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.222517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.222519:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.222521:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.222522:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.222523:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.222524:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.222526:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.222526:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.222530:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.222531:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.222534:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.222534:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.222535:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.222537:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.222537:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.222538:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.222538:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.222543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.222551:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.222553:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.222554:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.222555:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.222556:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.222558:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.223026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.223033:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.223036:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.223037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:4.2:1619677475.223038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:8.0:1619677475.223039:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.223041:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:8.0:1619677475.223042:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.223043:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677475.223045:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:8.0:1619677475.223045:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677475.223045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:21.0:1619677475.223047:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:8.0:1619677475.223047:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.223048:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677475.223048:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.223049:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.223050:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.223052:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:12.0:1619677475.223053:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:8.0:1619677475.223053:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677475.223054:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.223055:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.223056:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.223056:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677475.223057:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:20.0:1619677475.223058:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:12.0:1619677475.223059:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.223059:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.223059:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.223060:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:13.0:1619677475.223061:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:20.0:1619677475.223062:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677475.223062:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.223062:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:12.0:1619677475.223063:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.223063:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.223063:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677475.223064:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:12.0:1619677475.223064:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677475.223065:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96500963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.223067:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677475.223068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.223068:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000400:00000200:12.0:1619677475.223068:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.223069:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000400:00000200:12.0:1619677475.223069:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.223070:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677475.223071:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:20.0:1619677475.223072:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.223073:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.223073:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:12.0:1619677475.223073:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.223075:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:6.0:1619677475.223076:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:12.0:1619677475.223078:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.223078:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:6.0:1619677475.223078:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677475.223079:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:22.0:1619677475.223079:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:12.0:1619677475.223079:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:6.0:1619677475.223079:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.223080:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.223081:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:6.0:1619677475.223081:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.223082:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.223082:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.223084:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:6.0:1619677475.223084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.223085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:12.0:1619677475.223085:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677475.223086:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:6.0:1619677475.223086:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:9.2:1619677475.223087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677475.223091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:25.2:1619677475.223092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.223093:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.223096:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.223097:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.223097:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.223099:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.223100:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:13.0:1619677475.223100:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:26.0:1619677475.223102:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:13.0:1619677475.223102:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:22.0:1619677475.223103:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[5] from 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677475.223103:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:6.0:1619677475.223103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.223104:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.223105:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:13.0:1619677475.223105:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:22.0:1619677475.223106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.223418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.223423:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.223426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.223426:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.223428:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.223430:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.223431:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.223432:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.223434:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.223435:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.223436:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.223439:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.223441:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.223443:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.223444:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.223445:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.223447:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.223447:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.223448:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.223448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.223453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.223462:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.223464:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.223465:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.223466:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.223467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.223469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.223634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.223642:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.223645:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.223647:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.223649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.223649:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.223651:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.223652:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.223653:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.223654:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.223655:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.223658:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.223660:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.223663:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.223664:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.223665:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.223666:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.223667:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.223668:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.223670:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.223671:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.223672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.223677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.223680:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.223683:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.223683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.223684:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.223684:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.223686:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.223687:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.223688:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.223689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.223689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.223727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.223734:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.223736:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.223737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.223739:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.223741:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.223743:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.223744:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.223746:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.223747:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.223748:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.223752:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.223754:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.223757:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.223757:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.223758:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.223760:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.223761:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.223761:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.223762:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.223763:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.223766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677475.223768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677475.223773:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:2.0:1619677475.223774:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677475.223775:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.223776:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.223776:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.223776:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.223777:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677475.223778:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.223780:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.223781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.223792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.223796:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.223798:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677475.223801:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.223801:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.223802:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.223804:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.223805:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.223807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677475.223807:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.223807:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.223809:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.223811:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.223813:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677475.223815:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.223815:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.223816:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.223816:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677475.223818:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677475.223818:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.223819:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.223819:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677475.223820:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.223820:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677475.223822:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.223823:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.223824:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.223825:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677475.223825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:30.0:1619677475.223827:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.223827:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.223829:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.223832:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.223832:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:30.0:1619677475.223834:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.0:1619677475.223834:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.223835:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.223837:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.223837:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.223837:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:30.0:1619677475.223838:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.223838:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.223840:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677475.223840:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677475.223841:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.223842:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.223844:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.223844:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.223846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.223850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.223852:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.223855:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.223856:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.223856:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.223858:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.223858:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.223860:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677475.223860:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.223861:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.223862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677475.223918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.223925:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.223928:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.223929:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.223931:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.223933:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.223935:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.223936:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.223937:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.223938:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.223940:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.223943:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.223945:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.223948:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.223948:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.223949:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.223951:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.223951:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.223952:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.223953:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.223957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.223965:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.223967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.223968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.223968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.223971:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.223972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.224233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.224241:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.224244:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.224245:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.224247:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.224250:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.224251:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.224252:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.224253:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.224254:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.224256:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.224259:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.224261:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.224263:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.224264:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.224264:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.224266:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.224267:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.224267:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.224269:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.224269:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.224272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.224275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:13.0:1619677475.224280:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677475.224282:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:13.0:1619677475.224282:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.224283:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.224284:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.224285:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677475.224285:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677475.224286:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677475.224287:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.224288:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.224474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.224480:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.224483:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.224483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.224485:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.224488:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.224489:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.224490:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.224492:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.224493:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.224494:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.224497:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.224499:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.224502:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.224502:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.224503:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.224505:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.224505:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.224506:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.224507:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.224508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.224511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.224513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.224518:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.224519:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.224520:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.224521:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.224521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.224522:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.224522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.224524:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.224525:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.224526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.224531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.224538:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.224541:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.224542:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.224542:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.224543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.224544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.224578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.224586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677475.224589:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.224592:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.224594:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.224596:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.224596:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.224597:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677475.224598:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.224599:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.224600:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.224601:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.224602:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.224605:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.224606:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.224608:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.224608:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.224609:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677475.224610:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.224612:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.224612:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.224613:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.224616:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.224617:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.224617:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:30.0:1619677475.224617:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.224618:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.224619:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.224620:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.224621:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.224621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.224621:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.224622:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677475.224623:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.224623:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677475.224624:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677475.224624:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677475.224625:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:9.2:1619677475.224625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.224626:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677475.224626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677475.224630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677475.224632:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.224634:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.224635:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.224635:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.224637:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.224639:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:8.0:1619677475.224639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:18.0:1619677475.224641:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.224642:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.224643:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.224645:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.224649:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.224939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677475.224947:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.224949:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677475.224950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677475.224952:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.224955:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.224957:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677475.224958:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.224959:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.224960:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.224961:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677475.224964:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.224966:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677475.224969:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.224969:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677475.224970:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677475.224972:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677475.224972:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.224973:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677475.224973:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677475.224978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.224986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.224989:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.224990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.224990:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.224992:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.224995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.225224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.225232:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.225235:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.225237:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.225238:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.225240:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.225241:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.225242:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.225243:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.225244:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.225245:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.225248:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.225250:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.225252:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.225252:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.225253:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.225254:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.225255:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.225256:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.225256:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.225260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.225266:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.225269:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.225270:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.225270:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.225272:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.225274:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.225280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.225286:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.225289:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.225289:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.225290:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.225290:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.225291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.225357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.225363:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.225365:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.225366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.225368:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.225371:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.225372:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.225373:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.225375:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.225376:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.225377:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.225380:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.225382:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.225384:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.225385:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.225386:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.225387:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.225388:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.225388:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.225389:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.225394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.2:1619677475.225397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.225401:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.225402:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.225402:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.225403:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.225404:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.225405:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.225406:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.225407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.225408:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:21.0:1619677475.225409:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.225409:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.225411:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.225412:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.225413:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.225414:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.225415:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.225418:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.225420:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.225423:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.225423:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.225424:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.225426:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.225426:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.225427:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.225428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.225434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.225443:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.225445:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.225446:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.225446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.225448:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.225450:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677475.225671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.225677:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.225680:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.225681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.225682:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.225686:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.225687:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.225688:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.225689:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.225690:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.225691:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.225694:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.225695:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.225697:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.225698:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.225699:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.225701:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.225701:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.225702:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.225703:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.225704:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.225707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.225709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.225716:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.225717:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.225717:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:21.2:1619677475.225718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:18.0:1619677475.225718:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.225718:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.225719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.225720:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.225720:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.225721:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677475.225721:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677475.225726:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.225729:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.225731:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.225734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.225734:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.225735:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.225736:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.225738:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.225739:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.225740:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.225743:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.225744:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.225748:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.225748:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.225749:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.225751:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.225751:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.225752:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.225753:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.225757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.225764:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.225767:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.225768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.225768:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.225770:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.225773:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.226129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.226137:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.226140:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.226142:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.226144:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.226145:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.226146:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.226147:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.226149:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.226150:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.226151:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.226153:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.226155:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.226157:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.226157:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.226158:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.226159:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.226160:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.226161:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.226161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.226165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.226171:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.226174:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.226175:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.226176:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.226178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.226179:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.226181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.226188:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.226191:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.226192:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.226192:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.226193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.226193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.226203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.226208:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.226211:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.226211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.226213:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.226216:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.226217:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.226218:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.226219:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.226221:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.226222:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.226225:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.226227:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.226229:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.226230:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.226230:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.226232:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.226233:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.226233:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.226234:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.226238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.226246:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.226248:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.226249:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.226251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.226251:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.226253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.226491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677475.226497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.226499:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677475.226500:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677475.226502:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.226504:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.226506:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677475.226507:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.226508:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.226510:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.226510:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677475.226514:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.226515:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677475.226518:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.226519:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677475.226520:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677475.226521:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:6.0:1619677475.226522:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.226523:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.226523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677475.226528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.226535:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.226538:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.226539:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.226539:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.226542:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.226545:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677475.226670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.226678:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.226680:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.226681:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.226683:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.226685:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.226687:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.226688:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.226689:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.226690:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.226691:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.226694:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.226696:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.226699:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.226700:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.226700:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.226702:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.226703:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.226703:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.226704:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.226705:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.226708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.226710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.226716:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.226718:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.226719:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.226719:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.226720:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.226720:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.226721:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677475.226722:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.226722:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677475.226725:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.226934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.226939:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.226942:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.226944:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.226945:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.226946:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.226947:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.226949:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.226950:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.226951:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.226952:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.226955:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.226957:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.226960:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.226960:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.226961:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.226963:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.226963:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.226964:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.226964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.226970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.226978:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.226980:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.226980:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.226982:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.226983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.226984:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.226991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.226999:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.227001:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.227002:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.227002:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.227003:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.227004:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.227068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.227075:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.227077:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.227078:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.227080:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.227082:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.227083:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.227085:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.227086:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.227087:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.227088:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.227092:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.227093:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.227096:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.227096:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.227097:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.227099:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.227099:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.227100:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.227100:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.227104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.227112:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.227115:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.227115:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.227117:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.227118:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.227119:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.227191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.227199:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.227201:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.227202:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.227204:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.227207:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.227208:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.227210:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.227211:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.227212:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.227213:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.227217:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.227218:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677475.227220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677475.227221:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.227222:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.227223:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.227224:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.227225:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.227225:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.227226:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677475.227227:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677475.227228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.0:1619677475.227230:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:25.2:1619677475.227231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.227231:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.227232:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.227235:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.227236:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.227237:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.227237:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.227239:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:8.0:1619677475.227240:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.227241:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:8.0:1619677475.227241:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.227242:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:8.0:1619677475.227242:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677475.227243:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.227244:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.227246:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.227247:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.227249:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.227251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.227251:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.227252:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677475.227252:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.227253:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677475.227253:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677475.227254:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677475.227254:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000400:00000200:6.0:1619677475.227254:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677475.227256:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.227256:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.227257:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677475.227257:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.227258:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677475.227259:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.227260:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.227261:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.227262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677475.227262:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.227264:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:4.2:1619677475.227266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.227267:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.227268:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.227268:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.227270:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.227271:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.227271:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.227272:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.227272:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677475.227273:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:22.0:1619677475.227273:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.227273:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677475.227274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.227275:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677475.227275:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.227276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677475.227276:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.227277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.227277:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.227281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.227283:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.227286:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.227286:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.227288:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.227288:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.227289:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.227290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677475.227291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.227291:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.227292:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.227292:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.227298:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.227439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.227447:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.227450:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.227452:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.227454:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.227455:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.227456:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.227458:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.227459:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.227460:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.227461:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.227464:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.227466:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.227469:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.227470:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.227471:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.227473:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.227473:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.227474:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.227474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.227479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.227487:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.227489:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.227490:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.227490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.227492:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.227495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.227971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.227978:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.227981:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.227982:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.227984:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.227987:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.227988:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677475.227989:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.227991:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.227992:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.227993:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677475.227996:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.227997:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677475.228000:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.228001:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677475.228002:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677475.228003:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677475.228004:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.228005:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.228005:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.228010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.228018:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.228021:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.228021:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.228022:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.228024:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.228027:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.228102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.228108:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.228111:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.228111:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.228113:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.228116:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.228117:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.228118:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.228119:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.228121:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.228122:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.228125:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.228127:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.228129:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.228130:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.228131:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.228132:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.228133:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.228134:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.228135:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.228136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.228138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.228141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.228146:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.228146:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.228148:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.228149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.228149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.228149:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.228149:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.228151:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.228152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.228153:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.228160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.228167:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.228169:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.228170:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.228171:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.228172:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.228172:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.228431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.228439:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.228442:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.228443:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.228445:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.228448:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.228450:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.228451:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.228452:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.228453:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.228454:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.228458:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.228459:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.228462:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.228463:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.228463:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.228465:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.228465:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.228466:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.228467:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.228471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.228480:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.228483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.228483:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677475.228484:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.228486:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.228488:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677475.228648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.228653:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.228656:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.228657:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.228659:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.228661:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.228662:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.228663:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.228665:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.228666:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.228667:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.228670:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.228672:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.228675:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.228675:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.228676:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.228678:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.228678:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.228678:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.228679:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.228684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.228692:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.228695:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.228696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.228696:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.228698:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.228700:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.228792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.228800:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.228803:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.228805:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.228808:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.228808:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.228810:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.228811:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.228812:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.228813:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.228815:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.228818:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.228820:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.228822:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.228823:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.228824:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.228825:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.228826:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.228827:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.228829:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.228830:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.228831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.228835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.228837:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.228840:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.228841:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.228841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.228843:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.228843:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.228845:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.228846:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.228846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.228847:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.228969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.228974:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.228977:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.228978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.228979:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.228982:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.228983:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.228984:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.228986:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.228987:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.228988:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.228991:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.228992:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.228994:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.228995:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.228996:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.228997:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.228998:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.228999:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.228999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.229003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.229011:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.229013:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.229014:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.229015:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.229017:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.229017:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.2:1619677475.229213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.229222:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.229226:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.229228:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.229231:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.229234:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.229236:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.229238:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.229241:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.229242:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.229244:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.229250:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.229253:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.229257:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.229258:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.229259:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.229262:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.229263:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.229264:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.229265:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.229268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.229275:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.229278:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.229279:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.229281:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.229284:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.229285:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.229562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.229570:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.229573:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.229576:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.229577:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.229578:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.229579:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.229581:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.229582:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.229583:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.229584:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.229588:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.229589:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.229593:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.229594:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.229594:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.229596:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.229597:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.229598:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.229598:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.229603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.229610:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.229613:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.229614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.229615:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.229617:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.229620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.229652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.229660:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.229663:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.229665:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.229667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.229668:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.229669:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.229670:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.229672:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.229673:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.229674:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.229677:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.229678:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.229681:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.229681:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.229682:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.229684:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.229684:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.229685:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.229686:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.229689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.229696:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.2:1619677475.229698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:8.0:1619677475.229698:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.229699:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.229700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.229701:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.229703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677475.229704:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.229707:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.229708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.229710:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.229712:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.229714:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677475.229715:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.229717:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.229718:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.229719:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677475.229722:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.229723:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677475.229726:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.229727:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677475.229727:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.229729:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677475.229730:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.229731:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.229731:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677475.229735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.229742:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.229745:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.229746:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.229746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.229748:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.229750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.229973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.229978:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.229981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.229981:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.229983:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.229985:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.229986:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.229987:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.229989:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.229990:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.229991:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.229994:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.229996:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.229998:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.229999:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.230000:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.230001:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.230002:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.230002:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.230003:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.230007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.230015:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.230017:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.230018:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.230019:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.230021:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.230022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.230052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677475.230058:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.230060:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677475.230061:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677475.230063:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.230065:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.230067:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677475.230068:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.230070:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.230070:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.230071:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:22.0:1619677475.230075:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.230076:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677475.230078:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.230079:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:22.0:1619677475.230079:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677475.230081:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677475.230081:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.230082:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677475.230082:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677475.230087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.230094:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.230097:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.230098:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.230098:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.230100:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.230103:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.230255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.230263:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.230266:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.230268:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.230270:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.230270:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.230272:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.230273:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.230274:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.230275:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.230276:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.230279:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.230280:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.230282:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.230283:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.230284:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.230285:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.230286:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.230287:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.230288:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.230291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.230298:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.230301:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.230302:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.230303:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.230305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.230306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.230640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.230648:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.230651:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.230652:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.230654:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.230655:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.230655:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.230656:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.230657:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.230658:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.230659:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.230662:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.230663:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.230665:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.230665:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.230666:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.230667:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.230668:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.230669:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.230669:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.230674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.230681:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.230683:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.230684:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.230685:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.230687:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.230688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.230851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.230857:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.230860:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.230861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.230862:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.230865:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.230866:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.230868:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.230869:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.230870:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.230871:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.230874:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.230875:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.230878:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.230878:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.230879:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.230881:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.230881:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.230882:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.230882:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.230887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.230895:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.230897:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.230898:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.230899:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.230901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.230901:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.2:1619677475.231037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.231045:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.231048:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.231050:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.231051:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.231052:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.231053:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.231054:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.231055:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.231056:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.231057:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.231061:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.231062:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.231064:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.231065:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.231066:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.231067:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.231068:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.231069:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.231071:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.231071:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677475.231077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677475.231079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.231085:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.231087:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.231087:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.231088:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.231088:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.231088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.231089:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677475.231090:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.231090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.231091:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.231368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.231374:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.231377:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.231378:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.231380:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.231382:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.231384:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.231385:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.231386:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.231387:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.231388:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.231392:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.231393:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.231396:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.231397:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.231397:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.231399:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.231399:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.231400:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.231401:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.231406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.231414:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.231417:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.231418:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.231419:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.231420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.231421:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.231520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.231525:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.231527:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.231529:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.231530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.231532:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.231534:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.231535:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.231536:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.231537:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.231538:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.231541:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.231543:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.231545:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.231546:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.231547:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.231549:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.231549:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.231550:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.231551:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.231554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.231562:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.231564:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.231565:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.231566:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.231567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.231568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.231628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.231636:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.231639:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.231641:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.231644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.231644:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.231645:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.231647:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.231648:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.231649:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.231650:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.231653:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.231655:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.231658:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.231659:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.231660:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.231661:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.231662:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.231663:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.231665:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.231665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.231668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.231671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.231677:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677475.231679:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.231680:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.231681:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.231681:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.231682:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.231683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.231684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:22.0:1619677475.231684:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.231687:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.231689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.231696:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.231699:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.231700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.231701:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.231702:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.231703:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.231716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.231722:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.231724:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.231725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.231728:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.231730:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.231731:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.231733:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.231734:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.231735:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.231737:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.231740:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.231742:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.231744:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.231745:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.231746:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.231748:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.231748:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.231749:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.231750:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.231751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.231754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.231756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.231762:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.231764:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.231764:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.231765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.231766:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.231766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.231767:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.231769:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.231770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.231771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.231777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.231784:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.231787:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.231788:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.231789:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.231790:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.231791:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.232413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.232421:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.232424:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.232424:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.232427:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.232430:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.232431:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.232432:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.232434:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.232434:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.232436:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.232439:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.232440:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.232443:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.232443:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.232444:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.232446:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.232446:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.232447:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.232447:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.232452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.232459:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.232462:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.232462:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.232464:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.232466:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.232467:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.232468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.232475:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.232478:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.232479:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.232480:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.232483:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.232484:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.232486:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.232487:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.232488:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.232488:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.232491:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.232492:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.232495:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.232496:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.232497:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.232499:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.232499:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.232500:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.232500:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.232505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.232512:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.232515:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.232516:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.232516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.232519:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.232522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.232523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.232530:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.232533:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.232534:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.232534:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.232535:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.232536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.232610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.232617:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.232620:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.232621:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.232622:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.232625:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.232626:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.232627:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.232629:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.232630:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.232630:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.232633:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.232635:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.232638:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.232638:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.232639:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.232640:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.232641:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.232642:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.232642:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.232646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.232654:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.232657:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.232658:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.232658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.232661:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.232664:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.232712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.232716:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.232719:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.232721:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.232722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.232724:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.232725:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.232727:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.232728:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.232730:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.232731:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.232735:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.232737:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.232739:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.232740:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.2:1619677475.232741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.0:1619677475.232741:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.232743:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.232744:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.232745:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.232746:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.232747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677475.232747:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.2:1619677475.232749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677475.232750:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.232751:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.2:1619677475.232752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677475.232754:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.232756:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:6.0:1619677475.232756:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.232757:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:6.0:1619677475.232757:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.0:1619677475.232758:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677475.232758:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.232759:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:6.0:1619677475.232759:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.232760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.232760:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.232761:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.232762:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:6.0:1619677475.232762:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.232763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677475.232763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:6.0:1619677475.232764:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.232768:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.232770:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.2:1619677475.232771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:6.0:1619677475.232772:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.232773:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.232774:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.232776:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.232776:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.232777:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.232778:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.232779:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:25.2:1619677475.232782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.232782:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.232783:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.232784:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.232784:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.232786:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677475.232787:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.232789:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.232790:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.232792:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.232793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.232794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.232862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.232870:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.232873:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.232874:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.232875:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.232878:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.232879:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.232881:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.232882:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.232883:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.232884:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.232888:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.232889:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.232892:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.232893:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.232894:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.232895:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.232896:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.232897:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.232897:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.232901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.232908:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.232911:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.232912:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.232912:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.232915:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.232918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.233477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677475.233485:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.233488:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677475.233490:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.233493:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.233493:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677475.233494:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:21.2:1619677475.233495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:31.0:1619677475.233496:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.233497:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.233498:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.233499:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677475.233502:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.233503:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.233504:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.233504:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677475.233506:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.233507:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:30.0:1619677475.233507:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.233508:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.233509:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.233510:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000400:00000200:30.0:1619677475.233510:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.233512:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.233513:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.233514:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.233515:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.233516:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677475.233516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.233517:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677475.233519:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677475.233519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.233519:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.233521:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.233521:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.233522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.233523:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.233523:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.233524:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.233525:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.233525:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.233525:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.233525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.233527:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.233527:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.233528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677475.233528:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.233528:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.233529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677475.233530:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677475.233532:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677475.233533:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:25.2:1619677475.233533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.233534:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:6.0:1619677475.233541:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.233543:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.233544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.233545:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.233547:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.233549:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.233611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.233616:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.233618:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.233619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.233621:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.233623:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.233625:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.233626:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.233627:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.233628:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.233629:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.233632:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.233633:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.233636:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.233637:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.233637:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.233639:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.233639:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.233640:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.233640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.233645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.233652:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.233655:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.233656:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.233657:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.233659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.233659:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.233928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.233934:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.233937:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.233938:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.233940:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.233942:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.233943:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.233945:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.233946:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.233947:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.233948:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.233951:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.233953:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.233956:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.233957:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.233957:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.233959:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.233959:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.233960:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.233961:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.233966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.233973:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.233976:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.233977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.233977:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.233979:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.233981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.234481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.234487:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.234489:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.234490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.234491:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.234494:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.234495:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.234496:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.234497:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.234498:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.234499:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.234503:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.234504:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.234507:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.234508:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.234508:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.234510:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.234510:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.234511:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.234511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.234516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.234523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.0:1619677475.234529:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677475.234531:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.234531:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.234532:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.234534:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:21.0:1619677475.234534:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.234535:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.234536:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.234537:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.234539:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.234539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.234541:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.234542:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.234543:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.234544:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.234545:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.234549:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.234551:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.234553:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.234554:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.234555:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.234556:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.234557:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.234558:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.234559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.234562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.234570:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.234573:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.234574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.234574:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.234576:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.234578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.234820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.234826:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.234829:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.234831:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.234832:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.234835:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.234836:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.234837:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.234839:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.234840:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.234841:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.234844:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.234846:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.234848:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.234849:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.234850:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.234851:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.234852:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.234853:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.234853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.234857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.234865:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.234868:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.234869:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.234869:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.234871:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.234873:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.234878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.234885:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.234888:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.234889:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.234889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.234890:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.234891:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.235217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.235223:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.235226:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.235226:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.235228:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.235230:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.235232:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.235233:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.235234:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.235235:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.235236:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.2:1619677475.235237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677475.235239:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.235240:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.235241:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.235242:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.235242:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.235244:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.235244:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677475.235245:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677475.235245:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.235246:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677475.235248:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.235250:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677475.235250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677475.235252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.235252:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.235254:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.235255:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.235256:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.235257:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677475.235257:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:30.0:1619677475.235258:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677475.235259:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.235260:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.235261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:30.0:1619677475.235262:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677475.235263:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.235264:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677475.235264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677475.235267:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.235268:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.235268:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.235270:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.235270:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.235271:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.235271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677475.235272:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677475.235279:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.235282:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.235283:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.235283:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.235284:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.235285:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:25.2:1619677475.235286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.235293:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.235301:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.235301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.235302:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.235303:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.235306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.235307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.235314:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.235317:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.235318:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.235319:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.235319:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.235321:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.235511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.235517:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.235520:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.235521:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.235523:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.235525:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.235526:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.235528:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.235529:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.235530:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.235531:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.235534:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.235536:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.235538:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.235539:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.235539:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.235541:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.235541:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.235542:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.235543:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.235544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.235547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.235548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.235553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.235554:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.235555:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.235556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.235556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.235557:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.235557:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.235559:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.235560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.235561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.235570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.235577:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.235579:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.235580:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.235581:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.235582:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.235583:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.235613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.235618:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.235621:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.235623:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.235623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.235625:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.235627:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.235628:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.235629:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.235630:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.235631:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.235634:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.235636:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.235638:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.235639:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.235639:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.235641:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.235642:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.235642:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.235644:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.235644:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677475.235647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.235651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.235656:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.235657:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.235658:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.235658:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:20.0:1619677475.235658:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.235659:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.235660:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.235661:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.235661:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677475.235662:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.235769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.235777:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.235780:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.235782:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.235784:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.235784:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.235785:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.235786:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.235788:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.235789:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.235790:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.235793:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.235795:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.235797:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.235798:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.235798:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.235800:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.235801:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.235802:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.235802:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.235806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.235812:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.235815:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.235816:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.235816:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.235817:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.235819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.235827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.235833:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.235836:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.235836:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.235836:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.235837:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.235838:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.236123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.236129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.236132:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.236133:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.236135:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.236137:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.236138:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.236140:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.236141:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.236142:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.236143:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.236146:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.236147:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.236150:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.236151:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.236151:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.236153:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.236153:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.236155:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.236156:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.236156:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.236159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.236162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.236167:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677475.236169:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677475.236169:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.236170:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677475.236171:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677475.236171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:21.0:1619677475.236172:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.236174:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.236175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.236176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.236216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.236224:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.236227:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.236229:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.236231:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.236231:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.236233:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.236234:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.236235:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.236236:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.236237:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.236241:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.236243:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.236245:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.236246:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.236246:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.236248:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.236249:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.236250:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.236252:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.236252:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.236253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.236258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.236259:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.236262:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.236263:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.236263:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.236264:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.236266:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677475.236266:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.236269:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.236269:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.236270:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.236374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.236380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.236383:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.236383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.236386:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.236388:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.236389:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.236391:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.236392:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.236393:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.236394:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.236397:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.236399:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.236401:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.236402:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.236403:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.236405:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.236405:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.236406:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.236406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.236411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.236419:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.236421:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.236422:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.236424:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.236425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.236427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.236560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.236568:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.236571:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.236572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.236574:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.236577:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.236578:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.236579:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.236581:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.236582:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.236583:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.236586:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.236587:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.236590:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.236590:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.236591:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.236592:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.236593:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.236594:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.236594:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.236598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.236605:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.236607:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.236608:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.236608:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.236610:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.236612:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.236880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.236888:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.236891:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.236893:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.236895:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.236896:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677475.236896:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.236897:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.236898:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.236899:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.236900:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.236903:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.236905:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.236907:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.236907:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.236908:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.236910:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.236910:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.236911:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.236912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.236915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.236921:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.236923:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.236924:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.236925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.236926:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.236928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.237110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.237118:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.237121:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.237123:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.237125:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.237126:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.237127:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.237128:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.237129:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.237130:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.237131:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.237134:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.237136:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.237139:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.237140:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.237140:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.237142:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.237142:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.237143:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.237144:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.237148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.237155:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.237158:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.237159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.237159:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.237162:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.237164:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.237165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.237172:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.237174:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.237175:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.237175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.237177:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.237177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.237213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.237218:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.237221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.237221:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.237223:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.237225:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.237227:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.237227:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.237229:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.237230:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.237231:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.237233:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.237235:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.237237:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.237238:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.237239:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.237241:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.237241:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.237241:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.237242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.237247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.237255:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.237257:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.237258:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.237260:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.237261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.237262:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.237304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.237311:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.237314:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.237315:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.237317:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.237319:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.237321:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.237322:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.237324:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.237325:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.237326:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.237329:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.237331:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.237334:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.237334:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.237335:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.237337:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.237337:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.237338:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.237338:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.237343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.237351:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.237354:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.237355:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.237355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.237358:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.237360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.237490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.2:1619677475.237497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677475.237497:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677475.237500:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677475.237500:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.237501:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.237502:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.237503:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.237505:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.237506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.237506:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.237506:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.237507:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677475.237508:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677475.237508:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.237510:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.237510:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.237511:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.237512:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.237513:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.237513:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677475.237514:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.237519:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.237520:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.237522:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677475.237523:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677475.237524:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.237525:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.237525:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:20.0:1619677475.237526:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:10.0:1619677475.237526:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677475.237527:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677475.237527:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.237528:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.237528:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:20.0:1619677475.237529:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.237529:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677475.237529:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.0:1619677475.237530:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.237531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677475.237533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.2:1619677475.237535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677475.237539:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.237541:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.237542:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.237542:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677475.237543:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:26.0:1619677475.237544:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.237545:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:26.0:1619677475.237546:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:21.0:1619677475.237546:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.237548:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.237548:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.237550:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.237697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.237704:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.237707:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.237708:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.237709:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.237711:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.237713:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.237714:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.237716:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.237717:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.237718:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.237722:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.237723:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.237726:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.237727:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.237728:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.237729:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.237730:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.237731:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.237732:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.237733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.237735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.237738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.237743:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.237745:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.237745:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.237746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.237746:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.237746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677475.237747:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.237748:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.237749:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.237749:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.237751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677475.237753:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.237756:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.237757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.237758:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.237761:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.237763:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.237764:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.237765:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.237766:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.237767:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.237770:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.237772:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.237775:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.237775:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.237776:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.237778:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.237778:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.237779:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.237779:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.237784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.237791:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.237794:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.237795:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.237795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.237797:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.237800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.238256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.238261:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.238263:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.238264:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.238266:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.238268:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.238269:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.238270:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.238271:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.238272:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.238273:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.238276:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.238278:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.238280:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.238281:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.238281:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.238283:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.238283:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.238284:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.238284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.238289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.238299:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.238301:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.238302:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.238304:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.238305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.238306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.238577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677475.238583:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.238586:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677475.238588:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677475.238589:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.238592:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.238594:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677475.238595:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.238596:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.238597:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.238598:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677475.238602:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.238604:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677475.238607:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.238608:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677475.238608:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677475.238610:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677475.238611:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.238612:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677475.238612:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677475.238617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.238624:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.238627:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.238628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.238628:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.238630:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.238633:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677475.238636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677475.238642:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.238645:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.238646:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.238646:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.238647:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.238647:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677475.238673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.238678:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.238681:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.238683:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.238683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.238685:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.238686:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.238688:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.238689:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.238690:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.238691:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.238695:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.238696:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.238699:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.238700:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.238700:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.238702:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.238702:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.238703:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.238704:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.238708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.238716:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.238718:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.238719:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.238721:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.238722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.238723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.238727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.238735:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.238737:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.238738:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.238738:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.238739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.238740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.238769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.238777:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.238780:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.238782:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.238785:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.238785:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.238787:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.238788:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.238790:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.238791:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.238792:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.238795:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.238797:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.238800:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.238801:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.238801:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.238803:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.238804:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.238805:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.238806:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.238809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.238817:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.238820:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.238821:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.238821:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.238823:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.238826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.239138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.239143:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.239145:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.239146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.239148:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.239150:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.239151:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.239152:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.239153:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.239154:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.239156:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.239159:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.239161:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.239163:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.239164:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.239165:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.239166:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.239167:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.239168:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.239169:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.239169:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.239172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.239174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.239179:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.239180:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.239181:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.239182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.239182:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.239183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.239183:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.239184:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.239186:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.239187:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.239190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:5.2:1619677475.239191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.0:1619677475.239197:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677475.239199:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.0:1619677475.239200:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.239201:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.239201:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:1.0:1619677475.239201:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.0:1619677475.239202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.239203:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677475.239203:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.239204:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.239207:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.239208:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.239210:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.239211:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.239212:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.239213:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.239216:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.239218:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.239221:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.239222:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.239222:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.239224:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.239224:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.239225:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.239226:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.239226:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.239230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.239231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.239237:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677475.239237:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677475.239240:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.239240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677475.239240:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.239241:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.239241:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.239242:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677475.239243:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.239246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.239966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.239972:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.239975:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.239975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.239977:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.239979:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.239980:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.239981:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.239983:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.239984:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.239985:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.239988:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.239989:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.239991:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.239992:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.239993:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.239995:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.239995:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.239996:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.239996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.240001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.240010:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.240012:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.240013:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.240014:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.240015:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.240016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.240313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.240321:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.240324:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.240326:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.240329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.240329:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.240330:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.240331:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.240333:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.240334:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.240335:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.240338:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.240340:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.240342:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.240343:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.240343:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.240344:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.240345:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.240346:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.240347:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.240351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.240358:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.240360:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.240361:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.240363:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.240364:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.240365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.240526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.240534:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.240537:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.240537:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.240539:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.240541:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.240543:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.240544:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.240545:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.240546:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.240547:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.240550:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.240551:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.240553:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.240554:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.240555:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.240556:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.240557:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.240557:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.240558:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.240562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.240569:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.240572:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.240573:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.240574:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.240576:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.240577:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.240583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677475.240590:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.240592:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.240593:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.240594:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.240594:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.240595:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.240737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.240745:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.240748:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.240750:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.240752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.240752:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.240753:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.240754:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.240756:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.240757:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.240757:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.240761:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.240762:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.240765:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.240766:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.240767:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.240768:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.240769:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.240770:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.240772:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.240773:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.240774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.240778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.240782:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.240784:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.240785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.240786:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677475.240786:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.240788:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.240789:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.240790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.240791:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.240791:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.240792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.240799:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.240802:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.240803:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.240803:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.240804:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.240805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.240874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.240882:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.240885:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.240887:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.240889:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.240890:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.240891:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.240892:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.240894:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.240895:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.240895:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.240899:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.240900:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.240903:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.240904:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.240905:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.240906:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.240907:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.240908:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.240908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.240913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.240921:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.240923:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.240924:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.240926:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.240927:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.240929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.240931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.240938:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.240941:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.240942:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.240942:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.240943:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.240944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.240959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.240964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.240967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.240967:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.240969:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.240971:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.240973:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.240974:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.240975:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.240976:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.240977:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.240980:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.240982:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.240985:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.240985:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.240986:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.240987:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.240988:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.240989:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.240989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.240993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.241001:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.241003:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.241004:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.241005:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:4.2:1619677475.241005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:20.0:1619677475.241008:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.241009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.241013:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.241016:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677475.241018:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.241020:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677475.241021:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.241023:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677475.241024:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.241026:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.241027:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.241028:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.241032:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.241033:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677475.241037:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.241038:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:30.0:1619677475.241038:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.241040:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677475.241040:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.241041:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677475.241042:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677475.241046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.241054:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.241057:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.241058:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.241058:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.241060:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.241064:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.241068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.241076:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.241078:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.241079:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.241079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.241080:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.241081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.241310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.241315:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.241317:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.241319:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.241320:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.241322:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.241324:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.241325:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.241326:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.241327:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.241328:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.241330:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.241332:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.241335:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.241335:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.241336:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.241337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:1.0:1619677475.241338:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.241338:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.241338:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.241339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.241344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677475.241345:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.241348:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.241349:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.241351:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.241352:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:26.0:1619677475.241353:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.241354:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:26.0:1619677475.241355:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677475.241355:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.241356:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.241357:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:26.0:1619677475.241358:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.241358:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:26.0:1619677475.241359:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.241359:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:26.0:1619677475.241360:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677475.241363:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.241365:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677475.241368:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.241368:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677475.241369:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677475.241371:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677475.241371:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.241372:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.241372:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.241377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.241385:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.241387:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.241388:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.241390:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.241392:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.241393:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.241413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.241420:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.241423:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.241425:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.241427:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.241428:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.241429:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.241430:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.241431:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.241432:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.241433:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.241436:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.241437:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.241439:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.241440:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.241441:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.241442:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.241442:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.241444:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.241446:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.241446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.241447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.241452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.241457:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.241458:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.241458:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.241460:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.241460:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.241461:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.241462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.241463:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.241464:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.241465:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.241820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.241827:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.241830:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.241830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.241832:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.241835:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.241836:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.241838:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.241839:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.241840:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.241841:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.241844:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.241846:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.241849:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.241849:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.241850:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.241852:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.241852:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.241853:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.241853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.241858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.241865:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.241867:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.241868:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.241870:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.241871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.241872:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.241958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.241965:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.241968:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.241968:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.241971:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.241973:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.241975:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.241976:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.241978:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.241979:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.241980:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.241983:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.241984:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.241987:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.241988:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.241989:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.241990:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.241991:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.241992:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.241992:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.241996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.242003:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.242006:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.242007:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.242009:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.242010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.242012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.242023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.242030:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.242033:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.242034:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.242034:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.242035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.242036:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.242091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.242098:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.242101:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.242101:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677475.242103:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.242105:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.242107:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.242108:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.242109:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.242110:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.242111:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.242114:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.242116:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.242118:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.242119:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.242120:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.242122:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.242122:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.242123:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.242123:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.242128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.242135:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.242138:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.242139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.242139:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.242141:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.242144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.242145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.242152:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.242155:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.242156:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.242156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.242157:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.242158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.242458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.242466:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.242469:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.242471:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.242473:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.242473:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.242474:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.242476:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.242477:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.242478:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.242479:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.242482:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.242484:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.242487:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.242487:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.242488:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.242489:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.242490:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.242491:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.242492:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.242496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.242504:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.242507:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.242508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.242508:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.242511:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.242513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.242517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.242523:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.242525:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.242526:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.242528:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.242531:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.242533:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677475.242534:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.242535:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.242536:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.242537:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677475.242540:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.242542:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677475.242545:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.242545:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677475.242546:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.242548:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677475.242548:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.242549:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.242549:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677475.242553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.242561:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.242563:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.242564:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.242564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.242566:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.242569:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.242644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.242649:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.242651:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.242652:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.242654:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.242655:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.242657:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.242658:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.242659:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.242660:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.242661:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.242664:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.242666:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.242668:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.242669:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.242669:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.242671:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.242671:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.242672:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.242673:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.242674:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.242677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.242679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.242684:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.242684:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.242686:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.242686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.242686:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.242687:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.242687:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.242689:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.242691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.242692:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.242696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.242703:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.242705:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.242706:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.242707:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.242707:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.242708:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.242914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.242922:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.242924:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.242925:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.242927:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.242930:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.242932:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.242933:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.242934:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.242935:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.242936:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.242939:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.242941:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.242944:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.242945:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.242945:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.242947:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.242948:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.242948:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.242949:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.242953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.242960:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.242963:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.242964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.242964:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.242966:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.242969:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.243007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.243015:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.243018:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.243019:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.243020:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.243023:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.243025:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.243026:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.243027:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.243028:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.243029:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.243033:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.243034:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.243037:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.243038:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.243038:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.243040:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.243040:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.243042:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.243043:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.243043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.243045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.243049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.243053:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.243056:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677475.243056:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.243056:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.243057:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.243059:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.243059:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.243059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.243060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.243062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.243067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.243074:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.243077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.243077:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.243077:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.243079:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.243079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.243211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.243217:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.243219:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.243221:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.243222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.243224:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.243225:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.243227:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.243228:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.243229:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.243230:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.243234:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.243235:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.243238:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.243239:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.243239:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.243241:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.243242:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.243243:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.243243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.243248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.243256:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.243259:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.243260:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.243260:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.243262:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.243264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.243653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.243658:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.243661:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.243662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.243664:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.243666:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.243667:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.243669:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.243670:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.243671:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.243672:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.243675:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.243677:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.243679:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.243680:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.243681:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.243682:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.243683:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.243684:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.243684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.243689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.243697:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.243699:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.243700:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.243703:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.2:1619677475.243703:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:20.0:1619677475.243705:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.243708:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677475.243710:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.243713:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.243713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.243716:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.243718:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.243719:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.243721:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.243722:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.243723:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.243724:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.243727:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.243729:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.243731:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.243732:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.243733:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.243734:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.243735:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.243736:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.243736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.243740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.243747:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.243749:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.243750:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.243751:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.243752:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.243753:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:5.2:1619677475.243931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.2:1619677475.243937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (20)++
00000800:00000200:1.0:1619677475.243937:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.243940:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.243941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.243942:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.243943:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)++
00000800:00000200:21.0:1619677475.243945:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.2.63@o2ib3
00000400:00000200:1.0:1619677475.243945:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.243946:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677475.243948:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)++
00000400:00000200:1.0:1619677475.243948:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.243949:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.243950:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000400:00000200:1.0:1619677475.243950:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.243951:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.243951:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.243952:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:20.0:1619677475.243953:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.243954:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.243955:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.243956:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.243957:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.243958:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.243959:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677475.243959:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677475.243961:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.243962:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.243962:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000400:00000200:20.0:1619677475.243963:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:1.0:1619677475.243963:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:21.0:1619677475.243964:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677475.243964:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.243965:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.243965:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.0:1619677475.243966:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.243966:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.243966:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.243966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:20.0:1619677475.243967:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.243968:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:25.2:1619677475.243971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677475.243971:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.243972:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:20.0:1619677475.243972:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.243973:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.243974:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.243974:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:21.0:1619677475.243975:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (23)++
00000800:00000200:21.0:1619677475.243976:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (24)--
00000800:00000200:21.0:1619677475.243976:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)--
00000800:00000200:21.0:1619677475.243977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:20.0:1619677475.243977:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.243978:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:20.0:1619677475.243978:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677475.243978:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:9.2:1619677475.243980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677475.243980:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:20.2:1619677475.243981:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (21)++
00000800:00000200:22.0:1619677475.243982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.243982:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.243984:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:6.0:1619677475.243984:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.243985:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (22)++
00000800:00000200:20.0:1619677475.243986:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (23)--
00000800:00000200:9.2:1619677475.243986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677475.243986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677475.243987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:8.0:1619677475.243987:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677475.243988:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)++
00000800:00000200:21.0:1619677475.243991:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:8.0:1619677475.243991:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.243992:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.243992:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.243993:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.243993:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:18.0:1619677475.243994:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:8.0:1619677475.243994:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.243995:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.243996:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:21.0:1619677475.243997:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677475.243997:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.243997:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:21.0:1619677475.243998:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.244000:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.244000:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.244002:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.244003:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677475.244003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:8.0:1619677475.244005:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:21.0:1619677475.244006:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677475.244007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.244008:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677475.244008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.244009:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677475.244010:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.244011:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.2:1619677475.244011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (21)++
00000800:00000200:18.0:1619677475.244011:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677475.244011:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:21.0:1619677475.244012:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677475.244012:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (21)++
00000800:00000200:24.0:1619677475.244013:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.244013:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677475.244013:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.244013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.0:1619677475.244014:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (22)++
00000800:00000200:20.0:1619677475.244014:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)++
00000800:00000200:21.0:1619677475.244015:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (24)--
00000800:00000200:24.0:1619677475.244016:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.244016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)--
00000800:00000200:20.0:1619677475.244016:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:8.0:1619677475.244016:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.244017:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677475.244017:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:24.0:1619677475.244018:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677475.244018:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.244019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.244019:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:20.0:1619677475.244020:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.244021:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.244023:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.244024:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.244025:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.244026:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677475.244026:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677475.244029:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:20.0:1619677475.244029:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.244030:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.244030:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.244032:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.244032:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.244034:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.244035:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.244035:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.244036:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.244037:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.2:1619677475.244039:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (21)++
00000800:00000200:20.0:1619677475.244041:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (22)++
00000800:00000200:20.0:1619677475.244043:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e7f44800] (23)++
00000800:00000200:8.2:1619677475.244043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.244044:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (24)--
00000800:00000200:21.0:1619677475.244045:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (24)++
00000800:00000200:20.0:1619677475.244046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (24)--
00000800:00000200:21.0:1619677475.244047:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:20.0:1619677475.244047:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)--
00000400:00000200:21.0:1619677475.244049:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677475.244050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (22)++
00000400:00000200:21.0:1619677475.244051:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.244052:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)++
00000800:00000200:17.0:1619677475.244052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:21.0:1619677475.244053:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677475.244053:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e7f44800] (24)--
00000800:00000200:20.0:1619677475.244054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)--
00000800:00000200:20.0:1619677475.244054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:17.0:1619677475.244054:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:21.0:1619677475.244055:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.244055:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.244055:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:21.0:1619677475.244056:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.244057:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.244058:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:17.0:1619677475.244058:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.2:1619677475.244059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (21)++
00000800:00000200:17.0:1619677475.244060:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677475.244061:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)++
00000400:00000200:21.0:1619677475.244062:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.244063:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:21.0:1619677475.244065:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677475.244065:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.244068:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.244068:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:20.0:1619677475.244068:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.244069:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:20.0:1619677475.244069:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.244070:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.244071:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:20.0:1619677475.244072:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.244073:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.244074:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:25.2:1619677475.244077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:20.0:1619677475.244077:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.244078:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (23)++
00000800:00000200:20.0:1619677475.244078:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.244079:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (24)--
00000800:00000200:21.0:1619677475.244080:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)--
00000800:00000200:21.0:1619677475.244080:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)++
00000800:00000200:20.0:1619677475.244080:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.244081:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.244082:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:20.0:1619677475.244082:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.244083:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677475.244084:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.244084:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (23)++
00000800:00000200:31.0:1619677475.244085:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:20.0:1619677475.244085:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (24)--
00000800:00000200:20.0:1619677475.244085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)--
00000400:00000200:21.0:1619677475.244086:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.244086:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000400:00000200:21.0:1619677475.244087:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:31.0:1619677475.244088:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:21.0:1619677475.244088:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.244089:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.244089:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677475.244089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:21.0:1619677475.244091:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.244092:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.244092:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677475.244092:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:21.0:1619677475.244096:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.244097:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.244098:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677475.244098:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.244098:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.244100:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:20.0:1619677475.244100:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.244101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.0:1619677475.244101:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.244102:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677475.244102:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (22)++
00000800:00000200:20.2:1619677475.244103:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (21)++
00000800:00000200:21.0:1619677475.244104:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (23)--
00000800:00000200:21.0:1619677475.244105:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (22)++
00000800:00000200:20.0:1619677475.244105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.244106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677475.244107:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (23)--
00000800:00000200:20.0:1619677475.244107:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)++
00000800:00000200:30.0:1619677475.244108:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.0:1619677475.244108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)--
00000800:00000200:21.0:1619677475.244109:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)++
00000800:00000200:20.0:1619677475.244109:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:21.0:1619677475.244111:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:30.0:1619677475.244112:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.244113:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (23)++
00000400:00000200:20.0:1619677475.244113:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.244114:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (24)--
00000400:00000200:21.0:1619677475.244114:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.244114:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.244115:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.244116:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:20.0:1619677475.244116:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:20.0:1619677475.244117:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.244118:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:20.0:1619677475.244118:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.244119:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.244119:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.244121:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.244122:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.244122:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677475.244124:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.244125:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.244126:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:21.0:1619677475.244127:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677475.244127:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:21.0:1619677475.244129:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.244130:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.244131:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.244132:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:20.0:1619677475.244135:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.244136:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.244136:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.0:1619677475.244136:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677475.244138:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677475.244138:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677475.244139:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (23)++
00000800:00000200:20.0:1619677475.244140:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e7f44800] (24)++
00000800:00000200:21.0:1619677475.244141:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.244141:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (25)--
00000800:00000200:21.0:1619677475.244142:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:21.0:1619677475.244142:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.0:1619677475.244142:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (24)--
00000800:00000200:20.0:1619677475.244142:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)++
00000800:00000200:21.0:1619677475.244144:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:20.0:1619677475.244144:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:9.2:1619677475.244144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677475.244145:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (24)++
00000800:00000200:21.0:1619677475.244146:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (25)--
00000800:00000200:21.0:1619677475.244146:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (24)--
00000400:00000200:20.0:1619677475.244146:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.244147:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)++
00000800:00000200:21.0:1619677475.244148:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000400:00000200:20.0:1619677475.244149:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.244150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677475.244150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:21.0:1619677475.244151:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.244151:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:20.0:1619677475.244152:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.244153:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.244153:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.244154:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:20.0:1619677475.244154:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.244154:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.244156:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:13.0:1619677475.244157:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:8.0:1619677475.244157:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.244158:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.244160:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677475.244160:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.244160:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.244161:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.244162:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.244163:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:9.2:1619677475.244163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:21.0:1619677475.244164:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.244164:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.244165:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.244166:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.244167:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:21.0:1619677475.244169:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.244170:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.244170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677475.244171:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:20.0:1619677475.244171:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.244171:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.0:1619677475.244172:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:24.0:1619677475.244173:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677475.244173:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677475.244173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.244174:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677475.244174:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.244174:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:24.0:1619677475.244175:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677475.244175:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:21.0:1619677475.244175:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.0:1619677475.244175:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (24)++
00000800:00000200:24.0:1619677475.244176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677475.244176:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (25)--
00000800:00000200:20.0:1619677475.244176:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (24)--
00000800:00000200:20.0:1619677475.244177:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)++
00000800:00000200:21.0:1619677475.244178:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677475.244178:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (24)++
00000800:00000200:20.0:1619677475.244178:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e7f44800] (24)--
00000800:00000200:21.0:1619677475.244179:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (24)--
00000800:00000200:20.0:1619677475.244179:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (24)--
00000800:00000200:25.2:1619677475.244180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677475.244180:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:20.0:1619677475.244181:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)++
00000800:00000200:25.2:1619677475.244184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.0:1619677475.244184:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:21.0:1619677475.244187:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)++
00000400:00000200:20.0:1619677475.244187:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.244188:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:21.0:1619677475.244189:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000400:00000200:20.0:1619677475.244189:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.244190:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:20.0:1619677475.244191:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677475.244191:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.244192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:20.0:1619677475.244192:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677475.244192:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:20.0:1619677475.244193:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.244194:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.244194:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:20.0:1619677475.244195:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:12.0:1619677475.244197:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.244198:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.244198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:21.0:1619677475.244199:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.244200:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.244200:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:21.0:1619677475.244201:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.244202:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.244203:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677475.244203:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:21.0:1619677475.244204:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.244206:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.244207:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.244208:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677475.244211:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.244211:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:21.0:1619677475.244212:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.244212:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:20.0:1619677475.244212:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677475.244214:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:20.0:1619677475.244214:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677475.244215:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (23)++
00000800:00000200:21.0:1619677475.244216:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.244216:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e7f44800] (24)++
00000800:00000200:21.0:1619677475.244217:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:20.0:1619677475.244217:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (25)--
00000800:00000200:21.0:1619677475.244218:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677475.244218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (24)--
00000800:00000200:21.0:1619677475.244219:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.0:1619677475.244219:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)++
00000800:00000200:21.0:1619677475.244220:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (24)++
00000800:00000200:20.0:1619677475.244220:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:9.2:1619677475.244220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677475.244221:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (25)--
00000800:00000200:21.0:1619677475.244222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (24)--
00000800:00000200:21.0:1619677475.244222:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (23)++
00000400:00000200:20.0:1619677475.244222:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.244224:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000400:00000200:20.0:1619677475.244225:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.244226:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677475.244226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:20.0:1619677475.244227:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677475.244228:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.244228:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:20.0:1619677475.244229:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.244230:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677475.244230:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.244231:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:20.0:1619677475.244231:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677475.244232:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.244234:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.244234:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:10.0:1619677475.244235:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.244236:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677475.244237:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96500844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.244237:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.244237:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.244238:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.244238:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.244239:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.244240:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.244240:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.244241:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.244242:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677475.244244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:21.0:1619677475.244246:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.244247:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.244249:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677475.244251:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.244251:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677475.244251:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677475.244252:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677475.244253:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677475.244253:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.244253:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.244254:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.244254:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:21.0:1619677475.244255:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.244255:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677475.244255:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677475.244256:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (24)++
00000800:00000200:21.0:1619677475.244257:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.244257:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (25)++
00000800:00000200:20.0:1619677475.244257:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (25)--
00000800:00000200:10.0:1619677475.244257:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.244257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.0:1619677475.244258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (24)--
00000800:00000200:21.0:1619677475.244259:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (24)--
00000800:00000200:21.0:1619677475.244259:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (24)--
00000800:00000200:20.0:1619677475.244259:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (24)++
00000800:00000200:20.0:1619677475.244260:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e7f44800] (24)--
00000800:00000200:20.0:1619677475.244261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:21.0:1619677475.244262:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)--
00000800:00000200:8.2:1619677475.244263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:25.2:1619677475.244271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677475.244271:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.244272:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.244273:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.244275:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.244276:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.244277:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677475.244277:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:30.0:1619677475.244280:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.244282:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.244283:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.244284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.244286:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677475.244337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (20)++
00000800:00000200:21.0:1619677475.244343:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)++
00000800:00000200:21.0:1619677475.244346:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000400:00000200:21.0:1619677475.244348:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.244349:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000400:00000200:21.0:1619677475.244351:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.244352:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677475.244353:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.244355:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.244356:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.244356:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.244359:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.244360:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.244363:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.244364:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.244364:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.244366:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.244367:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (21)++
00000800:00000200:21.0:1619677475.244368:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e7f44800] (22)++
00000800:00000200:21.0:1619677475.244369:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (23)--
00000800:00000200:21.0:1619677475.244370:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:20.2:1619677475.244375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (21)++
00000800:00000200:20.0:1619677475.244378:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)++
00000800:00000200:20.0:1619677475.244380:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e7f44800] (23)--
00000800:00000200:20.0:1619677475.244380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000800:00000200:20.0:1619677475.244381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)--
00000800:00000200:25.2:1619677475.244383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.244391:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.244393:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.2:1619677475.244393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (20)++
00000400:00000200:31.0:1619677475.244394:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.244395:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)++
00000400:00000200:31.0:1619677475.244396:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.244397:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677475.244398:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:31.0:1619677475.244399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:20.0:1619677475.244400:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.244401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000400:00000200:20.0:1619677475.244403:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.244404:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.244406:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.244407:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.244408:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.244409:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.244411:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.244413:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.244416:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.244417:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.244417:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.244419:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.244420:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (21)++
00000800:00000200:20.0:1619677475.244421:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (22)--
00000800:00000200:20.0:1619677475.244421:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)--
00000800:00000200:20.2:1619677475.244424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (20)++
00000800:00000200:9.2:1619677475.244425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.244429:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)++
00000800:00000200:21.0:1619677475.244432:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:8.0:1619677475.244432:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:21.0:1619677475.244434:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.244434:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.244435:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.244435:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000400:00000200:21.0:1619677475.244436:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.244436:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.244438:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:8.0:1619677475.244438:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.244439:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.244441:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.244442:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.244443:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.244446:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.244448:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.0:1619677475.244448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677475.244450:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.244451:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.244451:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.244453:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.244453:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (21)++
00000800:00000200:21.0:1619677475.244454:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (22)--
00000800:00000200:21.0:1619677475.244455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)--
00000800:00000200:8.2:1619677475.244459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.244466:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.244469:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.2:1619677475.244470:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e7f44800] (20)++
00000800:00000200:21.0:1619677475.244472:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.244472:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.244473:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)++
00000400:00000200:20.0:1619677475.244474:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.244475:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.63@o2ib3
00000800:00000200:20.0:1619677475.244477:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:21.0:1619677475.244478:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.63@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.244478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (22)--
00000400:00000200:21.0:1619677475.244480:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.244482:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:21.0:1619677475.244483:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.244484:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.244485:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.244486:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:21.0:1619677475.244489:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.63@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.244490:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.0:1619677475.244492:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.244493:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:21.2:1619677475.244495:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:21.0:1619677475.244497:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677475.244499:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.0:1619677475.244499:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e7f44800] (21)++
00000800:00000200:21.0:1619677475.244500:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e7f44800] (22)--
00000800:00000200:22.0:1619677475.244501:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.244501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e7f44800] (21)--
00000800:00000200:22.0:1619677475.244504:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:8.2:1619677475.244505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.244505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.244506:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.244509:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.244510:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.244511:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.244513:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.244513:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:22.0:1619677475.244514:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.244515:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677475.244516:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.244517:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.244517:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:22.0:1619677475.244519:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677475.244519:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.244521:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677475.244522:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677475.244523:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.244524:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.244525:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.244526:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.244527:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.244527:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.244528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.244532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.244533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677475.244540:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677475.244541:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677475.244542:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.244543:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.244544:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.0:1619677475.244545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.244545:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.244546:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.244547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:20.0:1619677475.244547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677475.244550:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.244551:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.244553:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.244554:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.244555:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.244556:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.244560:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.244562:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.244565:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.244566:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.244566:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.244568:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.244568:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.244569:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.244570:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.244574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.244581:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.244584:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.244585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.244585:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.244587:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.244590:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.244632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.244640:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.244643:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.244645:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.244647:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.244648:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.244650:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.244651:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.244653:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.244653:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.244654:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.244657:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.244659:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.244661:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677475.244661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677475.244662:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.244662:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.244664:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.244664:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.244665:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.244666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677475.244666:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.244669:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:25.2:1619677475.244670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677475.244671:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.244671:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.244673:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.244675:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677475.244676:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:21.0:1619677475.244676:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.244678:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.244678:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.244679:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.244679:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.244680:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.244681:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.244683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677475.244683:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.244684:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.0:1619677475.244686:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.244688:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.244689:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.244689:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.244691:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.244692:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.244693:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.244695:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.244695:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.244697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.244700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.244703:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.244705:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677475.244705:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677475.244706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.244707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677475.244708:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.244709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.244709:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.244711:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.244713:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.244773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677475.244778:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.244781:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677475.244782:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.244784:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.244786:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.244787:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677475.244788:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.244790:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.244791:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.244792:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677475.244795:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.244797:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677475.244800:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.244800:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:17.0:1619677475.244801:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677475.244803:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677475.244803:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.244805:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677475.244806:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677475.244806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:21.2:1619677475.244811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.244816:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:25.2:1619677475.244818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:1.0:1619677475.244818:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677475.244819:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.244820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:31.0:1619677475.244825:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.244828:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.244829:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.244830:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.244832:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.244833:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.245352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.245358:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.245361:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.245362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.245363:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.245365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:22.0:1619677475.245366:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.245367:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.245368:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.245370:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.245371:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.245372:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.245373:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677475.245375:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.245376:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677475.245377:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:30.0:1619677475.245378:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.245379:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.245380:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.245380:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:31.0:1619677475.245381:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.245381:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677475.245381:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.245382:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.245382:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.245383:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.245383:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.245384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:30.0:1619677475.245385:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.245385:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.245387:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:25.2:1619677475.245389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.245390:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.245392:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.245394:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.245395:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.245396:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.245397:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.245397:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.245397:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677475.245398:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.245399:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.245399:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.245400:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.245402:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.245403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677475.245403:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.245404:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.245409:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.245412:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.245413:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.245414:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.245416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.245417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.245691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.245698:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.245701:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.245702:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.245703:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.245707:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.245708:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677475.245709:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.245711:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.245712:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.245713:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677475.245716:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.245718:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677475.245721:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.245722:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677475.245722:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677475.245724:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677475.245725:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.245725:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.245726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677475.245730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.245738:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.245741:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.245742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.245742:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.245744:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.245748:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.245896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.245903:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.245906:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.245908:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.245910:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.245911:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.245912:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.245913:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.245915:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.245916:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.245917:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.245919:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.245921:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.245923:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.245923:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.245924:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.245925:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.245926:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.245926:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.245927:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.245931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.245938:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.245940:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.245941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.245942:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.245944:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.245945:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.245954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.245960:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.245963:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.245964:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.245964:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.245965:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.245966:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.246303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.246311:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.246314:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.246316:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.246317:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.246319:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677475.246319:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.246320:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.246321:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.246322:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.246323:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.246326:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.246327:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.246331:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.246331:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.246332:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.246334:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.246334:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.246335:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.246335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.246341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.246349:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.246352:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.246353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.246353:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.246356:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.246359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.246364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.246369:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.246372:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.246372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.246375:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.246377:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.246378:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.246379:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.246380:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.246382:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.246382:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.246386:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.246388:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.246390:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.246391:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.246391:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.246393:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.246393:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.246394:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.246395:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.246396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.246399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.246401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.246406:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.246408:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.246408:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.246409:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.246410:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.246411:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.246412:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.246413:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.246414:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.246416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.246418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.246425:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.246428:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.246429:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.246430:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.246430:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.246431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.246878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.246885:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.246887:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.246888:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.246891:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.246893:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.246895:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.246896:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.246898:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.246899:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.246900:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.246903:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.246905:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.246907:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677475.246907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677475.246908:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.246909:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.246911:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.246911:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.246912:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.246912:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:26.0:1619677475.246914:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.246917:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.2:1619677475.246917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677475.246918:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.246919:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.246922:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.246923:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.246924:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.246925:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:26.0:1619677475.246926:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.246927:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.246928:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677475.246928:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.246928:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.246930:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.246931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:26.0:1619677475.246932:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.246933:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:26.0:1619677475.246935:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.2:1619677475.246936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:26.0:1619677475.246937:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.246938:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.246939:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.246941:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.246941:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.246942:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.246943:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.246944:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.0:1619677475.246944:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:25.2:1619677475.246947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677475.246947:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.246948:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.246948:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.246949:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:4.2:1619677475.246949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677475.246950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677475.246954:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.246957:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677475.246957:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677475.246958:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.246958:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.246959:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.246959:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677475.246961:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.246961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677475.246964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.247211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.247217:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.247219:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.247220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.247222:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.247224:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.247225:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.247226:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.247227:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.247229:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.247229:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.247233:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.247235:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.247237:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.247238:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.247239:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.247240:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.247241:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.247242:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.247242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.247247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.247254:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.247256:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.247257:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.247259:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.247259:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.247261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.247436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.247444:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.247447:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.247449:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.247452:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.247452:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.247453:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.247455:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.247456:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.247457:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.247458:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.247461:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.247463:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.247466:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.247467:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.247467:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.247469:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.247469:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.247471:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.247472:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.247473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.247475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.247479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.247482:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.247485:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.247486:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.247486:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.247487:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.247488:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.247489:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.247490:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677475.247490:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.247491:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.247751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.247759:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.247761:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.247762:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.247764:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.247766:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.247768:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.247769:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.247770:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.247771:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.247772:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.247775:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.247777:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.247780:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.247781:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.247781:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.247783:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.247783:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.247784:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.247784:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.247789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.247797:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.247800:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.247801:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.247801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.247804:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.247806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.247809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.247816:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.247818:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.247819:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.247820:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.247821:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.247821:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.247873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.247880:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.247883:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.247883:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.247885:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.247888:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.247889:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.247891:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.247892:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.247893:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.247894:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.247897:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.247898:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.247901:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.247902:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.247903:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.247905:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.247905:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.247907:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.247908:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.247908:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.247911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677475.247914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677475.247918:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677475.247919:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677475.247921:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:2.0:1619677475.247921:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:13.0:1619677475.247922:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.247922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.247923:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677475.247924:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.247925:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.247926:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.248098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.248104:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.248106:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.248107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.248109:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.248111:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.248112:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.248114:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.248115:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.248116:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.248117:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.248121:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.248122:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.248125:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.248125:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.248126:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.248128:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.248128:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.248129:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.248129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.248134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.248142:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.248144:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.248145:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.248146:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.248147:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.248149:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.248259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.248266:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.248269:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.248270:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.248271:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.248274:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.248276:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.248277:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.248279:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.248280:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.248281:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.248284:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.248286:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.248289:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.248289:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.248290:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.248292:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.248292:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.248298:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.248298:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.248299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.248306:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.248309:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.248310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.248310:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.248312:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.248315:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.248388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.248396:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.248399:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.248401:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.248403:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.248404:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.248405:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.248406:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.248408:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.248409:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.248409:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.248413:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.248414:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.248417:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.248418:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.248419:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.248420:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.248421:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.248422:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.248424:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.248425:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.248426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.248430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.248434:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.248436:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.248437:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.248437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.248438:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.248439:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.248441:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.248442:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.248443:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.248443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.248831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.248834:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.248837:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.248839:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.248840:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.248841:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.248842:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.248844:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.248845:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.248846:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.248847:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.248850:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.248852:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.248854:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.248855:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.248855:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.248857:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.248858:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.248858:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.248859:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.248863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.248871:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.248873:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.248874:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.248875:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.248876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.248877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.249113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.249119:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.249121:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.249122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.249124:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.249126:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.249127:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.249129:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.249130:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.249131:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.249132:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.249135:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.249137:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.249139:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.249140:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.249141:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.249142:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.249143:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.249144:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.249144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.249148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.249155:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.249157:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.249158:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.249160:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.249161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.249162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.249207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.249213:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.249216:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.249217:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.249219:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.249221:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.249222:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.249223:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.249225:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.249226:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.249227:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.249230:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.249232:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.249235:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.249235:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.249236:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.249238:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.249238:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.249239:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.249239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.249244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.249251:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.249254:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.249254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.249255:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.249257:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.249259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.249264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.249271:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.249274:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.249274:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.249275:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.249276:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.249276:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.249385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.249394:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.249397:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.249399:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.249401:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.249402:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.249403:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.249405:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.249406:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.249407:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.249408:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.249411:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.249413:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.249415:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.249416:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.249417:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.249418:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.249419:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.249420:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.249421:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.249425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.249432:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.249434:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.249435:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.249437:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.249438:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.249439:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.249954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.249960:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.249963:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.249963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.249965:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.249968:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.249969:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.249970:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.249972:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.249973:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.249974:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.249977:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.249979:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.249982:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.249982:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.249983:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.249985:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.249985:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.249986:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.249987:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.249988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.249991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.249992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.249998:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.250000:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.250000:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.250000:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.250001:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.250002:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.250003:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.250005:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.250005:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.250007:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.250010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.250017:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.250020:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.250021:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.250021:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.250022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.250023:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.250089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.250097:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.250100:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.250102:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.250104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.250104:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.250106:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.250107:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.250108:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.250109:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.250110:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.250114:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.250115:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.250118:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.250119:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.250120:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.250121:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.250122:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.2:1619677475.250122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.250123:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.250125:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677475.250126:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.250127:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677475.250128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.250129:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.250130:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.250130:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.250131:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.250132:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.250134:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.250134:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.250135:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.250135:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677475.250138:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.250138:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.250139:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.250139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.250140:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.250142:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.250142:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.250143:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.250143:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677475.250144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.250145:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.250145:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.250146:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.250147:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.250150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.250157:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.250159:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.250160:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.250162:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.250163:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.250164:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.250383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677475.250391:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.250394:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677475.250396:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.250398:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.250399:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677475.250400:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677475.250401:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.250403:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.250403:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.250404:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.250407:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.250409:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.250412:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.250413:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677475.250413:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.250415:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677475.250415:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.250416:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677475.250417:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677475.250422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.250431:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.250433:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.250434:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.250435:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.250437:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.250441:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.250581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.250587:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.250589:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.250591:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.250592:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.250593:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.250595:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.250596:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.250598:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.250598:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.250600:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.250603:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.250605:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.250607:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.250608:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.250609:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.250610:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.250611:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.250612:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.250612:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.250617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.250626:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.250629:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.250630:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.250630:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.250632:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.250634:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.250635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.250642:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.250645:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.250646:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.250646:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.250647:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.250648:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.250812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.250818:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.250820:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.250821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.250823:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.250825:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.250826:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.250828:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.250829:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.250830:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.250831:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.250834:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.250836:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.250838:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.250839:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.250840:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.250841:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.250842:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.250842:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.250843:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.250847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.250854:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.250856:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.250857:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.250859:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.250861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.250861:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.251230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.251237:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.251240:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.251241:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.251243:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.251246:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.251247:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:4.2:1619677475.251247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:10.0:1619677475.251249:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.251250:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.251251:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.251252:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677475.251254:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000400:00000200:10.0:1619677475.251255:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.251257:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677475.251257:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677475.251258:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:10.0:1619677475.251260:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677475.251260:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677475.251261:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.251262:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.251263:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:8.0:1619677475.251263:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.251264:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677475.251264:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677475.251265:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.251265:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:8.0:1619677475.251265:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.251266:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.251267:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.251268:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677475.251270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:8.0:1619677475.251272:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677475.251273:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.251276:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.251277:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:8.0:1619677475.251277:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.251278:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.251279:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.251280:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:8.0:1619677475.251280:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.251280:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:6.0:1619677475.251280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.251281:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.251281:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:22.0:1619677475.251283:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.251285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.251288:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677475.251290:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.251292:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.251293:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.251301:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.251303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677475.251303:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.251307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.251308:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.251310:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.251311:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.251312:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.251312:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.251313:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.251892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.251897:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.251900:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.251901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.251902:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.251904:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.251905:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.251906:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.251908:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.251909:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.251909:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.251912:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.251914:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.251916:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.251917:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.251918:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.251919:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.251920:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.251920:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.251921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.251926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.251934:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.251936:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.251937:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.251939:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.251940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.251941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.252034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.252040:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.252043:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.252044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.252045:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.252047:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.252049:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.252050:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.252052:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.252052:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.252054:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.252057:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.252059:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.252061:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.252062:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.252062:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.252064:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.252065:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.252065:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.252066:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.252071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.252078:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.252081:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.252081:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.252083:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.252085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.252085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.252210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.252219:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.252222:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.252224:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.252226:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.252227:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.252229:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.252230:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.252231:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.252232:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.252233:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.252237:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.252239:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.252242:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.252243:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.252244:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.252245:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.252246:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.252247:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.252247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.252251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.252259:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.252262:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.252263:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.252263:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.252265:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.252268:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.252340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.252348:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.252352:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.252354:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.252356:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.252357:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.252359:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.252360:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.252362:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.252363:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.252363:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.252367:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.252368:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.252371:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.252372:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.252373:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.252374:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.252375:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.252376:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.252376:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.252380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.252389:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:4.2:1619677475.252389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.0:1619677475.252391:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.252392:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.252394:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.252396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.252397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677475.252398:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.252402:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.252403:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.252405:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.252408:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.252410:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.252411:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.252413:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.252414:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.252415:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.252418:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.252420:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.252422:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.252423:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.252423:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.252425:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.252425:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.252426:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.252426:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.252431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.252440:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.252442:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.252443:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.252444:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.252446:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.252447:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.252452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677475.252458:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.252461:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.252461:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.252462:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.252462:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.252463:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.252565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.252572:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.252576:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:4.2:1619677475.252576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:24.0:1619677475.252578:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.252578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.252581:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.252583:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677475.252583:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677475.252584:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.252585:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.252585:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.252586:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677475.252586:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.252587:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.252588:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.252590:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.252590:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.252592:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.252592:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.252593:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.252594:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677475.252595:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.252595:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:10.0:1619677475.252595:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.252596:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677475.252596:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.252598:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.252598:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.252599:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.252599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:10.0:1619677475.252599:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.252601:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677475.252603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.252603:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.252604:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.252605:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:10.0:1619677475.252606:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:10.0:1619677475.252607:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.252608:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.252608:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677475.252611:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677475.252614:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:6.0:1619677475.252614:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677475.252615:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677475.252615:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.252616:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.252618:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.252618:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.252621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.252621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.252622:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.252628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.252634:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.252638:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.252639:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.252641:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.252644:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.252645:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.252646:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.252647:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.252649:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.252649:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.252653:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.252654:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.252656:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.252657:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.252658:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.252660:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.252660:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.252661:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.252662:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.252666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.252673:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.252675:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.252676:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.252676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.252678:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.252679:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.252686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.252694:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.252696:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.252697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.252697:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.252697:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.252698:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.252745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.252751:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.252753:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.252754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.252756:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.252758:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.252759:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.252761:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.252762:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.252763:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.252764:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.252767:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.252769:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.252771:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.252772:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.252773:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.252774:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.252775:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.252776:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.252776:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.252781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.252788:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.252790:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.252791:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.252793:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.252794:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.252795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.253114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.253120:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.253123:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.253123:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.253125:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.253128:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.253129:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.253130:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.253132:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.253133:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.253134:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.253137:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.253139:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.253141:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.253142:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.253143:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.253144:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.253145:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.253146:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.253146:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.253151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.253159:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.253161:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677475.253162:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.253163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.253164:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.253167:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.253560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.253568:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.253571:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.253572:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.253574:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.253577:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.253578:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.253580:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.253581:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.253582:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.253583:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.253587:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.253588:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.253591:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.253592:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.253593:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.253594:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.253595:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.253596:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.253598:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.253598:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.253601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.253604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.253609:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.253611:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.253612:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:21.2:1619677475.253612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.253612:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677475.253613:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677475.253613:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.253614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.253614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.253615:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:6.0:1619677475.253616:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.253617:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677475.253619:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.253619:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.253620:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.253621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000400:00000200:22.0:1619677475.253622:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.253624:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.253625:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.253627:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.253628:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.253629:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.253629:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.253631:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.253632:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.253633:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.253633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.253634:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.253634:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677475.253636:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.253639:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.253640:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.253640:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.253642:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.253642:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.253644:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.253644:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.253645:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.253648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.253650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.253655:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.253657:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.253657:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.253658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.253659:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.253659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.253660:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.253662:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.253663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.253664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.253667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.253674:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.253676:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.253677:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.253678:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.253679:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.253680:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.253842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.253851:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.253853:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.253855:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.253858:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.253858:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.253860:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.2:1619677475.253860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:31.0:1619677475.253861:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.253862:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.253863:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.253865:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.253868:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.253868:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.253869:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677475.253870:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.253871:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.253873:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.253873:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.253874:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.253875:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:30.0:1619677475.253875:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677475.253876:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:30.0:1619677475.253876:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.253877:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.253878:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.253879:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.253880:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.253882:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.253882:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677475.253882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.253883:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.253884:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.253884:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.253885:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.253887:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.253888:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.253888:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.253889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.253890:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.253890:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.253891:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.253892:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677475.253893:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.253895:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677475.253895:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677475.253896:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.253896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.253897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677475.253897:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.253898:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.253900:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.253900:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.253902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677475.253903:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.253906:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.253907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.253908:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.253910:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.253912:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.253916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677475.253922:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.253925:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677475.253926:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677475.253928:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.253930:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.253931:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.253933:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.253934:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.253935:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.253936:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.253939:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.253941:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.253942:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.253943:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.253944:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.253945:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.253946:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.253947:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677475.253948:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677475.253948:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677475.253951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677475.253954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.253959:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.253960:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.253961:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.253961:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677475.253961:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677475.253962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677475.253963:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.253963:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.253964:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677475.253964:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:5.2:1619677475.254399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.254405:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.254408:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.254409:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.254411:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.254413:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.254414:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.254416:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.254417:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.254418:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.254419:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.254422:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.254424:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.254427:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.254428:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.254428:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.254430:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.254431:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.254431:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.254432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.254437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.254445:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.254448:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.254449:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.254449:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.254451:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.254453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.254568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.254571:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.254574:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.254576:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.254578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.254578:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.254580:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.254581:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.254583:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.254583:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.254585:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.254588:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.254590:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.254592:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.254593:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.254593:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.254595:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.254596:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.254596:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.254597:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677475.254601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:8.2:1619677475.254603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.254608:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.254610:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677475.254611:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:21.0:1619677475.254612:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.254612:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.254612:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.254614:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.254614:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.254616:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.254617:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.254619:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.254619:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.254621:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.254623:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677475.254624:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.254624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.254624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677475.254626:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.254629:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.254629:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.254630:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.254632:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.254632:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.254633:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.254634:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.254638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677475.254643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:18.0:1619677475.254646:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677475.254649:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:18.0:1619677475.254649:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.254650:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.254650:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:22.0:1619677475.254652:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.254652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:18.0:1619677475.254653:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.254655:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.254656:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:22.0:1619677475.254657:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.254658:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.254659:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.254661:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.254662:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.254663:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.254666:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.254668:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.254670:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.254671:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.254671:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.254673:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.254673:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.254674:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.254675:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.254679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.254686:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.254689:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.254690:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.254691:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.254693:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.254693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.254931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.254940:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.254943:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.254945:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.254947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.254948:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.254949:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.254950:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.254952:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.254953:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677475.254953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:31.0:1619677475.254954:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.254957:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.254959:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.254960:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:31.0:1619677475.254962:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.254962:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:31.0:1619677475.254963:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.254963:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.254964:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:31.0:1619677475.254965:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:24.0:1619677475.254965:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677475.254966:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.254967:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.254968:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.254969:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000400:00000200:24.0:1619677475.254969:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677475.254970:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:24.0:1619677475.254970:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.254971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:24.0:1619677475.254971:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.254972:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.254973:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.254975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:24.0:1619677475.254976:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677475.254978:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.254979:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.254981:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.254981:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (23)++
00000800:00000200:6.0:1619677475.254981:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.254983:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677475.254983:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677475.254983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.254983:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.254985:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.254985:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.254986:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.254986:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677475.254986:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:30.0:1619677475.254987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677475.254987:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677475.254988:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677475.254988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:6.0:1619677475.254988:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:25.2:1619677475.254991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.254994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:22.0:1619677475.254998:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.255001:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677475.255001:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000400:00000200:22.0:1619677475.255002:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.255002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.255003:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.255003:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677475.255004:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677475.255005:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:22.0:1619677475.255006:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.255025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.255032:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.255035:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.255037:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.255039:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.255040:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.255041:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.255042:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.255044:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.255045:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.255046:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.255049:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.255050:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.255052:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.255053:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.255053:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.255055:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.255056:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.255056:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.255057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.255061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.255067:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.255069:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.255070:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.255070:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.255072:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.255075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.255515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.255521:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.255524:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.255525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.255526:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.255529:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.255530:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.255531:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.255533:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.255533:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.255535:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.255538:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.255540:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.255542:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.255543:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.255544:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.255545:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.255546:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.255547:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.255547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.255552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.255561:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.255563:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.255564:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.255565:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.255567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.255567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677475.256164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.256171:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.256174:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.256175:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.256177:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.256180:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.256181:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.256182:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.256184:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.256185:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.256186:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.256189:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.256191:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.256194:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.256195:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.256195:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.256197:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.256197:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.256198:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.256199:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.256203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.256211:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.256213:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.256214:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.256215:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.256217:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.256219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.256363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677475.256368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.256372:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.256373:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.256375:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677475.256377:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.256377:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:31.0:1619677475.256378:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.256379:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677475.256380:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.256381:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.256382:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.256383:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:6.0:1619677475.256383:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:31.0:1619677475.256385:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.256385:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.256387:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.256387:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.256389:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.256389:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.256390:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677475.256391:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.256396:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.256397:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.256399:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.256400:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.256401:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.256402:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:31.0:1619677475.256403:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.256403:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.256404:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.256405:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.256405:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.256405:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.256405:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.256406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677475.256407:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.256407:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.256408:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.256409:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.256411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677475.256416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.256419:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.256422:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.256423:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677475.256423:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.256424:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.256425:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677475.256425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.256426:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:22.0:1619677475.256427:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.256427:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.256429:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.256430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.256434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.256435:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.256437:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.256438:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.256438:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.256439:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.256440:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.256610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.256616:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.256619:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.256620:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.256622:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.256624:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.256625:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.256627:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.256628:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.256629:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.256630:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.256634:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.256636:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.256638:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.256639:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.256639:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.256641:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.256641:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.256643:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.256644:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.256644:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.256648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.256650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.256655:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.256657:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.256657:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677475.256658:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677475.256658:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677475.256660:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.256661:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.256662:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.256663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.256665:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.256669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.256676:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.256679:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.256680:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.256681:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.256681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.256682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.257351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.257359:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.257363:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.257365:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.257367:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.257368:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.257370:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.257371:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.257372:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.257373:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677475.257373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:30.0:1619677475.257374:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.257378:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.257379:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:6.0:1619677475.257379:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677475.257382:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.257382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.257382:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:30.0:1619677475.257383:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.257383:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:6.0:1619677475.257384:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.257385:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.257385:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.257386:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677475.257386:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.257387:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677475.257388:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.257389:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.257390:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.257391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:6.0:1619677475.257391:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.257392:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.257396:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.257397:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.257398:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:6.0:1619677475.257400:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.257401:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:6.0:1619677475.257401:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.257401:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.257402:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.257403:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.257403:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.257403:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677475.257404:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677475.257404:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:31.0:1619677475.257405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677475.257405:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.257406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.257409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.257411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.257415:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.257416:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.257417:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.257418:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.257419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.257419:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.257419:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.257421:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.257422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.257423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.257429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:4.2:1619677475.257431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.0:1619677475.257436:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677475.257438:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677475.257439:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.257440:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.257440:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.257441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677475.257441:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.257442:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677475.257442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677475.257444:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.257447:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.257448:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.257449:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.257451:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.257452:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.257453:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.257456:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.257458:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.257461:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.257462:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.257462:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.257464:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.257465:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.257466:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.257467:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.257468:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.257470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.257473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.257478:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.257480:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.257480:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.257481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.257482:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.257483:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.257483:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677475.257484:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.257485:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677475.257486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.257558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.257563:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.257565:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.257568:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.257568:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.257570:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.257572:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.257573:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.257574:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.257575:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.257576:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.257580:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.257582:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.257584:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.257585:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.257585:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.257587:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.257588:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.257589:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.257590:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.257591:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.257594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.257596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.257598:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.257600:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.257601:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.257602:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677475.257603:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.257605:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.257606:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.257607:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.257608:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.257611:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.257620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677475.257627:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.257630:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677475.257632:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677475.257633:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677475.257636:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.257637:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677475.257638:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.257640:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.257641:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.257642:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677475.257645:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677475.257647:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.257648:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.257649:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.257650:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.257651:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.257651:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.257652:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677475.257652:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677475.257656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.257662:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.257664:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.257665:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.257666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.257667:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.257669:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.258056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.258064:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.258067:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.258069:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.258071:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.258071:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.258072:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.258074:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.258075:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.258076:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.258077:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.258080:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.258081:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.258085:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.258085:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.258086:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.258087:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.258088:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.258089:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.258089:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.258094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.258101:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.258104:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.258105:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.258105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.258107:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.258110:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.258317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.258324:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677475.258325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.0:1619677475.258327:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.258328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.258330:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.258331:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677475.258332:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.258333:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.258334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:8.0:1619677475.258334:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.258335:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.258337:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.258337:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.258338:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.258339:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:6.0:1619677475.258339:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.258340:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.258342:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.258343:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.258344:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.258345:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.258349:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.258351:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.258352:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677475.258353:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.258355:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.258356:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.258356:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.0:1619677475.258357:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.258357:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677475.258358:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.258359:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.258359:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.0:1619677475.258360:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.258360:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.258360:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.258360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677475.258361:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.258362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.258365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677475.258370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.258373:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.258375:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.258376:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.258377:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677475.258378:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.258379:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677475.258379:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:22.0:1619677475.258380:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.258380:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677475.258381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.258383:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.258385:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.258386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677475.258394:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.258397:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677475.258399:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.258402:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.258402:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677475.258404:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677475.258405:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.258406:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.258408:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.258409:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677475.258412:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.258414:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677475.258417:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.258417:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677475.258418:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677475.258420:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677475.258420:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.258421:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677475.258422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677475.258426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.258434:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.258437:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.258439:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.258439:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.258441:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.258444:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.258683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.258691:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.258694:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.258696:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.258697:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.258699:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.258700:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.258701:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.258703:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.258704:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.258705:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.258708:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.258710:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.258712:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.258713:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.258713:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.258715:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.258715:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.258716:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.258717:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.258720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.258727:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.258729:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.258730:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.258730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.258732:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.258735:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.258831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.258838:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.258841:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.258841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.258843:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.258845:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.258846:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.258847:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.258849:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.258850:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.258851:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.258854:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.258856:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.258857:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.258858:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.258859:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.258860:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.258861:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.258862:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.258863:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.258864:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.258866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.258869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.258875:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.258875:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.258876:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.258876:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.258876:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.258877:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.258878:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.258878:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.258879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677475.258879:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.259205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.259210:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.259213:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.259214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.259216:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.259218:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.259219:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.259220:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.259222:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.259223:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.259224:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.259227:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.259229:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.259231:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.259232:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.259232:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.259234:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.259234:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.259235:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.259235:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.259240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.259248:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.259250:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.259251:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.259252:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.259253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.259255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.259281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.259287:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.259290:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.259290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.259293:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.259301:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.259302:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.259304:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.259305:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.259306:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.259307:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.259310:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.259312:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.259314:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.259315:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.259316:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.259317:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.259318:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.259319:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.259320:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.259320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.259324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.259326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677475.259332:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677475.259332:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.259334:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677475.259335:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677475.259335:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677475.259336:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.259336:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.259336:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677475.259338:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.259340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.259347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.259354:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.259356:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.259357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.259358:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.259358:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.259359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.259552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677475.259558:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.259560:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677475.259561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.259563:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.259565:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.259567:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677475.259568:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.259569:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.259570:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.259571:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677475.259574:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.259576:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677475.259579:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.259579:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:17.0:1619677475.259580:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677475.259582:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677475.259583:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.259583:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677475.259584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:25.2:1619677475.259588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.259598:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.259601:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677475.259603:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.259606:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677475.259606:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.259609:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677475.259695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.259703:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.259706:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.259708:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.259711:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.259711:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.259712:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.259713:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.259715:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.259716:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.259717:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.259721:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.259722:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.259725:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.259726:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.259726:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.259728:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.259729:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.259730:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.259732:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.259732:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.259733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.259738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.259741:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.259743:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.259744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.259744:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.259746:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.259746:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.259748:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677475.259748:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.259749:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.259750:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.259752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.259758:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.259761:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.259761:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.259762:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.259762:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.259763:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.260077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fe800] (20)++
00000800:00000200:1.0:1619677475.260083:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fe800] (21)++
00000800:00000200:1.0:1619677475.260086:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.58@o2ib3
00000800:00000200:17.0:1619677475.260087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fe800] (22)--
00000400:00000200:1.0:1619677475.260089:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.1.58@o2ib3 : GET - for me
00000400:00000200:1.0:1619677475.260092:0:19931:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.1.58@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:1.0:1619677475.260095:0:19931:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.1.58@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:1.0:1619677475.260096:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fe800] (21)++
00000800:00000200:1.0:1619677475.260097:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fe800] (22)++
00000800:00000200:1.0:1619677475.260098:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fe800] (23)--
00000400:00000200:1.0:1619677475.260100:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.1.58@o2ib3 local destination
00000800:00000200:21.2:1619677475.260104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fe800] (22)++
00000400:00000200:1.0:1619677475.260104:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.1.58@o2ib3(10.51.1.58@o2ib3:10.51.1.58@o2ib3) : REPLY try# 0
00000800:00000200:1.0:1619677475.260107:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.1.58@o2ib3
00000800:00000200:1.0:1619677475.260109:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918ce53e8d80] -> 10.51.1.58@o2ib3 (2) version: 12
00000800:00000200:17.0:1619677475.260110:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fe800] (23)++
00000800:00000200:1.0:1619677475.260110:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918cfd7fe800] (23)++
00000800:00000200:1.0:1619677475.260111:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fe800] (25)++
00000800:00000200:17.0:1619677475.260112:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fe800] (26)--
00000800:00000200:1.0:1619677475.260112:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918cfd7fe800] (26)--
00000800:00000200:1.0:1619677475.260112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fe800] (25)--
00000800:00000200:17.0:1619677475.260113:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fe800] (23)--
00000800:00000200:1.0:1619677475.260113:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fe800] (22)--
00000800:00000200:21.2:1619677475.260117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fe800] (21)++
00000800:00000200:17.0:1619677475.260122:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fe800] (22)++
00000800:00000200:17.0:1619677475.260124:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fe800] (23)--
00000400:00000200:17.0:1619677475.260125:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.260125:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fe800] (22)--
00000400:00000200:17.0:1619677475.260127:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.1.58@o2ib3: REPLY: OK
00000800:00000200:17.0:1619677475.260128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fe800] (21)--
00000800:00000200:20.2:1619677475.260178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.260185:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.260188:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.260189:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.260191:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.260192:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.260194:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.260195:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.260196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000400:00000200:18.0:1619677475.260197:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.260198:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.260199:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677475.260201:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000400:00000200:18.0:1619677475.260203:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677475.260204:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:18.0:1619677475.260205:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677475.260205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677475.260207:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.260208:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.260208:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:22.0:1619677475.260209:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.260209:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:22.0:1619677475.260211:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:18.0:1619677475.260211:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.260211:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:22.0:1619677475.260212:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.260212:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.260213:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:22.0:1619677475.260214:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.260215:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.260216:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:25.2:1619677475.260217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:22.0:1619677475.260219:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.260221:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677475.260223:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.260224:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:22.0:1619677475.260224:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677475.260225:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.260227:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.260227:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.260227:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677475.260227:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.260228:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000400:00000200:30.0:1619677475.260229:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.260229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:31.0:1619677475.260230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.260233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:8.2:1619677475.260233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.260237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:25.2:1619677475.260240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677475.260241:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:6.0:1619677475.260242:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:1.0:1619677475.260243:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.260244:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.260245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677475.260245:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.260246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677475.260247:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:1.0:1619677475.260247:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.260248:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.260250:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:1.0:1619677475.260250:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677475.260251:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:6.0:1619677475.260251:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.260252:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.260252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:6.0:1619677475.260252:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677475.260253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:6.0:1619677475.260254:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.260255:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.260256:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.260257:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.260260:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.260262:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.260264:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.260265:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.260265:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.260267:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.260267:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.260268:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.260269:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.260273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.260280:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.260282:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.260283:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.260285:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.260286:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.260287:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.260369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.260377:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.260380:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.260382:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.260384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.260385:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.260386:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.260388:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.260389:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.260390:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.260391:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.260394:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.260396:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.260399:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.260400:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.260401:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.260403:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.260403:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.260404:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.260405:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.260409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.260418:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.260421:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.260421:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.260422:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.260424:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.260427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.260428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.260435:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.260438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.260438:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.260439:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.260440:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.260440:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.260674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.260682:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.260685:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.260687:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.260689:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.260690:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.260691:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.260693:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.260694:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.260695:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.260696:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.260699:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.260700:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.260704:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.260704:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.260705:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.260707:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.260707:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.260708:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.260709:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.260713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.260722:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.260724:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.260725:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.260727:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.260728:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.260730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.260736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.260744:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.260746:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.260747:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.260748:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.260749:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.260750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.260967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.260975:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.260978:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.260978:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677475.260981:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.260983:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.260985:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.260986:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.260987:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.260988:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.260988:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.260991:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.260993:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.260996:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.260996:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.260997:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.260999:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.260999:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.260999:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.261000:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.261005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.261012:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.261015:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.261016:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.261016:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.261018:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.261021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.261094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.261099:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.261101:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.261102:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.261104:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.261106:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.261107:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.261108:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.261109:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.261110:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.261111:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.261114:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.261116:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.261118:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.261119:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.261120:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.261121:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.261122:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.261123:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.261124:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.261125:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.261127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.261130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.261135:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.261135:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.261137:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.261137:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.261138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.261138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.261138:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.261139:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.261141:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.261142:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.261146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.261154:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.261156:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.261157:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.261158:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.261158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.261159:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.261528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.261533:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.261535:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.261537:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.261538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.261539:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.261540:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.261542:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.261543:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.261544:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.261545:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.261548:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.261549:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.261551:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.261552:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.261553:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.261554:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.261555:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.261556:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.261556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.261561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.261568:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.261570:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.261571:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.261573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.261573:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.261575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.261814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.261822:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.261825:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.261827:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.261830:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.261830:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.261832:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.261833:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.261835:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.261836:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.261837:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.261840:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.261842:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.261845:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.261846:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.261847:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.261848:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.261849:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.261850:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.261851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.261854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.261862:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.261864:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.261866:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.261867:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.261868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.261870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.261939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.261945:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.261948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.261948:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.261950:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.261952:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.261954:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.261955:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.261956:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.261957:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.261959:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.261962:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.261964:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.261966:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.261967:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.261967:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.261969:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.261970:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.261970:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.261970:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.261976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.261983:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.261985:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.261986:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.261988:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.261989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.261990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.262146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.262153:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.262156:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.262157:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.262159:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.262161:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.262163:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677475.262164:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.262166:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.262167:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.262168:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677475.262171:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.262173:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677475.262176:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.262177:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677475.262177:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677475.262179:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677475.262179:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.262180:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.262180:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:9.2:1619677475.262185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.262193:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.262196:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.262197:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.262198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.262199:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.262203:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.262206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.262213:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.262216:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.262216:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.262217:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.262218:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.262219:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.262432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.262440:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.262443:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.262445:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.262448:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.262448:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.262449:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.262451:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.262452:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.262453:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.262454:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.262457:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.262459:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.262461:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.262462:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.262462:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.262464:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.262465:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.262466:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.262466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.262470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.262478:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.262480:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.262481:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.262483:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.262484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.262485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.262488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.262495:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.262498:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.262498:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.262499:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.262499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.262500:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.262708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.262716:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.262718:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.262719:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.262721:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.262724:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.262725:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.262726:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.262728:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.262729:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.262730:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.262733:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.262734:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.262737:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.262738:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.262738:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.262740:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.262740:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.262741:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.262741:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.262746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.262754:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.262756:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.262758:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.262758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.262760:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.262762:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.262944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.262949:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.262952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.262952:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.262955:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.262956:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.262958:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.262959:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.262960:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.262961:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.262962:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.262965:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.262967:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.262969:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.262970:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.262970:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.262972:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.262972:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.262973:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.262973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.262978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.262986:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.262988:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.262989:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.262990:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.262991:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.262992:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.263149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.263156:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.263158:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.263159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.263161:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.263164:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.263165:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.263166:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.263168:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.263169:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.263169:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.263172:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.263174:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.263176:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.263177:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.263178:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.263179:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.263180:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.263181:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.263181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.263187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.263194:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.263196:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.263197:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.263199:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.263200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.263201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.263336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:4.2:1619677475.263337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.0:1619677475.263342:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:24.0:1619677475.263344:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677475.263345:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:24.0:1619677475.263347:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:21.0:1619677475.263347:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.263348:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.0:1619677475.263348:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.263350:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.263350:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.263352:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.263353:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.263354:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.263355:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.263356:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.263356:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.263357:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.263357:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:24.0:1619677475.263359:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.263360:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.263361:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677475.263362:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.263364:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:24.0:1619677475.263366:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.263366:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.263367:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:24.0:1619677475.263368:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.263368:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.263369:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.263370:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.263370:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:24.0:1619677475.263371:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.263371:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.263371:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:24.0:1619677475.263372:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.263374:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.263375:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.2:1619677475.263375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677475.263376:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.263377:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.263381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.263383:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.263385:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.263386:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.263387:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.263388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:22.0:1619677475.263389:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.263389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677475.263391:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.263392:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.263393:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.263395:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.263398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.263725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.263730:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.263733:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.263735:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.263735:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.263737:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.263738:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.263740:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.263741:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.263742:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.263743:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.263746:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.263748:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.263750:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.263751:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.263752:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.263753:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.263754:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.263754:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.263755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.263759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.263767:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.263769:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.263770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.263770:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.263772:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.263774:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.263798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677475.263800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.0:1619677475.263801:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.263803:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677475.263805:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:20.0:1619677475.263805:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.263806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677475.263807:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:20.0:1619677475.263807:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.263809:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677475.263809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.263811:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.263811:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.263813:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.263814:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.263814:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.263815:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677475.263815:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677475.263817:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.263819:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.263819:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.263820:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.263821:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.263822:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677475.263824:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.263826:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.263827:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.263828:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677475.263829:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.263829:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.263829:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677475.263830:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.263831:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677475.263832:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.263832:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:22.0:1619677475.263833:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.263833:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.263833:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.263833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:22.0:1619677475.263834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.2:1619677475.263837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:25.2:1619677475.263838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.263845:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:1.0:1619677475.263845:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.263847:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.263848:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.263848:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.263849:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.263849:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.263850:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.263851:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:1.0:1619677475.263852:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.263853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677475.263855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.264412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677475.264420:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677475.264423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677475.264423:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000400:00000200:10.0:1619677475.264425:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.264428:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.264430:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677475.264431:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.264432:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.264433:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.264434:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677475.264438:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.264440:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677475.264442:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.264443:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677475.264444:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677475.264445:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677475.264445:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677475.264446:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677475.264447:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677475.264452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677475.264456:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677475.264458:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:8.0:1619677475.264459:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.264460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:8.0:1619677475.264461:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.264464:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:24.2:1619677475.264474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:20.2:1619677475.264475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:24.0:1619677475.264476:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.264479:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677475.264480:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677475.264480:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677475.264481:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677475.264481:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:2.0:1619677475.264482:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.264485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.264485:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000400:00000200:2.0:1619677475.264487:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677475.264488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:2.0:1619677475.264490:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.264491:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677475.264493:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.264494:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:2.0:1619677475.264494:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.264495:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.264496:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:17.0:1619677475.264497:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.264498:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.264501:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.264501:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677475.264503:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.264503:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:17.0:1619677475.264504:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.264506:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.264507:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677475.264507:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677475.264508:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677475.264508:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:17.0:1619677475.264509:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:2.0:1619677475.264509:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677475.264510:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677475.264511:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:17.0:1619677475.264512:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.264512:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.264513:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:17.0:1619677475.264514:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.264517:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.264517:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:9.2:1619677475.264517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677475.264518:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.264520:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.264520:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.264521:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.264521:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677475.264524:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.264526:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:25.2:1619677475.264526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.264527:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.264528:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.264530:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.264533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677475.264534:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.264536:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.264537:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.264538:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.264540:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.264542:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.264652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.264657:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.264660:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.264660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.264662:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.264665:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.264666:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.264667:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.264668:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.264670:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.264671:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.264674:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.264676:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.264678:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.264679:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.264679:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.264681:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.264681:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.264682:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.264683:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.264684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.264687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.264688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.2:1619677475.264692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677475.264693:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.264694:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.264695:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.264696:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.264696:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.264697:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.264697:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.264699:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.264700:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.264700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.264701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.264703:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.264705:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.264707:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.264708:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677475.264708:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.264709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000400:00000200:31.0:1619677475.264710:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.264711:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.264712:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.264713:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.264716:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.264716:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.264718:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.264719:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.0:1619677475.264719:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.264720:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677475.264721:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.264721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.264721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.264722:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.264723:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.264725:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.264725:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.264727:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.264729:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.264729:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.264730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.264735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.264737:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.264739:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.264740:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.264740:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.264742:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.264743:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677475.264744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.264745:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.264746:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.264747:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.264879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.264885:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.264887:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.264889:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.264890:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.264893:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.264894:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677475.264896:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.264897:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.264898:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.264899:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677475.264903:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.264905:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677475.264908:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.264909:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677475.264909:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677475.264911:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677475.264912:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.264912:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.264913:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677475.264917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.264923:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.264926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677475.264926:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.264927:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.264929:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.264932:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.264951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.264959:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.264962:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.264964:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677475.264966:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.264966:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.264968:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.264969:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.264970:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.264971:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.264972:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.264976:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677475.264978:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.264980:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.264981:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.264981:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.264983:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.264983:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.264984:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.264986:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.264986:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.264988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.264992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.264994:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.264997:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.264998:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.264998:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.265000:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.265000:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.265002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.265003:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.265003:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.265004:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.265717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.265723:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.265725:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.265726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.265728:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.265730:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.265732:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.265733:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.265734:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.265735:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.265737:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.2:1619677475.265737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:22.0:1619677475.265740:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.265742:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.265745:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.265745:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.265746:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.265747:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.265748:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677475.265748:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.265749:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677475.265750:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.265750:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.265750:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:31.0:1619677475.265753:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.265753:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.265754:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.265755:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.265755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.265757:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.265758:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.265759:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.265762:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.265764:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.0:1619677475.265764:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677475.265766:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.265766:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677475.265767:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:21.0:1619677475.265767:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.265768:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:21.0:1619677475.265769:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.265770:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.265770:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.265770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.265771:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.265771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.265772:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.265775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.265783:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.265786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.265786:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.265787:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.265789:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.265791:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.265792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.265799:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.265802:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.265803:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.265805:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.265806:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.265808:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.265809:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.265811:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.265811:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.265813:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.265816:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.265817:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.265820:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.265821:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.265821:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.265823:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.265823:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.265825:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.265826:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.265826:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.265829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.265832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.265837:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.265839:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.265840:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.265841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.265841:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677475.265841:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.265842:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677475.265842:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677475.265843:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.265846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.266330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.266337:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.266340:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.266341:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.266343:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.266346:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.266347:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.266348:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.266350:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.266351:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.266352:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.266355:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.266356:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.266359:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.266359:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.266360:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.266361:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.266362:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.266363:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.266364:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.266368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.266377:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.266380:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.266381:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.266381:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.266383:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.266386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.266387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.266394:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.266397:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.266398:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.266398:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.266399:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.266400:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.266546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.266552:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.266555:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.266555:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.266557:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.266560:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.266561:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.266562:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.266564:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.266565:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.266565:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.266569:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.266570:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.2:1619677475.266570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.266573:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.266574:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.266574:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.266576:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.266576:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.266577:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.266577:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677475.266578:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.266581:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.266583:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.266583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.266586:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.266586:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.266587:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.266589:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.266590:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.266590:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:30.0:1619677475.266591:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.266592:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677475.266593:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.266594:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.266596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.266596:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.266597:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.266599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.266600:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.266602:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.266603:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.266604:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.266605:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.266605:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.266606:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.266607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.266611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.266617:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.266620:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.266621:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.266622:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.266624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.266625:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.267135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.267143:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.267145:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.267147:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.267149:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.267150:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.267151:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.267152:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.267153:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.267154:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.267155:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.267158:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.267159:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.267161:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.267162:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.267162:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.267163:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.267164:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.267164:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.267165:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.267169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.267176:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.267179:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.267180:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.267181:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.267183:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.267183:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.267409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.267415:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.267418:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.267418:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.267421:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.267423:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.267424:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.267425:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.267427:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.267428:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.267429:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.267431:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.267433:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.267435:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.267436:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.267437:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.267438:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.267439:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.267439:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.267440:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.267445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.267453:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.267455:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.267456:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.267458:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.267459:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.267460:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.267772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677475.267778:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.267781:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677475.267781:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677475.267783:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.267786:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.267787:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677475.267788:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.267790:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.267791:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.267791:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677475.267794:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.267795:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677475.267798:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.267798:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677475.267799:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677475.267801:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677475.267801:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.267802:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677475.267802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677475.267807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.267814:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.267817:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.267818:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.267818:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.267820:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.267823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.267923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.267931:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.267934:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.267937:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.267938:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.267939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677475.267939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677475.267940:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.267941:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.267942:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.267943:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.267944:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677475.267946:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677475.267948:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.267949:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677475.267950:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677475.267950:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677475.267952:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.267953:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.267953:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677475.267953:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.267955:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.267955:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677475.267955:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.267957:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:10.0:1619677475.267957:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677475.267958:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:10.0:1619677475.267958:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.267959:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677475.267959:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.267960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677475.267960:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.267961:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.267964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677475.267965:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.267966:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.267969:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:10.0:1619677475.267969:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.267970:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.267971:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:10.0:1619677475.267971:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.267972:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.267972:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.267972:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.267973:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677475.267973:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.267973:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677475.267974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677475.267974:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.267974:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677475.267975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.267976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.267977:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.267978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.267986:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.267989:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.267990:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.267990:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.267992:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.267995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.268001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.268009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.268011:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.268012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.268013:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.268013:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.268014:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.268030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.268037:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.268040:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.268041:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.268042:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.268046:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.268047:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.268048:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.268050:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.268051:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.268052:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.268055:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.268057:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.268059:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.268060:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.268061:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.268062:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.268063:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.268064:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.268065:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.268066:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.268068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.268071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.268075:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.268078:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.268078:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.268079:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.268079:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.268080:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.268081:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677475.268081:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.268082:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677475.268084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.268474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.268480:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.268482:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.268483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.268485:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.268487:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.268488:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.268489:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.268491:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.268492:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.268493:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.268496:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.268498:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.268500:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.268501:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.268502:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.268503:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.268504:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.268505:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.268506:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.268506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.268510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.268512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.268517:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.268517:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.268519:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.268519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.268520:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.268520:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.268520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.268522:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.268524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.268524:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.268530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.268537:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.268539:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.268540:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.268541:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.268542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.268543:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.268800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.268806:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.268809:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.268810:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.268812:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.268815:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.268816:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.268818:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.268819:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.268820:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.268821:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.268824:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.268826:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.268829:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.268829:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.268830:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.268832:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.268832:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.268833:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.268833:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.268839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.268847:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.268849:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.268850:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.268851:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.268852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.268854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.268860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.268868:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.268870:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.268871:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.268872:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.268873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.268874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.269005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677475.269013:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677475.269016:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677475.269017:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677475.269019:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.269022:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.269023:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677475.269025:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.269026:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.269027:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.269028:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677475.269032:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.269033:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677475.269036:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.269036:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677475.269037:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677475.269038:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677475.269039:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677475.269040:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677475.269040:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:25.2:1619677475.269044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.269052:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.269054:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.269055:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.269057:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.269059:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.269060:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.269064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.269071:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.269074:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.269075:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.269076:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.269076:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.269077:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.269185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.269190:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.269192:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.269194:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.269195:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.269197:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.269198:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.269199:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.269200:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.269201:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.269202:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.269205:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.269207:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.269208:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.269209:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.269210:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.269211:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.269212:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.269213:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.269214:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.269217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.269225:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.269227:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.269228:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.269229:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.269230:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.269232:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677475.269244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.269249:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.269252:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.269253:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.269255:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.269257:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.269259:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.269260:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.269261:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.269262:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.269263:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.269266:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.269267:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.269270:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.269271:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.269271:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.269273:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.269273:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.269274:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.269274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.269279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.269286:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.269289:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.269290:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.269290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.269292:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.269299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.269300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.269308:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.269310:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.269311:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.269311:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.269312:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.269313:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.269366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:4.2:1619677475.269370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:22.0:1619677475.269371:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.269374:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.269375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.269376:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.269376:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.269378:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:22.0:1619677475.269379:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.269380:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677475.269381:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.269381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:22.0:1619677475.269382:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.269383:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.269384:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.269384:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.269385:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:22.0:1619677475.269386:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677475.269387:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.269388:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.269389:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.269390:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.269395:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.269396:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.269398:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.269399:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.269400:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.269401:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.269401:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.269402:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.269402:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.269403:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.269404:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.269404:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677475.269405:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.269405:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.269406:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.269406:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677475.269407:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.269407:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.269407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.269409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.269412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:25.2:1619677475.269414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.269417:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.269418:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677475.269419:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.269420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.269420:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.269420:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:6.0:1619677475.269420:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.269422:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:6.0:1619677475.269422:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.269423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.0:1619677475.269424:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.269424:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.269424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677475.269425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.269426:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.269428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.270221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.270226:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.270229:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.270231:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.270231:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.270233:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.270235:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.270236:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.270238:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677475.270239:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:21.0:1619677475.270242:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.270243:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677475.270245:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.270247:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:21.0:1619677475.270247:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.270248:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.270248:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.270250:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.270251:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.270252:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.270252:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:22.0:1619677475.270253:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.270254:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.0:1619677475.270254:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.270255:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.270255:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:22.0:1619677475.270256:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.270256:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:22.0:1619677475.270257:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.270258:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.270259:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.2:1619677475.270261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:22.0:1619677475.270262:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.270264:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.270266:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.270266:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.270267:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.270269:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.270269:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677475.270269:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677475.270270:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.270270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:17.0:1619677475.270272:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.270273:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.270273:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:25.2:1619677475.270275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:17.0:1619677475.270275:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.270278:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.270279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.270282:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.270284:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.270285:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.270286:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:20.0:1619677475.270287:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.270288:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.270289:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677475.270289:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.270290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.270290:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.270290:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.270291:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.270397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677475.270404:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.270407:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677475.270408:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677475.270410:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.270412:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.270413:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.270415:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.270416:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.270417:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.270418:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.270422:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.270424:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.270427:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.270428:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.270429:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.270430:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.270431:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.270432:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677475.270433:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:25.2:1619677475.270453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.270460:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.270463:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.270464:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.270466:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.270468:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.270469:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.270619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.270627:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.270630:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.270631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.270633:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.270635:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.270637:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.270638:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.270639:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.270640:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.270641:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.270644:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.270646:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.270649:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.270650:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.270651:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.270652:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.270653:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.270653:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.270654:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.270658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.270666:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.270669:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.270670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.270670:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.270672:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.270675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.270683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.270688:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.270691:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.270693:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.270693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.270695:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.270696:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.270698:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.270699:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.270700:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.270701:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.270704:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.270706:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.270708:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.270709:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.270709:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.270711:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.270711:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.270712:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.270713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.270717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.270724:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.270726:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.270727:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.270728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.270729:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.270732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.270832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.270838:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.270841:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.270842:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.270845:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.270847:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.270849:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.270850:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.270852:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.270853:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.270854:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.270857:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.270858:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.270861:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.270861:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.270862:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.270864:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.270864:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.270866:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.270867:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.270867:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677475.270873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.270879:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.270881:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.270881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677475.270881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.270882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677475.270890:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.270892:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.270893:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.270893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.270895:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.270897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.270912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.270921:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.270923:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.270926:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.270928:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.270928:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.270929:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.270931:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.270932:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.270933:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.270934:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.270938:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.270940:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.270943:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.270944:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.270944:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.270946:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.270947:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.270948:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.270948:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.270952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.270959:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.270961:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.270962:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.270963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.270964:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.270967:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.271245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.271251:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.271254:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.271255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.271257:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.271259:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.271260:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.271262:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.271263:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.271264:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.271265:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.271268:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.271270:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.271272:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.271273:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.271274:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.271275:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.271276:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.271277:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.271277:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.271281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.271289:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.271291:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.271292:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.271296:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.271300:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.271302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.271655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.271664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.271667:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.271669:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.271671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.271671:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.271673:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.271674:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.271675:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.271676:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.271677:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.271681:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.271683:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.271685:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.271685:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.271686:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.271687:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.271688:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.271689:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.271690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.271694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.271701:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.271703:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.271704:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.271705:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.271707:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.271708:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.272071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.272078:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.272080:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.272082:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.272084:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.272085:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677475.272085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.272086:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.272087:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.272088:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.272089:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.272092:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.272093:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.272094:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.272095:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.272096:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.272097:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.272097:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.272098:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.272099:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.272102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.272109:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.272112:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.272112:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.272114:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.272116:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.272116:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.2:1619677475.272129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.272135:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.272138:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.272139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.272141:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.272143:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.272145:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.272146:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.272147:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.272148:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.272149:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.272152:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.272154:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.272156:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.272157:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.272157:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.272159:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.272159:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.272161:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.272162:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.272162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.272165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.272167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.272172:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.272174:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.272174:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.272174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.272175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.272177:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.272177:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.272179:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.272180:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.272181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.272184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.272191:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.272194:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.272195:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.272196:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.272196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.272197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.272677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.272685:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.272687:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.272688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.272690:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.272693:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.272694:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.272696:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.272697:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.272698:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.272699:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.272702:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.272704:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.272707:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.272708:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.272708:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.272710:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.272711:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.272712:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.272712:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.272718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.272727:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.272730:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.272731:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.272731:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.272734:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.272735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.272739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.272740:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.272743:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.272744:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.272744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.272745:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.272746:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.272761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.272770:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.272772:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.272773:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.272775:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.272778:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.272779:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.272780:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.272781:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.272782:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.272783:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.272786:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.272788:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.272790:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.272791:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.272792:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.272793:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.272794:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.272795:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.272795:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.272800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.272808:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.272810:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.272812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.272812:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.272814:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.272817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.272957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:4.2:1619677475.272960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:22.0:1619677475.272962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.272964:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.272965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.272967:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.272967:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:22.0:1619677475.272969:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.272970:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.272970:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:22.0:1619677475.272971:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.272973:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.272974:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.272975:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.272976:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677475.272976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:22.0:1619677475.272977:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677475.272977:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.272979:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677475.272980:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.272981:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.272982:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.272983:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.272984:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677475.272984:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.272985:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:22.0:1619677475.272986:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.272987:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677475.272987:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.272987:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.272989:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677475.272989:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.272990:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.272990:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.272990:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.272991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677475.272992:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.272992:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.272992:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:30.0:1619677475.272993:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.272994:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96500704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.272995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677475.272995:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.272996:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.272996:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677475.272996:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:30.0:1619677475.272997:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.272998:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.272998:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677475.272998:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.272999:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.273000:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000400:00000200:30.0:1619677475.273001:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.273001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:20.0:1619677475.273002:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677475.273003:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:25.2:1619677475.273003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677475.273004:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677475.273005:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677475.273005:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.273006:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.273006:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.273008:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.273008:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.273008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.273008:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:4.2:1619677475.273008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677475.273010:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677475.273010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:13.0:1619677475.273011:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:30.0:1619677475.273012:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.273012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:13.0:1619677475.273013:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:9.2:1619677475.273013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:13.0:1619677475.273014:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.273015:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:13.0:1619677475.273016:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.273017:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.273018:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:21.2:1619677475.273018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677475.273018:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677475.273019:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:13.0:1619677475.273019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:8.0:1619677475.273020:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.273022:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.273023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.273023:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.273025:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.273026:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.273028:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677475.273028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.273029:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.273030:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.273033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.273039:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.273042:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.273042:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.273043:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.273043:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.273044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.273988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.273994:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:4.2:1619677475.273995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:6.0:1619677475.273997:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.273998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.273999:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.274001:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:6.0:1619677475.274001:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.274003:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:6.0:1619677475.274003:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.274005:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.274006:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.274006:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:6.0:1619677475.274006:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.274007:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.274008:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.274008:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.274012:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.274014:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677475.274014:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:21.0:1619677475.274015:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.274016:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.274017:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.274017:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:21.0:1619677475.274018:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.274018:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677475.274019:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.274019:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677475.274022:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.274023:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.274024:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.274024:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.274025:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.274026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.274026:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.274027:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.274028:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.274029:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.274030:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.274031:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.274031:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:21.0:1619677475.274033:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:25.2:1619677475.274035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.274035:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.274036:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.274037:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.274038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.274039:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677475.274043:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.274045:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.274046:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.274046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.274048:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.274050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.274468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.274476:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.274479:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.274480:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.274481:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.274484:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.274486:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.274487:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.274488:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.274489:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.274490:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.274494:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.274495:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.274498:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.274499:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.274500:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.274502:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.274502:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.274503:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.274503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.274518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.274527:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.274530:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.274531:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.274531:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.274533:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.274536:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677475.274548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.274553:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.274556:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.274557:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.274559:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.274560:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.274562:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.274563:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.274564:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.274565:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.274566:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.274569:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.274571:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.274574:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.274574:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.274575:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.274577:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.274577:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.274578:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.274578:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.274584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.274592:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.274594:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.274595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.274595:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.274597:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.274600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.274840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.274845:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.274848:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.274849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.274851:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.274853:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.274854:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.274855:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.274857:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.274857:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.274859:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.274862:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.274864:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.274866:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.274867:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.274868:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.274869:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.274870:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.274870:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.274871:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.274876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.274883:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.274886:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.274886:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.274888:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.274889:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.274890:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.275415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.275423:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.275425:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.275426:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.275428:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.275431:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.275432:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.275433:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.275435:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.275436:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.275437:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.275440:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.275442:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.275445:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.275446:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.275446:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.275448:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.275448:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.275449:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.275450:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.275455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.275463:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.275466:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.275467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.275467:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.275469:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.275471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.275616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.275627:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.275631:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.275635:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.275636:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.275639:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.275641:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.275643:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.275646:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.275647:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.275649:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.275655:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.275659:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.275662:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.275663:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.275664:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.275666:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.275667:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.275669:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.275670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.275672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.275679:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.275682:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.275683:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.275683:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.275685:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.275687:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.275698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.275703:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.275706:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.275706:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.275708:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.275711:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.275712:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.275713:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.275715:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.275716:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.275717:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.275720:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.275722:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.275725:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.275725:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.275726:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.275728:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.275728:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.275729:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.275730:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.275731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.275734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.275735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.275740:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.275742:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.275743:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.275743:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.275744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.275745:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.275746:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.275747:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.275749:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.275749:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:25.2:1619677475.275755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.275762:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.275765:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.275766:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.275766:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.275767:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.275768:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.276168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677475.276170:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.276172:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677475.276175:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.276176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677475.276177:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.276179:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.276180:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.276181:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.276182:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.276183:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.276187:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.276189:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.276191:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.276192:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.276192:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.276194:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.276195:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.276195:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677475.276196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677475.276200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.276205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:22.0:1619677475.276208:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.276210:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.276211:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.276211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.276212:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:22.0:1619677475.276213:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.276215:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:22.0:1619677475.276215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677475.276216:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.276218:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.276221:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.276223:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.276224:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.276225:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.276226:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.276227:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.276231:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.276232:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.276235:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.276236:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.276237:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.276238:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.276239:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.276240:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677475.276241:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.276241:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677475.276244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.276247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677475.276253:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677475.276254:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:21.0:1619677475.276255:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.276256:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.276256:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.276257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677475.276258:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.276258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.276258:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677475.276261:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.276397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.276405:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.276408:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.276410:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.276413:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.276414:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:31.0:1619677475.276415:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.276415:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.276417:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.276418:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.276418:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.276422:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.276423:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.276425:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.276426:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.276427:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.276428:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.276429:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.276430:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.276430:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.276435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.276442:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.276444:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.276445:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.276445:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.276447:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.276449:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677475.276466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677475.276473:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677475.276476:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677475.276477:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677475.276479:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.276481:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.276482:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677475.276484:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.276485:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.276486:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.276487:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677475.276490:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.276492:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677475.276494:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.276495:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677475.276496:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677475.276498:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677475.276498:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677475.276499:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677475.276499:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677475.276504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.276511:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.276513:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677475.276514:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.276515:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.276517:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.276520:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677475.276588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.276591:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.276593:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.276596:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.276597:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.276598:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.276599:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.276601:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.276602:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.276603:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.276604:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.276608:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.276610:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.276612:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.276613:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.276614:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.276615:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.276616:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.276617:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.276618:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.276619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.276622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.276624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.276629:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677475.276630:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.276631:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.276632:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677475.276632:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677475.276633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677475.276634:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.276634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.276636:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.276638:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.276695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.276700:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.276703:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.276704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.276706:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.276708:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.276709:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.276711:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.276712:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.276713:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.276714:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.276717:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.276719:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.276721:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.276722:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.276723:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.276724:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.276725:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.276726:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.276726:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.276730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.276738:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.276740:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.276741:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.276742:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.276743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.276744:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.276918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.276926:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.276929:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.276931:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.276932:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.276933:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.276935:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.276936:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.276937:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.276938:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.276939:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.276943:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.276944:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.276947:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.276948:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.276948:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.276950:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.276951:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.276952:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.276952:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.276956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.276962:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.276964:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.276965:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.276965:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.276967:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.276969:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.277583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.277589:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.277591:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.277592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.277594:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.277597:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.277598:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.277599:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.277600:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.277602:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.277603:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.277606:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.277608:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.277610:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.277611:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.277612:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.277614:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.277614:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.277615:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.277615:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.277620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677475.277627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.0:1619677475.277632:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.277634:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.277635:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.277635:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:20.0:1619677475.277636:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.277638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677475.277638:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.277639:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677475.277640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677475.277641:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.277644:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.277646:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.277647:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.277648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677475.277649:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.277650:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.277651:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.277654:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.277655:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.277657:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.277658:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.277659:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677475.277659:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.277660:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.277661:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.277661:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.277662:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677475.277663:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.277663:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:31.0:1619677475.277664:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.277664:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677475.277664:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:30.0:1619677475.277665:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.277665:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677475.277667:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.277667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.277668:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.277668:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677475.277670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677475.277672:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.277673:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.277675:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.277675:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.277676:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.277676:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.277677:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.277678:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677475.277678:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.277678:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.277679:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.277679:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.277680:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:26.0:1619677475.277680:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.277680:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677475.277681:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:26.0:1619677475.277681:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.277681:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.277682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.277683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677475.277684:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.277687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:21.2:1619677475.277687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.277692:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.277694:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.277695:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.277695:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677475.277696:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.277696:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.277697:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:22.0:1619677475.277697:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.277698:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677475.277698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.277699:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677475.277699:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.277700:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677475.277700:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.277700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677475.277701:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.278106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.278114:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.278117:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.278118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.278119:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.278122:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.278124:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.278125:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.278126:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.278127:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.278128:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.278131:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.278133:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.278136:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.278136:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.278137:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.278139:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.278139:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.278140:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.278140:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.278145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.278152:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.278154:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.278155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.278156:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.278158:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.278161:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.278256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.278264:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.278267:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.278267:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.278269:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.278272:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.278273:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.278274:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.278276:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.278276:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.278278:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.278281:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.278283:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.278286:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.278287:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.278288:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.278290:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.278290:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.278291:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.278292:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.278296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.278303:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.278306:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.278307:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.278309:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.278310:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.278312:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.278464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.278470:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.278473:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.278474:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.278475:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.278478:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.278479:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.278481:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.278482:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.278483:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.278484:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.278487:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.278489:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.278492:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.278492:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.278493:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.278495:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.278495:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.278496:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.278496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.278501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.278509:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.278511:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.278512:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.278513:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.278514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.278515:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.278569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.278573:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.278575:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.278577:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.278578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.278580:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.278582:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.278583:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.278584:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.278585:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.278586:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.278590:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.278591:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.278594:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.278595:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.278596:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.278598:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.278599:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.278599:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.278600:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.278604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.278611:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.278614:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.278615:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.278615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.278618:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.278621:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.278663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.278669:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.278672:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.278673:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.278675:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.278677:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.278679:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.278680:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.278681:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.278682:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.278683:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.278686:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.278687:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.278690:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.278691:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.278691:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.278693:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.278693:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.278694:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.278694:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.278699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.278707:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.278709:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.278710:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.278712:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.278712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.278714:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.279289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.279299:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.279302:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.279304:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.279306:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.279306:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.279308:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.279309:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.279311:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.279311:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.279313:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.279316:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.279318:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.279321:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.279322:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.279322:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.279324:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.279325:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.279326:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.279326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.279331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.279339:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.279342:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.279343:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.279344:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.279346:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.279347:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.279481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.279487:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.279490:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.279491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.279493:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.279495:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.279496:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.279497:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.279499:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.279500:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.279501:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.279504:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.279506:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.279508:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.279509:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.279509:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.279511:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.279512:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.279513:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.279514:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.279514:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.279518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.279519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.279524:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.279526:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.279527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.279527:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.279528:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.279529:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.279529:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.279531:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.279532:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.279533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.279538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.279546:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.279548:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.279549:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.279550:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.279551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.279552:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.279744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.279752:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.279755:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.279756:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.279758:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.279761:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.279762:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.279764:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.279765:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.279766:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.279767:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.279770:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.279772:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.279775:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.279776:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.279776:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.279778:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.279779:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.279779:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.279780:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.279784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.279792:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.279795:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.279795:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.279796:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.279798:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.279800:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.279805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.279812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.279815:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.279815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.279816:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.279817:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.279817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.279840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.279848:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.279851:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.279852:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.279854:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.279856:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.279858:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.279859:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.279861:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.279861:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.279863:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.279866:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.279868:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.279870:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.279871:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.279872:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.279873:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.279874:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.279875:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.279877:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.279877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.279879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.279882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677475.279886:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677475.279888:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677475.279889:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677475.279889:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677475.279890:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.279890:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.279890:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:6.0:1619677475.279892:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.279892:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677475.279894:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.280315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.280320:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.280323:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.280324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.280326:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.280328:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.280329:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.280330:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.280332:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.280333:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.280334:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.280337:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.280339:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.280342:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.280342:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.280343:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.280345:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.280345:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.280345:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.280346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.280351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.280359:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.280362:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.280362:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.280364:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.280365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.280366:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677475.280841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.280848:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.280851:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.280852:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.280852:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.280853:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.280854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.280994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.281003:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.281005:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.281008:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.281010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.281011:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.281012:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.281013:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.281014:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.281016:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.281017:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.281020:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.281022:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.281024:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.281025:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.281025:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.281027:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.281027:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.281028:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.281029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.281032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.281039:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.281042:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.281042:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.281044:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.281046:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.281047:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.281051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.281058:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.281060:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.281061:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.281062:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.281062:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.281063:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.281180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.281185:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.281188:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.281189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.281191:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.281193:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.281194:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.281195:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.281197:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.281198:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.281199:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.281202:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.281203:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.281206:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.281206:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.281207:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.281209:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.281209:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.281210:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.281210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.281215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.281223:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.281225:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.281226:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.281227:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.281228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.281229:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.281421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.281427:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.281430:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.281431:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.281433:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.281435:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.281436:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677475.281438:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.281439:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.281440:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.281441:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677475.281445:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.281446:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.281449:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.281449:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677475.281450:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.281452:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.281452:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.281453:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.281453:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677475.281459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.281467:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.281469:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.281470:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.281470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.281473:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.281475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.281838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.281846:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.281849:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.281851:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.281853:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.281854:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.281855:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.281856:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.281857:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.281858:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.281860:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.281863:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.281865:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.281867:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.281868:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.281868:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.281870:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.281870:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.281871:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.281872:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.281875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.281882:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.281885:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.281885:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.281887:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.281889:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.281889:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.282139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.282147:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.282150:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.282151:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.282153:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.282156:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.282157:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.282158:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.282160:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.282161:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.282162:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:21.2:1619677475.282164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:26.0:1619677475.282165:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.282167:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677475.282169:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677475.282170:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.282171:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.282171:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677475.282172:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:26.0:1619677475.282173:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.282173:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:22.0:1619677475.282173:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677475.282174:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.282174:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:6.0:1619677475.282175:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.282177:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.282178:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.282179:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677475.282180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:6.0:1619677475.282181:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.282182:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.282183:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.282187:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.282188:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:6.0:1619677475.282188:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.282190:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.282191:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.282191:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.282192:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.282192:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677475.282193:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.282194:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:6.0:1619677475.282194:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.282194:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.282195:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:6.0:1619677475.282195:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.282196:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:6.0:1619677475.282196:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.282197:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.282197:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.282198:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (24)--
00000800:00000200:20.0:1619677475.282198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (24)--
00000800:00000200:21.0:1619677475.282199:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.282200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:25.2:1619677475.282201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.282208:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.282210:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.282211:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.282213:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.282214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.282215:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.282236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.282246:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.282250:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:13.0:1619677475.282254:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.282254:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677475.282257:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.282260:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677475.282262:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.282264:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.282266:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.282268:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677475.282273:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.282276:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677475.282280:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.282281:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677475.282282:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677475.282284:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677475.282285:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.282287:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.282288:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.282290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.282297:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.282300:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.282301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677475.282301:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.282303:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.282305:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677475.282312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677475.282318:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.282321:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677475.282321:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.282322:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.282323:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.282323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677475.282353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.282357:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.282360:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.282362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.282362:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.282364:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.282366:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.282367:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.282369:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.282370:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.282371:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.282374:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.282376:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.282379:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.282380:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.282380:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.282382:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.282383:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.282384:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.282385:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.282386:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.282388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.282392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677475.282395:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.282397:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.282398:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.282398:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677475.282398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.282400:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.282400:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.282401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.282402:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677475.282403:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.282428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.282433:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.282436:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.282438:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.282438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.282440:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.282442:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.282443:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.282445:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.282446:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.282447:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.282450:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.282452:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.282454:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.282455:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.282455:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.282457:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.282458:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.282459:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.282459:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.282463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.282471:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.282474:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.282475:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.282475:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.282477:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.282480:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.282500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.282508:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.282511:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.282512:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.282514:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.282516:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.282517:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.282519:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.282520:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.282521:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.282522:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.282525:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.282527:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.282530:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.282531:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.282531:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.282533:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.282533:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.282534:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.282535:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.282539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.282547:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.282550:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.282550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.282551:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.282553:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.282555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.282742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.282745:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.282748:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.282750:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.282751:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.282752:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.282754:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677475.282755:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.282756:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.282757:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.282758:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.282761:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.282763:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.282766:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.282766:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.282767:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.282769:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677475.282769:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.282770:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.282771:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.282775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.282782:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.282785:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.282786:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.282787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.282788:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.282791:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.282880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677475.282883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.0:1619677475.282888:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677475.282890:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.0:1619677475.282891:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.282892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677475.282893:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.282893:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.282896:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.282896:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.282899:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.282900:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.282901:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677475.282902:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677475.282902:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.282903:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.282904:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.282905:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.282905:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.282907:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.282907:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677475.282908:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.282914:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.282915:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.282916:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677475.282918:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677475.282919:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.282920:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.282920:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677475.282921:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.282922:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677475.282922:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.282922:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677475.282923:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.282924:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.282924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:8.0:1619677475.282924:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.282925:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.282927:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:9.2:1619677475.282928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677475.282928:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.282928:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.282931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.282934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677475.282935:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.282938:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.282938:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.282939:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:18.0:1619677475.282940:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.282940:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.282942:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.282942:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:2.0:1619677475.282942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677475.282943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.282943:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.282944:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.282945:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677475.282945:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.282946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677475.282948:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.282964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.282972:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.282974:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.282975:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.282975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.282976:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.282977:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.283031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.283036:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.283038:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.283039:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.283041:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.283043:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.283044:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.283045:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.283047:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.283048:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.283049:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.283052:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.283054:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.283057:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.283057:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.283058:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.283060:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.283060:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.283061:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.283062:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.283062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.283066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.283067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.283073:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.283074:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.283075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.283075:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.283076:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.283077:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.283078:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.283079:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.283080:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.283082:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.283086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.283093:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.283095:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.283097:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.283097:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.283098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.283099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.283265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.283273:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.283276:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.283279:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.283281:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.283281:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.283282:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.283283:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.283285:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.283286:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.283287:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.283290:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.283292:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.283299:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.283300:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.283301:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.283302:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.283303:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.283305:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.283307:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.283307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677475.283313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:9.2:1619677475.283319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.283322:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.283325:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.283325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.283326:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677475.283326:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.283329:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.283330:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.283330:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.283332:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.283334:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.283342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.283348:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.283350:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.283351:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.283351:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.283352:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.283352:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.283906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.283912:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.283914:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.283915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.283917:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.283919:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.283921:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.283922:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.283923:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.283924:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.283925:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.283928:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.283930:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.283932:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.283933:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.283933:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.283935:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.283935:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.283936:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.283937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.283941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.283949:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.283952:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.283952:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.283954:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.283955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.283956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.284117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.284124:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.284126:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.284127:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.284129:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.284131:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.284133:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.284134:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.284135:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.284136:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.284137:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.284140:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.284142:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.284144:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.284145:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.284146:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.284148:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.284148:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.284149:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.284150:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.284154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.284163:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.284165:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.284166:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.284166:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.284168:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.284171:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.284393:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.284399:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.284402:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.284402:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.284405:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.284407:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.284409:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677475.284410:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.284412:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.284413:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.284414:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677475.284417:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.284419:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677475.284421:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.284422:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:22.0:1619677475.284423:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677475.284424:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:22.0:1619677475.284425:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.284426:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.284426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.2:1619677475.284431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.284440:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.284442:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.284443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.284444:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.284446:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.284448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.284453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.284460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.284462:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.284463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.284464:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.284465:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.284465:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677475.284571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.284576:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.284578:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.284579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.284581:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.284583:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.284584:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.284585:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.284587:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.284588:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.284589:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.284592:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.284594:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.284596:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.284597:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.284598:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.284599:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.284600:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.284600:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.284601:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.284606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.284614:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.284616:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.284617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.284617:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.284619:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.284621:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.284648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.284656:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.284659:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.284659:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.284661:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.284664:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.284665:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.284667:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.284668:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.284669:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.284670:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.284674:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.284676:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.284679:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.284679:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.284680:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.284682:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.284682:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.284683:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.284685:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.284685:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.284687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.284691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.284695:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.284698:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677475.284698:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.284698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.284699:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.284700:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677475.284701:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.284701:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.284702:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677475.284704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.284832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.284839:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.284842:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.284843:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.284845:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.284847:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.284849:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.284850:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.284851:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.284852:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.284853:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.284856:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.284857:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.284860:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.284861:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.284861:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.284863:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.284864:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.284864:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.284865:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677475.284869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.284878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.284880:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.284881:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.284883:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.284883:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.284885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.284907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.284913:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.284915:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.284916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.284918:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.284920:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.284922:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.284923:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.284925:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.284925:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.284927:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.284929:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.284931:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.284934:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.284934:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.284935:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.284936:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.284937:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.284938:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.284938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.284942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.284950:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.284952:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.284953:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.284954:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.284956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.284956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.285482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.285490:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.285493:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.285496:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.285498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.285498:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.285500:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.285501:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.285502:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.285503:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.285504:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.285508:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.285509:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.285512:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.285513:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.285513:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.285515:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.285516:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.285517:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.285517:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.285521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.285526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:8.0:1619677475.285528:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.285531:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.285532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.285532:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.285534:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000400:00000200:8.0:1619677475.285534:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.285536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.285537:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677475.285538:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.285541:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.285541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677475.285542:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677475.285543:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.285544:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.285545:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.285546:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677475.285549:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.285551:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.285554:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.285555:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677475.285555:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677475.285557:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677475.285558:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.285559:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:31.0:1619677475.285561:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677475.285561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:9.2:1619677475.285563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.285567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:2.0:1619677475.285571:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.285573:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:30.0:1619677475.285575:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000800:00000200:18.0:1619677475.285575:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.285575:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.285577:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000400:00000200:2.0:1619677475.285577:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.285578:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677475.285579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:2.0:1619677475.285580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.285612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.285619:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.285622:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.285623:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.285624:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.285628:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.285629:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.285630:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.285632:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.285632:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.285633:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.285636:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.285638:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.285641:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.285641:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.285642:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.285644:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.285644:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.285645:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.285645:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.285649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.285658:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.285661:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.285661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.285662:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.285664:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.285667:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.285750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.285760:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.285763:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.285767:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.285767:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.285770:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.285773:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:13.0:1619677475.285775:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.285777:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.285779:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.285780:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.2:1619677475.285785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:13.0:1619677475.285786:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.285789:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:22.0:1619677475.285791:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:13.0:1619677475.285793:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.285794:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:13.0:1619677475.285794:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677475.285794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:13.0:1619677475.285795:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:22.0:1619677475.285796:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.285798:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:22.0:1619677475.285799:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.285799:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000400:00000200:22.0:1619677475.285800:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:13.0:1619677475.285800:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000400:00000200:22.0:1619677475.285801:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677475.285801:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:22.0:1619677475.285803:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.285804:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677475.285804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:22.0:1619677475.285805:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.285808:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.285810:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.285813:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.285813:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.285814:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.285814:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677475.285816:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.285816:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.285817:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.285817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:18.0:1619677475.285818:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.285820:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.285820:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:25.2:1619677475.285822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677475.285825:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677475.285826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.285830:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677475.285830:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.285832:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.285833:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.285833:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:21.0:1619677475.285835:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.285835:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677475.285836:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.285836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.285837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677475.285838:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.285839:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.285841:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.286206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.286213:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.286216:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.286219:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.286220:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.286222:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.286223:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.286225:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.286226:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.286227:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.286228:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.286232:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.286234:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.286237:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.286238:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.286239:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.286240:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.286241:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.286242:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.286242:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.286246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.286254:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.286256:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.286257:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.286259:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.286260:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.286262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.286628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.286634:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.286637:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.286638:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.286640:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.286642:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.286643:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.286644:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.286645:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.286647:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.286648:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.286651:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.286652:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.286655:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.286655:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.286656:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.286658:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.286658:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.286659:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.286660:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.286660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.286664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.286666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.286671:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.286673:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.286673:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.286673:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.286674:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.286675:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.286676:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.286678:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.286679:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.286680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.286682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.286689:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.286692:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.286693:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.286693:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.286694:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.286695:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.287000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.287009:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.287012:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.287014:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.287017:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.287017:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.287018:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.287020:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.287021:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.287022:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.287023:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.287026:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.287028:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.287030:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.287030:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.287031:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.287032:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.287033:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.287034:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.287035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.287038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.287045:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.287047:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.287048:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.287049:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.287051:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.287053:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.287379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.287385:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.287389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.287389:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677475.287391:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.287393:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.287395:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.287396:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.287398:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.287399:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.287400:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.287403:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.287405:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.287407:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.287408:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.287409:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.287411:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.287411:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.287412:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.287413:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.287414:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.287417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.287419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.287425:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.287426:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677475.287427:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677475.287427:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.287428:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677475.287429:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.287429:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.287431:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.287432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.287434:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.287639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.287645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.287647:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.287648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.287650:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.287652:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.287653:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.287655:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.287656:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.287657:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.287658:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.287661:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.287663:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.287665:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.287666:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.287667:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.287668:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.287669:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.287669:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.287670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.287675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.287683:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.287685:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.287686:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.287688:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.287689:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.287690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.287761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.287769:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.287771:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.287773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.287774:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.287777:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.287779:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.287780:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.287781:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.287782:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.287783:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.287786:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.287788:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.287791:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.287792:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.287792:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.287794:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.287794:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.287795:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.287796:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.287800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.287807:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:4.2:1619677475.287809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:22.0:1619677475.287810:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.287811:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.287811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.287814:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.287817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677475.287817:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.287820:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.287821:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.287822:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.287825:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.287826:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.287828:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.287829:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.287830:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.287831:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.287834:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.287835:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.287838:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.287839:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.287839:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.287841:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.287841:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.287842:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.287842:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.287847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.287854:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.287857:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.287857:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.287858:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.287860:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.287862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.288502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.288507:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.288510:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.288511:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.288512:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.288515:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.288516:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.288517:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.288518:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.288519:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.288520:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.288524:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.288526:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.288528:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.288529:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.288530:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.288531:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.288532:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.288532:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.288533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.288538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.288547:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.288549:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.288550:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.288551:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.288552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.288554:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.288663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.288668:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.288671:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.288672:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.288674:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.288676:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.288678:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.288679:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.288681:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.288682:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.288683:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.288686:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.288688:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.288691:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.288692:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.288693:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.288695:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.288696:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.288697:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.288697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.288701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.288710:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.288714:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.288715:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.288718:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677475.288718:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.288722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677475.288841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.288844:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.288846:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.288848:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.288849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.288851:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.288853:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.288854:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.288856:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.288857:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.288858:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.288862:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.288864:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.288866:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.288867:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.288868:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.288869:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.288870:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.288870:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.288871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.288876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.288885:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.288887:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.288888:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.288888:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.288890:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.288893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.289406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.289412:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.289415:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.289416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.289418:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.289420:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.289421:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.289423:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.289424:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.289425:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.289426:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.289429:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.289430:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.289432:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.289433:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.289434:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.289435:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.289436:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.289436:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.289437:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.289442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.289451:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.289453:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.289454:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.289455:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.289456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.289457:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.289519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.289527:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.289530:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.289530:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.289532:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.289536:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.289537:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.289539:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.289540:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.289541:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.289542:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.289545:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.289547:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.289550:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.289551:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.289551:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.289553:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.289554:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.289555:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.289555:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.289560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.289567:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.289570:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.289571:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.289571:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.289573:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.289576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.289578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.289584:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.289587:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.289588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.289588:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.289589:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:21.2:1619677475.289590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677475.289590:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677475.289598:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.289601:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.289603:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.289605:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.289605:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.289606:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.289607:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.289608:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.289609:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.289610:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.289613:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.289615:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.289617:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.289618:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.289618:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.289619:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.289620:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.289622:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.289623:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.289624:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.289625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.289630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.289632:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.289635:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.289636:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.289636:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.289638:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.289638:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.289640:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677475.289640:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.289641:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.289642:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677475.289864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.289871:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.289873:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.289874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.289876:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.289878:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.289880:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.289881:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.289882:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.289883:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.289884:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.289888:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.289889:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.289892:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.289893:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.289893:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.289895:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.289896:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.289896:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.289897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.289901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.289909:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.289911:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.289912:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.289912:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.289915:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.289917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.290344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.290348:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.290351:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.290353:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.290354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.290355:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.290357:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.290358:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.290359:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.290360:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.290361:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.290364:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.290366:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.290368:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.290369:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.290369:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.290371:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.290372:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.290373:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.290373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.290378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.290386:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.290388:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.290389:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.290389:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.290391:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.290394:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.290438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.290444:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.290446:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.290447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.290449:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.290451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.290452:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.290453:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.290455:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.290456:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.290457:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.290460:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.290462:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.290464:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.290465:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.290465:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.290467:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.290467:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.290468:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.290469:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.290470:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.290473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.290475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.290480:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.290481:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.290482:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.290482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.290483:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.290484:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.290485:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.290487:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.290487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.290489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.290498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.290505:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.290508:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.290509:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.290509:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.290510:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.290511:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.290606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:10.0:1619677475.290613:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677475.290616:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677475.290617:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:10.0:1619677475.290619:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.290622:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.290624:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677475.290625:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.290626:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.290627:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.290628:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.290632:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.290633:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677475.290636:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.290637:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677475.290638:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677475.290639:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677475.290640:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677475.290641:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677475.290641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:9.2:1619677475.290646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.290654:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.290657:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.290658:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.290658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.290660:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.290664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.290742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.290750:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.290753:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.290755:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.290757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.290757:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.290759:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.290760:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.290761:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.290762:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.290763:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.290766:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.290768:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.290770:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.290770:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.290771:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.290772:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.290773:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.290774:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.290774:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.290778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.290785:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.290787:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.290788:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.290789:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.290791:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.290792:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.290997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.291004:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.291007:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.291008:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.291010:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.291012:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.291014:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677475.291015:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.291016:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.291017:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.291018:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677475.291021:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.291022:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677475.291024:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.291025:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677475.291026:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677475.291027:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677475.291028:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.291028:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.291029:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.291033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.291039:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.291042:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.291043:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.291044:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.291046:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.291047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.291296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.291302:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.291305:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.291306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.291308:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.291310:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.291312:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.291313:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.291314:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.291315:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.291316:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.291320:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.291321:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.291324:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.291324:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.291325:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.291327:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.291327:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.291328:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.291328:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.291333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677475.291336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.0:1619677475.291342:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.291344:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677475.291344:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677475.291345:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.291346:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.291347:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677475.291347:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.291348:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677475.291348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677475.291350:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.291352:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.291354:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.291355:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.291356:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.291357:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.291358:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.291362:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.291363:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.291366:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.291367:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.291368:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.291369:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.291370:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.291371:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.291371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.291375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.291384:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.291386:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.291387:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.291388:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.291390:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.291393:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.291406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677475.291414:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.291417:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677475.291419:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.291421:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.291421:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677475.291423:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677475.291424:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.291425:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.291426:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.291427:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677475.291430:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.291432:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677475.291434:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.291435:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:31.0:1619677475.291436:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677475.291437:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:31.0:1619677475.291438:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.291439:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:31.0:1619677475.291441:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:31.0:1619677475.291442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:25.2:1619677475.291443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.2:1619677475.291447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:13.0:1619677475.291450:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677475.291453:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677475.291453:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.291455:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000400:00000200:13.0:1619677475.291455:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.291457:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:13.0:1619677475.291457:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:30.0:1619677475.291458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:12.0:1619677475.291458:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:30.0:1619677475.291459:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:25.2:1619677475.291463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:13.0:1619677475.291469:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677475.291472:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677475.291472:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677475.291473:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677475.291474:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677475.291475:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677475.291625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.291632:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.291635:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.291637:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.291639:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.291639:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.291640:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.291642:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.291643:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.291644:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.291645:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.291648:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.291650:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.291652:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.291652:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.291653:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.291654:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.291655:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.291656:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.291656:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.291660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.291666:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.291669:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.291670:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.291670:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.291672:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.291674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.292126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.292132:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.292135:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.292135:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.292137:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.292140:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.292141:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.292142:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.292144:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.292145:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.292145:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.292148:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.292150:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.292152:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.292153:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.292153:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.292155:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.292155:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.292156:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.292157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.292161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.292168:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.292171:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.292171:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.292173:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.292175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.292175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.292502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.292511:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.292514:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.292516:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.292518:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.292518:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.292520:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.292521:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.292522:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.292523:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.292524:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.292528:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.292529:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.292531:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.292532:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.292533:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.292534:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.292535:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.292536:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.292536:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.292541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.292547:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.292550:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.292551:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.292552:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.292554:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.292555:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.292557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.292564:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.292566:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.292567:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.292567:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.292568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.292569:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.292652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.292660:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.292663:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.292665:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.292668:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.292668:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.292669:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677475.292670:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.292672:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.292673:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.292673:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677475.292676:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.292678:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677475.292680:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.292681:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677475.292681:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677475.292683:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677475.292684:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.292684:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.292685:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677475.292689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.292695:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.292698:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.292698:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.292700:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.292702:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.292703:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.292718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.292724:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.292727:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.292727:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.292728:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.292728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.292729:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677475.292801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.292810:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.292815:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.292817:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.292821:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.292825:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.292827:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.292829:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.292832:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.292834:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.292836:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.292841:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.292844:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.292847:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.292848:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.292849:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.292851:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.292852:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.292853:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.292854:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.292857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.292866:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.292870:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.292871:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.292872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.292874:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.292876:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.292976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.292984:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.292987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.292987:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677475.292990:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.292992:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.292993:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.292995:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.292996:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.292997:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.292998:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.293002:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.293003:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.293006:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.293007:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.293008:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.293010:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.293010:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.293010:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.293011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.293016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.293024:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.293027:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.293028:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.293028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.293030:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.293033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.293034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.293041:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.293043:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.293044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.293044:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.293045:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.293046:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.293086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.293092:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.293094:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.293096:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.293097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.293099:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.293101:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.293102:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.293103:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.293104:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.293105:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.293108:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.293110:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.293112:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.293113:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.293113:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.293115:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.2:1619677475.293116:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:21.0:1619677475.293119:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.293120:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:8.2:1619677475.293121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677475.293122:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.293122:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.293124:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:22.0:1619677475.293125:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.293125:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.293126:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:22.0:1619677475.293127:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.293127:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.293127:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.293128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000400:00000200:22.0:1619677475.293129:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.293130:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677475.293130:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000400:00000200:22.0:1619677475.293131:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.293131:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.293132:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:22.0:1619677475.293133:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.293133:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000400:00000200:22.0:1619677475.293134:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.293135:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.293135:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677475.293135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.293136:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:22.0:1619677475.293138:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.293139:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.2:1619677475.293141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:22.0:1619677475.293142:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.293142:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.293143:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.293145:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.293145:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.293146:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.293146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.293149:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:25.2:1619677475.293151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.293151:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.293152:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.293153:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.293154:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.293155:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.293156:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.293158:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.293159:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.293161:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.293162:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.293163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.293267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677475.293273:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.293276:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677475.293277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677475.293279:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677475.293281:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.293283:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677475.293284:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.293285:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.293287:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.293287:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677475.293291:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.293293:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677475.293295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677475.293299:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.293300:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677475.293301:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.293303:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677475.293303:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.293303:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.293304:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677475.293304:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:26.0:1619677475.293306:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.293306:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.293309:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:25.2:1619677475.293309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677475.293312:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.293313:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.293315:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.293316:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.293317:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.293317:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:26.0:1619677475.293318:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677475.293319:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.293320:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677475.293321:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:6.0:1619677475.293321:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.293323:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:6.0:1619677475.293323:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:26.0:1619677475.293325:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.293325:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677475.293326:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.293327:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.293329:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.293329:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.293330:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.293331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.293335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.293341:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.293344:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.293345:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.293346:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.293348:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.293349:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.293518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.293525:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.293527:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.293529:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.293530:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.293533:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.293535:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.293536:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.293537:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.293538:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.293539:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.293542:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.293543:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.293546:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.293547:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.293547:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.293549:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.293550:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.293551:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.293552:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.293553:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.293555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.293558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.293562:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.293565:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.293565:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.293566:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.293566:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.293567:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.293568:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677475.293568:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.293569:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.293572:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.293980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.293986:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.293988:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.293989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.293991:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.293993:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.293994:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.293995:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.293996:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.293997:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.293998:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.294002:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.294003:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.294006:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.294006:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.294007:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.294009:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.294009:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.294010:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.294011:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.294012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.294015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.294017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.294022:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.294024:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.294024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.294024:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.294025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.294027:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.294027:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.294029:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.294030:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.294031:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.294041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.294048:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.294051:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.294052:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.294052:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.294053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.294054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.294302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.294307:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.294310:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.294312:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:4.2:1619677475.294312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:21.0:1619677475.294313:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.294314:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677475.294314:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.294316:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.294317:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.294318:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.294319:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.294319:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.294322:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.294323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:21.0:1619677475.294323:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.294325:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.294326:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.294328:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.294328:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.294329:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677475.294329:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.294330:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:10.0:1619677475.294330:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:21.0:1619677475.294331:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:10.0:1619677475.294331:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.294332:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.294332:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:10.0:1619677475.294333:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.294334:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.294335:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677475.294337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677475.294338:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.294340:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.294343:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.294344:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.294344:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.294345:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.294346:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.294346:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.294347:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677475.294347:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.294348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.294348:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.294348:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.294351:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:25.2:1619677475.294352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.294355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.294357:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.294359:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.294359:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.294360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.294361:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.294363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.294820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.294825:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.294828:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.294828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.294830:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.294832:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.294834:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.294835:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.294836:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.294837:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.294838:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.294841:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.294843:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.294845:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.294846:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.294846:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.294848:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.294848:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.294849:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.294850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.294854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.294862:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.294864:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.294865:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.294867:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.294868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.294869:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.295040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.295045:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.295048:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.295050:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.295050:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.295052:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.295053:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.295055:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.295056:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.295057:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.295058:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.295061:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.295063:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.295065:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.295066:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.295066:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.295068:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.295069:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.295069:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.295070:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.295075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.295084:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.295086:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.295087:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.295087:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.295089:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.295092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.295093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.295101:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.295103:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.295104:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.295105:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.295105:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.295106:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677475.295115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.295120:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.295123:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.295124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.295125:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.295128:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.295129:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.295130:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.295131:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.295132:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.295133:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.295137:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.295138:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.295141:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.295142:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.295143:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.295144:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.295145:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.295145:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.295146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.295150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.295158:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.295160:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.295161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.295161:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.295164:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.295166:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.295650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.295658:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.295661:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.295663:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.295665:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.295666:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.295666:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.295668:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.295669:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.295670:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.295671:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.295674:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.295676:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.295678:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.295679:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.295680:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.295681:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.295682:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.295683:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.295685:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.295685:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.295688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.295691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.295695:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.295697:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.295698:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.295699:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677475.295699:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.295700:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.295701:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.295702:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.295703:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677475.295703:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.295707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.295713:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.295715:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.295716:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.295717:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.295717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.295718:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.295817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.295823:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.295827:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.295828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.295829:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.295831:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.295833:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.295834:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.295836:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.295836:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.295838:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.295841:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.295843:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.295845:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.295846:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.295847:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.295848:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.295849:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.295850:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.295850:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.295854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.295862:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.295864:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.295865:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.295867:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.295868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.295869:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.295877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.295883:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.295887:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.295888:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.295890:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.295891:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.295893:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.295894:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.295896:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.295897:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.295898:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.295902:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.295904:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.295907:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.295908:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.295908:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.295910:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.295911:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.295912:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.295913:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.295914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.2:1619677475.295919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:25.2:1619677475.295925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.295925:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.295927:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.295927:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.295928:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:31.0:1619677475.295932:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.295935:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.295936:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.295938:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.295939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.295941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677475.296081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.296087:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.296090:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.296091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.296093:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.296095:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.296097:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:2.0:1619677475.296098:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.296100:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.296101:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.296101:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:2.0:1619677475.296104:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.296106:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:2.0:1619677475.296109:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.296110:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:2.0:1619677475.296110:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677475.296112:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:2.0:1619677475.296112:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.296113:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677475.296114:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677475.296114:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677475.296118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677475.296120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:26.0:1619677475.296125:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:18.0:1619677475.296125:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677475.296128:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677475.296128:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:26.0:1619677475.296129:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:2.0:1619677475.296129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:26.0:1619677475.296130:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.296130:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.296132:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.296135:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.296342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.296350:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.296353:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.296355:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.296357:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.296357:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.296358:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.296359:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.296361:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.296362:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.296363:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.296366:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.296368:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.296370:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.296371:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.296371:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.296373:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.296373:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.296374:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.296375:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.296379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.296386:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.296389:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.296389:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.296390:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.296391:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.296393:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.296673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.296679:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.296681:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.296682:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.296684:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.296687:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.296688:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.296689:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.296690:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.296691:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.296693:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.296695:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.296697:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.296699:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.296700:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.296701:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.296702:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.296703:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.296703:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.296704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.296709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.296717:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.296719:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.296720:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.296721:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.296723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.296724:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.296994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.297002:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.297005:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.297007:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.297009:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.297009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.297011:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.297012:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.297014:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.297014:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.297016:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.297019:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.297021:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.297023:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.297024:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.297025:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.297026:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.297027:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.297028:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.297028:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.297032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.297038:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.297041:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.297042:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.297042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.297044:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.297046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.297534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.297540:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.297543:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.297544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.297545:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.297548:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.297550:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.297551:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.297552:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.297553:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.297554:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.297558:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.297559:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.297561:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.297562:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.297563:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.297564:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.297565:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.297566:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.297567:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.297567:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.297571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.297573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.297578:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.297579:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.297580:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.297580:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.297581:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.297581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.297582:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.297583:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.297585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.297586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.297592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.297600:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.297602:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.297603:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.297604:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.297604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.297605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.297893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.297901:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.297903:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.297904:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.297906:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.297909:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.297910:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.297912:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.297913:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.297914:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.297915:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.297918:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.297920:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.297923:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.297924:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.297924:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.297926:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.297926:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.297928:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.297929:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.297930:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.297932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.297935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.297939:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.297942:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.297942:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.297943:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.297943:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.297945:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.297946:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677475.297946:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.297947:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677475.297948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.297951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.297958:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.297960:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.297961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.297961:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.297962:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.297963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.297995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.298001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:30.0:1619677475.298003:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.298006:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.298006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:30.0:1619677475.298009:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.298009:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:31.0:1619677475.298011:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.298011:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.298011:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.298011:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:30.0:1619677475.298013:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.298014:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.298014:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.298016:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.298016:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.298017:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.298018:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.298020:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:21.0:1619677475.298020:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.298021:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.298022:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.298024:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.298026:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:21.0:1619677475.298028:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677475.298029:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.298030:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.298030:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677475.298031:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.298032:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.298032:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.298033:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.298033:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.298034:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677475.298034:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.298035:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.298037:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.298037:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:9.2:1619677475.298037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677475.298038:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.298038:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.298042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.298047:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.298049:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.298050:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.298051:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.298051:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.298052:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.298053:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677475.298054:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.298055:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.298056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677475.298057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.298057:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.298113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.298120:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.298123:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.298124:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.298126:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.298128:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.298130:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.298131:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.298132:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.298133:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.298134:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.298138:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.298140:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.298143:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.298143:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.298144:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.298146:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.298146:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.298147:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.298147:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.298152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.298159:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.298162:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.298163:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.298163:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.298165:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.298168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.298212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677475.298218:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.298222:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677475.298222:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677475.298225:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677475.298227:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.298229:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677475.298230:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.298231:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.298232:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.298233:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.298237:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.298239:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.298241:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.298242:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677475.298243:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677475.298245:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677475.298245:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.298247:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:17.0:1619677475.298248:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:17.0:1619677475.298248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:9.2:1619677475.298250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.298253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:8.0:1619677475.298257:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:1.0:1619677475.298259:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:24.0:1619677475.298260:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.298260:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.298261:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.298261:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677475.298261:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677475.298262:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000400:00000200:8.0:1619677475.298263:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.298266:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.298431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.298434:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.298437:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.298439:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.298440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.298442:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.298443:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.298445:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.298446:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.298447:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.298448:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.298451:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.298452:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.298454:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.298455:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.298455:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.298457:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.298458:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.298459:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.298460:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.298460:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.298464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.298467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.298470:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.298471:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.298472:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677475.298472:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.298473:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677475.298473:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.298474:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.298475:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.298476:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.298476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677475.298601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.298607:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.298609:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.298610:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.298612:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.298614:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.298615:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.298617:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.298618:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.298619:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.298620:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.298623:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.298625:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.298627:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.298628:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.298628:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.298630:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.298630:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.298631:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.298632:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.298637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.298645:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.298647:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.298648:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.298650:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.298650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.298652:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.299154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.299162:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.299165:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.299166:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.299168:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.299171:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.299172:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.299174:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.299175:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.299176:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.299177:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.299180:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.299182:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.299185:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.299185:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.299186:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.299188:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.299188:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.299189:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.299189:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.299194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.299201:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.299204:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.299205:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.299205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.299207:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.299210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.299212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.299218:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.299221:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.299222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.299222:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.299223:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.299224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.299397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cefbce000] (20)++
00000800:00000200:17.0:1619677475.299404:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cefbce000] (21)++
00000800:00000200:17.0:1619677475.299408:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.68@o2ib3
00000800:00000200:1.0:1619677475.299408:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cefbce000] (22)--
00000400:00000200:17.0:1619677475.299411:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.0.68@o2ib3 : GET - for me
00000400:00000200:17.0:1619677475.299413:0:19930:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.0.68@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:17.0:1619677475.299415:0:19930:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.0.68@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:17.0:1619677475.299416:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cefbce000] (21)++
00000800:00000200:17.0:1619677475.299418:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cefbce000] (22)--
00000400:00000200:17.0:1619677475.299420:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.0.68@o2ib3 local destination
00000400:00000200:17.0:1619677475.299424:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.0.68@o2ib3(10.51.0.68@o2ib3:10.51.0.68@o2ib3) : REPLY try# 0
00000800:00000200:17.0:1619677475.299426:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.0.68@o2ib3
00000800:00000200:17.0:1619677475.299428:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918ce53e8c80] -> 10.51.0.68@o2ib3 (2) version: 12
00000800:00000200:17.0:1619677475.299429:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918cefbce000] (21)++
00000800:00000200:17.0:1619677475.299430:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cefbce000] (22)++
00000800:00000200:17.0:1619677475.299432:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918cefbce000] (23)--
00000800:00000200:17.0:1619677475.299432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cefbce000] (22)--
00000800:00000200:21.2:1619677475.299436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cefbce000] (21)++
00000800:00000200:1.0:1619677475.299442:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cefbce000] (22)++
00000800:00000200:1.0:1619677475.299443:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cefbce000] (23)--
00000400:00000200:1.0:1619677475.299444:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.299445:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cefbce000] (22)--
00000400:00000200:1.0:1619677475.299446:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.0.68@o2ib3: REPLY: OK
00000800:00000200:1.0:1619677475.299448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cefbce000] (21)--
00000800:00000200:21.2:1619677475.299470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.299476:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.299478:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.299479:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.299481:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.299483:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.299484:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.299485:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.299487:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.299488:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.299489:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.299492:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.299493:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.299496:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.299496:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.299497:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.299499:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.299499:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.299500:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.299500:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.299505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.299513:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.299516:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.299516:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.299518:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.299520:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.299521:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.299665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.299671:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.299674:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.299675:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.299678:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.299681:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.299682:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.299683:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.299685:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.299686:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.299687:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.299690:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.299692:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.299693:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.299694:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.299695:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.299696:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.299697:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.299698:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.299698:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.299703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.299709:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.299711:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.299712:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.299713:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.299714:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.299715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.299856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.299862:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.299865:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.299866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.299868:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.299870:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.299871:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.299873:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.299874:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.299875:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.299876:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.299879:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.299881:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.299883:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.299884:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.299885:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.299886:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.299886:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.299887:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.299888:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.299892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.299901:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.299903:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.299904:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.299905:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.299906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.299908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.300180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.300189:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.300192:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.300194:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.300196:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.300196:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.300198:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.300199:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.300200:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.300201:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.300202:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.300206:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.300208:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.300210:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.300211:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.300211:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.300213:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.300213:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.300215:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.300216:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.300217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.300219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.300222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.300227:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.300229:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.300229:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.300231:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.300231:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.300232:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.300233:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.300233:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.300234:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.300235:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677475.300323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.300329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.300332:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.300333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.300335:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.300337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:1.0:1619677475.300337:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.300338:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.300340:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.300341:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.300342:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.300343:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:1.0:1619677475.300343:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.300346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.300346:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:1.0:1619677475.300347:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.300349:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:6.0:1619677475.300350:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.300351:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.300352:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.300352:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.300353:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677475.300354:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.300355:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.300355:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.300355:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:6.0:1619677475.300356:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.300356:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000400:00000200:6.0:1619677475.300357:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.300357:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000400:00000200:6.0:1619677475.300358:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677475.300358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.300361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677475.300362:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677475.300362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677475.300363:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.300366:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.300367:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677475.300368:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:6.0:1619677475.300368:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.300370:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677475.300370:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.300370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:6.0:1619677475.300370:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.300371:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677475.300371:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677475.300371:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:22.0:1619677475.300372:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.300372:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.300373:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:22.0:1619677475.300374:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.300374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:25.2:1619677475.300376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.300376:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677475.300384:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.300386:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.300387:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.300388:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.300390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.300391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.301419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.301426:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.301428:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.301429:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.301431:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.301433:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.301434:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.301435:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.301437:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.301438:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.301439:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.301442:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.301443:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.301445:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.301446:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.301446:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.301447:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.301448:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.301449:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.301450:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.301450:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.301454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.301455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.301460:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.301462:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.301462:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.301463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.301464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.301465:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.301465:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.301467:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.301469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.301469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:25.2:1619677475.301477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.301485:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.301487:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.301488:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.301488:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.301489:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.301490:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.301543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.301548:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.301550:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.301552:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.301553:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.301554:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.301556:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.301557:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.301558:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.301559:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.301560:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.301563:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.301564:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.301565:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.301566:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.301567:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.301568:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.301569:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.301570:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.301571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.301575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.301583:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.301585:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.301586:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.301588:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.301589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.301590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.301660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.301667:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.301670:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.301671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.301672:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.301675:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.301676:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.301678:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.301679:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.301680:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.301681:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.301684:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.301685:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.301688:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.301688:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.301689:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.301690:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.301691:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.301692:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.301693:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.301697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.301705:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.301707:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677475.301708:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.301709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.301711:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.301713:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.301962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.301970:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.301973:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.301975:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.301977:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.301977:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.301979:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.301980:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.301981:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.301982:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.301983:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.301987:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.301988:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.301990:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.301991:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.301992:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.301994:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.301994:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.301995:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.301996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.301999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.302006:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.302009:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.302009:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.302011:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.302013:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.302014:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.302083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.302088:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.302091:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.302093:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.302094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.302096:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.302097:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.302099:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.302100:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.302101:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.302102:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.302105:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.302106:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.302109:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.302109:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.302110:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.302112:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.302112:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.302113:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.302114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.302118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.302126:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.302128:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.302129:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.302129:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.302131:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.2:1619677475.302134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:17.0:1619677475.302134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677475.302142:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.302144:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.302146:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.302148:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.302148:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.302150:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.302151:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.302152:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.302153:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.302154:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.302157:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.302159:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.302162:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.302163:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.302164:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.302166:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.302166:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.302167:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.302168:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.302172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.302179:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.302182:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.302183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.302183:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.302185:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.302189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.302288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.302293:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.302298:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.302303:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677475.302305:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.302307:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.302308:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.302309:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.302310:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.302312:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.302312:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.302316:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.302317:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.302320:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.302321:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.302321:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.302323:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.302323:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.302324:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.302324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.302330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.302338:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.302340:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.302341:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.302342:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.302344:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.302345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.302687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.302695:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.302698:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.302701:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.302703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.302704:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.302705:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.302707:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.302708:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.302709:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.302710:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.302713:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.302715:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.302718:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.302719:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.302719:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.302721:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.302721:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.302722:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.302723:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.302726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.302733:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.302736:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.302737:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.302737:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.302739:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.302741:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.302745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.302751:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.302754:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.302754:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.302754:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.302755:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.302756:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.302935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.302943:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.302946:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.302946:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.302948:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.302951:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.302952:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.302953:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.302955:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.302956:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.302956:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.302960:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.302961:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.302964:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.302965:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.302965:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.302967:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.302967:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.302968:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.302968:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.302973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.302981:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.302983:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.302984:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.302984:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.302986:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.302989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.303015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.303022:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.303024:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.303025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.303027:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.303030:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.303031:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.303032:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.303034:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.303035:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.303036:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.303039:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.303040:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.303043:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.303044:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.303044:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.303046:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.303047:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.303047:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.303048:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.303052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.303059:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.303062:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.303063:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.303063:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.303065:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.303067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.303071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.303078:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.303080:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.303081:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.303082:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.303083:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.303083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.303138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.303143:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.303145:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.303146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.303148:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.303150:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.303151:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.303152:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.303153:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.303154:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.303155:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.303159:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.303160:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.303163:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.303164:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.303164:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.303166:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.303166:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.303167:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.303167:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.303172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.303180:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.303182:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.303183:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.303185:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.303185:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.303187:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.303718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.303725:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.303728:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.303731:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.303733:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.303733:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.303734:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677475.303736:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.303737:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.303738:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.303739:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677475.303743:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.303744:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677475.303747:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.303747:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677475.303748:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677475.303750:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677475.303750:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.303752:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677475.303754:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677475.303754:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:25.2:1619677475.303756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.303760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:30.0:1619677475.303764:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.303767:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.303767:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.303767:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:30.0:1619677475.303769:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.303769:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677475.303770:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:30.0:1619677475.303771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677475.303771:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:31.0:1619677475.303772:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.303941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.303948:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.303951:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.303952:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.303954:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.303956:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.303958:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.303959:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.303960:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.303961:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.303962:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.303965:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.303966:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.303969:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.303970:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.303971:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.303972:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.303973:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.303974:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.303975:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.303976:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.303979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.303981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.303986:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677475.303988:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.303989:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677475.303990:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677475.303990:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.303990:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.303991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.303992:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.303992:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.303995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.304160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.304165:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.2:1619677475.304167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.304168:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.304168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.304170:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.304172:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.304174:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677475.304174:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.304174:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.304176:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.304177:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.304178:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:22.0:1619677475.304178:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.304181:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.304181:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.304182:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.304183:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.304185:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677475.304185:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:30.0:1619677475.304186:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.304187:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.304188:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.304188:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.304188:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.304189:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.304190:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.304190:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.304190:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.304191:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.304192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:30.0:1619677475.304193:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.304195:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:25.2:1619677475.304196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677475.304197:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.304198:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.304199:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.304200:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.304200:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.304202:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.304202:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.304204:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:25.2:1619677475.304206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677475.304206:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.304207:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.304208:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.304209:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.304211:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.304212:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.304215:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.304216:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.304218:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.304219:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.304220:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.304297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.304303:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.304306:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.304307:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.304309:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.304311:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.304313:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677475.304314:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.304315:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.304316:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.304317:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677475.304320:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.304322:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677475.304324:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.304325:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677475.304325:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677475.304327:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677475.304328:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.304328:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.304329:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677475.304336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.304344:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.304347:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.304348:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.304348:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.304350:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.304352:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677475.304354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677475.304362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.304364:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677475.304365:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.304366:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.304366:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.304367:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.304469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.304477:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.304480:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.304482:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.304485:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.304485:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.304486:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.304488:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.304489:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.304490:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.304491:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.304494:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.304496:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.304500:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.304500:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.304501:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.304503:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.304503:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.304504:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.304505:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.304519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.304528:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.304531:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.304531:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.304532:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.304534:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.304537:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.305018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.305023:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.305026:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.305027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.305029:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.305031:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.305032:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.305033:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.305034:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.305035:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.305036:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.305040:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.305041:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.305043:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.305044:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.305045:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.305046:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.305047:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.305048:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.305049:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.305050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.305053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.305055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.305060:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.305061:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.305062:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.305062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.305063:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.305064:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.305065:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.305066:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.305068:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.305069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.305072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.305080:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.305082:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.305083:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.305084:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.305085:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.305086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.305151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.305155:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.305157:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.305160:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.305161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.305161:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.305163:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.305164:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.305165:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.305166:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.305167:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.305171:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.305172:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.305175:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.305175:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.305176:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.305177:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.305178:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.305179:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.305180:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.305181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677475.305186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.305191:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:25.2:1619677475.305194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.305194:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.305194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.305195:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:22.0:1619677475.305202:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.305204:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.305205:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.305205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.305207:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.305210:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.305336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.305344:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.305347:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.305349:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.305352:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.305352:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.305353:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.305355:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.305356:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.305357:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.305358:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.305361:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.305362:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.305365:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.305366:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.305366:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.305368:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.305368:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.305370:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.305371:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.305372:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.305374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.305378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.305381:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.305383:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.305384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.305384:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.305385:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.305386:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.305388:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677475.305388:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.305389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.305390:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:5.2:1619677475.305495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.305502:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.305504:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.305505:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.305508:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.305510:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.305511:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.305512:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.305514:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.305515:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.305516:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.305519:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.305521:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.305523:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.305524:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.305525:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.305526:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.305527:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.305527:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.305528:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.305532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.305540:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.305543:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.305544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.305544:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.305546:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.305548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.305857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.305862:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.305865:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.305865:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.305868:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.305870:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.305871:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.305872:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.305873:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.305875:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.305876:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.305879:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.305880:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.305883:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.305883:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.305884:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.305886:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.305886:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.305887:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.305887:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.305892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.305899:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.305902:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.305902:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.305904:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.305905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.305906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.306342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.306350:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.306353:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.306356:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.306358:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.306359:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.306360:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.306362:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.306363:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.306364:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.306365:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.306368:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.306370:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.306374:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.306375:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.306375:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.306377:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.306378:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.306379:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.306379:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.306384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.306392:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.306395:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.306396:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.306396:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.306398:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.306401:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.306903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.306908:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.306910:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.306911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.306913:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.306915:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.306916:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.306918:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.306919:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.306920:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.306921:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.306924:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.306926:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.306929:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.306930:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.306930:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.306932:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.306932:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.306933:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.306933:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.306938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.306946:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.306948:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.306949:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.306951:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.306952:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.306953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.306983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.306989:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.306992:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.306993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.306995:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.306998:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.306999:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.307000:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.307002:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.307003:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.307004:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.307007:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.307009:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.307012:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.307012:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.307013:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.307015:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.307016:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.307017:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.307017:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.307021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.307029:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.307031:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.307032:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.307034:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.307037:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.307038:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.307043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.307051:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.307054:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.307056:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.307058:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.307058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.307059:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.307060:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.307062:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.307063:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.307064:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.307067:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.307068:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.307071:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.307071:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.307072:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.307074:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.307074:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.307075:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.307076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.307079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.307087:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.307089:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.307090:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.307090:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.307092:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.307094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.307649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.307656:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.307658:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.307659:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.307661:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.307664:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.307665:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.307667:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.307668:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.307669:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.307670:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.307673:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.307675:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.307676:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.307677:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.307678:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.307679:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.307680:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.307681:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.307683:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.307683:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.307686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.307689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.307695:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.307696:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.307696:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.307697:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.307697:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.307697:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.307698:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.307699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677475.307699:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.307700:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.307783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.307789:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.307792:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.307793:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.307795:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.307797:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.307798:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.307800:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.307801:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.307802:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.307803:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.307806:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.307808:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.307810:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.307811:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.307811:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.307813:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.307813:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.307814:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.307815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.307819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.307827:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.307829:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.307830:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.307832:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.307832:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.307834:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.307964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.307972:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.307975:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.307976:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.307977:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.307980:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.307982:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.307983:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.307984:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.307985:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.307986:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.307990:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.307991:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.307994:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.307995:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.307996:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.307997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.307998:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.307998:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.307999:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.307999:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.308004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.308005:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.308007:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677475.308009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:30.0:1619677475.308010:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.308011:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677475.308012:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.308012:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:31.0:1619677475.308013:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.308013:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.308014:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.308014:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.308015:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.308015:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:6.0:1619677475.308015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677475.308016:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.308017:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.308018:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.308018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.308018:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677475.308019:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.308020:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:20.0:1619677475.308020:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.308022:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:25.2:1619677475.308024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000400:00000200:20.0:1619677475.308024:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.308025:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677475.308026:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.308027:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.308028:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:20.0:1619677475.308028:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.308031:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.308031:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.308032:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.308032:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:20.0:1619677475.308032:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677475.308033:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.308034:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.308034:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.308034:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.308034:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677475.308035:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.308036:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677475.308036:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.308037:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.308037:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.308037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.308038:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.308039:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:9.2:1619677475.308039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677475.308040:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.308041:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.308042:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.308043:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677475.308045:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:9.2:1619677475.308047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.308048:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.308050:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.308051:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.308052:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.308053:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.308054:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.308054:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.308055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:24.0:1619677475.308055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.308056:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.308059:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.308066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.308072:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.308075:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.308075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.308076:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.308077:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.308077:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.308333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.2:1619677475.308339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677475.308341:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.308344:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.0:1619677475.308345:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677475.308346:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.308347:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677475.308348:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.308348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:21.0:1619677475.308349:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.308350:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:4.2:1619677475.308350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677475.308351:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.308352:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.308352:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.308352:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677475.308354:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.308354:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677475.308356:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.308356:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.308358:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.308358:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:21.0:1619677475.308359:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.308360:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.308361:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677475.308362:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.308362:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677475.308363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677475.308365:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.308365:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.308366:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.308369:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.308369:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:10.0:1619677475.308369:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.308370:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.308370:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.308371:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.0:1619677475.308371:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677475.308371:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:31.0:1619677475.308372:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.308372:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:10.0:1619677475.308372:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96500554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.308373:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:26.0:1619677475.308373:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:21.0:1619677475.308373:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.308374:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:10.0:1619677475.308374:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.308375:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:21.0:1619677475.308375:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677475.308375:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.308376:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677475.308376:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677475.308377:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.308377:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677475.308377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:26.0:1619677475.308379:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.308381:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.308381:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677475.308381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:26.0:1619677475.308383:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677475.308383:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.308384:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96500553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.308384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:26.0:1619677475.308386:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.308386:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.308387:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677475.308390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.308390:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:26.0:1619677475.308391:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.308391:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.308392:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:10.0:1619677475.308392:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.308393:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.308393:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.308393:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:30.0:1619677475.308394:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:17.0:1619677475.308394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.308394:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.308395:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:10.0:1619677475.308395:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:30.0:1619677475.308396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:26.0:1619677475.308396:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.308396:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:1.0:1619677475.308396:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.308397:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.308397:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.308397:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:26.0:1619677475.308398:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677475.308398:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:10.0:1619677475.308399:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677475.308400:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677475.308401:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (25)--
00000800:00000200:25.2:1619677475.308401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.308402:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.308402:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (25)--
00000800:00000200:26.0:1619677475.308403:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.308403:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.308403:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.308405:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.308405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.308407:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.308409:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.308409:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.308409:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.308410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.308412:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.308414:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.308418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677475.308422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.308427:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.308429:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.308429:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.308430:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677475.308431:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.308432:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.308433:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.308433:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.308434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.308434:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.308435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677475.308435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:21.2:1619677475.308648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.308653:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.308655:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.308656:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.308658:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.308660:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.308661:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.308662:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.308664:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.308665:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.308666:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.308669:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.308671:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.308673:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.308674:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.308675:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.308676:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.308677:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.308678:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.308679:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.308679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.308683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.308684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.308689:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.308690:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.308691:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.308692:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.308692:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.308693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.308693:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.308695:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.308696:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.308697:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.308703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.308710:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.308713:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.308714:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.308715:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.308715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.308716:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.309162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677475.309170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.309172:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.309175:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.309176:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.309178:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.309180:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.309181:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.309183:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.309184:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.309185:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.309188:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.309190:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.309193:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.309193:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.309194:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.309195:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.309196:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.309197:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677475.309198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.309201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.309208:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.309210:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.309211:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.309212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.309213:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.309216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.309267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.309274:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.309277:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.309277:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.309279:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.309282:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.309284:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.309285:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.309286:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.309287:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.309288:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.309290:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.309292:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.309301:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.309301:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.309302:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.309303:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.309304:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.309305:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.309305:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.309310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.309318:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.309320:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.309321:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.309321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:12.0:1619677475.309323:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.309325:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.309326:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.0:1619677475.309327:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.309329:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:25.2:1619677475.309330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:21.0:1619677475.309332:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.309332:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.309334:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.309336:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:13.0:1619677475.309336:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:21.0:1619677475.309338:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.309339:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:13.0:1619677475.309339:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000400:00000200:21.0:1619677475.309340:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677475.309340:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:21.0:1619677475.309341:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:13.0:1619677475.309341:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.309341:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.309343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:21.0:1619677475.309344:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.309346:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.309348:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.309349:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.309350:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.309351:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.309352:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.309353:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.309353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.309357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.309364:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.309366:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.309368:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.309368:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.309370:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.309373:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.309407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.309415:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.309418:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.309420:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.309422:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.309422:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.309423:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.309424:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.309426:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.309427:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.309428:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.309431:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.309433:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.309435:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.309436:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.309436:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.309438:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.309438:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.309439:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.309440:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.309443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.309450:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.309452:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.309453:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.309454:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.309456:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.309457:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.309462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.309468:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.309471:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.309471:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.309472:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.309473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.309473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.309650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.309657:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.309659:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.309660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.309662:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.309665:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.309666:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.309667:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.309669:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.309670:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.309671:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.309674:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.309676:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.309678:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.309679:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.309680:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.309682:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.309682:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.309683:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.309683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.309688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.309695:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.309697:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.309698:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.309699:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.309700:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.309702:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.310208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.310215:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.310218:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.310220:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.310222:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.310222:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.310224:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.310225:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.310226:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.310228:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.310228:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.310232:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.310233:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.310236:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.310237:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.310238:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.310240:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.310240:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.310241:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.310242:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.310256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.310264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.310267:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.310267:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.310268:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.310270:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.310273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.310361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a6a00] (20)++
00000800:00000200:8.0:1619677475.310369:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)++
00000800:00000200:8.0:1619677475.310372:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.3.8@o2ib3
00000800:00000200:24.0:1619677475.310373:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)--
00000400:00000200:8.0:1619677475.310375:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.51@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.8@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.310378:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.310379:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.51@o2ib7 local destination
00000400:00000200:8.0:1619677475.310381:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.310382:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.310383:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.310384:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.51@o2ib7
00000400:00000200:8.0:1619677475.310387:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.51@o2ib7(10.0.10.51@o2ib7:10.0.10.51@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.310389:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.51@o2ib7
00000800:00000200:8.0:1619677475.310392:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3b0f56480] -> 10.0.10.51@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.310392:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:8.0:1619677475.310393:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:8.0:1619677475.310395:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:8.0:1619677475.310395:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a6a00] (21)++
00000800:00000200:8.0:1619677475.310396:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a6a00] (22)--
00000800:00000200:8.0:1619677475.310397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)--
00000800:00000200:24.2:1619677475.310401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:26.0:1619677475.310408:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)++
00000800:00000200:26.0:1619677475.310410:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2fa00] (23)--
00000400:00000200:26.0:1619677475.310411:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.310411:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)--
00000400:00000200:26.0:1619677475.310413:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.51@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.310415:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (21)--
00000800:00000200:24.2:1619677475.310427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2fa00] (20)++
00000800:00000200:10.0:1619677475.310433:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (21)++
00000800:00000200:26.0:1619677475.310436:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:10.0:1619677475.310436:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.51@o2ib7
00000800:00000200:10.0:1619677475.310437:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (22)++
00000800:00000200:10.0:1619677475.310437:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2fa00] (22)--
00000800:00000200:10.0:1619677475.310438:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2fa00] (21)--
00000800:00000200:4.2:1619677475.310491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a6a00] (20)++
00000800:00000200:24.0:1619677475.310498:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)++
00000800:00000200:21.2:1619677475.310500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:24.0:1619677475.310501:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.8@o2ib3
00000800:00000200:8.0:1619677475.310502:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)--
00000400:00000200:24.0:1619677475.310503:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.8@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.310505:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.310506:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677475.310507:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:24.0:1619677475.310508:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.310508:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.310509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677475.310511:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.310512:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677475.310512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a6a00] (21)++
00000400:00000200:24.0:1619677475.310513:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.310513:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:6.0:1619677475.310514:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.310515:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.310517:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.310519:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.310520:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.310520:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)++
00000400:00000200:6.0:1619677475.310520:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.310521:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.8@o2ib3
00000400:00000200:6.0:1619677475.310522:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677475.310523:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:8.0:1619677475.310524:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.8@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.310526:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.310526:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.310527:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:24.0:1619677475.310527:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:8.0:1619677475.310527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.310529:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:8.0:1619677475.310529:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:6.0:1619677475.310529:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677475.310531:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:10, s:-96500542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.310531:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.310532:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:8.0:1619677475.310533:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.310533:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.310533:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.310534:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677475.310535:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a6a00] (23)++
00000400:00000200:8.0:1619677475.310535:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:6.0:1619677475.310535:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677475.310536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:6.0:1619677475.310536:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.310536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677475.310537:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a6a00] (24)--
00000800:00000200:24.0:1619677475.310537:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (23)--
00000800:00000200:24.0:1619677475.310538:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)--
00000400:00000200:8.0:1619677475.310539:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.310541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677475.310541:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677475.310543:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:8.0:1619677475.310543:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.310544:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677475.310544:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677475.310546:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677475.310546:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.310546:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677475.310546:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a6a00] (21)++
00000800:00000200:8.0:1619677475.310547:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a6a00] (22)--
00000800:00000200:8.0:1619677475.310547:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)--
00000400:00000200:12.0:1619677475.310548:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.310549:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:12.0:1619677475.310550:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677475.310551:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:13.0:1619677475.310551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:9.2:1619677475.310551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:21.0:1619677475.310552:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.310553:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.310554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.310555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677475.310558:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.310560:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.310561:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.310561:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.310563:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.310565:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677475.310577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a6a00] (20)++
00000800:00000200:24.0:1619677475.310584:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)++
00000800:00000200:24.0:1619677475.310587:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.8@o2ib3
00000800:00000200:8.0:1619677475.310588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)--
00000400:00000200:24.0:1619677475.310589:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.8@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.310591:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.310593:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677475.310594:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.310596:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.310597:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.310598:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677475.310601:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.310603:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677475.310605:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.310606:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677475.310607:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677475.310608:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677475.310609:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a6a00] (21)++
00000800:00000200:24.0:1619677475.310610:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c773a6a00] (22)++
00000800:00000200:24.0:1619677475.310611:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a6a00] (23)--
00000800:00000200:24.0:1619677475.310612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)--
00000800:00000200:8.2:1619677475.310614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677475.310617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a6a00] (21)++
00000800:00000200:17.0:1619677475.310623:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:8.0:1619677475.310624:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)++
00000800:00000200:17.0:1619677475.310626:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:8.0:1619677475.310626:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c773a6a00] (23)--
00000800:00000200:1.0:1619677475.310626:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.310627:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.310627:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)--
00000800:00000200:8.0:1619677475.310628:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)--
00000400:00000200:17.0:1619677475.310629:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.310632:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677475.310684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:4.2:1619677475.310684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a6a00] (20)++
00000800:00000200:1.0:1619677475.310690:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677475.310691:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)++
00000800:00000200:1.0:1619677475.310692:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.310693:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677475.310694:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.8@o2ib3
00000800:00000200:8.0:1619677475.310694:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)--
00000400:00000200:1.0:1619677475.310696:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.310697:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.8@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.310698:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.310699:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:24.0:1619677475.310700:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.310700:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.310702:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677475.310702:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.310704:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.310704:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.310705:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677475.310706:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.310707:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.310709:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677475.310709:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.310712:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677475.310713:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.310715:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677475.310715:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.310716:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.310716:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677475.310718:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.310718:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:1.0:1619677475.310718:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.310718:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:24.0:1619677475.310719:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677475.310719:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.310719:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677475.310721:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:24.0:1619677475.310721:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a6a00] (21)++
00000800:00000200:24.0:1619677475.310722:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a6a00] (22)--
00000800:00000200:24.0:1619677475.310722:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)--
00000800:00000200:25.2:1619677475.310724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:25.2:1619677475.310728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677475.310732:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.310734:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:12.0:1619677475.310734:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:22.0:1619677475.310735:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.310735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.310737:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.310737:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.310738:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.310739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:12.0:1619677475.310740:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.310741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.310742:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.310783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a6a00] (20)++
00000800:00000200:8.0:1619677475.310789:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)++
00000800:00000200:8.0:1619677475.310792:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.8@o2ib3
00000800:00000200:24.0:1619677475.310793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)--
00000400:00000200:8.0:1619677475.310795:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.8@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.310798:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.310799:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:8.0:1619677475.310800:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.310801:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.310802:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.310803:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:8.0:1619677475.310806:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.310808:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:8.0:1619677475.310810:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.310810:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:8.0:1619677475.310811:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677475.310812:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:8.0:1619677475.310812:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a6a00] (21)++
00000800:00000200:8.0:1619677475.310813:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a6a00] (22)--
00000800:00000200:8.0:1619677475.310813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)--
00000800:00000200:24.2:1619677475.310817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.310820:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677475.310822:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677475.310822:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.310823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a6a00] (20)++
00000400:00000200:24.0:1619677475.310824:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.310826:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:8.0:1619677475.310826:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:24.0:1619677475.310827:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)++
00000800:00000200:24.0:1619677475.310829:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.8@o2ib3
00000800:00000200:8.0:1619677475.310830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)--
00000400:00000200:24.0:1619677475.310831:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.8@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.310832:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.310834:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677475.310835:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.310836:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.310837:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.310837:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677475.310840:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.310842:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677475.310843:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.310844:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:24.0:1619677475.310845:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677475.310846:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:24.0:1619677475.310846:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a6a00] (21)++
00000800:00000200:24.0:1619677475.310847:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a6a00] (22)--
00000800:00000200:24.0:1619677475.310848:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)--
00000800:00000200:25.2:1619677475.310851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.310858:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.310861:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.310861:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.310863:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.310865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.310866:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.310873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a6a00] (20)++
00000800:00000200:8.0:1619677475.310880:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)++
00000800:00000200:8.0:1619677475.310883:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.8@o2ib3
00000800:00000200:24.0:1619677475.310884:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)--
00000400:00000200:8.0:1619677475.310885:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.8@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.310888:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.310889:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:8.0:1619677475.310890:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.310891:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.310892:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.310893:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677475.310895:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.310897:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.310898:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.310898:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.310899:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.310900:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.310900:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a6a00] (21)++
00000800:00000200:8.0:1619677475.310901:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919c773a6a00] (22)++
00000800:00000200:8.0:1619677475.310903:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a6a00] (23)--
00000800:00000200:8.0:1619677475.310903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)--
00000800:00000200:9.2:1619677475.310905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.310908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919c773a6a00] (21)++
00000800:00000200:24.0:1619677475.310913:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.310914:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.310915:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.310916:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.310917:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.310918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677475.310918:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)++
00000800:00000200:24.0:1619677475.310919:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919c773a6a00] (23)++
00000800:00000200:8.0:1619677475.310919:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919c773a6a00] (23)--
00000800:00000200:8.0:1619677475.310919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (23)--
00000800:00000200:24.0:1619677475.310921:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.8@o2ib3
00000800:00000200:8.0:1619677475.310921:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (22)--
00000800:00000200:9.2:1619677475.310922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:24.0:1619677475.310923:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.8@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.310924:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.310926:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677475.310927:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.310929:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.310929:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:24.0:1619677475.310930:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.310930:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677475.310931:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:8.0:1619677475.310931:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.310932:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.310932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.310933:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:24.0:1619677475.310935:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.8@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.310936:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677475.310938:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.310939:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677475.310939:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677475.310941:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677475.310941:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919c773a6a00] (21)++
00000800:00000200:24.0:1619677475.310942:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919c773a6a00] (22)--
00000800:00000200:24.0:1619677475.310942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919c773a6a00] (21)--
00000800:00000200:25.2:1619677475.310946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.310954:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.310957:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.310957:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.310959:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.310961:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.310962:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.311134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677475.311141:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677475.311144:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677475.311145:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677475.311147:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.311149:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.311150:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677475.311152:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.311153:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.311154:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.311154:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677475.311157:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.311159:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 584 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.311162:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.311163:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677475.311163:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677475.311165:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677475.311165:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677475.311166:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677475.311167:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:25.2:1619677475.311171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.311178:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.311181:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.311182:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.311184:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.311184:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.311187:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.311274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.311281:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.311284:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.311286:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.311288:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.311289:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.311290:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.311291:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.311293:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.311300:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.311301:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.311304:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.311306:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.311308:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.311309:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.311309:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.311310:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.311311:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.311312:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.311314:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.311315:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.311317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.311321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.311326:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.311327:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.311328:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.311329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677475.311330:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.311330:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.311331:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.311331:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.311332:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.311333:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.311389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.311396:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.311398:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.311399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.311401:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.311403:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.311405:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.311406:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.311407:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.311408:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.311409:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.311412:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.311414:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.311416:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.311417:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.311418:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.311419:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.311420:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.311420:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.311421:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.311425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.311433:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.311435:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.311436:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.311438:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.311439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.311439:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:25.2:1619677475.311447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.311455:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.311457:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.311458:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.311459:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.311459:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.311460:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.312194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.312198:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.312200:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.312202:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.312203:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.312205:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.312206:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677475.312208:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.312209:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.312210:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.312211:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.312214:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.312216:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.312218:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.312219:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.312219:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.312221:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677475.312221:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.312222:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.312223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.312227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.312236:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.312238:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.312240:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.312240:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.312241:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.312244:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.312247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.312254:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.312257:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.312258:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.312258:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.312259:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.312260:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.312275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.312283:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.312286:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.312288:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.312289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.312290:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.312292:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.312297:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.312298:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.312299:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.312300:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.312303:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.312305:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.312308:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.312309:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.312310:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.312311:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.312312:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.312313:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.312313:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.312318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.312326:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.312329:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.312330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.312331:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.312333:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.312335:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.312338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.312345:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.312348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.312348:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.312349:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.312350:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.312350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.312370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.312379:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.312382:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.312384:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.312386:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.312386:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.312388:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.312389:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.312390:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.312391:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.312392:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.312396:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.312397:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.312399:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.312400:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.312400:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.312401:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.312402:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.312403:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.312404:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.312407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.312414:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.312416:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.312417:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.312418:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.312420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.312421:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.312436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.312441:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.312444:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.312444:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.312446:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.312449:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.312450:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.312451:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.312453:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.312454:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.312455:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.312458:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.312460:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.312462:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.312463:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.312464:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.312465:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.312466:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.312467:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.312468:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.312468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.312471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.312473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:4.2:1619677475.312476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (20)++
00000800:00000200:6.0:1619677475.312478:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.312480:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.312481:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.312481:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (21)++
00000800:00000200:6.0:1619677475.312481:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.312482:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.312482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.312483:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.312483:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000400:00000200:21.0:1619677475.312485:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:20.0:1619677475.312486:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : GET - for me
00000800:00000200:21.0:1619677475.312487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.312488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:20.0:1619677475.312489:0:19934:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.70@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:21.0:1619677475.312490:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000400:00000200:20.0:1619677475.312491:0:19934:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.70@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:20.0:1619677475.312492:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (21)++
00000800:00000200:20.0:1619677475.312493:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (22)--
00000400:00000200:20.0:1619677475.312495:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.70@o2ib3 local destination
00000400:00000200:20.0:1619677475.312499:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.70@o2ib3(10.51.5.70@o2ib3:10.51.5.70@o2ib3) : REPLY try# 0
00000800:00000200:20.0:1619677475.312501:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.70@o2ib3
00000800:00000200:20.0:1619677475.312503:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9197c81a8280] -> 10.51.5.70@o2ib3 (2) version: 12
00000800:00000200:20.0:1619677475.312504:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919476fc2600] (21)++
00000800:00000200:20.0:1619677475.312504:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919476fc2600] (22)++
00000800:00000200:20.0:1619677475.312506:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919476fc2600] (23)--
00000800:00000200:20.0:1619677475.312507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000800:00000200:4.2:1619677475.312512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (21)++
00000800:00000200:21.0:1619677475.312517:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (22)++
00000800:00000200:21.0:1619677475.312519:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919476fc2600] (23)--
00000400:00000200:21.0:1619677475.312520:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.312522:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.70@o2ib3: REPLY: OK
00000800:00000200:20.0:1619677475.312522:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000800:00000200:21.0:1619677475.312524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (21)--
00000800:00000200:21.2:1619677475.312702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.312708:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.312711:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.312712:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.312714:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.312717:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.312718:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.312719:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.312721:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.312722:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.312723:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.312726:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.312727:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.312730:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.312731:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.312731:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.312733:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.312733:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.312734:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.312734:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.312740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.312748:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.312750:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.312751:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.312753:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.312754:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.312755:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.312760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:4.2:1619677475.312767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.312769:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.312771:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.312771:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.312772:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.312773:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.312773:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.312773:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.312774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:20.0:1619677475.312775:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.312776:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.312777:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.312779:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.312780:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.312781:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.312782:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.312783:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.312786:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.312788:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.312790:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.312791:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.312792:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.312793:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.312794:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.312794:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.312795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.312799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.312807:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.312809:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.312810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.312810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.312812:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.312814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.312899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.312907:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.312910:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.312911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.312912:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.312915:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.312916:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.312917:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.312919:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.312920:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.312921:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.312924:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.312925:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.312928:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.312929:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.312929:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.312931:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.312932:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.312932:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.312933:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.312934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.312937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.312939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.312944:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.312947:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.312947:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.312948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.312948:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.312949:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.312950:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677475.312950:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.312951:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677475.312953:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.313142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.313149:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.313152:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.313153:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.313155:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.313158:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.313160:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.313161:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.313162:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.313163:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.313164:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.313167:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.313169:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.313171:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.313172:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.313173:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.313175:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.313175:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.313176:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.313176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.313181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.313188:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.313190:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.313191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.313192:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.313194:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.313196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.313289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.313298:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.313301:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.313302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.313303:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.313306:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.313307:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.313308:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.313309:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.313310:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.313311:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.313314:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.313315:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.313318:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.313319:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.313319:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.313321:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.313321:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.313322:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.313322:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.313327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.313332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.313337:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.313338:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.313339:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.313340:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.313340:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.313342:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.313343:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.313346:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.313346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.313347:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:6.0:1619677475.313347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.313348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.313349:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.313349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:22.0:1619677475.313350:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.313351:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.313352:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.313356:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.313357:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.313358:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.313360:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677475.313360:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.313361:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:30.0:1619677475.313362:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.313362:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.313363:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.313364:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.313365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.313365:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.313365:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.313365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:30.0:1619677475.313366:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.313367:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.313369:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.313369:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.313369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.313371:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.313373:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.313375:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.313377:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.313377:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677475.313378:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.313379:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677475.313379:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677475.313380:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677475.313380:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.313381:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.313382:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.313382:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677475.313382:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.313383:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.313384:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.313386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.313392:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.313395:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.313396:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.313398:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.313399:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.313400:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.313404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.313411:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.313414:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.313414:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.313416:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.313418:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.313420:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.313421:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.313422:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.313423:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.313425:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.313428:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.313430:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.313433:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.313433:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.313434:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.313436:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.313436:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.313438:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.313438:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.313442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.313449:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.313451:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.313452:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.313454:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.313455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.313457:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.313463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.313469:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.313472:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.313473:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.313474:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.313475:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.313476:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.313967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.313974:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.313977:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.313979:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.313981:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.313981:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.313983:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.313984:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.313985:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.313986:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.313987:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.313991:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.313993:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.313996:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.313997:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.313997:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.313999:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.314000:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.314000:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.314001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.314005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.314013:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.314015:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.314016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.314017:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.314019:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.314022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.314025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.314032:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.314034:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.314035:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.314036:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.314036:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.314037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.314242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.314247:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.314250:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.314251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.314252:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.314254:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.314256:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.314257:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.314258:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.314259:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.314260:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.314263:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.314264:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.314267:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.314267:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.314268:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.314270:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.314270:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.314271:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.314271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.2:1619677475.314272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677475.314276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.314280:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.314283:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677475.314283:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:31.0:1619677475.314285:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.314285:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.314286:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.314287:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.314288:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.314288:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.314289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677475.314290:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677475.314290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.314291:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.314292:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.314299:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.314300:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.314304:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.314305:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.314307:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.314308:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.314309:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.314310:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.314311:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.314312:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.314312:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.314317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.314324:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.314327:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.314328:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.314329:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.314331:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.314332:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.314335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.314342:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.314344:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.314345:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.314346:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.314346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.314347:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.314597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.314602:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.314605:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.314607:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.314607:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.314609:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.314611:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.314612:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.314614:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.314615:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.314616:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.314619:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.314621:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.314623:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.314624:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.314625:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.314627:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.314627:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.314628:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.314629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.314634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.314642:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.314645:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.314646:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.314646:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.314648:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677475.314650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.314653:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.314655:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.314656:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.314658:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.314658:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.314658:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.314659:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.315276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677475.315277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.315283:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.0:1619677475.315284:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.315286:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.315287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677475.315288:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.315288:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:22.0:1619677475.315290:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.315291:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.315298:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.315299:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.315300:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.315301:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.315302:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.315303:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.315303:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677475.315304:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677475.315304:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.315306:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.315307:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.315308:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677475.315309:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.315311:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677475.315312:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.315314:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.315314:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.315315:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677475.315315:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677475.315317:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.315317:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.315318:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.315318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677475.315318:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.315318:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.315319:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.315321:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.315321:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.315322:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.315322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.315323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.2:1619677475.315327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677475.315331:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.315333:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.315334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.315335:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:20.0:1619677475.315336:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.315337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.315338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:17.0:1619677475.315338:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.315339:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.315339:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.315341:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.315344:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:25.2:1619677475.315345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.2:1619677475.315347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:21.0:1619677475.315352:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.315354:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.315355:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:1.0:1619677475.315355:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677475.315356:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.315357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.315358:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677475.315358:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.315359:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.315359:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.315360:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.315361:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.315983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.315991:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.315993:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.315996:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.315998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.315998:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.315999:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.316000:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.316002:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.316003:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.316004:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.316008:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.316009:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.316012:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.316013:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.316013:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.316015:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.316016:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.316017:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.316019:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.316019:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.316021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.316025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.316029:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.316032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.316032:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.316033:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677475.316033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.316035:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677475.316035:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.316036:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.316037:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.316038:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.316125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677475.316132:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.316135:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677475.316138:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.316139:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677475.316140:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.316142:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.316144:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.316145:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.316146:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.316147:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.316150:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.316151:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.316153:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.316154:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.316155:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.316156:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.316157:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.316158:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677475.316158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.316162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.316168:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.316171:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.316172:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.316172:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.316173:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.316176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.316196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.316201:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.316204:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.316205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.316207:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.316209:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.316210:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.316211:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.316213:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.316214:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.316215:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.316218:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.316220:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.316222:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.316223:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.316224:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.316225:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.316226:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.316227:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.316228:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.316228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.316231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.316234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.316239:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.316240:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.316241:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.316241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.316242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.316243:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.316244:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.316246:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.316246:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.316248:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.316300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.316306:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.316309:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.316309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.316311:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.316313:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.316315:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.316316:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.316317:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.316318:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.316319:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.316323:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.316324:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.316327:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.316328:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.316328:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.316330:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.316330:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.316331:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.316332:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.316337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.316345:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.316348:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.316349:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.316349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.316351:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.316353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.316701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.316710:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.316713:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.316715:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.316718:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.316718:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.316720:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.316721:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.316722:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.316723:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.316724:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.316727:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.316729:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.316731:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.316732:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.316733:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.316734:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.316734:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.316736:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.316737:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.316738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.316740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.316743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.316748:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.316750:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.316750:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.316752:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.316752:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.316753:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.316753:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.316754:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.316755:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.316756:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677475.317045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.317052:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.317054:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.317056:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.2:1619677475.317057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:2.0:1619677475.317058:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.317060:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.317062:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:2.0:1619677475.317062:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.317063:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.317065:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:2.0:1619677475.317065:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.317066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:2.0:1619677475.317066:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.317068:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677475.317069:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.317071:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.317071:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.317072:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677475.317073:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:6.0:1619677475.317074:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.317075:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.317076:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677475.317076:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.317076:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:6.0:1619677475.317077:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677475.317077:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.317079:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.317079:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:4.2:1619677475.317080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:2.0:1619677475.317080:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000400:00000200:6.0:1619677475.317081:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.317081:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:6.0:1619677475.317083:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:9.2:1619677475.317085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677475.317085:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.317085:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.317086:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677475.317088:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:6.0:1619677475.317088:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.317088:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.317089:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.317089:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677475.317091:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677475.317092:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:18.0:1619677475.317092:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:8.0:1619677475.317093:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.317094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.317094:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.317095:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.317096:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.317096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.317097:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.317098:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677475.317099:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:8.0:1619677475.317099:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.317101:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677475.317101:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.317101:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.317103:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:8.0:1619677475.317103:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:20.0:1619677475.317104:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.317106:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.317107:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677475.317107:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.317109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677475.317110:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.317113:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.317114:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.317114:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.317116:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.317116:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677475.317117:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677475.317118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677475.317122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.317130:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.317133:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.317134:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.317134:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.317136:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.317139:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.317161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.317170:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.317174:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.317178:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.317180:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.317181:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.317184:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677475.317186:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.317188:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.317190:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.317191:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677475.317196:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.317199:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677475.317202:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.317204:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:12.0:1619677475.317204:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:12.0:1619677475.317207:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:12.0:1619677475.317208:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.317209:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.317210:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677475.317212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.317219:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.317222:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.317223:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.317224:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.317226:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.317227:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.317560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.317568:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.317571:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.317573:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.317574:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.317575:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677475.317575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.317576:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.317578:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.317579:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.317580:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.317582:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.317584:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.317586:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.317587:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.317588:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.317589:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.317590:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.317591:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.317591:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.317596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.317603:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.317605:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.317606:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.317607:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.317608:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.317611:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.317734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.317742:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.317744:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.317746:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.317748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.317749:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.317750:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.317751:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.317753:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.317754:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.317755:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.317759:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.317760:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.317763:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.317764:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.317764:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.317766:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.317767:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.317767:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.317768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.317772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.317780:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.317783:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.317784:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.317784:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.317786:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.317789:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.317834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.317840:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.317842:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.317844:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.317845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.317847:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.317849:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.317850:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.317852:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.317853:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.317853:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.317856:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.317858:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.317861:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.317861:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.317862:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.317864:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.317865:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.317866:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.317867:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.317867:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677475.317868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:9.2:1619677475.317870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677475.317875:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677475.317876:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:26.0:1619677475.317879:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.0:1619677475.317879:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.317880:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677475.317880:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:8.0:1619677475.317880:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.317882:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.317882:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:26.0:1619677475.317885:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.317886:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677475.317886:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677475.317888:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.317889:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.317890:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.317891:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:9.2:1619677475.317893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:26.0:1619677475.317894:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.317895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:26.0:1619677475.317896:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.317898:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.317899:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:24.0:1619677475.317899:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.317899:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:26.0:1619677475.317900:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.317901:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677475.317901:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:26.0:1619677475.317902:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677475.317902:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.317902:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.317902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.317902:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:26.0:1619677475.317903:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677475.317903:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.317903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677475.317904:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:24.0:1619677475.317904:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:25.2:1619677475.317908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.317915:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.317918:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.317919:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.317919:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.317921:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.317924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.317926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.317932:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.317935:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.317936:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.317936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.317937:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.317938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.318122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.318126:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.318129:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.318130:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677475.318130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:6.0:1619677475.318131:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.318133:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.318134:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.318135:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.318136:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.318137:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677475.318137:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.318138:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.318140:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.318141:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677475.318143:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.318144:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.318145:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:10.0:1619677475.318147:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.318147:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677475.318148:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:6.0:1619677475.318148:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.318149:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:10.0:1619677475.318150:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.318150:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677475.318151:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.318151:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.318151:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677475.318152:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.318152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:10.0:1619677475.318153:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677475.318156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:10.0:1619677475.318157:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.318158:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.318161:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.318162:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.318162:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.318164:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677475.318164:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.318165:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677475.318166:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677475.318166:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:20.0:1619677475.318167:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.318167:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:20.0:1619677475.318168:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.318168:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677475.318169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:25.2:1619677475.318170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.318171:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.318172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:4.2:1619677475.318176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.318177:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.318179:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.318180:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.318181:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.318181:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.318182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.318182:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:26.0:1619677475.318183:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677475.318184:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.318185:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677475.318185:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.318186:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677475.318186:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:8.0:1619677475.318187:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.318188:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677475.318189:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.318190:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.318192:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.318193:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.318194:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.318197:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.318199:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.318201:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.318202:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.318203:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.318204:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.318205:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.318205:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.318206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.318210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.318217:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.318220:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.318221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.318221:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.318223:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.318226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.318234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.318240:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.318243:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.318244:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.318244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.318245:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.318246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.318667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.318673:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.318676:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.318677:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.318679:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.318681:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.318683:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.318684:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.318685:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.318686:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.318687:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.318690:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.318692:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.318694:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.318695:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.318696:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.318697:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.318698:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.318699:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.318700:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.318701:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677475.318705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677475.318706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.318712:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677475.318713:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677475.318714:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.318715:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.318715:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677475.318716:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.318717:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.318717:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.318719:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.318721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.318799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.318808:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.318811:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.318813:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.318815:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.318816:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.318818:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.318819:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.318820:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.318822:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.318823:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.318826:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.318828:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.318830:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.318831:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.318831:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.318833:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.318833:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.318834:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.318835:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.318839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.318846:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.318849:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.318849:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.318851:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.318853:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.318854:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.319006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.319013:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.319016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.319016:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.319018:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.319020:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.319022:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.319023:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.319025:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.319026:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.319027:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.319030:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.319032:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.319034:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.319035:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.319036:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.319037:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.319038:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.319039:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.319039:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.319043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.319051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.319054:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.319054:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.319056:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.319057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.319058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.319072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.319079:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.319081:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.319082:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.319083:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.319084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.319085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.319786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.319794:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.319797:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.319799:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.319801:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.319802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.319803:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.319804:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.319806:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.319807:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.319808:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.319811:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.319813:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.319816:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.319817:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.319818:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.319819:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.319820:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.319821:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.319822:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.319826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.319835:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.319837:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.319838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.319839:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.319841:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.319844:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.319892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.319897:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.319899:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.319900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.319902:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.319904:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.319905:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.319906:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.319908:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.319909:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.319910:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.319913:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.319915:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.319917:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.319918:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.319919:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.319920:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.319921:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.319922:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.319923:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.319923:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.319926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.319928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.319933:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.319934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.319935:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.319935:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.319936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.319936:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.319937:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.319938:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.319940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.319941:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.320389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.320395:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.320398:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.320399:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.320401:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.320403:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.320404:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.320405:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.320407:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.320408:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.320409:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.320412:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.320414:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.320416:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.320417:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.320418:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.320419:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.320420:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.320421:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.320421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.320426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.320434:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.320436:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.320437:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.320439:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.320440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.320441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.320883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.320890:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.320893:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.320894:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.320896:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.320899:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.320901:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.320902:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.320903:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.320904:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.320905:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.320909:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.320910:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.320913:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.320914:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.320915:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.320916:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.320917:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.320917:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.320918:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677475.320933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.2:1619677475.320933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.320939:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677475.320942:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677475.320942:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.320943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.320944:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.320945:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.320946:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.320947:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.320947:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.320948:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677475.320950:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.320950:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.320952:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.320953:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:6.0:1619677475.320953:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.320954:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.320957:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.320959:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.320961:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.320962:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.320963:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.320964:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.320965:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.320965:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.320966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.320971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.320979:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.320981:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.320982:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.320984:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.320985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.320986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.321766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.321774:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.321777:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.321780:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.321782:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.321782:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.321784:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.321785:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.321787:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.321788:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.321789:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.321792:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.321794:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.321797:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.321798:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.321798:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.321800:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.321801:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.321802:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.321802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.321807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.321815:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.321818:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.321819:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.321819:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.321821:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.321824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.321837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.321842:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.321845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.321845:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.321847:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.321849:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.321851:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.321852:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.321853:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.321854:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.321855:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.321857:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.321859:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.321862:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.321862:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.321863:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.321865:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.321865:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.321866:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.321866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.321871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.321878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.321880:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.321881:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.321882:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.321884:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.2:1619677475.321886:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.0:1619677475.321889:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.321894:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.321897:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.321899:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.321901:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.321902:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.321903:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.321905:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.321906:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.321907:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.321908:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.321911:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.321912:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.321914:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.321915:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.321916:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.321917:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.321918:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.321919:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.321919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.321923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.321929:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.321932:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.321933:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.321934:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.321936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.321937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.321948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.321955:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.321958:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.321959:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.321959:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.321960:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.321961:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.321970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.321976:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.321979:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.321980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.321982:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.321984:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.321986:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677475.321987:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.321988:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.321989:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.321990:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677475.321994:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.321995:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677475.321999:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.322000:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677475.322000:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677475.322002:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677475.322003:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.322004:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677475.322005:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677475.322005:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677475.322008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.2:1619677475.322012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:26.0:1619677475.322017:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:17.0:1619677475.322018:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677475.322020:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:26.0:1619677475.322021:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:17.0:1619677475.322021:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.322022:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:26.0:1619677475.322023:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.322023:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.322027:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.322030:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677475.322263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.322271:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.322273:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.322276:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677475.322278:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.322278:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.322279:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:13.0:1619677475.322281:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.322282:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.322283:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.322284:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677475.322287:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.322289:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677475.322291:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.322292:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:13.0:1619677475.322298:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:13.0:1619677475.322300:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677475.322300:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.322301:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677475.322302:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.322307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.322316:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.322318:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677475.322319:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.322320:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.322322:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.322325:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.322640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.322649:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.322654:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.322655:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.322658:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.322662:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.322664:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.322666:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.322668:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.322670:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.322672:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.322677:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.322680:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.322683:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.322685:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.322685:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.322687:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.322688:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.322689:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.322691:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.322692:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.322693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.322697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.322701:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.322704:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677475.322705:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677475.322705:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.322705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.322707:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.322708:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677475.322708:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.322709:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677475.322710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.322711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.322716:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.322718:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.322719:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.322721:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.322723:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.322724:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.322725:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.322726:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.322728:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.322729:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.322732:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.322734:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.322736:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.322737:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.322737:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.322739:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.322739:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.322740:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.322740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.322745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.322752:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.322755:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.322755:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.322757:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.322758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.322759:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.322763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.322770:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.322773:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.322774:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.322774:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.322775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.322776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.322825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.322833:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.322836:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.322838:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.322840:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.322841:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.322842:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.322843:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.322844:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.322845:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.322846:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.322849:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.322851:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.322854:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.322855:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.322855:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.322857:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.322857:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.322859:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.322860:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.322861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.322862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.322867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.322869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.322872:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.322873:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.322873:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.322874:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.322875:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.322877:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.322878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677475.322878:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.322879:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.322951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.322959:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.322961:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.322962:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.322964:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.322966:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.322967:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.322969:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.322970:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.322971:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.322972:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.322974:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.322976:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.322979:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.322980:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.322980:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.322982:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.322982:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:4.2:1619677475.322982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677475.322983:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.322983:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.322988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.322989:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.322991:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.322993:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.322994:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.322995:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:24.0:1619677475.322997:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.322998:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677475.322998:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:24.0:1619677475.322999:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.322999:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.322999:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:24.0:1619677475.323001:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.323002:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.323002:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677475.323003:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.323007:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.323009:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.323009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:24.0:1619677475.323012:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.323012:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.323013:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.323014:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.323015:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.323016:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.323018:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.323018:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.323020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.323024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.323028:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.323030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.323030:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.323031:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.323031:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.323033:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.323033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677475.323033:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.323034:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677475.323036:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.323039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:4.2:1619677475.323043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:22.0:1619677475.323046:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.323049:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.323049:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.323049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.323050:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.323050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:22.0:1619677475.323051:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677475.323052:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.323053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.323055:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.323058:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.323059:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677475.323060:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.323062:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.323063:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.323063:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677475.323066:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.323068:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677475.323069:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.323070:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677475.323070:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.323072:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.323072:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.323073:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.323073:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.323078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.323084:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.323086:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.323086:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.323087:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.323088:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.323090:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.323731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.323737:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.2:1619677475.323738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.323740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.323740:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.323742:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.323744:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.323746:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677475.323746:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.323747:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.323748:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.323749:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.323750:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677475.323751:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.323754:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.323754:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677475.323755:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.323756:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.323757:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.323758:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.323759:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.323760:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.323760:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.323760:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.323761:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.323762:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.323763:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:6.0:1619677475.323763:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.323763:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.323764:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.323765:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:31.0:1619677475.323766:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.323766:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677475.323768:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:25.2:1619677475.323769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.323770:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677475.323770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.323771:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.323772:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.323773:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.323774:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.323775:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.323775:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677475.323775:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.323777:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.323777:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.323778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.323779:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.323779:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:9.2:1619677475.323779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:21.0:1619677475.323780:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.323782:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.323783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.323784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677475.323786:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.323788:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.323789:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.323789:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.323792:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.323794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.323824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.323832:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.323834:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.323837:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.323838:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.323839:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.323841:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.323842:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.323843:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.323844:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.323845:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.323848:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.323850:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.323853:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.323853:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.323854:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.323855:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.323856:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.323858:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.323859:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.323860:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.323862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.323865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.323870:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.323872:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.323873:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.323873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.323874:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.323876:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677475.323876:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.323877:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.323878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.323879:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.324252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.324259:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.324262:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.324263:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.324265:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.324267:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.324268:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.324269:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.324271:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.324272:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.324273:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.324276:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.324277:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.324280:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.324280:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.324281:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.324283:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.324283:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.324284:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.324285:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.324286:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.324289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.324291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:12.0:1619677475.324298:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677475.324299:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.324301:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.324301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.324302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:12.0:1619677475.324302:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.324303:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.324306:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.324307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.324309:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.324627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.324633:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.324636:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.324636:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.324638:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.324641:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.324642:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.324643:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.324645:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.324646:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.324647:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.324650:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.324652:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.324654:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.324654:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.324655:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.324657:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.324657:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.324658:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.324658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.324663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.324671:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.324673:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.324674:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.324676:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.324677:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.324679:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.324802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.324809:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.324811:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.324812:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.324814:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.324817:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.324818:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.324820:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.324821:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.324822:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.324823:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.324826:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.324828:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.324831:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.324831:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.324832:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.324834:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.324835:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.324835:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.324836:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.324840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.324847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:4.2:1619677475.324847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (20)++
00000800:00000200:13.0:1619677475.324849:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.324850:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.324852:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.324853:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (21)++
00000800:00000200:12.0:1619677475.324853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677475.324855:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.5.70@o2ib3
00000800:00000200:13.0:1619677475.324855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.324858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:21.0:1619677475.324858:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.324859:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000400:00000200:21.0:1619677475.324860:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.324862:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.324863:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.324865:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677475.324865:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:21.0:1619677475.324866:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.324867:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:12.0:1619677475.324868:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.324869:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:4.2:1619677475.324869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (21)++
00000400:00000200:21.0:1619677475.324870:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.324870:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.324871:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677475.324871:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677475.324872:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.324872:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (22)++
00000800:00000200:20.0:1619677475.324874:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000800:00000200:21.0:1619677475.324876:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.324877:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.324877:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677475.324877:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.324879:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677475.324879:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.324880:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (23)++
00000800:00000200:21.0:1619677475.324882:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (24)--
00000800:00000200:21.0:1619677475.324882:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (23)--
00000400:00000200:20.0:1619677475.324882:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:20.0:1619677475.324883:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.324884:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000400:00000200:20.0:1619677475.324884:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.324885:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677475.324885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677475.324886:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:20.0:1619677475.324890:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.324892:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.324892:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:20.0:1619677475.324894:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.324895:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677475.324895:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:20.0:1619677475.324895:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:24.0:1619677475.324896:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.324896:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.324897:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:24.0:1619677475.324898:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.324898:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (21)++
00000800:00000200:20.0:1619677475.324899:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (22)--
00000800:00000200:20.0:1619677475.324899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (21)--
00000800:00000200:24.0:1619677475.324901:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.324903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (20)++
00000800:00000200:9.2:1619677475.324904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677475.324908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (21)++
00000800:00000200:21.0:1619677475.324911:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000800:00000200:18.0:1619677475.324911:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000400:00000200:21.0:1619677475.324913:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.324913:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:20.0:1619677475.324914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000400:00000200:18.0:1619677475.324914:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.324914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:21.0:1619677475.324916:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.324917:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:18.0:1619677475.324917:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.324919:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.324920:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.324921:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.324922:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677475.324922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (21)++
00000800:00000200:18.0:1619677475.324924:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.0:1619677475.324925:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (22)++
00000400:00000200:21.0:1619677475.324926:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.324927:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000800:00000200:21.0:1619677475.324928:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677475.324929:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.324931:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677475.324931:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.324932:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677475.324932:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:20.0:1619677475.324933:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:21.0:1619677475.324934:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:20.0:1619677475.324934:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.324935:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.324936:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.324937:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.324938:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (23)++
00000800:00000200:21.0:1619677475.324939:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (24)--
00000800:00000200:25.2:1619677475.324940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677475.324940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (23)--
00000400:00000200:20.0:1619677475.324940:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.324941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000800:00000200:20.0:1619677475.324942:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.324944:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.324945:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.324945:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.324947:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:12.0:1619677475.324947:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:20.0:1619677475.324948:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (21)++
00000800:00000200:20.0:1619677475.324949:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919476fc2600] (22)++
00000800:00000200:12.0:1619677475.324949:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:20.0:1619677475.324950:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (23)--
00000800:00000200:20.0:1619677475.324950:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000400:00000200:12.0:1619677475.324951:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.324952:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.324953:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:8.2:1619677475.324953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:12.0:1619677475.324955:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:4.2:1619677475.324955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (21)++
00000800:00000200:21.0:1619677475.324961:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (22)++
00000800:00000200:1.0:1619677475.324962:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.324963:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919476fc2600] (23)--
00000800:00000200:21.0:1619677475.324964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000800:00000200:1.0:1619677475.324964:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677475.324965:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (21)++
00000800:00000200:17.0:1619677475.324965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.324965:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.324966:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000800:00000200:20.0:1619677475.324966:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000400:00000200:1.0:1619677475.324968:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.324969:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.324971:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:21.0:1619677475.324973:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.324975:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:21.0:1619677475.324976:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.324978:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.324979:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.324980:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:21.0:1619677475.324984:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.324985:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.324988:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.324989:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.0:1619677475.324990:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677475.324990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (21)++
00000800:00000200:21.0:1619677475.324992:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.0:1619677475.324992:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (22)++
00000800:00000200:21.0:1619677475.324993:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (23)--
00000800:00000200:21.0:1619677475.324994:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000800:00000200:20.0:1619677475.324995:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (21)++
00000800:00000200:20.0:1619677475.324997:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000800:00000200:25.2:1619677475.324998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:20.0:1619677475.324999:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.325000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000400:00000200:20.0:1619677475.325001:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.325003:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677475.325004:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.325005:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:20.0:1619677475.325005:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.325006:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.325007:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.325008:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.325009:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.325010:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.325011:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.325012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677475.325012:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.325015:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.0:1619677475.325016:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.325017:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:4.2:1619677475.325017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (21)++
00000800:00000200:20.0:1619677475.325018:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677475.325020:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677475.325021:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (22)++
00000800:00000200:21.0:1619677475.325022:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (23)++
00000800:00000200:20.0:1619677475.325022:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (23)--
00000800:00000200:20.0:1619677475.325023:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (23)--
00000800:00000200:21.0:1619677475.325024:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000800:00000200:20.0:1619677475.325024:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000800:00000200:25.2:1619677475.325026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677475.325026:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.325028:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.325030:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.325032:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.325033:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:21.0:1619677475.325033:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.325034:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.325035:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.325036:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.325037:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.325038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (21)++
00000400:00000200:30.0:1619677475.325039:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.325039:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.325040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.0:1619677475.325041:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.325042:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (22)++
00000800:00000200:30.0:1619677475.325043:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.0:1619677475.325043:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000800:00000200:21.0:1619677475.325044:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.325045:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.325046:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:20.0:1619677475.325046:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.325048:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.325048:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (23)++
00000400:00000200:20.0:1619677475.325048:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.325049:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (24)--
00000800:00000200:21.0:1619677475.325050:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (23)--
00000400:00000200:20.0:1619677475.325050:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:21.0:1619677475.325051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (22)++
00000400:00000200:20.0:1619677475.325051:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.325052:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000400:00000200:20.0:1619677475.325053:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.325054:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677475.325054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:20.0:1619677475.325055:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.325058:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.325060:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.325061:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.325061:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:21.0:1619677475.325062:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:20.0:1619677475.325062:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.325064:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677475.325064:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:21.0:1619677475.325065:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677475.325065:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.325065:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.325066:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.325067:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677475.325068:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.325070:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.325070:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:21.0:1619677475.325071:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.325071:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.325072:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.325073:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:20.0:1619677475.325073:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.325074:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (23)++
00000800:00000200:21.0:1619677475.325076:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.325076:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919476fc2600] (24)++
00000800:00000200:21.0:1619677475.325077:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:21.0:1619677475.325077:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677475.325077:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (25)--
00000800:00000200:20.0:1619677475.325078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (24)--
00000800:00000200:20.0:1619677475.325078:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (23)++
00000800:00000200:21.0:1619677475.325079:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677475.325079:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (24)++
00000800:00000200:8.2:1619677475.325079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.325080:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (25)--
00000800:00000200:20.0:1619677475.325080:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000800:00000200:21.0:1619677475.325081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (24)--
00000800:00000200:21.0:1619677475.325082:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (23)++
00000400:00000200:20.0:1619677475.325082:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.325083:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919476fc2600] (24)--
00000800:00000200:21.0:1619677475.325084:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (23)--
00000800:00000200:21.0:1619677475.325084:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (22)++
00000400:00000200:20.0:1619677475.325084:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.325085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:20.0:1619677475.325085:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.0:1619677475.325086:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000400:00000200:20.0:1619677475.325087:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.325087:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:20.0:1619677475.325088:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.325089:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677475.325089:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:20.0:1619677475.325090:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.325090:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.325090:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.325093:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.325093:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:20.0:1619677475.325094:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.325095:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.325096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.325096:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.325097:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.325097:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:21.0:1619677475.325098:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.325098:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.325099:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.325100:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.325101:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677475.325101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:21.0:1619677475.325102:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.325102:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.325104:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (23)++
00000400:00000200:21.0:1619677475.325105:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:20.0:1619677475.325105:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (24)--
00000800:00000200:20.0:1619677475.325105:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (23)--
00000400:00000200:21.0:1619677475.325106:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.325106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000400:00000200:21.0:1619677475.325107:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.325108:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677475.325108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:21.0:1619677475.325109:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677475.325112:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.325114:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.325116:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.325116:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.325117:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:4.2:1619677475.325117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (21)++
00000800:00000200:21.0:1619677475.325118:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.325118:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.325119:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.325119:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.325120:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (22)++
00000400:00000200:20.0:1619677475.325120:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.325121:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (23)--
00000800:00000200:21.0:1619677475.325122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000800:00000200:20.0:1619677475.325122:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.325123:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.325124:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (21)++
00000800:00000200:20.0:1619677475.325127:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000800:00000200:8.2:1619677475.325127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:20.0:1619677475.325129:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.325130:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.325131:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (22)++
00000400:00000200:20.0:1619677475.325132:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:21.0:1619677475.325133:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000400:00000200:20.0:1619677475.325134:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.325135:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.325136:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.325137:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:21.0:1619677475.325140:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.325141:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.325143:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.325144:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:21.0:1619677475.325145:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:21.0:1619677475.325146:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.325147:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.325148:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.325149:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677475.325152:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.325153:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.325153:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:20.0:1619677475.325153:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677475.325155:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677475.325155:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.0:1619677475.325156:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (23)++
00000800:00000200:20.0:1619677475.325157:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919476fc2600] (24)++
00000800:00000200:21.0:1619677475.325158:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.325158:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (25)--
00000800:00000200:21.0:1619677475.325159:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:20.0:1619677475.325159:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (24)--
00000800:00000200:21.0:1619677475.325160:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677475.325160:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.325161:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677475.325161:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:8.2:1619677475.325161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677475.325162:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (23)++
00000400:00000200:20.0:1619677475.325162:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.325163:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (24)--
00000800:00000200:21.0:1619677475.325164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (23)--
00000800:00000200:21.0:1619677475.325164:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (22)++
00000400:00000200:20.0:1619677475.325164:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.325166:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000800:00000200:20.0:1619677475.325166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.325167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677475.325167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:21.0:1619677475.325168:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.325168:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (23)++
00000800:00000200:20.0:1619677475.325169:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919476fc2600] (24)--
00000800:00000200:20.0:1619677475.325169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (23)--
00000800:00000200:1.0:1619677475.325169:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:21.0:1619677475.325170:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.325170:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000800:00000200:1.0:1619677475.325171:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:21.0:1619677475.325172:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677475.325172:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.325173:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.325173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:21.0:1619677475.325174:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.325174:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:1.0:1619677475.325175:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.325176:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.325176:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.325176:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.325177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.325177:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.325180:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677475.325180:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677475.325182:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.0:1619677475.325183:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.325183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.0:1619677475.325184:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677475.325185:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (22)++
00000400:00000200:8.0:1619677475.325185:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.325185:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677475.325188:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (24)--
00000800:00000200:8.0:1619677475.325188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677475.325188:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:21.0:1619677475.325189:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (21)++
00000800:00000200:17.0:1619677475.325189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:9.2:1619677475.325189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677475.325189:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (24)++
00000800:00000200:21.0:1619677475.325190:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (22)--
00000800:00000200:1.0:1619677475.325190:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (24)--
00000800:00000200:21.0:1619677475.325191:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (21)--
00000800:00000200:1.0:1619677475.325191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.2:1619677475.325194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.325196:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.325198:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.325199:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.325200:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.325201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.325202:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677475.325202:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.325205:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.325206:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.325206:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.325208:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.325210:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.325226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (20)++
00000800:00000200:20.0:1619677475.325229:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (21)++
00000800:00000200:20.0:1619677475.325232:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000400:00000200:20.0:1619677475.325234:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.325235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000400:00000200:20.0:1619677475.325236:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.325238:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677475.325239:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.325241:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.325242:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.325243:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677475.325246:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.325247:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677475.325250:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.325251:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:20.0:1619677475.325251:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677475.325253:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677475.325254:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (21)++
00000800:00000200:20.0:1619677475.325255:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (22)--
00000800:00000200:20.0:1619677475.325255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (21)--
00000800:00000200:25.2:1619677475.325259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.325266:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.325269:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.325270:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.325272:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.325273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.325275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.325287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (20)++
00000800:00000200:21.0:1619677475.325293:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (21)++
00000800:00000200:21.0:1619677475.325300:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000800:00000200:20.0:1619677475.325300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000400:00000200:21.0:1619677475.325302:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.325305:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.325307:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677475.325308:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.325309:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.325310:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.325311:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677475.325314:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.325316:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677475.325319:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.325319:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677475.325320:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677475.325322:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677475.325322:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (21)++
00000800:00000200:21.0:1619677475.325324:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919476fc2600] (22)++
00000800:00000200:21.0:1619677475.325325:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (23)--
00000800:00000200:21.0:1619677475.325326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000800:00000200:9.2:1619677475.325328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:4.2:1619677475.325331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (21)++
00000800:00000200:20.0:1619677475.325334:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (22)++
00000800:00000200:20.0:1619677475.325336:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919476fc2600] (23)--
00000800:00000200:10.0:1619677475.325336:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:20.0:1619677475.325337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000800:00000200:20.0:1619677475.325338:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (21)--
00000800:00000200:26.0:1619677475.325339:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.325339:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.325340:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.325342:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.325345:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677475.325350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.325356:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.325358:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.325359:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.325359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.325360:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.325360:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677475.325368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919476fc2600] (20)++
00000800:00000200:20.0:1619677475.325371:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919476fc2600] (21)++
00000800:00000200:20.0:1619677475.325374:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.70@o2ib3
00000400:00000200:20.0:1619677475.325376:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.70@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.325377:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (22)--
00000400:00000200:20.0:1619677475.325379:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.325380:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677475.325382:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.325383:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.325384:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.325385:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677475.325389:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.325390:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677475.325393:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.325394:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677475.325394:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677475.325396:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677475.325397:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919476fc2600] (21)++
00000800:00000200:20.0:1619677475.325397:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919476fc2600] (22)--
00000800:00000200:20.0:1619677475.325398:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919476fc2600] (21)--
00000800:00000200:25.2:1619677475.325402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.325409:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.325412:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.325413:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.325415:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.325417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.325418:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.325505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.325512:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.325515:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.325516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.325517:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.325520:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.325521:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.325522:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.325523:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.325524:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.325525:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.325529:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.325530:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.325532:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.325533:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.325534:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.325535:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.325536:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.325536:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.325537:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.325541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.325549:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.325551:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.325552:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.325553:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.325554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.325556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.325561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.325569:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.325571:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.325574:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.325575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.325577:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.325577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:31.0:1619677475.325578:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.325580:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.325581:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.325582:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.325583:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.325584:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:31.0:1619677475.325586:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.325587:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:31.0:1619677475.325588:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.325588:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.325590:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.325591:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677475.325591:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.325592:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.325593:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677475.325593:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.325594:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.325595:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.325595:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:10.0:1619677475.325595:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.325596:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.325597:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.325597:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:10.0:1619677475.325597:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.325600:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.325601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.325601:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.325604:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.325605:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.325605:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.325607:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.325607:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.325608:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:6.0:1619677475.325608:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.325610:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.325610:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:6.0:1619677475.325611:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.325612:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.325612:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.325613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:6.0:1619677475.325615:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:4.2:1619677475.325616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:6.0:1619677475.325617:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:13.0:1619677475.325620:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677475.325623:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:13.0:1619677475.325623:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.325624:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.325625:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:13.0:1619677475.325625:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.325626:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677475.325627:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.325628:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677475.325628:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677475.325687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.325694:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.325697:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.325697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.325700:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.325702:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.325703:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.325704:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.325705:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.325707:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.325707:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.325710:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.325712:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.325714:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.325715:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.325715:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.325717:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.325718:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.325719:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.325719:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.325723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.325731:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.325733:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.325734:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.325734:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.325736:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.325737:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.326233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.326241:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.326244:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.326245:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.326247:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.326249:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.326251:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.326252:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.326253:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.326254:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.326255:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.326258:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.326260:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.326263:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.326264:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.326265:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.326266:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.326267:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.326268:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.326268:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.326272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.326281:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.326284:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.326285:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.326285:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.326288:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.326290:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.326327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.326334:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.326337:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.326337:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.326340:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.326341:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.326343:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.326344:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.326345:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.326346:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.326347:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.326350:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.326351:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.326354:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.326355:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.326355:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.326357:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.326357:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.326358:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.326359:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677475.326363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.326372:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.326374:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.326375:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.326376:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.326378:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.326379:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.326534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.326540:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.326543:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.326544:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.326546:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.326548:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.326549:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.326551:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.326552:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.326553:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.326554:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.326557:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.326558:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.326561:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.326562:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.326562:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.326564:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.326565:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.326565:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.326566:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.326571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.326579:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.326581:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.326582:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.326583:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.326585:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.326586:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.326589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.326596:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.326599:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.326600:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.326600:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.326601:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.326602:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.327374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.327380:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.327383:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.327384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.327385:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.327388:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.327389:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.327390:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.327391:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.327392:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.327393:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.327396:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.327398:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.327400:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.327400:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.327401:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.327403:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.327403:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.327404:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.327405:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.327406:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.327409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.327411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.327416:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.327417:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.327418:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.327419:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.327419:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.327420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.327420:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.327422:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.327423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.327424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.327519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.327527:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.327529:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.327530:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.327532:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.327535:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.327536:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.327538:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.327539:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.327540:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.327541:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.327544:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.327545:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.327548:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.327549:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.327550:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.327552:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.327552:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.327552:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.327553:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.327558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.327565:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.327568:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.327568:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.327569:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.327571:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.327573:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.327765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.327771:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.327774:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.327775:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.327777:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.327779:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.327781:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.327782:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.327783:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.327784:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.327785:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.327789:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.327791:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.327793:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.327794:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.327795:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.327796:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.327797:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.327798:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.327799:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.327800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.327803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.327805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677475.327810:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677475.327811:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.327812:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:22.0:1619677475.327813:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:17.0:1619677475.327813:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677475.327814:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.327814:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.327815:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677475.327816:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.327818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.327826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.327833:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.327835:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.327836:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.327836:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.327838:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.327838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.328190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.328197:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.328200:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.328201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.328203:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.328206:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.328207:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.328209:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.328210:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.328211:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.328212:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.328215:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.328217:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.328220:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.328221:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.328221:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.328223:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.328224:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.328224:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.328225:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.328229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.328236:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.328239:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.328239:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.328240:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.328242:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.328244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.328255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.328260:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.328263:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.328263:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.328265:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.328266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:6.0:1619677475.328267:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.328268:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.328269:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.328271:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.328271:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.328272:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.328275:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.328276:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.328277:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.328279:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.328280:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677475.328281:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:6.0:1619677475.328281:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.328282:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.328283:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.328283:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:30.0:1619677475.328284:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.328284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677475.328287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.328288:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.328288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.328290:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.328292:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.328299:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:30.0:1619677475.328301:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.328302:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:30.0:1619677475.328303:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.328303:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.328305:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.328305:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.328305:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:30.0:1619677475.328311:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.328311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677475.328314:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.328314:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.328318:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.328318:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677475.328319:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.328320:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.328321:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.328322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677475.328323:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.328324:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:24.0:1619677475.328324:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.328324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.328325:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.328327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:24.0:1619677475.328327:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.328329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677475.328329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:24.0:1619677475.328329:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.328330:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.328331:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.328332:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677475.328332:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.328333:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.328335:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.328336:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.328337:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:24.0:1619677475.328338:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.328340:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.328340:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.328340:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.328341:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677475.328342:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:24.0:1619677475.328343:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.328343:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.328344:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.328344:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677475.328345:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96500441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.328345:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.328346:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677475.328347:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.328347:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677475.328348:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677475.328348:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:30.0:1619677475.328349:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.328350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677475.328351:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.328354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:30.0:1619677475.328356:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.328356:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.328358:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677475.328359:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.328360:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.328360:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (24)++
00000800:00000200:6.0:1619677475.328360:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677475.328361:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677475.328361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.328361:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.328362:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.328363:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.328364:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.328365:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.328366:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677475.328366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.328368:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.328369:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.328371:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.328372:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.328374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.328381:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.328383:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.328384:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.328384:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.328386:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.328389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.328436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:10.0:1619677475.328443:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:10.0:1619677475.328446:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.22@o2ib3
00000800:00000200:26.0:1619677475.328447:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:10.0:1619677475.328449:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.328451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:10.0:1619677475.328451:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.328453:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677475.328454:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.328455:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.328456:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.328457:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:1.0:1619677475.328458:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:10.0:1619677475.328460:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.328461:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.328462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:10.0:1619677475.328462:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677475.328464:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677475.328464:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.328465:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677475.328466:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:1.0:1619677475.328466:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.328467:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.328468:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000400:00000200:1.0:1619677475.328468:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677475.328469:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (22)++
00000400:00000200:1.0:1619677475.328469:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.328470:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:10.0:1619677475.328471:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:1.0:1619677475.328471:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.328472:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677475.328473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:1.0:1619677475.328473:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.328476:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.328477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (21)++
00000800:00000200:1.0:1619677475.328478:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.328480:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.328481:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:26.0:1619677475.328482:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:1.0:1619677475.328482:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.328483:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:26.0:1619677475.328484:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:1.0:1619677475.328484:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:26.0:1619677475.328485:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.328485:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000800:00000200:1.0:1619677475.328485:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:10.0:1619677475.328486:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (23)--
00000800:00000200:1.0:1619677475.328486:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.328486:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:26.0:1619677475.328487:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.328487:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:10.0:1619677475.328488:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.328489:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:8.2:1619677475.328489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677475.328490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:21.2:1619677475.328491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677475.328497:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677475.328497:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.328499:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677475.328500:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677475.328500:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677475.328501:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.328501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677475.328502:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.328503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.328504:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.328928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (20)++
00000800:00000200:26.0:1619677475.328935:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677475.328938:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.14.22@o2ib3
00000800:00000200:10.0:1619677475.328939:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677475.328941:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.22@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.328943:0:19941:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 14880
00000800:00000200:26.0:1619677475.328945:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677475.328946:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (22)++
00000800:00000200:26.0:1619677475.328947:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (23)--
00000800:00000200:26.0:1619677475.328947:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000800:00000200:20.2:1619677475.328948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677475.328952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (21)++
00000800:00000200:21.0:1619677475.328953:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.328955:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.328957:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.328959:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677475.328959:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000400:00000200:21.0:1619677475.328960:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.328961:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (23)--
00000400:00000200:21.0:1619677475.328962:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:10.0:1619677475.328962:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (22)--
00000400:00000200:21.0:1619677475.328963:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.328964:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.328965:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.328966:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:4.2:1619677475.328968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f787000] (21)++
00000400:00000200:21.0:1619677475.328969:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.328970:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.328973:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.328973:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.328974:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.328975:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f787000] (22)++
00000800:00000200:21.0:1619677475.328976:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.328976:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677475.328977:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.14.22@o2ib3
00000800:00000200:21.0:1619677475.328977:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:10.0:1619677475.328978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (23)--
00000800:00000200:26.0:1619677475.328979:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f787000] (22)--
00000400:00000200:26.0:1619677475.328979:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.328979:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.328979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677475.328981:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677475.328982:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677475.328982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:26.0:1619677475.328983:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.328984:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677475.328984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:26.0:1619677475.328985:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677475.328987:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:26.0:1619677475.328988:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.328989:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677475.328989:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677475.328990:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 14880 bytes in 256 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677475.328990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.328990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677475.328991:0:19941:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 14880
00000800:00000200:26.0:1619677475.328992:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.328992:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:26.0:1619677475.328993:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:17.0:1619677475.328993:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.328994:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677475.328994:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.328995:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:17.0:1619677475.328995:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.328996:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f787000] (21)++
00000800:00000200:26.0:1619677475.328996:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f787000] (22)--
00000800:00000200:26.0:1619677475.328997:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f787000] (21)--
00000800:00000200:17.0:1619677475.328998:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:9.2:1619677475.329001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.329007:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.329010:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677475.329011:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:9.2:1619677475.329022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.329027:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.329030:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677475.329030:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.329032:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.329034:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.329035:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:9.2:1619677475.329044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.329050:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.329052:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.329053:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.329053:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.329054:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.329056:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.329305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.329311:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.329313:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.329314:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.329316:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.329318:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.329320:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.329321:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.329322:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.329323:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677475.329323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:22.0:1619677475.329324:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.329327:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.329327:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:22.0:1619677475.329329:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.329329:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:22.0:1619677475.329331:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.329332:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:20.0:1619677475.329332:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.329333:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.329333:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:22.0:1619677475.329334:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.329334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.329335:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.329336:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.329336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.329336:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677475.329337:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.329339:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.329340:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.329341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677475.329341:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677475.329345:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.329347:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.329348:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.329350:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.329350:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.329350:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.329350:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677475.329351:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:21.0:1619677475.329352:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.329353:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.329354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.329354:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.329354:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.329355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:21.0:1619677475.329356:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.329359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.329367:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.329370:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.329371:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.329372:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.329374:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.329375:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.330036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.330043:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.330047:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.330049:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.330050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.330051:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.330052:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.330054:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.330055:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.330056:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.330057:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.330060:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.330062:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.330065:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.330066:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.330066:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.330068:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.330069:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.330070:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.330070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.330074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.330082:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.330085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.330085:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.330086:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.330088:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.330091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.330174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.330179:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.330182:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.330183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.330185:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.330187:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.330188:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.330189:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.330191:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.330192:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.330193:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.330196:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.330198:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.330201:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.330201:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.330202:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.330204:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.330204:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.330205:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.330205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.330210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.330217:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.330220:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.330221:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.330222:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.330223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.330225:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.330228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.330235:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.330238:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.330239:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.330239:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.330240:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.330241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.331020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.331026:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.331029:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.331030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.331032:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.331034:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.331035:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.331036:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.331037:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.331039:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.331039:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.331042:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.331044:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.331046:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.331046:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.331047:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.331048:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.331049:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.331050:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.331051:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.331052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.331055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.331058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.331063:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.331065:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.331065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.331065:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.331066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.331067:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.331067:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.331069:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.331070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.331071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.331195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.331203:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.331207:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.331209:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.331212:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.331216:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.331218:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.331220:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.331222:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.331224:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.331226:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.331231:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.331234:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.331238:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.331239:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.331240:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.331243:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.331244:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.331245:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.331246:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.331249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.331257:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.331259:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.331260:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.331262:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.331264:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.331266:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:5.2:1619677475.331359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677475.331367:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677475.331372:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:17.0:1619677475.331373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:1.0:1619677475.331376:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677475.331380:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.331382:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677475.331385:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.331387:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.331389:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.331390:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677475.331395:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677475.331398:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677475.331402:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.331403:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677475.331404:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.331407:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.331408:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677475.331409:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677475.331410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677475.331413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.331420:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.331423:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.331424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.331425:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.331427:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.331430:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.331635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.331643:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.331646:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.331648:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.331651:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.331651:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.331653:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.331654:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.331655:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.331656:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.331657:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.331660:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.331662:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.331664:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.331665:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.331666:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.331667:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.331668:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.331669:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.331669:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.331673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.331680:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.331682:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.331683:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.331684:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.331686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.331687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.331693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.331700:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.331702:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.331703:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.331703:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.331704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.331705:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.331795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.331803:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.331805:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.331806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.331807:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.331810:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.331812:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.331813:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.331814:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.331815:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.331816:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.331819:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.331821:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.331824:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.331824:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.331825:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.331827:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.331827:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.331828:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.331828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.331833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.331843:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.331847:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.331849:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.331849:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.2:1619677475.331852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:17.0:1619677475.331853:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.331859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.331862:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.331865:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.331866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.331867:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.331869:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.331870:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.332027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.332032:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.332035:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.332036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.332038:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.332040:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.332042:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.332043:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.332044:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.332045:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.332046:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.332050:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.332051:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.332054:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.332055:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.332055:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.332057:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.332057:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.332058:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.332058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.332063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.332070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677475.332071:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.332073:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.332074:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.332075:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.332077:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677475.332077:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.332078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.332080:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.332082:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.332084:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.332084:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.332086:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.332087:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.332089:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.332090:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.332091:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.332094:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.332096:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.332099:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.332100:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.332100:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.332102:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.332103:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.332104:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.332106:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.332106:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.332108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.332112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.332116:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.332118:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.332119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.332120:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.332120:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.332122:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677475.332122:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.332123:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.332124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.332125:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.332352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.332360:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.332363:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.332363:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.332365:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.332367:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.332368:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.332369:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.332371:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.332372:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.332373:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.332376:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.332378:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.332380:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.332381:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.332382:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.332383:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.332384:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.332385:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.332385:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.332390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.332398:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.332401:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.332402:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.332402:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.332404:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.332407:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.332908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.332913:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.332916:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.332917:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.332918:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.332921:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.332922:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.332923:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.332924:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.332925:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.332926:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.332929:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.332930:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.332932:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.332933:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.332934:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.332936:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.332936:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.332937:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.332937:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.332942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.332950:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.332952:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.332953:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.332954:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.332955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.332956:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.332964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.332972:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.332974:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.332975:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.332977:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.332980:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.332981:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.332982:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.332983:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.332984:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.332985:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.332988:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.332990:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.332993:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.332994:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.332994:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.332996:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.332996:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.332997:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.332998:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.333002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.333009:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.333012:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.333013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.333013:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.333016:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.333018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.333020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.333027:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.333030:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.333030:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.333031:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.333032:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.333032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.333188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.333196:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.333199:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.333200:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.333201:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.333205:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.333206:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.333207:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.333209:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.333210:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.333210:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.333213:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.333215:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.333217:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.333218:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.333219:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.333220:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.333221:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.333222:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.333223:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.333224:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.333226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.333229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.333234:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.333236:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.333236:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.333237:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.333238:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.333239:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.333239:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677475.333239:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.333240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677475.333242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.333486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.333494:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.333497:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.333499:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.333502:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.333502:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.333504:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.333505:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.333506:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.333507:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.333508:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.333512:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.333513:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.333517:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.333517:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.333518:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.333520:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.333520:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.333521:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.333522:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.333526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.333534:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.333537:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.333538:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.333538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.333541:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.333543:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.333661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.333668:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.333671:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.333671:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.333674:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.333677:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.333678:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.333679:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.333681:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.333682:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.333683:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.333686:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.333688:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.333689:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.333690:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.333691:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.333692:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.333693:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.333694:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.333694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.333698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.333705:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.333708:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.333708:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.333709:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.333710:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.333711:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.333719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.333726:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.333729:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.333729:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.333730:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.333730:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677475.333730:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.2:1619677475.333807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.333812:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.333815:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.333816:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.333818:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.333820:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.333821:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.333822:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.333824:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.333825:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.333826:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.333829:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.333831:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.333833:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.333834:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.333835:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.333836:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.333837:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.333837:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.333838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.333842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.333850:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.333852:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.333853:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.333854:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.333856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.333856:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.333863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.333870:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.333872:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.333873:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.333874:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.333875:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.333876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.334709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.334717:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.334720:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.334722:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.334725:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.334725:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.334727:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.334728:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.334729:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.334730:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.334731:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.334735:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.334737:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.334740:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.334741:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.334741:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.334743:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.334744:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.334745:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.334745:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.334750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.334758:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.334761:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.334762:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.334762:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.334764:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.334767:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.334839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.334844:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.334847:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.334847:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.334849:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.334851:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.334852:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.334853:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677475.334855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:6.0:1619677475.334855:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.334856:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.334857:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.334860:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677475.334860:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.334862:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.334863:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677475.334864:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.334865:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:21.0:1619677475.334866:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.334866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677475.334866:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.334868:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.334868:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677475.334869:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.334869:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.334870:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:21.0:1619677475.334871:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677475.334871:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677475.334872:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.334874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677475.334874:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677475.334875:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677475.334878:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.334879:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677475.334880:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.334881:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.334882:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:21.0:1619677475.334882:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.334882:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.334883:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.334883:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.334884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.334884:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.334885:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.334887:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.334887:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.334888:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.334888:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.334888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677475.334890:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.334891:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.334892:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.334892:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.334897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.334906:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.334908:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.334910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.334910:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.334912:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.334915:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.334917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.334924:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.334927:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.334928:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.334928:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.334929:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.334929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.335689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.335697:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677475.335697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.335700:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.335702:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.335703:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.335704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.335705:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.335706:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:30.0:1619677475.335707:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677475.335707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:30.0:1619677475.335708:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.335710:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.335710:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.335712:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.335712:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.335713:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:6.0:1619677475.335714:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.335715:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.335717:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.335718:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.335719:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.335720:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.335722:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.335724:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.335724:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.335725:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.335726:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677475.335726:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.335727:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.335728:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677475.335728:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.335729:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677475.335729:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.335730:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677475.335731:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:6.0:1619677475.335731:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.335732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677475.335732:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.335732:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.335733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.335734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.335737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677475.335738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.335741:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.335743:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.335744:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677475.335744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677475.335745:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677475.335746:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.335746:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.335747:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.335748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.335749:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.0:1619677475.335749:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677475.335750:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677475.335750:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677475.335751:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.335752:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.335752:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.335753:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:20.0:1619677475.335754:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.335755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:2.0:1619677475.335756:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.335757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:2.0:1619677475.335759:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.335760:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.335762:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.335763:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.335764:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.335765:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.335768:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.335770:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.335773:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.335774:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.335774:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.335776:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.335776:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.335777:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.335778:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.335783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.335791:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.335794:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.335795:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.335795:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.335797:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.335799:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.336466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.336474:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.336477:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.336480:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.336482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.336482:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.336483:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.336485:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.336486:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.336487:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.336488:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.336491:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.336493:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.336495:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.336496:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.336496:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.336498:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.336498:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.336499:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.336500:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.336504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.336511:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.336514:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.336514:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.336516:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.336518:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.336518:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.336621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.336627:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.336630:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.336631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.336633:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.336635:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.336636:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.336638:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.336639:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.336640:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.336641:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.336644:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.336646:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.336649:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.336649:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.336650:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.336651:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.336652:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.336653:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.336653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:4.2:1619677475.336656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677475.336657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677475.336664:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677475.336665:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677475.336666:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:21.0:1619677475.336667:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677475.336667:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:21.0:1619677475.336668:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.336670:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.336670:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.336671:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.336672:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677475.336673:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.336674:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.336675:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.336677:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.336678:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.336679:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.336681:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.336683:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.336686:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.336687:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.336688:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.336689:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.336690:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.336690:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.336691:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.336695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.336703:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.336706:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.336706:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.336707:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.336709:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.336711:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.336831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:21.2:1619677475.336835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:18.0:1619677475.336838:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.336841:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677475.336841:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:31.0:1619677475.336842:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:18.0:1619677475.336844:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.336845:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:18.0:1619677475.336846:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.336847:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.336848:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.336849:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.336849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:18.0:1619677475.336849:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.336851:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677475.336851:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.336852:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.336852:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677475.336853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:18.0:1619677475.336853:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.336854:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.336855:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.336856:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677475.336858:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.336859:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:18.0:1619677475.336860:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.336862:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.336862:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:31.0:1619677475.336864:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.336864:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.336865:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677475.336865:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:17.0:1619677475.336865:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.336867:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677475.336867:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:17.0:1619677475.336867:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.336868:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.336869:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677475.336869:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:17.0:1619677475.336869:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677475.336870:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677475.336870:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.336871:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000400:00000200:17.0:1619677475.336871:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.336872:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000400:00000200:17.0:1619677475.336872:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.336873:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677475.336874:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.336874:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.336875:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.336876:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:17.0:1619677475.336877:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.336879:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.336881:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.336882:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.336883:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:25.2:1619677475.336884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677475.336884:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.336885:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.336886:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.336886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.336891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:22.0:1619677475.336892:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677475.336894:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677475.336895:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000400:00000200:22.0:1619677475.336896:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.336896:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677475.336897:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.336898:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.336899:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:6.0:1619677475.336899:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.336901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677475.336901:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.336902:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.336902:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.336903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:20.0:1619677475.336904:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.336905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.336905:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.336906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.336912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:6.0:1619677475.336912:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.336915:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.336916:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.336916:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.336918:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.336918:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677475.336919:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.336922:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.336923:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.336923:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.336924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.336925:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.337098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.337106:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.337108:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.337110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.337111:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.337115:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.337116:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.337117:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.337119:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.337120:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.337121:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.337124:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.337126:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.337129:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.337129:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.337130:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.337132:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.337132:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.337133:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.337133:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.337137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.337145:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.337148:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.337149:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.337149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.337151:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.337154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.337403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.337411:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.337414:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.337416:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.337419:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.337419:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.337421:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.337422:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.337423:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.337424:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.337425:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.337428:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.337429:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.337432:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.337432:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.337433:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.337434:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.337435:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.337436:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.337436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.337441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.337448:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.337450:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.337451:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.337451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.337453:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.337456:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.337694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.337700:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.337703:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.337703:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.337705:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.337707:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.337709:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.337710:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.337711:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.337712:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.337713:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.337716:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.337718:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.337720:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.337721:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.337722:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.337723:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.337724:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.337725:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.337725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.337730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.337738:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.337740:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.337740:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.337742:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.337743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.337744:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.337751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.337758:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.337761:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.337762:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.337762:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.337763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.337764:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.337788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.337796:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.337799:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.337800:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.337801:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.337804:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.337806:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.337807:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.337808:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.337809:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.337810:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.337814:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.337815:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.337818:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.337819:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.337820:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.337821:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.337822:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.337823:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.337824:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.337825:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.337827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.337830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.337835:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.337837:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.337838:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.337838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.337839:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677475.337839:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.337840:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.337841:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.337841:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.337844:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.338018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.338024:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.338027:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.338028:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.338029:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.338032:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.338033:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.338035:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.338036:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.338037:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.338038:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.338042:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.338044:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.338046:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.338047:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.338047:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.338049:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.338049:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.338050:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.338050:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.338055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.338055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.338062:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:6.0:1619677475.338063:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.338065:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:6.0:1619677475.338065:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677475.338066:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:22.0:1619677475.338066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.338067:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.338068:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.338069:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.338071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:10.0:1619677475.338072:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.338073:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.338074:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.338076:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.338077:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.338078:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.338081:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.338083:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.338085:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.338086:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.338087:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.338088:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:4.2:1619677475.338088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:10.0:1619677475.338089:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.338089:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.338090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.338094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677475.338095:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.338098:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.338099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.338100:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.338102:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:8.0:1619677475.338103:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.338104:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.338105:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.338105:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:12.0:1619677475.338106:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.338106:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677475.338108:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:8.0:1619677475.338108:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.338108:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:13.0:1619677475.338109:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:8.0:1619677475.338109:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.338112:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.338114:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.338117:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.338118:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.338118:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.338120:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.338120:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.338121:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.338122:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.338126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.338133:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.338136:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.338136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.338137:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.338139:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.338142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.338145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.338152:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.338155:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.338156:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.338156:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.338157:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.338158:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.338517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.338524:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.338527:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.338528:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.338530:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.338533:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.338534:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677475.338535:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.338537:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.338538:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.338538:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677475.338542:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.338543:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677475.338546:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.338547:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677475.338547:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.338549:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.338549:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.338550:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.338551:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.338556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.338565:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.338567:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.338568:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.338569:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.338571:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.338574:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.338633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.338639:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.338642:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.338643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.338645:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.338647:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.338648:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.338650:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.338651:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.338652:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.338653:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.338656:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.338658:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.338660:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.338661:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.338662:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.338663:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.338664:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.338665:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.338666:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.338667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.338670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.338672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.338677:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.338678:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.338679:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.338679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.338680:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.338680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.338681:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.338683:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.338684:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.338685:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.338922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.338930:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.338933:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.338935:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.338938:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.338938:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.338940:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.338941:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.338942:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.338943:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.338944:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.338947:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.338949:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.338952:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.338953:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.338953:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.338955:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.338956:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.338957:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.338959:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.338959:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.338961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.338965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.338969:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.338972:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.338972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.338973:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.338973:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.338975:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.338976:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.338976:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.338977:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.338978:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.339097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:10.0:1619677475.339104:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677475.339107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677475.339107:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:10.0:1619677475.339110:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.339112:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.339114:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.339115:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.339116:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.339117:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.339118:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.339122:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.339123:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.339126:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.339127:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.339127:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.339129:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.339130:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677475.339131:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:10.0:1619677475.339132:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677475.339133:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:8.2:1619677475.339136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.339138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:21.0:1619677475.339143:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677475.339145:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:21.0:1619677475.339146:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677475.339147:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000400:00000200:21.0:1619677475.339147:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.339148:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:21.0:1619677475.339148:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.339149:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:20.0:1619677475.339149:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.339152:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.339154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.339161:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.339164:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.339165:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.339165:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.339166:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.339167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.339252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.339256:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.339258:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.339260:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.339261:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.339262:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.339264:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.339265:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.339267:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.339268:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.339269:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.339271:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.339273:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.339275:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.339276:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.339277:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.339279:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.339279:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.339280:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.339282:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677475.339282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677475.339284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.339287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.339292:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.339293:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.339298:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.339298:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:21.0:1619677475.339299:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.339299:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.339300:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:21.0:1619677475.339301:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.339301:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.339303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.339494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.339500:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.339502:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.339503:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.339505:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.339507:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.339508:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.339509:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.339511:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.339512:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.339513:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.339516:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.339518:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.339520:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.339521:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.339521:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.339523:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.339523:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.339524:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.339524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.339529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.339537:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.339539:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.339540:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.339542:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.339543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.339544:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.340035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.340043:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.340046:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.340048:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.340050:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.340051:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.340052:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.340054:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.340055:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.340056:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.340057:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.340060:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.340062:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.340064:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.340065:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.340066:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.340067:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.340068:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.340069:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.340070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.340073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.340080:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.340083:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.340084:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.340084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.340086:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.340088:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.340539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.340544:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.340547:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.340548:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.340549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.340551:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.340552:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.340553:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.340555:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.340556:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.340557:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.340560:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.340562:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.340564:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.340565:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.340566:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.340568:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.340568:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.340569:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.2:1619677475.340571:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:21.0:1619677475.340574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.340575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677475.340577:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.340580:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.340581:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.340583:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.340583:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:22.0:1619677475.340585:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.340585:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.340586:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.340587:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:17.0:1619677475.340587:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:22.0:1619677475.340588:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.340589:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.340590:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.340591:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.340591:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:22.0:1619677475.340592:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.340596:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.340598:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.340600:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.340601:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.340601:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.340603:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.340603:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.340604:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.340605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.340610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.340617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677475.340617:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.340620:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.340620:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.340622:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.340624:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.340625:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677475.340625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.340628:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.340631:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.340633:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.340633:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.340634:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.340636:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.340637:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.340638:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.340639:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677475.340642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:31.0:1619677475.340643:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.340644:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.340647:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.340648:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677475.340648:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:31.0:1619677475.340649:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.340650:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.340651:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:2.0:1619677475.340651:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:31.0:1619677475.340652:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:18.0:1619677475.340652:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:31.0:1619677475.340653:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:2.0:1619677475.340654:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.340656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:2.0:1619677475.340656:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.340658:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.340659:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.340660:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.340661:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.340662:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677475.340664:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.340666:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:2.0:1619677475.340666:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.340667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.340668:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.340668:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.340669:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.340670:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.340670:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.340671:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677475.340673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677475.340673:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.340673:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.340675:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.340676:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.340676:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.340679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.340682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.340688:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.340689:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.340689:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:18.0:1619677475.340690:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.340690:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.340690:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.340691:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.340692:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.340692:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677475.340694:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.340696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b7a00] (20)++
00000800:00000200:17.0:1619677475.340703:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (21)++
00000800:00000200:17.0:1619677475.340706:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.3.20@o2ib3
00000800:00000200:1.0:1619677475.340706:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)--
00000400:00000200:17.0:1619677475.340709:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.340711:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.340712:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677475.340714:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.340715:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.340716:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.340717:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.340721:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.340722:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.340725:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.340726:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677475.340727:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677475.340729:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677475.340729:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (21)++
00000800:00000200:17.0:1619677475.340730:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (22)--
00000800:00000200:17.0:1619677475.340730:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (21)--
00000800:00000200:9.2:1619677475.340735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.340740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b7a00] (20)++
00000800:00000200:8.0:1619677475.340742:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.340744:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.340745:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.340745:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (21)++
00000400:00000200:8.0:1619677475.340746:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.340748:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)++
00000400:00000200:8.0:1619677475.340748:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.340748:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:17.0:1619677475.340750:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:8.0:1619677475.340751:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:1.0:1619677475.340751:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.340753:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.340754:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.340755:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.340755:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677475.340757:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.340758:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.340759:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.340760:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.340762:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677475.340763:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.340764:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677475.340766:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.340766:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.340767:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.340767:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.340770:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.340771:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:17.0:1619677475.340772:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.340772:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677475.340774:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.340774:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.340775:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:17.0:1619677475.340776:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.340776:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:17.0:1619677475.340777:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.340777:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)--
00000800:00000200:17.0:1619677475.340778:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.340778:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)++
00000800:00000200:1.0:1619677475.340779:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:17.0:1619677475.340780:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.340780:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:9.2:1619677475.340780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677475.340781:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)--
00000400:00000200:1.0:1619677475.340781:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.340782:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)--
00000800:00000200:17.0:1619677475.340783:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)--
00000400:00000200:1.0:1619677475.340783:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.340785:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:25.2:1619677475.340786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.340786:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:1.0:1619677475.340786:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.340787:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677475.340788:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:1.0:1619677475.340788:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.340788:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.340789:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.340789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.340792:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:1.0:1619677475.340792:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.340793:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.2:1619677475.340793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b7a00] (21)++
00000800:00000200:1.0:1619677475.340794:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.340795:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677475.340796:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.340797:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.340797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.340797:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.340797:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677475.340798:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000400:00000200:22.0:1619677475.340799:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.340799:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)++
00000800:00000200:1.0:1619677475.340800:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677475.340801:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:22.0:1619677475.340802:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:9.2:1619677475.340802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677475.340802:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (23)++
00000400:00000200:17.0:1619677475.340804:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.340804:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b7a00] (24)++
00000800:00000200:1.0:1619677475.340805:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (25)--
00000800:00000200:25.2:1619677475.340806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:17.0:1619677475.340806:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.340806:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:1.0:1619677475.340807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)++
00000400:00000200:17.0:1619677475.340808:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:8.0:1619677475.340808:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.340808:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000400:00000200:17.0:1619677475.340809:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.340811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:17.0:1619677475.340811:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.340811:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.340812:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.340812:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000400:00000200:17.0:1619677475.340813:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:8.0:1619677475.340813:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.340813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:6.0:1619677475.340813:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677475.340815:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.340816:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.340817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:17.0:1619677475.340817:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.340817:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.340817:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.340820:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:6.0:1619677475.340820:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:1.0:1619677475.340820:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:1.0:1619677475.340821:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.340823:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.340824:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.340824:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:25.2:1619677475.340826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.340828:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677475.340828:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.340829:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.340830:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677475.340830:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677475.340830:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.340830:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:17.0:1619677475.340832:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:6.0:1619677475.340832:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.340833:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)++
00000800:00000200:6.0:1619677475.340833:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.340834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.340834:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.340835:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (25)--
00000800:00000200:6.0:1619677475.340835:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.340835:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:17.0:1619677475.340836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:6.0:1619677475.340836:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677475.340836:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677475.340837:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:17.0:1619677475.340838:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:9.2:1619677475.340838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677475.340838:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:17.0:1619677475.340839:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)--
00000800:00000200:17.0:1619677475.340839:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)++
00000800:00000200:1.0:1619677475.340839:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:1.0:1619677475.340840:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (23)--
00000800:00000200:1.0:1619677475.340840:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)--
00000800:00000200:17.0:1619677475.340841:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:1.0:1619677475.340841:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)++
00000800:00000200:1.0:1619677475.340842:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000400:00000200:17.0:1619677475.340843:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.340844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:1.0:1619677475.340845:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.340846:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.340846:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.340847:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677475.340847:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:17.0:1619677475.340848:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.340850:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.340850:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.340851:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:17.0:1619677475.340851:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:10.0:1619677475.340851:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.340853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677475.340853:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.340853:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677475.340854:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:1.0:1619677475.340854:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96500381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.340855:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.340855:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.340857:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677475.340857:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.340858:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677475.340858:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.340858:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:1.0:1619677475.340858:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.340859:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677475.340862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:17.0:1619677475.340862:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.340863:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (21)++
00000400:00000200:1.0:1619677475.340863:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.340864:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677475.340864:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:1.0:1619677475.340865:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.340866:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:17.0:1619677475.340867:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:17.0:1619677475.340867:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:17.0:1619677475.340868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)--
00000800:00000200:1.0:1619677475.340868:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.340869:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)++
00000800:00000200:1.0:1619677475.340869:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.340869:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.340870:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:25.2:1619677475.340871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:1.0:1619677475.340871:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:17.0:1619677475.340872:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.340872:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:1.0:1619677475.340873:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b7a00] (24)++
00000800:00000200:1.0:1619677475.340874:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (25)--
00000400:00000200:17.0:1619677475.340875:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.340875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:1.0:1619677475.340875:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:1.0:1619677475.340876:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000400:00000200:17.0:1619677475.340877:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.2:1619677475.340877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:17.0:1619677475.340878:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.340879:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:17.0:1619677475.340879:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.340880:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.340881:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:17.0:1619677475.340881:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.340882:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.340883:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.340885:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.340885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.0:1619677475.340885:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:1.0:1619677475.340885:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.340886:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.340886:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:21.0:1619677475.340887:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.340888:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.340888:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:1.0:1619677475.340888:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.340890:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:1.0:1619677475.340890:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.340891:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:1.0:1619677475.340892:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.340893:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677475.340895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677475.340897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:1.0:1619677475.340897:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.340898:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:17.0:1619677475.340898:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.340899:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677475.340899:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677475.340899:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677475.340899:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677475.340901:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.340901:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.340902:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677475.340902:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677475.340902:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.340902:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677475.340903:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.340903:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:17.0:1619677475.340903:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)++
00000800:00000200:4.2:1619677475.340903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:1.0:1619677475.340903:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.340904:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (25)--
00000800:00000200:17.0:1619677475.340904:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:17.0:1619677475.340905:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:1.0:1619677475.340905:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.340905:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)++
00000800:00000200:17.0:1619677475.340906:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:1.0:1619677475.340906:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (25)--
00000800:00000200:9.2:1619677475.340907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677475.340907:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:1.0:1619677475.340907:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:1.0:1619677475.340908:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b7a00] (24)--
00000400:00000200:17.0:1619677475.340909:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.340909:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)--
00000800:00000200:1.0:1619677475.340910:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)++
00000800:00000200:25.2:1619677475.340911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.340911:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:26.0:1619677475.340912:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:17.0:1619677475.340912:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.340913:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677475.340913:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.340914:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677475.340914:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677475.340916:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:24.0:1619677475.340916:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:17.0:1619677475.340916:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.340917:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677475.340917:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.0:1619677475.340917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.340918:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.340919:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:17.0:1619677475.340919:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677475.340920:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.340921:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:1.0:1619677475.340921:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.340922:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.340922:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:26.0:1619677475.340923:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.340923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:17.0:1619677475.340924:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.340924:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:22.0:1619677475.340925:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:26.0:1619677475.340926:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.340926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:1.0:1619677475.340926:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.340927:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:1.0:1619677475.340927:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.340929:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677475.340929:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.340930:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:26.0:1619677475.340931:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.340932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:8.0:1619677475.340932:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:26.0:1619677475.340934:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.340934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000400:00000200:1.0:1619677475.340934:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.340935:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.340936:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.340936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677475.340936:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.340937:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:8.0:1619677475.340937:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677475.340937:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:26.0:1619677475.340938:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:17.0:1619677475.340938:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677475.340938:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.340938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677475.340940:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677475.340940:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.340941:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:6.0:1619677475.340941:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.340941:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:1.0:1619677475.340941:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677475.340942:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:17.0:1619677475.340942:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)--
00000800:00000200:17.0:1619677475.340943:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)++
00000800:00000200:22.0:1619677475.340944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.340944:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.340944:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:1.0:1619677475.340944:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (23)++
00000400:00000200:26.0:1619677475.340945:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.340945:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:1.0:1619677475.340945:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b7a00] (24)++
00000800:00000200:25.2:1619677475.340946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:6.0:1619677475.340946:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.340946:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (25)--
00000800:00000200:6.0:1619677475.340947:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.340947:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677475.340947:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:1.0:1619677475.340948:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:26.0:1619677475.340949:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:17.0:1619677475.340949:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.340950:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:9.2:1619677475.340951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677475.340952:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677475.340952:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.340953:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.340954:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:17.0:1619677475.340954:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677475.340955:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:17.0:1619677475.340955:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96500375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677475.340955:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:17.0:1619677475.340956:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677475.340957:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.340957:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000400:00000200:17.0:1619677475.340957:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.340958:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:12.0:1619677475.340958:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:26.0:1619677475.340959:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.340960:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:12.0:1619677475.340960:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.340961:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:9.2:1619677475.340962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:17.0:1619677475.340963:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:12.0:1619677475.340963:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.340964:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:13.0:1619677475.340965:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677475.340966:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.340966:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.340966:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.340967:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677475.340968:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:2.0:1619677475.340969:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.340970:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.340970:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:1.0:1619677475.340970:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:17.0:1619677475.340971:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000400:00000200:1.0:1619677475.340971:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96500374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.340972:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677475.340973:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:1.0:1619677475.340973:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.340974:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:10.0:1619677475.340975:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.340975:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677475.340976:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:1.0:1619677475.340976:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:17.0:1619677475.340978:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)++
00000800:00000200:2.0:1619677475.340978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:17.0:1619677475.340979:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (25)--
00000400:00000200:10.0:1619677475.340979:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.340980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (24)--
00000400:00000200:1.0:1619677475.340980:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677475.340981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.340982:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:1.0:1619677475.340983:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677475.340984:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:10.0:1619677475.340984:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:17.0:1619677475.340986:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.340987:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677475.340988:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.340988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677475.340988:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677475.340988:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:17.0:1619677475.340990:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677475.340990:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:17.0:1619677475.340991:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.340993:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.340994:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.340994:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.340995:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677475.340995:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)++
00000800:00000200:1.0:1619677475.340996:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (25)--
00000800:00000200:1.0:1619677475.340996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:9.2:1619677475.340997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677475.340998:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:17.0:1619677475.340999:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.340999:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:26.0:1619677475.341000:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677475.341000:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.341001:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:17.0:1619677475.341001:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:1.0:1619677475.341001:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.341002:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.341003:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:1.0:1619677475.341003:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677475.341004:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:17.0:1619677475.341004:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.341004:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677475.341004:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:17.0:1619677475.341005:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.341005:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:1.0:1619677475.341005:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:17.0:1619677475.341007:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:1.0:1619677475.341007:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.341008:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)++
00000800:00000200:10.0:1619677475.341008:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.341009:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677475.341009:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (25)--
00000400:00000200:10.0:1619677475.341009:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.341009:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.341010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:17.0:1619677475.341011:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:1.0:1619677475.341011:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:17.0:1619677475.341012:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:17.0:1619677475.341012:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b7a00] (24)--
00000400:00000200:1.0:1619677475.341012:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.341013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)--
00000400:00000200:10.0:1619677475.341013:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:8.2:1619677475.341013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677475.341014:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)++
00000400:00000200:1.0:1619677475.341014:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.341015:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000400:00000200:1.0:1619677475.341015:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.341016:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677475.341019:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:17.0:1619677475.341020:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.341020:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.341021:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:1.0:1619677475.341022:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677475.341023:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:17.0:1619677475.341023:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.341024:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:20.0:1619677475.341025:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.341025:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.341027:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.341027:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.341027:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.341027:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:1.0:1619677475.341027:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:17.0:1619677475.341029:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.341029:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:17.0:1619677475.341030:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.341031:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677475.341033:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:1.0:1619677475.341034:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b7a00] (24)++
00000800:00000200:25.2:1619677475.341035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:17.0:1619677475.341035:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.341035:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (25)--
00000800:00000200:20.0:1619677475.341036:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:1.0:1619677475.341036:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:17.0:1619677475.341037:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677475.341037:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:1.0:1619677475.341039:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:17.0:1619677475.341040:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.341041:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:17.0:1619677475.341041:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:1.0:1619677475.341041:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.341042:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:17.0:1619677475.341043:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:17.0:1619677475.341044:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)++
00000400:00000200:1.0:1619677475.341044:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.341045:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:17.0:1619677475.341045:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (25)--
00000400:00000200:1.0:1619677475.341045:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.341046:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.341046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:17.0:1619677475.341046:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)++
00000400:00000200:1.0:1619677475.341047:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.341048:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:31.0:1619677475.341049:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.2:1619677475.341049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:1.0:1619677475.341049:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.341050:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:1.0:1619677475.341051:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.341051:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.341054:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:17.0:1619677475.341054:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.341056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:1.0:1619677475.341056:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.341057:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:17.0:1619677475.341057:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.341058:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.341059:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:21.0:1619677475.341060:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.341060:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.341060:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.341062:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.341063:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677475.341063:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:17.0:1619677475.341063:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.341064:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.2:1619677475.341065:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.341066:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.341067:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:1.0:1619677475.341067:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.341068:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.341068:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:21.0:1619677475.341068:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.341068:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677475.341068:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:17.0:1619677475.341069:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.341070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677475.341070:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.341071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:17.0:1619677475.341071:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677475.341071:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.341071:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)++
00000800:00000200:1.0:1619677475.341072:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (25)--
00000800:00000200:30.0:1619677475.341073:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677475.341073:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (24)--
00000800:00000200:17.0:1619677475.341074:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.341074:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:17.0:1619677475.341075:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677475.341075:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b7a00] (24)--
00000400:00000200:30.0:1619677475.341076:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.341076:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:9.2:1619677475.341076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.341076:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)--
00000800:00000200:31.0:1619677475.341077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677475.341077:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)++
00000800:00000200:17.0:1619677475.341078:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677475.341078:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000400:00000200:30.0:1619677475.341079:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.341079:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (23)++
00000800:00000200:17.0:1619677475.341079:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (24)--
00000400:00000200:30.0:1619677475.341080:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677475.341080:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (23)--
00000800:00000200:17.0:1619677475.341081:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)--
00000400:00000200:1.0:1619677475.341081:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.341082:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96500368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.341083:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677475.341083:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:30.0:1619677475.341084:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677475.341084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:1.0:1619677475.341084:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.341085:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:30.0:1619677475.341086:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677475.341086:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.341086:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:8.0:1619677475.341087:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:1.0:1619677475.341088:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:32, s:-96500367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.341089:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:1.0:1619677475.341090:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.341091:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.341091:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.341092:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.341094:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677475.341094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677475.341094:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.341097:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:1.0:1619677475.341097:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.341099:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.341099:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.341100:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.341100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677475.341101:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.341102:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:1.0:1619677475.341102:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677475.341103:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.341103:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.341104:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:1.0:1619677475.341104:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (23)++
00000800:00000200:30.0:1619677475.341105:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.341107:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:18.0:1619677475.341107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:1.0:1619677475.341107:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (23)--
00000800:00000200:1.0:1619677475.341107:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (21)++
00000800:00000200:30.0:1619677475.341108:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.341108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.341109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (22)++
00000800:00000200:1.0:1619677475.341109:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b7a00] (22)++
00000800:00000200:1.0:1619677475.341109:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (23)--
00000800:00000200:1.0:1619677475.341110:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)--
00000800:00000200:21.2:1619677475.341114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.341119:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (23)++
00000800:00000200:21.2:1619677475.341119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b7a00] (21)++
00000800:00000200:31.0:1619677475.341121:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (24)--
00000400:00000200:31.0:1619677475.341122:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.341122:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677475.341123:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.341123:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.341124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677475.341124:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)++
00000800:00000200:30.0:1619677475.341125:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (23)++
00000800:00000200:31.0:1619677475.341126:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (24)--
00000800:00000200:30.0:1619677475.341126:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (24)--
00000800:00000200:31.0:1619677475.341127:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:30.0:1619677475.341127:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.341127:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b7a00] (23)--
00000800:00000200:17.0:1619677475.341128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)--
00000800:00000200:31.0:1619677475.341129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677475.341129:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.341129:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b7a00] (21)++
00000800:00000200:30.0:1619677475.341131:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:1.0:1619677475.341131:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.20@o2ib3
00000800:00000200:17.0:1619677475.341132:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (22)--
00000400:00000200:1.0:1619677475.341133:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.20@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.341136:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.341137:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.341139:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.341140:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.341141:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.341142:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.341145:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.341147:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.341149:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.341150:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.341150:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.341152:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.341152:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b7a00] (21)++
00000800:00000200:1.0:1619677475.341153:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b7a00] (22)--
00000800:00000200:1.0:1619677475.341154:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b7a00] (21)--
00000800:00000200:25.2:1619677475.341158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.341166:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.341168:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.341169:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.341169:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.341171:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.341174:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.341497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.341502:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.341504:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.341505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.341507:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.341509:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.341510:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.341512:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.341513:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.341514:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.341515:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.341518:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.341520:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.341523:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.341523:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.341524:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.341526:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.341526:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.341527:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.341527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.341533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.341541:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.341543:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.341544:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.341546:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.341547:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677475.341547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.0:1619677475.341549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.341552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677475.341557:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677475.341559:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677475.341562:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:21.0:1619677475.341562:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.341563:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.341563:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677475.341563:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.0:1619677475.341564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.341565:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677475.341566:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.341569:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.341572:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.341574:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.341576:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.341578:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.341579:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.341585:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.341588:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.341592:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.341594:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.341595:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.341597:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.341598:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.341599:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.341601:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.341602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.341603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.341607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.341611:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.341613:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.341614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.341615:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.341616:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677475.341617:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.341619:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:10.0:1619677475.341620:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.341622:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.341623:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:4.2:1619677475.341684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677475.341692:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.341696:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677475.341697:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677475.341698:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677475.341701:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.341703:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.341704:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.341705:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.341706:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.341707:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.341710:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677475.341712:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.341715:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.341716:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.341716:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.341718:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.341718:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.341719:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677475.341719:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677475.341724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.341731:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.341734:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.341734:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.341735:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.341737:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.341740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.341827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.341836:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.341839:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.341841:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.341843:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.341843:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.341845:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.341846:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.341847:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.341848:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.341849:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.341853:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.341854:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.341857:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.341857:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.341858:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.341860:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.341860:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.341861:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.341862:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.341865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.341872:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.341875:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.341875:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.341877:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.341879:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.341880:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.341955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.341962:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.341964:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.341965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.341967:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.341969:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.341971:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677475.341972:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.341973:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.341974:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.341975:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677475.341978:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.341979:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677475.341981:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.341982:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677475.341983:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677475.341984:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677475.341985:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.341985:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.341986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.341990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.2:1619677475.341992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:30.0:1619677475.341997:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.0:1619677475.341997:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:30.0:1619677475.342000:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:21.0:1619677475.342000:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:30.0:1619677475.342001:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.342003:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:30.0:1619677475.342004:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.342004:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.342005:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.342006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:21.0:1619677475.342006:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.342008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000400:00000200:21.0:1619677475.342008:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677475.342009:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.342010:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.342011:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.342012:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.342015:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677475.342016:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.342018:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.342018:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.342019:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.342020:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.342021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.0:1619677475.342021:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.342021:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.342022:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.0:1619677475.342022:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.342024:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.342024:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.342025:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677475.342025:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677475.342030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.342037:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.342039:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.342040:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.342042:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.342043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.342044:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.342122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.342129:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.342132:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.342133:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.342134:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.342137:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.342138:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.342140:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.342141:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.342142:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.342143:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.342146:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.342148:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.342151:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.342151:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.342152:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.342154:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.342155:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.342156:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.342157:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.342157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.342160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677475.342162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.342168:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677475.342168:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677475.342170:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.342171:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677475.342171:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677475.342172:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677475.342172:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.342174:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.342175:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.342176:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.342180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677475.342187:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.342189:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.342190:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.342191:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.342191:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.342192:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.342424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.342430:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.342433:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.342433:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677475.342434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:22.0:1619677475.342435:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.342437:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.342439:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.342440:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.342441:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.342442:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.342443:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:22.0:1619677475.342443:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.342446:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:22.0:1619677475.342447:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.342449:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.342450:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677475.342450:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.342451:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.342452:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.342453:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:22.0:1619677475.342453:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:31.0:1619677475.342454:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.342454:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.342455:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677475.342456:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.342456:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.342456:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.342457:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:31.0:1619677475.342458:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.342458:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.342459:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677475.342461:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.342462:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.342463:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.2:1619677475.342464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.342466:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.342468:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.342468:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:6.0:1619677475.342469:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:31.0:1619677475.342470:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.342471:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677475.342471:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677475.342472:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.342472:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.342472:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.342472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.342473:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.342474:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.342474:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.342476:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.2:1619677475.342477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.342478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.342479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.342479:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.342482:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.342483:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.342485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.342485:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.342488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.342494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.342501:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.342504:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.342505:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.342505:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.342506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.342507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.342781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.342788:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.342791:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.342792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.342794:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.342797:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.342798:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.342800:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.342801:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.342802:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.342802:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.342805:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.342807:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.342810:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.342811:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.342811:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.342813:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.342813:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.342814:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.342814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.342819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.342827:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.342830:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.342831:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.342831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.342833:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.342836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.343141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.343150:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.343152:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.343154:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.343155:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.343158:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.343160:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.343161:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.343162:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.343163:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.343164:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.343167:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.343168:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.343171:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.343172:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.343172:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.343174:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.343175:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.343175:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.343176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.343180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.343188:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.343191:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.343192:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.343192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.343194:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.343197:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.343415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.343423:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.343425:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.343426:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.343428:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.343431:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.343432:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677475.343433:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.343434:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.343435:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.343436:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677475.343439:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.343441:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677475.343442:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.343443:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677475.343443:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677475.343445:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677475.343445:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.343446:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.343448:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.343448:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.343451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677475.343455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677475.343459:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.343462:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:26.0:1619677475.343462:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:30.0:1619677475.343463:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.343464:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.343465:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.343465:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:30.0:1619677475.343466:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677475.343466:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677475.343467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.343499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.343505:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.343508:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.343509:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.343511:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.343513:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.343514:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.343516:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.343517:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.343518:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.343519:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.343522:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.343524:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.343526:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.343527:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.343528:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.343529:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.343530:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.343530:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.343531:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.343535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.343543:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.343545:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.343546:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.343547:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.343549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.343549:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677475.343662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.343670:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.343673:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.343674:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.343676:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.343679:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.343680:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.343681:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.343683:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.343684:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.343685:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.343688:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.343690:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.343693:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.343694:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.343694:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.343696:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.343696:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.343698:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.343699:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.343700:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.343702:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.343705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.343710:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.343712:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.343712:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.343713:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.343714:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677475.343714:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.343715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.343716:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677475.343716:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.343718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.343720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.343727:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.343730:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.343731:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.343731:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.343732:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.343732:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.343919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.343926:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.343929:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.343931:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.343933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.343934:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.343935:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.343937:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.343938:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.343939:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.343940:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.343943:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.343944:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.343947:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.343948:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.343948:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.343950:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.343950:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.343951:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.343952:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.343955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.343962:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.343964:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.343965:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.343965:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.343967:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.343969:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.344188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.344194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.344196:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.344197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.344199:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.344202:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.344204:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677475.344205:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.344206:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.344207:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.344208:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677475.344211:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.344213:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.344216:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.344217:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677475.344217:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677475.344219:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677475.344220:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.344220:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.344221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677475.344225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.344231:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.344234:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.344235:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.344235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.344237:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.344240:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.344457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.344463:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.344466:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.344467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.344468:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.344471:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.344472:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.344473:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.344474:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.344475:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.344476:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.344479:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.344480:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.344482:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.344483:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.344483:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.344485:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.344485:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.344486:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.344486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.344492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.344501:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.344503:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.344504:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.344505:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.344507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.344508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.344540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.344548:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.344551:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.344553:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.344555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.344556:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.344558:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.344559:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.344561:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.344562:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.344563:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.344566:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.344568:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.344571:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.344572:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.344572:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.344574:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.344575:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.344575:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.344576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.344580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.344588:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.344590:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.344592:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.344592:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.344594:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.344597:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.345342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:4.2:1619677475.345342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:6.0:1619677475.345348:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.345350:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:26.0:1619677475.345351:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:22.0:1619677475.345351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677475.345353:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.345354:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:6.0:1619677475.345354:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.345355:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.345357:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.345357:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.345358:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.345359:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.345360:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.345361:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677475.345361:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.345362:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677475.345363:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.345364:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.345366:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.345367:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.345368:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.345370:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:26.0:1619677475.345372:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.345372:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.345373:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677475.345374:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.345374:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.345376:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.345376:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677475.345377:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.345377:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.345377:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:26.0:1619677475.345378:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677475.345378:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.345379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677475.345380:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.345381:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.345382:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.345382:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.345382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677475.345387:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677475.345387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.345390:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.345390:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:30.0:1619677475.345392:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.345392:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.345393:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.345394:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.345395:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.345395:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.345396:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:20.0:1619677475.345396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:1.0:1619677475.345396:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.345397:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677475.345398:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.345399:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.345399:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:30.0:1619677475.345400:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677475.345400:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.345400:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.345401:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:1.0:1619677475.345402:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.345403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000400:00000200:30.0:1619677475.345405:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.345406:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677475.345407:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.345409:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.345410:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.345411:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677475.345411:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677475.345412:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.345413:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.345413:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.345414:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.345414:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.345414:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.345415:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.345416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.345416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:9.2:1619677475.345418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.345425:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.345427:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.345428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.345428:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.345431:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.345433:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.345511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677475.345518:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.345520:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677475.345521:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677475.345523:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677475.345526:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.345527:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.345528:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.345529:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.345530:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.345531:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.345534:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677475.345536:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.345539:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.345540:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.345540:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.345542:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.345542:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.345543:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677475.345544:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677475.345548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.345556:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.345558:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.345559:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.345559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.345562:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.345565:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.345567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.345573:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.345576:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.345577:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.345577:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.345578:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.345579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.345959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.345967:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.345970:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.345972:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.345974:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.345974:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.345976:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.345977:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.345979:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.345979:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.345980:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.345984:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.345986:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.345988:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.345988:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.345989:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.345990:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.345991:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.345993:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.345994:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.345995:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.345996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.346000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.346005:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.346007:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.346008:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.346009:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.346009:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.346010:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.346010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.346011:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.346012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.346013:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677475.346146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.346152:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.346155:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.346156:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.346158:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.346161:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.346163:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:2.0:1619677475.346164:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.346165:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.346166:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.346167:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677475.346170:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.346172:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:2.0:1619677475.346175:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.346176:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677475.346177:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677475.346179:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677475.346179:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.346180:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.346181:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677475.346184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.346191:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.346194:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.346194:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.346195:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.346197:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.346200:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.346209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.346215:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.346218:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.346219:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.346219:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.346220:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.346221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.346262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.346269:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.346272:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.346274:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.346276:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.346276:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.346278:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.346279:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.346280:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.346281:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.346282:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.346286:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.346287:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.346291:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.346292:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.346292:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.346299:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.346300:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.346301:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.346302:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.346303:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.346306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.346309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.346314:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.346317:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.346318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.346318:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.346318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.346321:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677475.346321:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.346322:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.346323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.346324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.346426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677475.346434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:10.0:1619677475.346434:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.346437:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.346437:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677475.346439:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.346440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677475.346442:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.346442:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.346443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677475.346444:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677475.346445:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.346446:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.346447:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.346447:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.346448:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.346449:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677475.346449:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.346450:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.346451:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.346452:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.346453:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.346458:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.346460:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.346461:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.346463:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.346464:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.346465:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.346465:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677475.346466:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.346467:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.346467:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.346467:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.346468:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:10.0:1619677475.346469:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.346469:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:10.0:1619677475.346470:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.346470:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.346470:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677475.346471:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.346471:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.346473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.346475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677475.346477:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:20.0:1619677475.346480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.346482:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.346482:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.346483:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.346484:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.346485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.346485:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.346486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.346486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677475.346487:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:20.0:1619677475.346488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677475.346489:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.346489:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.346490:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.346492:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.346495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.346684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.346692:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.346695:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.346697:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.346698:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.346700:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.346701:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.346702:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.346704:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.346705:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.346706:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.346709:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.346711:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.346713:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.346714:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.346714:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.346716:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.346716:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.346717:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.346718:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.346721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.346728:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.346731:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.346731:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.346733:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.346735:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.346736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.346830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.346835:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.346838:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.346840:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.346841:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.346842:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.346844:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.346845:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.346847:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.346848:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.346849:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.346852:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.346854:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.346857:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.346858:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.346858:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.346860:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.346860:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.346861:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.346862:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.346867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.346876:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.346878:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.346879:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.346879:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.346881:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.346883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.346886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.346894:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.346896:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.346897:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.346897:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.346898:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.346899:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.347336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.347342:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.347345:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.347346:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.347347:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.347349:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.347351:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.347352:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.347353:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.347354:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.347355:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.347358:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.347359:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.347362:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.347362:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.347363:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.347365:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.347365:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.347366:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.347366:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.347371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.347379:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.347382:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.347382:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.347384:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.347385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.347386:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.347810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.347818:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.347821:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.347823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.347824:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.347827:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.347828:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.347830:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.347831:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.347832:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.347833:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.347836:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.347838:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.347841:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.347842:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.347842:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.347844:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.347845:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.347845:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.347846:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.347850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.347859:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.347862:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.347863:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.347863:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.347866:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.347868:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.348016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.348024:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.348027:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.348028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.348030:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.348033:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.348034:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.348035:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.348037:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.348038:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.348038:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.348041:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.348043:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.348046:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.348046:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.348047:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.348049:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.348049:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.348050:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.348050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.348055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.348062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.348065:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.348066:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.348066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.348069:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.348071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.348250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.348255:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.348258:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.348258:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.348260:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.348262:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.348263:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.348264:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.348266:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.348267:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.348268:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.348271:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.348273:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.348275:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.348276:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.348277:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.348278:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.348279:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.348279:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.348280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.348285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.348292:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.348298:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.348299:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.348299:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677475.348301:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.348303:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.348321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.348329:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.348332:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.348334:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.348336:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.348336:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.348338:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.348339:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.348340:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.348341:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.348342:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.348346:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.348347:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.348350:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.348351:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.348352:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.348354:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.348354:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.348355:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.348356:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.348360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.348368:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.348371:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.348372:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.348372:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.348374:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.348377:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.348802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677475.348808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677475.348809:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.348812:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.348813:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677475.348814:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:24.0:1619677475.348815:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.348817:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:24.0:1619677475.348818:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.348818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:24.0:1619677475.348819:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.348821:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.348821:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.348823:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.348824:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.348824:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.348825:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.348825:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.348827:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.348828:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.348829:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.348829:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.348830:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.348832:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.348834:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.348835:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.348836:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.348836:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.348836:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.348838:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.348838:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.348839:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677475.348839:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.348840:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677475.348840:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.348840:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677475.348842:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677475.348843:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.348844:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.348844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.348844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.348852:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)++
00000800:00000200:1.0:1619677475.348854:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (24)--
00000800:00000200:17.0:1619677475.348855:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (24)++
00000400:00000200:1.0:1619677475.348855:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.348856:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (24)--
00000400:00000200:17.0:1619677475.348856:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.348858:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:1.0:1619677475.348858:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.348861:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.348861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.348862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.349226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.349233:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.349236:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.349238:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.349239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:30.0:1619677475.349240:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.349241:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.349243:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.349244:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.349245:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.349245:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:31.0:1619677475.349246:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.349248:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:17.0:1619677475.349248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.349248:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:1.0:1619677475.349251:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.349253:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.349255:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.349256:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:1.0:1619677475.349256:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677475.349258:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677475.349258:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.349259:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:1.0:1619677475.349259:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.349260:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:1.0:1619677475.349260:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.349261:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.349262:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.349262:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.349263:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.349264:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677475.349265:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677475.349267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677475.349267:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677475.349270:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.349270:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677475.349271:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.349273:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.349273:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:8.0:1619677475.349274:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677475.349274:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.349275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.0:1619677475.349276:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.349277:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.349278:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.349279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:8.0:1619677475.349280:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.349283:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677475.349285:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.349286:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.349287:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.349287:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.349289:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.349291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677475.349293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.349299:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.349302:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.349303:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.349305:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.349308:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.349309:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.349310:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.349311:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.349312:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.349313:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677475.349316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:17.0:1619677475.349316:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.349317:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.349320:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.349320:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.349321:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.349322:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677475.349323:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.349323:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.349324:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.349324:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:6.0:1619677475.349324:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.349325:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.349327:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.349329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677475.349329:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.349330:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.349331:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.349333:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.349334:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.349336:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.349338:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.349340:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.349340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.349340:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.349342:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.349342:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.349344:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.349345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.349345:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.349346:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.349346:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.349347:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.349348:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.349348:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.349349:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.349349:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.349349:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.349350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.349350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677475.349351:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.349354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.349362:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.349364:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.349365:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.349367:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.349368:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.349369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.349379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.349386:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.349389:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.349390:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.349390:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.349391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.349392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.349396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.349402:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.349405:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.349406:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.349407:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.349410:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.349411:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677475.349412:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.349414:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.349415:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.349416:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677475.349419:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.349420:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.349422:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.349423:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.349423:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.349425:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.349425:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.349426:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.349426:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677475.349432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.349439:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.349442:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.349442:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.349443:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.349444:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.349446:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.349458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.349465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.349468:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.349469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.349469:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.349469:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.349470:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.350264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.350270:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.350273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.350273:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.350276:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.350278:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.350279:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.350280:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.350282:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.350283:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.350284:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.350287:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.350289:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.350291:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.350292:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.350298:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.350300:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.350301:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.350301:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.350302:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.350303:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677475.350304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677475.350307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.350308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677475.350313:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.350314:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:30.0:1619677475.350316:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677475.350316:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.350316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.350317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.350317:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:30.0:1619677475.350318:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.350319:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:30.0:1619677475.350320:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.350320:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.350321:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.350322:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:21.0:1619677475.350322:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.350323:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:30.0:1619677475.350324:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.350325:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.350325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677475.350326:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.350327:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.350330:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.350332:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.350335:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.350336:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.350336:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.350338:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.350339:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.350340:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.350340:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.350344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.350351:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.350354:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.350355:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.350356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.350357:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.2:1619677475.350357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:24.0:1619677475.350359:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.350364:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.350367:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.350369:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.350371:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.350372:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.350373:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.350374:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.350376:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.350376:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.350377:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.350381:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.350382:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.350385:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.350386:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.350387:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.350388:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.350389:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.350390:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.350390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.350394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.350402:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.350406:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.350407:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.350407:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.350409:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.350412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.350414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.350421:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.350424:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.350424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.350425:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.350426:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.350426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.350436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.350441:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.350443:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.350445:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.350446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.350448:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.350449:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677475.350451:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.350452:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.350453:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.350454:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.350458:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.350459:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.350462:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.350463:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.350463:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.350465:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677475.350466:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.350466:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.350467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.350471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.350480:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.350483:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.350484:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.350485:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.350486:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.350488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.350564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.350570:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.350572:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.350573:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.350575:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.350576:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.350578:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.350579:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.350580:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.350581:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.350582:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.350585:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.350587:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.350589:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.350589:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.350590:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.350591:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.350592:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.350593:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.350593:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.350597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.350604:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.350606:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.350607:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.350608:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.350608:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.350610:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.350616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.350624:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.350626:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.350627:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.350627:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.350628:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.350628:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.350882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.350889:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.350892:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.350895:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.350896:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.350898:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.350899:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.350901:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.350902:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.350903:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.350904:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.350907:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.350909:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.350911:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.350912:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.350913:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.350914:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.350915:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.350916:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.350918:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.350918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.350920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.350924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.350926:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.350929:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.350930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.350930:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.350932:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.350932:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.350934:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677475.350934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.350935:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.350936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677475.351031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677475.351039:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.351041:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677475.351043:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.351045:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.351046:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677475.351047:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677475.351048:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.351049:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.351050:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.351051:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677475.351054:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.351056:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677475.351059:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.351059:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:31.0:1619677475.351060:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:31.0:1619677475.351061:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677475.351062:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.351063:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677475.351063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677475.351067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.351075:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.351078:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.351079:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.351079:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.351081:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.351083:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.351125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677475.351132:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.351135:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677475.351136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.351138:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.351140:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.351141:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.351142:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.351143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:17.0:1619677475.351144:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.351145:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.351146:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.351148:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:17.0:1619677475.351149:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.351151:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.351151:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.351152:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:17.0:1619677475.351153:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.351153:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.351154:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:6.0:1619677475.351154:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.351156:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.351156:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:6.0:1619677475.351156:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.351157:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:17.0:1619677475.351158:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677475.351159:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677475.351159:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:6.0:1619677475.351159:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.351160:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.351161:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677475.351162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:6.0:1619677475.351162:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.2:1619677475.351164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000400:00000200:6.0:1619677475.351166:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.351167:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.351170:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.351170:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:1.0:1619677475.351170:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.351171:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:6.0:1619677475.351171:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677475.351171:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.351172:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000400:00000200:1.0:1619677475.351172:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.351173:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:6.0:1619677475.351173:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.351173:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677475.351174:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677475.351174:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.351174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:1.0:1619677475.351174:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.351175:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:1.0:1619677475.351176:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:25.2:1619677475.351179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.351187:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.351189:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.351190:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.351192:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.351192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.351194:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.351202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.351208:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.351211:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.351212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.351214:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.351216:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.351217:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.351219:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.351220:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.351221:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.351222:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.351225:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.351227:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.351229:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.351230:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.351231:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.351232:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.351233:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.351233:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.351234:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.351238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.351246:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.351248:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.351249:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.351251:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.351252:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.351254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.351324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.351332:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.351334:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.351335:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.351337:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.351340:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.351341:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.351342:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.351343:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.351344:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.351345:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.351348:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.351350:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.351353:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.351354:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.351355:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.351356:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.351357:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.351357:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.351358:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.351363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.351371:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.351374:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.351375:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.351375:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.351377:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.351380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.351976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.351984:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.351987:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.351989:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.351991:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.351992:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.351993:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.351994:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.351996:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.351997:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.351998:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.352001:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.352003:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.352006:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.352007:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.352007:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.352009:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.352009:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.352010:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.352011:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.352015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.352022:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.352025:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.352026:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.352026:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.352028:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.352031:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.352201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.352205:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.352208:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.352209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.352211:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.352213:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.352214:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.352215:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.352216:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.352217:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.352218:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.352221:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.352223:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.352225:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.352226:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.352226:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.352228:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.352229:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.352229:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.352229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.352234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.352242:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.352244:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.352245:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.352247:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.352248:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.352249:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.352438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677475.352447:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.352449:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677475.352451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677475.352452:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.352455:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.352457:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677475.352458:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.352459:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.352460:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.352461:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677475.352465:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.352467:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677475.352469:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.352470:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677475.352470:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677475.352472:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677475.352473:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.352474:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677475.352475:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677475.352476:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:4.2:1619677475.352482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.352489:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:25.2:1619677475.352490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677475.352491:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677475.352491:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677475.352492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:31.0:1619677475.352497:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.352499:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.352500:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.352502:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.352504:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.352505:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.352692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.352700:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.352702:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.352704:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.352707:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.352707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.352708:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.352709:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.352710:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.352711:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.352712:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.352715:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.352717:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.352719:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.352720:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.352720:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.352722:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.352722:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.352723:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.352724:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.352727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.352734:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.352736:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.352738:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.352738:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.352740:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.352742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.352746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.352752:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.352755:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.352756:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.352756:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.352757:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.352757:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.352878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.352885:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.352888:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.352889:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.352891:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.352893:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.352894:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.352896:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.352897:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.352898:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.352898:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.352901:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.352903:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.352906:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.352907:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.352907:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.352909:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.352909:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.352910:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.352910:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.352917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.352924:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.352927:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.352928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.352928:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.352930:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.352933:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.353009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.353016:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.353019:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.353020:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.353021:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.353024:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.353025:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.353027:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.353028:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.353029:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.353030:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.353033:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.353034:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.353037:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.353038:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.353038:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.353040:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.353040:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.353041:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.353042:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.353046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.353053:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.353056:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.353056:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.353057:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.353059:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.353062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.353160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.353165:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.353168:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.353169:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.353171:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.353172:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.353174:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.353175:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.353176:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.353177:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.353178:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.353182:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.353183:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.353186:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.353187:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.353187:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.353189:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.353190:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.353190:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.353190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.353195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.353203:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.353205:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.353206:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.353207:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.353209:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.353209:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.353215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.353223:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.353225:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.353226:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.353227:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.353227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.353229:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.353264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.353270:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.353273:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.353274:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.353276:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.353278:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.353280:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.353281:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.353282:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.353283:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.353284:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.353288:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.353290:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.353293:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.353294:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.353300:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.353302:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.353302:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.353303:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.353304:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.353308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.353316:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.353319:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.353320:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.353322:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.353323:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.353325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677475.353815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.353821:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.353824:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.353826:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.353826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.353829:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.353830:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.353832:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.353833:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.353834:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.353835:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.353838:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.353839:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.353842:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.353842:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.353843:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.353845:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.353845:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677475.353848:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.0:1619677475.353851:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.2:1619677475.353851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.353852:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.353853:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677475.353855:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.353858:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:20.2:1619677475.353858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677475.353860:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:31.0:1619677475.353861:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.353861:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:30.0:1619677475.353862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.353863:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.353863:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677475.353863:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:20.0:1619677475.353864:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677475.353864:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.353864:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.353865:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677475.353865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677475.353866:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.353867:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.353868:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.353869:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.353869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:31.0:1619677475.353870:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.353872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000400:00000200:31.0:1619677475.353873:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.353875:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.353878:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:31.0:1619677475.353879:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.353880:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.353880:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.353880:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.353881:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:31.0:1619677475.353882:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.353883:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:17.0:1619677475.353883:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.353884:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000400:00000200:17.0:1619677475.353885:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.353886:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.353886:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:17.0:1619677475.353886:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:25.2:1619677475.353888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:17.0:1619677475.353888:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.353889:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.353890:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.353891:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:21.2:1619677475.353892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:17.0:1619677475.353894:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.353896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677475.353896:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677475.353899:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.353899:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677475.353899:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.353900:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.353900:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677475.353900:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677475.353900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.353902:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:17.0:1619677475.353902:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677475.353902:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:30.0:1619677475.353903:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677475.353903:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.353903:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:30.0:1619677475.353904:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:17.0:1619677475.353904:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:22.0:1619677475.353906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.353908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:9.2:1619677475.353909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:6.0:1619677475.353916:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.353917:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:6.0:1619677475.353918:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.353919:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.353920:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677475.353920:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:6.0:1619677475.353920:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:26.0:1619677475.353921:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.353921:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.353921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:26.0:1619677475.353923:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.353927:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.354056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.354062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.354065:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.354065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.354068:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.354070:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.354071:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.354073:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.354074:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.354075:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.354076:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.354079:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.354081:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.354083:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.354084:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.354084:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.354086:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.354087:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.354088:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.354089:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.354089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.354092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.354094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.354099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.354099:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.354101:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.354101:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.354102:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.354103:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.354103:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.354104:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.354106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.354107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.354128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.354135:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.354138:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.354140:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.354142:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.354143:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.354145:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.354146:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.354147:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.354148:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.354149:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.354153:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.354155:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.354157:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.354158:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.354159:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.354160:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.354161:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.354162:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.354162:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.354166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.354172:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.354175:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.354176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.354176:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.354178:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.354180:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.354313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.354320:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.354323:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.354324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.354325:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.354328:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.354329:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.354330:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.354332:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.354332:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.354333:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.354336:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.354338:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.354340:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.354341:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.354341:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.354343:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.354343:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.354344:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.354344:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.354349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.354357:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.354360:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.354360:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.354362:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.354364:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.354365:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.354875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.354883:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.354886:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.354888:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.354890:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.354890:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.354892:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.354893:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.354894:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.354895:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.354896:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.354899:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.354901:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.354904:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.354905:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.354905:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.354907:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.354908:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.354908:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.354909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677475.354914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.354922:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.354925:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.354926:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.354927:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.354928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.354931:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.355127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.355135:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.355138:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.355140:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.355141:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.355143:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.355144:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.355145:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.355147:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.355148:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.355149:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.355152:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.355153:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.355155:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.355156:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.355156:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.355158:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.355158:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.355159:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.355159:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.355163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.355168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.355170:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.355173:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.355173:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.355173:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677475.355175:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.355176:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677475.355176:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.355177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677475.355178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:22.0:1619677475.355178:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.355181:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.355182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000400:00000200:22.0:1619677475.355182:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.355184:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.355185:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.355186:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.355187:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.355188:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:22.0:1619677475.355190:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.355191:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.355192:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:22.0:1619677475.355192:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.355193:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.355193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.355195:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:22.0:1619677475.355195:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.355196:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.355196:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.355198:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.355198:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.355199:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.355199:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.355204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.355211:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.355213:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.355214:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.355216:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.355217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.355218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.355760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.355768:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.355770:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.355771:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.355773:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.355776:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.355777:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.355778:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.355780:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.355781:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.355782:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.355785:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.355787:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.355790:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.355790:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.355791:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.355793:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.355793:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.355794:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.355794:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.355799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.355807:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.355810:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.355811:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.355811:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.355814:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.355816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.356048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.356057:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.356060:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.356062:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.356063:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.356064:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.356065:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.2:1619677475.356065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:31.0:1619677475.356066:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.356067:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.356068:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.356069:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.356070:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677475.356072:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.356073:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:31.0:1619677475.356074:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677475.356074:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.356076:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.356077:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.356078:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:22.0:1619677475.356078:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.356079:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.356079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:22.0:1619677475.356080:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677475.356081:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.356081:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:22.0:1619677475.356081:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.356082:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677475.356082:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.356083:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:22.0:1619677475.356083:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.356084:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.356084:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.356086:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:8.2:1619677475.356087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677475.356088:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.356090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677475.356091:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.356091:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:21.0:1619677475.356092:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.356093:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:22.0:1619677475.356094:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.356094:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:21.0:1619677475.356094:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.356094:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.356095:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677475.356096:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.356096:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.356097:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.356097:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677475.356097:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96500293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.356098:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.356098:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.356099:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.356100:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.356101:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677475.356103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677475.356104:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.356104:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.356106:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.356106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677475.356107:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.356108:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.356108:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.356109:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.356109:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677475.356109:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.356110:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.356111:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.356112:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.356113:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.356113:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.0:1619677475.356113:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.356114:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.356117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.356124:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.356127:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.356128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.356128:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.356130:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.356132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.356132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.356139:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.356141:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.356143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.356144:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.356147:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.356148:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.356150:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.356151:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.356152:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.356153:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.356156:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.356157:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.356160:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.356161:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.356162:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.356163:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.356164:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.356165:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.356167:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.356167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.356170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.356173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.356178:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677475.356180:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.356181:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.356182:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.356182:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677475.356182:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.356183:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.356184:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677475.356184:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.356187:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.356353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.356361:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.356364:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.356366:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.356369:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.356369:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.356370:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.356372:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.356373:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.356374:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.356375:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.356378:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.356379:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.356381:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.356382:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.356382:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.356384:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.356385:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.356386:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.356388:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.356388:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.356390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.356394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.356399:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.356400:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.356401:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.356403:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.356403:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.356404:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.356404:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.356405:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.356406:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.356407:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.356967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.356973:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.356976:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.356977:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.356979:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.356981:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.356982:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.356984:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.356985:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.356986:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.356987:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.356990:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.356991:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.356993:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.356994:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.356995:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.356996:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.356997:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.356997:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.356998:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.357002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.357010:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.357012:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.357013:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.357015:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.357016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.357017:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.357024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.357031:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.357034:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.357035:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.357035:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.357036:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.357037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.357598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.357604:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.357607:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.357608:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.357610:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.357613:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.357614:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677475.357615:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.357617:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.357618:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.357619:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677475.357622:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.357624:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.357626:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.357627:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677475.357628:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.357629:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.357630:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.357631:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.357631:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677475.357636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.357643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:6.0:1619677475.357644:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.357647:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.357648:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.357648:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.357650:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.357651:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.357653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.357654:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.357656:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.357658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.357658:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.357659:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.357661:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.357662:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.357663:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.357664:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.357668:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.357669:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.357673:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.357673:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.357674:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.357676:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.357676:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.357677:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.357678:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.357682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.357689:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.357691:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.357692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.357693:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.357695:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.357698:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.357990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.357997:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.358000:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.358001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.358003:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.358006:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.358008:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.358009:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.358010:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.358011:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.358012:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.358012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:24.0:1619677475.358016:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.358017:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.358017:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677475.358020:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.358020:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:24.0:1619677475.358021:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.358021:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677475.358022:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.358023:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.358023:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.358024:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.358024:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.358025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677475.358025:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.358026:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.358027:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.358029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677475.358029:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.358029:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.358031:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.358035:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.358037:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.358038:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.358039:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.358040:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.358040:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.358041:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.358041:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:22.0:1619677475.358042:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.358043:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.358044:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.358045:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.358046:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.358047:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.358047:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.358047:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.358049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.358052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677475.358054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.358057:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.358059:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.358059:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.358060:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.358061:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.358061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.358061:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.358062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:22.0:1619677475.358063:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.358063:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.358064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677475.358065:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.358066:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.358066:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.358067:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.358068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.358198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.358206:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.358209:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.358209:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677475.358212:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.358214:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.358216:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.358217:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.358218:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.358219:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.358220:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.358224:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.358225:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.358228:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.358229:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.358229:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.358231:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.358231:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.358233:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.358234:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.358234:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.358237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.358240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.358245:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.358247:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677475.358248:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.358248:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.358248:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.358250:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.358251:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677475.358251:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.358252:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677475.358254:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.358597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.358605:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.358608:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.358610:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.358613:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.358614:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.358615:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.358616:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.358617:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.358618:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.358619:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.358622:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.358624:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.358625:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.358626:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.358627:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.358628:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.358629:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.358630:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.358630:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.358635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.358641:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.358643:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.358644:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.358646:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.358647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.358649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.358950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.358956:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.358959:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.358960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.358962:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.358964:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.358966:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.358967:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.358968:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.358969:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.358970:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.358973:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.358975:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.358977:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.358978:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.358979:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.358980:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.358981:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.358981:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.358982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.358986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.358994:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.358996:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.358997:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.358999:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.359000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.359001:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.359159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677475.359167:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.359169:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677475.359171:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.359173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677475.359174:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.359175:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677475.359177:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.359178:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.359179:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.359180:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677475.359184:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.359185:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.359189:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.359190:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677475.359190:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677475.359192:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677475.359193:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.359194:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677475.359194:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.359199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.359208:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.359213:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.359214:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.359214:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.359218:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.359223:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.359317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.359324:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.359327:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.359330:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.359331:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.359333:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.359335:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.359336:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.359337:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.359338:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.359339:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.359343:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.359345:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.359348:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.359349:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.359349:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.359351:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.359351:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.359353:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.359354:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.359355:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.359357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.359361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.359365:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.359368:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.359369:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.359369:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.359370:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.359371:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.359372:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677475.359372:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.359373:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.359374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.359516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.359520:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.359522:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677475.359525:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.359525:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.359527:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.359529:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677475.359530:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.359531:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.359532:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.359533:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.359537:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.359538:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.359541:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.359542:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.359542:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.359544:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677475.359545:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.359546:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.359547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.359550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.359560:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.359564:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.359566:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.359566:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.359569:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.359572:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.359693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.359700:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.359703:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.359705:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.359707:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.359709:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.359710:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.359712:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.359713:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.359714:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.359715:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.359719:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.359720:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.359723:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.359723:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.359724:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.359725:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.359726:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.359727:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.359728:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.359731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.359738:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.359740:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.359741:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.359743:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.359744:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.359745:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.359831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.359836:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.359839:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.359840:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.359842:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.359844:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.359845:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.359847:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.359848:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.359849:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.359850:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.359853:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.359854:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.359857:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.359857:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.359858:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.359860:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.359860:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.359861:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.359861:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.359866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.359873:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.359875:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.359876:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.359878:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.359878:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.359880:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.360168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.360171:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.360174:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.360176:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.360176:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.360179:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.360180:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.360181:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.360182:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.360183:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.360184:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.360187:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.360189:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.360191:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.360192:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.360193:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.360194:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.360195:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.360196:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.360196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.360201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.360210:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.360212:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.360213:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.360214:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.360215:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.360218:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.360407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677475.360413:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.360415:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677475.360416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677475.360418:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.360421:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.360422:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677475.360423:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.360425:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.360426:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.360427:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677475.360429:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.360431:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.360433:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.360434:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677475.360435:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677475.360436:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677475.360437:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.360438:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677475.360439:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677475.360439:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:4.2:1619677475.360440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:8.2:1619677475.360443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.360445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:21.0:1619677475.360450:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:22.0:1619677475.360451:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:17.0:1619677475.360451:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.360452:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:22.0:1619677475.360453:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677475.360453:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677475.360454:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:21.0:1619677475.360454:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.360454:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.360455:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.360455:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.2:1619677475.360456:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:21.0:1619677475.360457:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.360457:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.360459:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677475.360459:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:8.2:1619677475.360459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:21.0:1619677475.360460:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.360461:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.360462:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677475.360462:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:21.0:1619677475.360463:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.360463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.360464:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:18.0:1619677475.360466:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.360466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:21.0:1619677475.360467:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.360467:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.360468:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.360468:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:18.0:1619677475.360469:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.360470:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.360470:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.360470:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:18.0:1619677475.360471:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.360473:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677475.360473:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:21.0:1619677475.360474:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:18.0:1619677475.360474:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.360475:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.360476:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:18.0:1619677475.360476:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.360476:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.360477:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.360478:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.360478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:18.0:1619677475.360478:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.360481:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.360483:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:9.2:1619677475.360483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:18.0:1619677475.360486:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.360487:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.360488:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677475.360489:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:18.0:1619677475.360490:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.360490:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.360491:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.360492:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.0:1619677475.360492:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.360493:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.360493:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.360495:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.360496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677475.360498:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:13.0:1619677475.360503:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.360506:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.360507:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.360508:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.360511:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.360512:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.360517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.360525:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.360528:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.360529:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.360531:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.360534:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.360536:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.360537:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.360538:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.360539:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.360540:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.360544:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.360545:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.360548:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.360549:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.360550:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.360551:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.360552:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.360553:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.360555:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.360555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.360557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.360561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.360565:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677475.360568:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.360568:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:20.2:1619677475.360568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:6.0:1619677475.360568:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.360569:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.360570:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.360571:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677475.360571:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.360572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677475.360574:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:2.0:1619677475.360574:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.360577:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.360578:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.360579:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.360582:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.360584:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.360585:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.360587:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.360588:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.360589:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.360592:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.360594:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.360596:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.360597:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.360597:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.360599:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.360600:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.360601:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.360601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.360605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.360612:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.360615:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.360615:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.360616:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.2:1619677475.360617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000400:00000200:18.0:1619677475.360617:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.360619:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677475.360623:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.360626:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.360627:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.360628:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.360631:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.360632:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.360634:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.360635:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.360636:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.360637:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.360640:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.360641:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.360644:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.360645:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.360645:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.360647:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.360647:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.360648:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.360649:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.360653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.360661:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.360663:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.360664:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.360666:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.360667:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.360668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.360674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.360682:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.360684:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.360685:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.360686:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.360687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.360687:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677475.360690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.360696:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.360699:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.360700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.360702:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.360704:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.360705:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.360707:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.360708:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.360709:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.360710:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.360713:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.360714:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.360717:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.360717:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.360718:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.360719:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.360720:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.360721:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.360721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.360725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.360733:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.360735:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.360736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.360737:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.360739:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.360741:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.360744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:4.2:1619677475.360748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:6.0:1619677475.360751:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.360753:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.360754:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.360754:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.360755:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677475.360755:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.360756:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677475.360758:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.360758:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677475.360760:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.360763:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.360765:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.360766:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.360767:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.360768:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.360769:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.360773:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.360775:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.360777:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.360778:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.360779:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.360781:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.360781:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.360782:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.360782:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.360791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.360799:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.360801:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.360802:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.360803:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.360804:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.360807:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.360882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.360887:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.360890:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.360891:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.360893:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.360894:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.360896:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.360897:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.360899:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.360899:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.360901:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.360904:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.360906:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.360908:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.360909:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.360910:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.360912:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.360912:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.360913:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.360913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.360917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.360925:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.360927:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.360928:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.360929:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.360931:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.360932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.360938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.360945:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.360947:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.360948:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.360949:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.360949:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.360951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.361018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.361026:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.361028:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.361031:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.361032:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.361033:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.361034:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.361035:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.361036:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.361038:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.361039:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.361042:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.361044:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.361046:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.361047:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.361048:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.361049:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.361050:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.361051:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.361051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.361055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.361062:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.361065:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.361065:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.361067:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.361069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.361070:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.361074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.361081:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.361083:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.361084:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.361084:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.361085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.361085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677475.361096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:1.0:1619677475.361102:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677475.361105:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677475.361105:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000400:00000200:1.0:1619677475.361108:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677475.361110:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.361112:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.361113:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.361114:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.361115:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.361116:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.361119:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677475.361121:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.361122:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.361123:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.361123:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.361125:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.361125:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:1.0:1619677475.361127:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677475.361128:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677475.361128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:8.2:1619677475.361131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.361135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677475.361140:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.361141:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.361141:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.361141:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000800:00000200:1.0:1619677475.361142:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000400:00000200:17.0:1619677475.361143:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.361143:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:1.0:1619677475.361144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.361145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677475.361145:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:21.2:1619677475.361365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.361373:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.361376:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.361378:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.361380:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.361380:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.361382:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.361383:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.361385:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.361385:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.361387:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.361390:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.361392:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.361395:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.361396:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.361396:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.361398:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.361399:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.361399:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.361400:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.361404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.361413:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.361415:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.361416:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.361417:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.361419:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.361421:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.361423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.361430:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.361433:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.361434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.361434:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.361435:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.361435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.361657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7ffa00] (20)++
00000800:00000200:1.0:1619677475.361663:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7ffa00] (21)++
00000800:00000200:1.0:1619677475.361666:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.69@o2ib3
00000800:00000200:17.0:1619677475.361667:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7ffa00] (22)--
00000400:00000200:1.0:1619677475.361669:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.361671:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.361672:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677475.361674:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.361675:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.361676:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.361677:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677475.361680:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.361681:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677475.361684:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.361685:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677475.361685:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.361687:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.361688:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7ffa00] (21)++
00000800:00000200:1.0:1619677475.361689:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7ffa00] (22)++
00000800:00000200:1.0:1619677475.361690:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7ffa00] (23)--
00000800:00000200:1.0:1619677475.361690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7ffa00] (22)--
00000800:00000200:9.2:1619677475.361693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.361695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7ffa00] (21)++
00000800:00000200:24.0:1619677475.361700:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:17.0:1619677475.361701:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7ffa00] (22)++
00000800:00000200:17.0:1619677475.361702:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7ffa00] (23)--
00000800:00000200:24.0:1619677475.361703:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:17.0:1619677475.361703:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7ffa00] (22)--
00000400:00000200:24.0:1619677475.361704:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.361704:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677475.361705:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7ffa00] (21)--
00000400:00000200:24.0:1619677475.361706:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.361709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.361801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.361806:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.361809:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.361809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.361811:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.361814:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.361815:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.361816:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.361818:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.361819:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.361820:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.361823:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.361825:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.361827:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.361828:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.361829:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.361830:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.361831:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.361832:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.361833:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.361833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.361837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.361838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.361843:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.361845:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.361845:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.361846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.361847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.361848:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.361848:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.361850:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.361851:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.361852:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.362060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.362067:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.362070:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.362070:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.362072:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.362075:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.362076:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.362078:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.362079:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.362080:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.362081:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.362085:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.362086:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.362089:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.362090:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.362091:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.362093:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.362093:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.362094:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.362094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.362099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.362106:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.362109:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.362110:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.362110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.362113:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.362116:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.362324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.362331:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.362334:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.362335:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.362337:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.362340:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.362341:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.362343:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.362344:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.362345:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.362346:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.362349:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.362351:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.362353:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.362354:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.362354:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.362356:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.362356:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.362357:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.362357:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.362362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.362370:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.362372:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.362372:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.362374:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.362376:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.362377:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.362475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.362482:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.362485:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.362487:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.362488:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.362489:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.362491:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677475.362492:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.362493:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.362495:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.362495:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677475.362498:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.362500:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677475.362503:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.362504:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:13.0:1619677475.362505:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:13.0:1619677475.362506:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677475.362507:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.362507:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677475.362508:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:8.2:1619677475.362513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.362522:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.362524:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.362525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.362526:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.362528:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.362531:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.362680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.362686:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.362689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.362689:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.362692:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.362694:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.362695:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.362696:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.362698:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.362699:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.362700:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.362703:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.362705:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.362707:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.362708:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.362708:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.362710:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.362711:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.362711:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.362712:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.362716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.362724:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.362726:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.362727:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.362728:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.362730:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.362731:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677475.362910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.362916:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.362919:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.362920:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.362920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.362921:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.362922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.363020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.363027:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.363030:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.363030:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677475.363032:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677475.363033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:10.0:1619677475.363035:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.363036:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.363037:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.363039:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.363040:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.363040:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:10.0:1619677475.363041:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.363043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.363043:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:10.0:1619677475.363045:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.363046:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.363047:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.363048:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.363050:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.363050:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:8.0:1619677475.363050:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:10.0:1619677475.363051:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:8.0:1619677475.363051:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.363052:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.363053:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.363053:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677475.363053:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677475.363054:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.363054:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677475.363054:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.363057:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.363058:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677475.363059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.363061:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.363062:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.363063:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:8.0:1619677475.363064:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (24)--
00000800:00000200:8.0:1619677475.363065:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.363065:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677475.363066:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:8.0:1619677475.363066:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.363069:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677475.363069:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000400:00000200:22.0:1619677475.363070:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.363070:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:6.0:1619677475.363071:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.363073:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.363073:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.363077:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.363077:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.363078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.363159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.363167:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.363170:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.363172:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.363174:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.363175:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.363177:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.363178:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.363180:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.363180:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.363181:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.363185:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.363186:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.363189:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.363190:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.363190:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.363192:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.363193:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.363194:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.363194:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.363198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.363205:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.363208:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.363209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.363209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.363211:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.363214:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.363218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.363226:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.363228:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.363229:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.363229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.363231:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.363231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.363294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677475.363302:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.363305:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677475.363306:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677475.363308:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.363311:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.363312:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677475.363314:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.363315:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.363316:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.363317:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677475.363320:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.363321:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677475.363324:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.363325:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677475.363325:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677475.363327:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677475.363327:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.363328:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677475.363328:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677475.363334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.363342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.363345:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.363346:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.363347:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.363349:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.363352:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.363357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.363365:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.363367:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.363368:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.363369:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.363370:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.363370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.363710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.363716:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.363719:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.363721:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.363721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.363723:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.363725:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.363726:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.363728:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.363729:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.363730:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.363733:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.363735:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.363738:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.363739:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.363739:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.363741:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.363742:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.363742:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.363743:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:21.2:1619677475.363765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.363771:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.363773:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.363774:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.363776:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.363778:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.363779:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.363780:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.363782:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.363783:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.363784:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.363787:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.363789:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.363791:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.363792:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.363793:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.363794:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.363795:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.363795:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.363796:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.363800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.363808:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.363810:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.363811:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.363812:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.363814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.363814:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677475.363827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.363834:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.363837:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.363838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.363838:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.363840:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.363843:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.363853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.363858:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.363861:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.363862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.363863:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.363866:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.363867:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677475.363869:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.363870:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.363871:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.363871:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677475.363874:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.363876:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677475.363879:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.363879:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677475.363880:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677475.363882:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677475.363882:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.363883:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677475.363884:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677475.363885:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:25.2:1619677475.363888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.2:1619677475.363890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.363895:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:12.0:1619677475.363896:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:6.0:1619677475.363897:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677475.363897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.363898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:12.0:1619677475.363899:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.363900:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.363901:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.363903:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.363904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.363905:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.363905:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.363906:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.363906:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.363907:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.364026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.364034:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.364036:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.364039:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.364041:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.364042:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.364043:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.364045:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.364046:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.364047:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.364048:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.364052:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.364053:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.364055:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.364056:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.364057:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.364058:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.364059:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.364060:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.364062:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.364062:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.364064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.364068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.364072:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.364073:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.364074:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.364076:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.364076:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.364076:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.364077:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.364078:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.364079:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.364080:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.364653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.364658:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.364661:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.364662:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.364664:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.364667:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.364668:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.364670:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.364671:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.364672:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.364673:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.364676:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.364678:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.364680:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.364681:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.364682:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.364683:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.364684:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.364684:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.364685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.364690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.364699:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.364701:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.364702:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.364704:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.364705:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.364706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.364707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.364715:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.364717:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.364718:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.364719:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.364719:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.364721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.364825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.364833:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.364836:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.364838:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.364840:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.364841:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.364842:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.364843:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.364845:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.364846:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.364847:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.364850:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.364852:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.364854:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.364855:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.364855:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.364857:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.364857:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.364858:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.364859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.364863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.364869:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.364872:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.364872:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.364874:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.364876:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.364876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.2:1619677475.365251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.365259:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.365261:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.365263:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.365265:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.365265:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.365267:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.365268:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.365269:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.365270:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.365271:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.365273:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.365275:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.365277:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.365278:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.365279:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.365280:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.365281:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.365282:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.365282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.365286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.365294:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.365298:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.365301:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.365302:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.365303:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.365306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.365354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.365361:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.365364:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.365365:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.365367:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.365369:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.365371:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.365372:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.365373:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.365374:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.365375:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.365378:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.365380:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.365383:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.365383:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.365384:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.365386:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.365386:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.365386:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.365387:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.365392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.365400:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.365402:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.365403:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.365404:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.365406:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.365408:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.365536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.365541:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.365543:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.365544:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.365546:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.365548:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.365549:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.365550:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.365552:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.365552:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.365554:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.365557:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.365559:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.365561:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.365562:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.365562:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.365564:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.365565:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.365566:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.365567:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.365567:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.365570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.365572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.365577:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.365579:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.365579:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.365580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.365581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.365582:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.365583:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.365584:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.365585:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.365586:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.365914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.365922:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.365925:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.365927:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.365929:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.365930:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.365931:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.365932:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.365934:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677475.365934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677475.365935:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.365936:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677475.365937:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.365939:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677475.365940:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.365942:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.365942:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.365943:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.365945:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677475.365945:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.365946:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.365946:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:20.0:1619677475.365946:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677475.365948:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.365948:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677475.365948:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.365949:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677475.365949:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.365950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677475.365950:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.365951:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:25.2:1619677475.365953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:20.0:1619677475.365955:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.365957:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.365959:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.365960:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:20.0:1619677475.365960:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.365960:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.365962:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:20.0:1619677475.365962:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:30.0:1619677475.365963:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.365963:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.365964:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.365964:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677475.365965:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.365967:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.365967:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.2:1619677475.365969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.365974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677475.365978:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.365980:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.365981:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.365981:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677475.365981:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:2.0:1619677475.365983:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:17.0:1619677475.365984:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.365985:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677475.365986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:2.0:1619677475.365987:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.365989:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.365990:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.365992:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.365993:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.365994:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.365995:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.365999:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.366001:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.366003:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.366004:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.366005:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.366006:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.366007:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.366009:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.366010:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.366010:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677475.366012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677475.366015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677475.366020:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677475.366021:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677475.366022:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677475.366023:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:1.0:1619677475.366023:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.366024:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677475.366024:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.366025:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677475.366025:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.366028:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.366388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.366393:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.366396:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.366398:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.366398:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.366400:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.366402:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.366403:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.366404:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.366405:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.366406:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.366409:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.366411:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.366413:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.366414:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.366414:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.366416:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.366417:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.366418:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.366419:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.366420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.366423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.366425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.366428:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677475.366430:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.366431:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.366431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.366432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677475.366433:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.366434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.366434:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.366436:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.366438:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.366602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.366608:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.366610:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.366611:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.366613:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.366615:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.366616:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.366617:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.366619:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.366620:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.366621:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.366624:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.366626:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.366628:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.366629:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.366629:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.366631:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.366631:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.366632:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.366632:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.366637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.366644:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.366647:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.366647:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.366649:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.366650:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.366651:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.366798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:10.0:1619677475.366806:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677475.366808:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677475.366809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:10.0:1619677475.366811:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.366814:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.366816:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677475.366817:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.366818:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.366819:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.366820:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677475.366824:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.366825:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.366828:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.366829:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677475.366830:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677475.366831:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677475.366832:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677475.366833:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677475.366833:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:8.2:1619677475.366838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.366847:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.366849:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.366850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.366851:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.366853:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.366855:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.366857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.366864:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.366866:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.366867:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.366868:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.366868:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.366869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.367319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.367324:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.367327:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.367329:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.367330:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.367331:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.367333:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.367334:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.367336:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.367337:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.367338:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.367341:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.367343:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.367345:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.367346:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.367347:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.367348:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.367349:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.367350:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.367350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.367355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.367363:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.367365:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.367366:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.367367:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.367368:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.367370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.367447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.367456:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.367459:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.367461:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.367463:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.367463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.367465:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.367466:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.367468:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.367468:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.367469:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.367473:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.367474:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.367477:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.367478:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.367479:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.367479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.367481:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.367481:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.367483:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.367485:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.367485:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677475.367485:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677475.367487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.367488:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.367491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677475.367491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:6.0:1619677475.367491:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.367493:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:6.0:1619677475.367494:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.367495:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.367496:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.367496:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.367497:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.367498:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.367498:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.367499:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677475.367499:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.367500:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.367501:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.367502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.367503:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.367504:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.367504:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.367505:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.367506:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.367508:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.367509:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.367510:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.367512:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.367512:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.367513:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.367513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.367518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.367525:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.367527:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.367528:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.367530:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.367530:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.367532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.367647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.367654:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.367657:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.367658:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.367660:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.367663:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.367664:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.367666:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.367667:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.367668:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.367669:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.367672:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.367674:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.367677:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.367678:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.367678:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.367680:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.367680:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.367681:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.367683:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.367683:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.367686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.367689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.367693:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.367696:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677475.367696:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.367696:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.367697:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.367698:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.367698:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677475.367699:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.367699:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.367702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.367703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.367710:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.367712:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.367713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.367713:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.367714:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.367715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.367858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:12.0:1619677475.367867:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.367871:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677475.367875:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677475.367875:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.103@o2ib7 : GET - for me
00000400:00000200:12.0:1619677475.367878:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.103@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677475.367881:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.103@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677475.367884:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.367885:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:12.0:1619677475.367888:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:12.0:1619677475.367894:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677475.367897:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:12.0:1619677475.367899:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.367901:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.367902:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.367903:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677475.367904:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:25.2:1619677475.367909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677475.367916:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677475.367918:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677475.367919:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.367920:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: REPLY: OK
00000800:00000200:13.0:1619677475.367922:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677475.367923:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677475.368349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.368355:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.368358:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.368358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.368360:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.368363:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.368364:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.368366:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.368367:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.368368:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.368369:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.368372:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.368374:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.368376:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.368377:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.368378:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.368380:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.368380:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.368380:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.368381:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.368386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.368394:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.368397:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.368397:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.368399:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.368401:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.368401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.368406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.368413:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.368415:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.368416:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:4.2:1619677475.368416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:21.0:1619677475.368417:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.368418:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.368419:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677475.368424:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.368426:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.368427:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.368430:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.368432:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.368434:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.368435:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.368436:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.368437:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.368439:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.368442:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.368443:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.368446:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.368447:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.368447:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.368449:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.368449:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.368450:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.368450:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.368455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.368462:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.368465:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.368466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.368466:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.368468:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.368471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.368654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.368662:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.368664:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.368666:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.368667:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.368670:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.368671:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.368672:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.368674:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.368675:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.368675:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.368678:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.368680:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.368682:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.368683:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.368684:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.368685:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.368686:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.368686:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.368687:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.368691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.368699:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.368701:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.368702:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.368703:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.368705:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.368707:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.369222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.369230:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.369233:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.369235:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.369237:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.369238:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.369239:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.369241:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.369242:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.369243:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.369244:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.369247:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.369249:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.369250:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.369251:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.369252:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.369253:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.369254:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.369254:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.369255:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.369259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.369265:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.369268:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.369269:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.369270:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.369272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.369273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.369277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.369284:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.369286:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.369287:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.369288:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.369288:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.369289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.369422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.369426:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.369429:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.369431:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.369432:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.369433:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.369434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:20.0:1619677475.369434:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.369436:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.369437:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.369438:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.369439:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:6.0:1619677475.369439:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.369442:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.369443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.369443:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.369446:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:6.0:1619677475.369446:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.369448:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.369450:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.369450:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.0:1619677475.369451:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:6.0:1619677475.369451:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.369452:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677475.369452:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.369453:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.369454:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.369454:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000400:00000200:6.0:1619677475.369454:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.369455:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.369456:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677475.369458:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.369459:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:9.2:1619677475.369460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677475.369462:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.369462:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.369463:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.369465:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.369465:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.369466:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.2:1619677475.369467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.0:1619677475.369467:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:6.0:1619677475.369467:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.369468:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677475.369470:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:25.2:1619677475.369471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677475.369471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.369471:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.369473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677475.369473:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.369476:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.369476:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.369478:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677475.369478:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.369480:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.369480:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:31.0:1619677475.369481:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.369481:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.369482:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:31.0:1619677475.369483:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.369483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677475.369483:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.369483:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.369484:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.369485:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.369486:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.369486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677475.369487:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.369487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.369488:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.369489:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.369493:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.369495:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.369498:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.369499:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.369499:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.369501:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.369502:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.369503:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.369503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.369507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.369514:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.369517:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.369518:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.369518:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.369520:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.369523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.369776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.369784:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.369787:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.369788:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.369789:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.369792:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.369794:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.369795:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.369796:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.369797:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.369798:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.369801:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.369803:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.369805:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.369806:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.369807:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.369808:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.369809:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.369809:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.369810:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.369814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.369822:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.369824:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.369825:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.369826:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.369828:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.369830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.370316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.370321:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.370323:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.370324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.370326:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.370328:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.370329:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.370330:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.370332:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.370333:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.370334:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.370337:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.370339:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.370341:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.370342:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.370342:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.370344:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.370344:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.370345:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.370345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.370350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.370358:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.370361:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.370361:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.370363:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.370365:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.370365:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.370786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.370793:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.370795:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.370797:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.370799:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.370801:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.370802:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.370803:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.370804:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.370806:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.370807:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.370810:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.370812:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.370816:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.370816:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.370817:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.370819:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.370819:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.370820:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.370821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.370825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.370832:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.370835:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.370836:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.370838:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.370839:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.370841:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.370952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.370960:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.370962:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.370965:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.370966:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.370967:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.370968:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.370970:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.370971:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.370972:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.370973:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.370976:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.370978:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.370981:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.370981:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.370982:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.370984:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.370984:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.370986:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.370987:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.370988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.370989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.370994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.370996:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.370999:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.371000:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.371000:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.371001:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.371002:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.371004:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677475.371004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.371005:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.371006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.371100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.371105:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.371108:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.371110:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.371111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.371113:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.371115:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.2:1619677475.371115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:21.0:1619677475.371116:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.371117:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.371118:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.371119:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.371123:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.371123:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:21.0:1619677475.371124:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.371128:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.371128:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:21.0:1619677475.371129:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.371129:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677475.371129:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:21.0:1619677475.371131:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.371132:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.371132:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.371133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:18.0:1619677475.371133:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.371136:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.371137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:18.0:1619677475.371138:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.371141:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.371143:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.371143:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:18.0:1619677475.371145:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.371146:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.371146:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.371147:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.371147:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.371150:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:18.0:1619677475.371152:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.371152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677475.371156:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.371159:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.371161:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.371162:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.371163:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.371164:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.371166:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.371167:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:21.2:1619677475.371168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.2:1619677475.371170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677475.371173:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.371176:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.371177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.371179:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.371179:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.371181:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:22.0:1619677475.371182:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.371182:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.371183:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:1.0:1619677475.371183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:22.0:1619677475.371184:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.371185:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.371186:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.371187:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677475.371187:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:22.0:1619677475.371188:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.371192:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.371194:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.371196:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.371197:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.371198:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.371199:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.371200:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.371201:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.371201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.371206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.371213:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.371215:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.371216:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.371218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.371218:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.371220:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.371279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.371287:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.371290:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.371292:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.371294:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.371301:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.371302:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.371304:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.371305:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.371306:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.371307:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.371311:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.371313:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.371316:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.371317:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.371317:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.371319:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.371320:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.371321:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.371321:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.371326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.371334:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.371337:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.371338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.371338:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.371340:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.371343:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.371345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.371352:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.371355:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.371355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.371356:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.371357:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.371357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.371553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.371560:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.371562:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.371563:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.371565:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.371568:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.371570:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677475.371571:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.371572:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.371573:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.371574:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677475.371578:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.371579:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677475.371583:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.371584:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:1.0:1619677475.371584:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677475.371587:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677475.371587:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.371588:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677475.371589:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677475.371590:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:25.2:1619677475.371593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.2:1619677475.371596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.371601:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:13.0:1619677475.371602:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:17.0:1619677475.371604:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677475.371604:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.371605:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:13.0:1619677475.371606:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.371607:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.371609:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.371612:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.371613:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677475.372067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.372074:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.372077:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.372078:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.372079:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.372082:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.372084:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.372085:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.372087:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.372087:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.372089:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.372092:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.372093:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.372095:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.372096:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.372097:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.372098:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.372099:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.372100:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.372100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.372105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.372112:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.372114:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.372115:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.372115:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:5.2:1619677475.372116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:18.0:1619677475.372117:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.372118:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:1.0:1619677475.372122:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.372125:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.372126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.372128:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.372130:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.372132:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.372133:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.372134:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.372135:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.372136:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:1.0:1619677475.372139:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.372141:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:1.0:1619677475.372143:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.372144:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.372145:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.372146:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.372147:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.372148:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.372149:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.372149:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.372152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.372154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:6.0:1619677475.372159:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677475.372160:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.372162:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677475.372162:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.372163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.372163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.372164:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:6.0:1619677475.372164:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.372166:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.372168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.372198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.372203:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.372206:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.372206:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.372208:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.372211:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.372212:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.372213:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.372215:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.372215:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.372217:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.372220:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.372222:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.372224:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.372224:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.372225:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.372227:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.372227:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.372228:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.372228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.372233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.372241:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.372243:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.372244:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.372245:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.372246:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.372248:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.372253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.372260:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.372263:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.372264:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.372264:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.372265:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.372266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.372525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.372528:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.372530:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.372532:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.372533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.372535:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.372536:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.372538:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.372539:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.372540:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.372541:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.372544:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.372546:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.372549:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.372550:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.372550:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.372552:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.372553:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.372553:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.372554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677475.372559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.2:1619677475.372559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677475.372568:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677475.372568:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:31.0:1619677475.372571:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677475.372571:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.372572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.372572:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.372573:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.372574:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.372575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.372576:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.372577:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:31.0:1619677475.372578:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.372579:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.372580:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.372581:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.372582:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.372586:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.372588:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.372590:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.372591:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.372591:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.372593:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.372593:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.372594:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.372595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.372599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.372606:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.372609:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.372609:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.372611:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.372613:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.372614:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.373146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.373152:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.373155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.373155:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.373157:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.373159:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.373161:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.373162:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.373163:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.373164:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.373165:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.373168:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.373168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677475.373170:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.373172:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.373173:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.373173:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677475.373175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.373175:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.373176:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677475.373177:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677475.373177:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.373178:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.373178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677475.373179:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.373180:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.373181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:24.0:1619677475.373183:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.373183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677475.373184:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.373186:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.373187:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.373188:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.373188:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:24.0:1619677475.373189:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.373190:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.373191:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.373191:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:24.0:1619677475.373192:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.373192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.373193:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:24.0:1619677475.373194:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.373194:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.373195:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.373196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677475.373198:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.373198:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.373198:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677475.373199:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.373201:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.373201:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.373202:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.373203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.373207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.373214:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.373217:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.373218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.373218:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.373221:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.373223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.373268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.373276:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.373278:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.373281:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.373282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.373283:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.373285:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.373286:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.373287:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.373289:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.373289:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.373296:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.373297:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.373300:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.373301:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.373302:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.373303:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.373304:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.373305:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.373305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677475.373309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:25.2:1619677475.373310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.373316:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:22.0:1619677475.373317:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677475.373318:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:22.0:1619677475.373320:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:18.0:1619677475.373320:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:22.0:1619677475.373321:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.373321:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:2.0:1619677475.373321:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.373323:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:2.0:1619677475.373324:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.373325:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677475.373326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:2.0:1619677475.373326:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.373328:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.373329:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.373330:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.373333:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.373335:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.373338:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.373338:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.373339:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.373341:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.373341:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.373342:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677475.373343:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677475.373344:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.2:1619677475.373347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677475.373349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.373355:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:17.0:1619677475.373356:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:18.0:1619677475.373357:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677475.373358:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:17.0:1619677475.373358:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677475.373359:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:17.0:1619677475.373359:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.373359:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.373361:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.373363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.373365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:4.2:1619677475.373372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677475.373372:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.373374:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.373375:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.373376:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.373377:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.373377:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677475.373379:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.373382:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.373383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.373385:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.373388:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.373390:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.373391:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.373392:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.373393:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.373394:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.373398:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.373399:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.373402:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.373403:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.373403:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.373405:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.373406:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.373406:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.373407:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.373411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.373419:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.373421:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.373422:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.373422:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.373425:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.373427:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.373675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.373679:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.373682:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.373684:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.373685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.373686:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.373688:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677475.373689:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.373690:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.373691:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.373692:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.373696:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.373697:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.373700:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.373700:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.373701:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.373702:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677475.373703:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.373704:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.373705:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677475.373706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677475.373709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.373711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.373716:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:2.0:1619677475.373716:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677475.373719:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:2.0:1619677475.373719:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677475.373720:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:2.0:1619677475.373720:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.373721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677475.373721:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.373722:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.373725:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.373888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677475.373896:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.373899:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.373901:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.373903:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.373904:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.373905:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677475.373907:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.373908:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.373909:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.373910:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677475.373913:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.373915:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677475.373918:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.373919:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677475.373920:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677475.373922:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677475.373922:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.373924:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677475.373925:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677475.373926:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677475.373928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677475.373931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:18.0:1619677475.373936:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.373939:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:31.0:1619677475.373940:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000400:00000200:18.0:1619677475.373940:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.373940:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677475.373942:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000400:00000200:18.0:1619677475.373942:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.373943:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677475.373944:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:18.0:1619677475.373946:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677475.373949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677475.373957:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.373959:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677475.373960:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.373961:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.373962:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.373962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.374019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.374024:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.374027:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.374028:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.374030:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.374032:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.374033:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.374035:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.374036:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.374037:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.374038:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.374041:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.374042:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.374045:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.374045:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.374046:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.374048:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.374049:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.374049:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.374050:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.374054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.374062:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.374064:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.374065:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.374067:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.374067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.374069:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.374267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.374274:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.374277:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.374280:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.374281:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.374283:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.374284:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.374286:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.374287:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.374288:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.374289:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.374292:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.374298:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.374300:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.374301:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.374301:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.374303:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.374303:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.374304:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.374305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.374309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.374317:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.374319:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.374320:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.374322:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.374324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.374325:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.374559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.374567:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.374569:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.374570:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.374572:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.374574:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.374576:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.374577:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.374578:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.374579:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.374580:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.374583:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.374585:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.374588:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.374589:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.374589:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.374591:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.374591:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.374592:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.374593:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.374597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.374605:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.374607:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.374608:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.374608:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.374611:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.374613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.374616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:21.2:1619677475.374617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:22.0:1619677475.374622:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.374623:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:22.0:1619677475.374625:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.374626:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677475.374626:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:6.0:1619677475.374626:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.374627:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.374627:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:22.0:1619677475.374628:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:17.0:1619677475.374629:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.374631:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.374633:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.374634:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.374635:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.374636:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.374637:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.374640:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.374642:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.374644:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.374645:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.374646:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.374647:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.374648:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.374649:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.374650:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.374651:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.374654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.374656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.374661:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:20.0:1619677475.374662:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.374663:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677475.374664:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677475.374664:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677475.374665:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.374665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677475.374667:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.374668:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.374669:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.375048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.375056:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.375059:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.375061:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.375063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.375063:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.375065:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.375066:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.375067:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.375068:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.375069:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.375072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:30.0:1619677475.375073:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.375075:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.375078:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.375078:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.375078:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677475.375079:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.375080:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:30.0:1619677475.375081:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.375081:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.375081:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677475.375083:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677475.375083:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.375084:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.375085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677475.375085:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.375087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677475.375087:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.375088:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.375090:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677475.375091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:6.0:1619677475.375091:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.375093:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.375096:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.375097:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.375097:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.375098:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.375099:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677475.375099:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.375100:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.375101:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.375102:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677475.375102:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.375103:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.375103:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.375103:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.375104:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.375105:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.375106:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.375106:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.375106:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.375107:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.375107:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.375108:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.375108:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.375109:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.375109:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.375112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.375120:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.375122:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.375122:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.375124:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.375125:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.375126:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.375884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.375892:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.375895:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.375897:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.375899:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.375900:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.375901:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.375903:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.375904:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.375905:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.375906:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.375909:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.375911:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.375914:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.375915:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.375915:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.375917:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.375917:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.375918:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.375919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.375923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.375930:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.375933:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.375934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.375934:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.375936:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.375939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.375964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.375971:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.375974:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.375975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.375976:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.375979:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.375980:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.375981:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.375982:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.375983:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.375984:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.375987:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.375988:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.375991:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.375992:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.375992:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.375994:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.375995:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.375995:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.375996:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.376000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.376009:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.376012:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.376012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.376013:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.376015:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.376018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.376026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.376032:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.376035:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.376036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.376038:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.376040:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.376042:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.376043:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.376044:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.376045:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.376046:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.376050:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.376051:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.376054:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.376054:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.376055:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.376057:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.376057:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.376058:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.376059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.376063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.376071:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.376073:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.376074:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.376076:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.376077:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.376078:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.376083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.376090:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.376093:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.376094:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.376094:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.376095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.376096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.376676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.376684:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.376687:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.376689:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.376691:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.376691:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.376693:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.376694:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.376695:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.376696:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.376698:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.376701:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.376703:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.376705:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.376706:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.376707:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.376709:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.376709:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.376711:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.376713:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.376713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.376715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.376719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.376722:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.376725:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.376726:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.376726:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.376728:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.376728:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.376730:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677475.376730:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.376731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.376732:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.376733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.376739:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.376742:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.376743:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.376743:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.376743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.376744:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.376937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.376943:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.376945:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.376946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.376948:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.376951:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.376952:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.376954:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.376954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:22.0:1619677475.376955:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.376956:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.376957:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.376960:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.376961:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.376961:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.376964:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677475.376964:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.376965:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.376966:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.376967:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.376967:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.376968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677475.376968:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677475.376969:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.376969:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.376970:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:31.0:1619677475.376971:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677475.376971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677475.376972:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.376973:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.376974:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.376974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.376975:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.376975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677475.376979:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.376981:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.376981:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.376983:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.376983:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:31.0:1619677475.376984:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.376984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.376984:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677475.376985:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.376985:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.376985:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.376986:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.376987:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.376988:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:21.0:1619677475.376988:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.376989:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.376989:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.0:1619677475.376989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.376990:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.376993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.377000:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.377003:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.377004:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.377004:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.377007:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.377009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.377738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.377745:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.377748:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.377748:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.377750:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.377753:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.377755:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.377756:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.377757:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.377759:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.377760:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.377763:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.377765:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.377768:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.377769:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.377769:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.377771:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.377772:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.377773:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.377773:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.377778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.377787:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.377791:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.377792:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.377795:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677475.377795:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.377798:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.377832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.377840:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.377843:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.377843:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.377846:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.377849:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.377850:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.377851:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.377852:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.377853:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.377854:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.377857:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.377859:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.377862:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.377863:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.377863:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.377865:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.377866:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.377867:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.377868:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.377869:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.377872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.377875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677475.377880:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.377882:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:6.0:1619677475.377882:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.377883:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.377884:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:6.0:1619677475.377884:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.377885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.377886:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:6.0:1619677475.377886:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.377889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.377978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.377984:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.377986:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.377987:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.377989:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.377991:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.377992:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.377993:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.377995:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.377996:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.377997:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.378000:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.378001:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.378004:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.378004:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.378005:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.378007:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.378007:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.378008:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.378008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.378013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.378020:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.378022:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.378023:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.378024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677475.378024:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.378027:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.378028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.378032:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.378035:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.378038:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.378040:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.378041:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.378042:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.378044:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.378045:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.378046:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.378047:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.378050:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.378052:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.378053:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.378054:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.378055:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.378056:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.378057:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.378058:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.378058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.378063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.378070:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.378072:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.378073:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.378075:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.378076:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.378077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.378187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.378195:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.378197:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.378198:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.378200:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.378203:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.378204:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.378205:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.378206:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.378207:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.378208:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.378211:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.378213:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.378215:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.378216:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.378217:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.378219:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.378219:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.378220:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.378221:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.378222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.378225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.378227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.378232:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.378235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.378235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.378235:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.378236:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.378237:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.378237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.2:1619677475.378238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:6.0:1619677475.378238:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.378239:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677475.378240:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677475.378245:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.378249:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.378251:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.378255:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.378259:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.378261:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.378263:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.378265:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.378267:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.378268:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.378273:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.378276:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.378280:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.378281:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.378282:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.378284:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.378285:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.378286:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.378287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677475.378291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.378301:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.378304:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.378305:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.378305:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.378307:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.378309:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.378521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.378529:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.378532:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.378533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.378535:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.378538:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.378539:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.378541:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.378542:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.378543:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.378544:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.378547:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.378548:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.378552:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.378552:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.378553:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.378555:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.378555:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.378557:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.378558:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.378558:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.378562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.378564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.378570:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.378571:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.378573:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677475.378573:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677475.378574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677475.378574:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677475.378575:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.378575:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.378577:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.378579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.378582:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.378584:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.378585:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.378586:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.378587:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.378587:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.378588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.378621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.378629:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.378631:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.378633:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.378636:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.378636:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.378637:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.378638:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.378640:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.378640:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.378641:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.378645:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.378646:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.378649:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.378650:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.378650:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.378652:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.378653:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.378654:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.378654:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.378658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.378666:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.378669:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.378670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.378670:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.378672:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.378675:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.378866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.378871:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.378874:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.378875:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.378877:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.378879:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.378880:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.378881:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.378883:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.378884:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.378885:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.378888:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.378890:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.378892:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.378893:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.378893:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.378895:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.378896:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.378896:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.378897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.378901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.378908:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.378911:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.378911:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.378913:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.378915:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.378915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.378943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.378948:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.378951:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.378953:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.378953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.378956:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.378957:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.378958:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.378960:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.378961:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.378962:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.378965:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.378967:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.378970:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.378970:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.378971:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.378973:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.378974:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.378975:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.378976:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.378977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.378979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.378981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.378984:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.378986:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.378987:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677475.378987:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677475.378988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.378990:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.378991:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.378991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.378993:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.378995:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.379363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.379371:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.379374:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.379374:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.379377:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.379379:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.379381:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.379382:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.379384:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.379385:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.379386:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.379389:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.379391:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.379394:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.379394:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.379395:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.379397:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.379397:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.379398:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.379399:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.379400:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.379403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.379406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.379411:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.379413:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.379414:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.379415:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.379415:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677475.379415:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.379416:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.379417:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.379417:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.379420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.379536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.379544:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.379547:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.379550:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.379552:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.379552:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.379554:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.379555:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677475.379555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:31.0:1619677475.379556:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.379557:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.379558:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677475.379559:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:31.0:1619677475.379561:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.379561:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.379564:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:31.0:1619677475.379565:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:20.0:1619677475.379565:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677475.379567:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.379567:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.379568:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:20.0:1619677475.379568:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.379569:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:20.0:1619677475.379569:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677475.379570:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677475.379570:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.379571:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.379571:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677475.379572:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.379573:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.379574:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677475.379576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:20.0:1619677475.379577:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.379579:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.379581:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.379582:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:20.0:1619677475.379582:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.379582:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.379584:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:20.0:1619677475.379584:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677475.379585:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.379585:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.379586:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000400:00000200:30.0:1619677475.379587:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.379587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:31.0:1619677475.379588:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.379590:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.379590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677475.379594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677475.379597:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.379600:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.379601:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:22.0:1619677475.379601:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.379601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.379603:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.379604:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.379605:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:22.0:1619677475.379605:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677475.379606:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.379607:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.379607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.379735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.379740:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.379743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.379743:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.379745:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.379747:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.379748:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.379750:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.379751:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.379752:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.379753:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.379755:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.379757:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.379759:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.379760:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.379761:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.379762:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.379763:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.379764:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.379764:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.379768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.379776:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.379778:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.379779:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.379780:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.379781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.379783:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.379787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.379794:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.379797:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.379798:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.379799:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.379799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.379800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.380628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.380636:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.380639:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.380641:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.380644:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.380644:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.380645:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.380647:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.380648:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.380649:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.380650:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.380653:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.380655:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.380658:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.380659:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.380659:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.380661:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.380662:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.380663:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.380663:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.380668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.380676:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.380679:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.380680:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.380680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.380683:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.380685:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.380796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.380801:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.380804:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.380805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.380807:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.380809:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.380810:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.380811:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.380812:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.380813:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.380814:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.380818:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.380819:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.380822:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.380822:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.380823:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.380825:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.380825:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.380826:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.380827:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.380828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.380831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.380833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.380838:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.380838:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.380839:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.380840:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.380840:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.380841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.380841:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.380843:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.380844:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.380845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.381548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.381556:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.381559:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.381561:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.381564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.381564:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.381564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:30.0:1619677475.381566:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.381567:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.381568:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.381569:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.381569:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677475.381570:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:21.0:1619677475.381572:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677475.381574:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.381575:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677475.381576:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.381578:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:21.0:1619677475.381578:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.381580:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:30.0:1619677475.381581:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.381582:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:21.0:1619677475.381582:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.381583:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:21.0:1619677475.381583:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.381584:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.381585:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.381585:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:21.0:1619677475.381585:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.381586:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.381587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677475.381589:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.381591:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:9.2:1619677475.381591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.0:1619677475.381594:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.381594:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.381595:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.381597:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.381597:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.381598:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.0:1619677475.381598:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.0:1619677475.381599:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.0:1619677475.381601:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.381602:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.381602:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.381603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:8.0:1619677475.381604:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.381607:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.381609:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.381612:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.381613:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.381615:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.381616:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.381618:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.381624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.381630:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.381633:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.381634:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.381635:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.381635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.381637:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.381657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.381663:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.381666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.381666:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.381669:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.381671:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.381672:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.381674:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.381675:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.381676:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.381677:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.381681:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.381683:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.381685:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.381686:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.381686:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.381688:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.381688:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.381689:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.381690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.381694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.381701:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.381703:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.381704:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.381706:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.381707:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.381708:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.382160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.382166:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.382169:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.382170:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.382172:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.382174:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677475.382175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:1.0:1619677475.382175:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:1.0:1619677475.382176:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.382177:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.382179:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.382180:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:18.0:1619677475.382182:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:1.0:1619677475.382183:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.382185:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.382185:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:1.0:1619677475.382185:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:18.0:1619677475.382187:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.382188:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.382189:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:1.0:1619677475.382189:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:18.0:1619677475.382190:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.382191:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677475.382191:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.382191:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.382192:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677475.382193:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.382193:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:18.0:1619677475.382194:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.382196:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.382197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:18.0:1619677475.382197:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677475.382200:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.382202:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.382204:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:18.0:1619677475.382205:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.382205:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.382206:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:18.0:1619677475.382206:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677475.382207:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.382207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:18.0:1619677475.382208:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677475.382208:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.382209:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:22.0:1619677475.382210:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.382210:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:22.0:1619677475.382212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.382225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.382232:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.382233:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.382234:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.382235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.382236:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.382238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.382247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.382254:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.382256:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.382257:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.382257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.382258:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.382259:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.382374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677475.382380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.382382:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677475.382384:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.382385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677475.382387:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.382389:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.382390:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.382391:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.382392:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.382393:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.382397:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.382398:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.382401:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.382401:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.382402:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.382403:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.382404:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.382405:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677475.382405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:4.2:1619677475.382405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.2:1619677475.382412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:24.0:1619677475.382417:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677475.382420:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:24.0:1619677475.382422:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.0:1619677475.382422:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.382423:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.382423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:20.0:1619677475.382425:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.382426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:24.0:1619677475.382427:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.382428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:24.0:1619677475.382431:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.382433:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.382435:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.382437:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.382439:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.382441:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.382446:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.382449:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.382453:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.382454:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.382455:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.382457:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.382458:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.382459:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.382460:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.382474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.382482:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.382485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.382485:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.382486:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.382488:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.382491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.382512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.382520:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.382524:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.382526:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.382527:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677475.382527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:31.0:1619677475.382528:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.382529:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.382531:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.382532:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.382532:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677475.382533:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.382534:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.382535:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.382536:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677475.382537:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.382539:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677475.382541:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.382543:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.382543:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:22.0:1619677475.382543:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.382544:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677475.382544:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.382545:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.382546:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.382546:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:22.0:1619677475.382547:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.382548:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.382548:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.382549:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677475.382549:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.382550:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677475.382552:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.382554:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.2:1619677475.382555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.382556:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.382557:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.382558:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.382559:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.382560:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.382561:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.382561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677475.382563:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:25.2:1619677475.382564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.382565:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.382566:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.382567:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.382568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.382571:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.382574:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.382575:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.382575:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:6.0:1619677475.382575:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.382577:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.382578:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.382578:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.382579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:21.0:1619677475.382580:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.382581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.382582:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.382968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.382976:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677475.382976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:30.0:1619677475.382978:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.382980:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.382982:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677475.382982:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:30.0:1619677475.382983:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.382985:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:2.0:1619677475.382985:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.382986:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677475.382987:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.382988:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:30.0:1619677475.382989:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.382990:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.382990:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.382991:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:2.0:1619677475.382992:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.382993:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.382995:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.382996:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677475.382996:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.382998:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:2.0:1619677475.382998:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677475.383002:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.383002:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:2.0:1619677475.383002:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677475.383003:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.383004:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:2.0:1619677475.383004:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677475.383005:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.383006:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:2.0:1619677475.383007:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.383008:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:2.0:1619677475.383008:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.383008:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677475.383009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.383010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:2.0:1619677475.383010:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.383010:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.383011:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.383012:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.2:1619677475.383014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677475.383015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.383021:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:31.0:1619677475.383022:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677475.383023:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677475.383023:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:31.0:1619677475.383025:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.383025:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677475.383025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.383025:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.383026:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677475.383026:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.383027:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.383027:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.383027:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.383029:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.383030:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677475.383032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.383549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.383558:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.383560:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.383561:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.383564:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.383566:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.383566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677475.383568:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.383569:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.383570:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.383571:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.383572:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.383573:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677475.383575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677475.383576:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.383576:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.383577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677475.383578:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.383579:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.383581:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677475.383581:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.383582:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.383582:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.383583:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677475.383583:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:26.0:1619677475.383584:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.383584:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:24.0:1619677475.383584:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677475.383585:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:24.0:1619677475.383585:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.383586:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677475.383586:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:24.0:1619677475.383586:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.383588:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.383588:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.383591:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.383591:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.383591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:24.0:1619677475.383591:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.383592:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.383593:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96500166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.383593:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.383595:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.383596:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.383596:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.383597:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677475.383597:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.383597:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (23)++
00000800:00000200:22.0:1619677475.383599:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000400:00000200:31.0:1619677475.383600:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.383600:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (25)--
00000800:00000200:24.0:1619677475.383600:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.383601:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677475.383602:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677475.383602:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.383602:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000400:00000200:22.0:1619677475.383603:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.383603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.383604:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.383605:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.383605:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:22.0:1619677475.383605:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.383606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677475.383607:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.383607:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.383608:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.383609:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677475.383609:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.383611:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:25.2:1619677475.383612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677475.383613:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.383614:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.383614:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.383615:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.383617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.383619:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.383621:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.383622:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.383624:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.383624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.383625:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.383626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677475.383631:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.383634:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.383635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.383635:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.383637:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.383637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.383693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.383699:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.383702:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.383702:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.383705:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.383707:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.383708:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.383710:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.383711:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.383712:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.383713:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.383716:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.383718:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.383720:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.383720:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.383721:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.383723:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.383723:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.383725:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.383726:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.383726:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.383729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.383731:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.383738:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.383739:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.383739:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.383740:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.383740:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.383740:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.383741:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.383742:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.383743:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677475.383743:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.383754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.383761:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.383764:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.383765:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.383767:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.383770:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.383771:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.383773:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.383774:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.383775:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.383776:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.383778:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.383780:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.383783:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.383784:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.383785:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.383786:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.383787:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.383787:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.383788:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.383792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.383800:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.383802:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.383803:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.383804:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.383806:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.383807:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677475.383881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.383884:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.383886:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.383889:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.383889:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.383891:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.383892:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.383894:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.383895:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.383896:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.383897:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.383899:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.383901:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.383903:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.383904:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.383905:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.383907:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.383907:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.383908:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.383908:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.383913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.383921:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.383924:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.383925:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.383925:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.383927:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.383929:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.384236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.384245:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.384249:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.384251:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.384254:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.384257:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.384260:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.384262:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.384264:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.384266:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.384267:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.384273:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.384276:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.384279:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.384281:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.384282:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.384284:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.384285:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.384286:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.384286:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.384290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.2:1619677475.384294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.384300:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.384300:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.384301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.384302:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677475.384303:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.384305:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.384305:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.384308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.384308:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.384309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:6.0:1619677475.384309:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677475.384310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.384311:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.384312:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.384314:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.384315:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.384318:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.384320:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.384322:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.384323:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.384324:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.384325:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.384326:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.384326:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.384327:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.384332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.384340:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.384342:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.384343:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.384344:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.384346:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.384346:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:25.2:1619677475.384356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.384363:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.384366:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.384367:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.384367:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.384368:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.384370:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.384414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.384422:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.384425:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.384427:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.384429:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.384429:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.384431:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.384432:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.384433:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.384434:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.384435:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.384439:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.384441:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.384443:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.384443:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.384444:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.384446:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.384446:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.384447:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.384447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.384451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.384458:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.384460:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.384461:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.384463:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.384464:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.384465:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.384786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.384793:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.384796:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.384798:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.384799:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.384800:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.384801:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.384802:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.384803:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.384804:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.384805:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.384808:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.384810:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.384813:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.384813:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.384814:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.384816:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.384816:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.384817:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.384817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.384833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.384840:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.384843:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.384844:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.384844:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.384846:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.384848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.385672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.385676:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.385679:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.385681:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.385682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.385683:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.385684:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.385686:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.385687:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.385688:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.385689:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.385692:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.385694:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.385697:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.385697:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.385698:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.385700:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.385700:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.385701:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.385702:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.385703:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.385706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.385708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.385713:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677475.385714:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677475.385716:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.385716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:24.0:1619677475.385717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.385717:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677475.385717:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.385718:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.385720:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.385723:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.385724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.385731:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.385733:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.385734:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.385734:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.385735:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.385736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.386298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.386304:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.386306:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.386307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.386309:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.386311:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.386312:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.386314:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.386315:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.386316:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.386317:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.386320:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.386322:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.386324:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.386325:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.386325:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.386327:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.386328:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.386329:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.386330:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.386330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.386334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.386336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.386341:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.386343:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.386344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.386344:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.2:1619677475.386344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:22.0:1619677475.386345:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.386346:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.386346:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.386348:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.386350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.386351:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677475.386351:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.386356:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.386358:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.386360:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.386364:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.386366:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.386369:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.386371:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.386372:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.386374:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.386380:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.386383:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.386386:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.386388:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.386389:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.386391:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.386392:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.386393:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.386394:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677475.386398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.386409:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.386413:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.386414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.386414:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.386417:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:17.0:1619677475.386418:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.386418:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)++
00000800:00000200:1.0:1619677475.386420:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (24)--
00000800:00000200:1.0:1619677475.386421:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.386422:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.386423:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.387084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677475.387090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.387095:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.387097:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.387099:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.387099:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.387100:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.387102:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.387103:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.387104:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.387105:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.387106:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.387107:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.387108:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.387111:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.387111:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.387112:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.387113:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.387114:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.387115:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.387115:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.387116:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.387119:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.387120:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:30.0:1619677475.387120:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.387121:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.387121:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.387123:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.387123:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.387124:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.387124:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.387125:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.387125:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.387125:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.387126:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677475.387126:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.387127:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.387128:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.387128:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.387129:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.387132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.387138:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:25.2:1619677475.387139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.387141:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.387141:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.387142:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.387144:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.387146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677475.387147:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.387149:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.387150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.387151:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.387153:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.387156:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.387862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.387867:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.387870:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.387870:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.387872:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.387874:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.387876:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.387877:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.387878:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.387879:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.387880:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.387883:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.387885:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.387887:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.387888:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.387889:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.387890:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.387891:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.387892:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.387892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.387897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.387905:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.387907:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.387908:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.387910:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.387911:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.387912:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.388030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.388038:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.388041:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.388042:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.388043:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.388047:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.388048:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.388049:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.388051:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.388052:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.388053:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.388056:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.388058:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.388061:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.388062:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.388062:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.388064:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.388064:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.388065:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.388065:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.388080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.388087:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.388089:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.388090:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.388090:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.388093:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.388095:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.388223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.388231:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.388234:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.388236:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.388238:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.388239:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.388240:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.388241:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.388243:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.388243:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.388244:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.388247:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.388249:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.388251:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.388252:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.388253:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.388254:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.388255:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.388256:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.388258:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.388258:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.388260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.388264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.388266:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.388269:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.388270:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.388270:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.388272:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.388272:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.388274:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677475.388274:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.388275:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.388276:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.388280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.388286:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.388288:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.388289:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.388289:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.388290:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.388290:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.388540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.388548:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.388550:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.388552:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.388554:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.388556:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.388557:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.388558:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.388560:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.388561:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.388562:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.388565:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.388567:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.388570:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.388570:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.388571:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.388573:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.388573:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.388574:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.388574:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.388582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.388591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.388593:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.388594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.388595:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.388597:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.388600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.388610:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.388617:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.388620:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.388621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.388621:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.388622:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.388623:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.388656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.388662:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.388665:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.388666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.388668:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.388670:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.388672:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.388673:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.388674:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.388675:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.388676:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.388680:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.388681:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.388684:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.388684:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.388685:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.388686:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.388687:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.388688:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.388688:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.388693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.388702:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.388704:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.388705:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.388707:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.388709:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.388709:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.2:1619677475.388714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.388722:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.388725:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.388726:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.388727:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.388727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.388728:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.388829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.388836:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.388838:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.388839:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.388842:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:4.2:1619677475.388842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000400:00000200:18.0:1619677475.388844:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.388845:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.388847:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.388848:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.388849:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.388850:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:13.0:1619677475.388850:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000400:00000200:18.0:1619677475.388854:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:13.0:1619677475.388854:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:18.0:1619677475.388857:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:13.0:1619677475.388857:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.388859:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:18.0:1619677475.388860:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:13.0:1619677475.388860:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.388861:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.388861:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:13.0:1619677475.388862:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:18.0:1619677475.388863:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:13.0:1619677475.388863:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.388864:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:13.0:1619677475.388864:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677475.388865:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.388866:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:13.0:1619677475.388866:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677475.388867:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:13.0:1619677475.388867:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:25.2:1619677475.388869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:13.0:1619677475.388870:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677475.388872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677475.388872:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677475.388874:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.388875:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677475.388875:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:30.0:1619677475.388877:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:13.0:1619677475.388877:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677475.388878:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:2.0:1619677475.388878:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:13.0:1619677475.388879:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:30.0:1619677475.388880:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:13.0:1619677475.388880:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:2.0:1619677475.388880:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:13.0:1619677475.388881:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:2.0:1619677475.388881:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:30.0:1619677475.388882:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.388882:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677475.388883:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:9.2:1619677475.388883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:31.0:1619677475.388885:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.388887:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.388887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:26.0:1619677475.388891:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.388895:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.388896:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.388896:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:10.0:1619677475.388896:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.388900:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.388900:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677475.388901:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.388902:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:26.0:1619677475.388904:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.388914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.388921:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.388924:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.388926:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.388928:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.388929:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.388930:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.388931:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.388932:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.388933:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.388934:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.388937:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.388939:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.388941:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.388942:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.388942:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.388944:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.388944:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.388945:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.388945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.388949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.388956:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.388958:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.388959:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.388961:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.388962:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.388963:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.389045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.389054:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.389058:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.389060:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.389063:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.389066:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.389068:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.389070:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.389073:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.389074:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.389076:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.389081:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.389084:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.389087:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.389089:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.389089:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.389092:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.389092:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.389093:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.389094:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.389098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.389106:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.389109:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.389109:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.389110:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.389112:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.389115:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.389117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.389124:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.389127:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.389128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.389128:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.389129:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.389130:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.389130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.389138:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.389140:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.389142:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.389143:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.389146:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.389148:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.389149:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.389150:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.389151:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.389152:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.389156:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.389157:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.389160:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.389161:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.389161:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.389163:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.389164:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.389165:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.389166:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.389167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.389169:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.389172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.389177:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.389179:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.389180:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.389181:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677475.389181:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.389181:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.389182:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.389183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677475.389183:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.389186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.389325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.389333:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.389336:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.389338:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.389340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.389340:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.389342:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.389343:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.389344:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.389345:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.389346:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.389349:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.389351:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.389353:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.389354:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.389355:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.389356:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.389357:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.389358:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.389358:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.389363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.389371:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.389373:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.389374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.389375:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.389377:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.389379:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.389804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.389809:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.389812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.389812:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.389814:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.389816:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.389817:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.389818:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.389820:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677475.389821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:6.0:1619677475.389821:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.389822:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.389824:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677475.389825:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.389826:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677475.389827:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:20.0:1619677475.389829:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.389829:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.389830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677475.389830:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:20.0:1619677475.389831:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.389831:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677475.389833:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677475.389833:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.389833:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:20.0:1619677475.389834:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.389834:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.389834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.389835:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.389836:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.389837:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:25.2:1619677475.389839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677475.389841:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.389843:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.389845:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.389845:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.389846:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.389847:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.389847:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.389849:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677475.389849:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.389849:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.389850:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.389850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:21.0:1619677475.389852:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.389852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677475.389854:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.2:1619677475.389855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.389856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.389857:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.389860:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.389862:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.389864:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677475.389864:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:31.0:1619677475.389865:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.389866:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:17.0:1619677475.389866:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.389867:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.389867:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:31.0:1619677475.389868:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.389870:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.389870:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.389871:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.389872:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677475.389872:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:31.0:1619677475.389876:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677475.389876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677475.389878:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.389880:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.389881:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.389881:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.389883:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.389883:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.389884:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677475.389884:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677475.389885:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677475.389886:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.389887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.389887:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:25.2:1619677475.389889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:1.0:1619677475.389889:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.389889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677475.389895:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.389898:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.389899:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.389901:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.389902:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.389903:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.389908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.389915:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.389918:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.389919:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.389919:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.389920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.389921:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677475.390083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677475.390089:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677475.390091:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677475.390092:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677475.390094:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677475.390096:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.390097:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.390098:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.390100:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.390101:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.390102:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.390105:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.390107:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.390108:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.390109:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.390109:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.390111:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.390111:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677475.390112:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677475.390112:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:8.2:1619677475.390117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.390124:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.390126:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.390127:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.390127:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.390128:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.390130:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.390158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.390162:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.390165:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.390167:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.390168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.390169:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.390171:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.390172:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.390173:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.390174:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.390175:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.390179:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.390180:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.390183:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.390184:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.390185:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.390186:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.390187:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.390188:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.390189:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.390192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.390199:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.390201:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.390203:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.390203:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.390205:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.390208:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.390613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.390619:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.390622:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.390623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.390625:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.390627:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.390629:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.390630:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.390631:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.390632:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.390633:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.390637:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.390638:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.390640:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.390641:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.390641:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.390643:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.390643:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.390644:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.390645:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.390650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.390657:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.390660:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.390660:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.390661:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.390662:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.390664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.390670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.390677:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.390680:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.390681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.390681:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.390681:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.390682:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.390823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677475.390832:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.390834:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677475.390837:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.390839:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677475.390839:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.390840:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677475.390842:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.390843:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.390844:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.390845:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.390848:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.390850:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.390853:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.390854:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677475.390855:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.390856:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677475.390857:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.390858:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677475.390859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677475.390862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.390870:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.390873:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.390874:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.390874:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.390876:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.390879:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.391121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.391129:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.391132:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.391134:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.391135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.391137:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.391138:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.391140:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.391141:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.391142:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.391143:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.391147:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.391148:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.391151:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.391152:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.391152:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.391154:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.391154:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.391156:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.391158:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.391158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.391160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.391164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.391168:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.391170:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.391171:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.391172:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.391172:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.391174:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677475.391174:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.391175:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.391176:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.391177:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.391791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.391796:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.391799:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.391800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.391802:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.391804:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.391805:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.391806:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.391807:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.391808:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.391809:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.391812:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.391814:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.391816:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.391817:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.391817:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.391819:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.391820:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.391820:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.391820:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.391826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.391834:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.391837:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.391837:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.391839:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.391840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.391842:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.391851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.391859:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.391861:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.391862:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.391863:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.391863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.391865:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.392732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.392740:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.392743:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.392745:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.392747:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.392748:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.392749:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.392750:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.392752:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.392753:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.392754:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.392757:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.392758:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:5.2:1619677475.392760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677475.392761:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.392762:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.392763:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.392764:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.392765:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.392766:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.392766:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677475.392766:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.392769:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:9.2:1619677475.392770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677475.392770:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.392772:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.392774:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.392776:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:24.0:1619677475.392777:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677475.392777:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.392778:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.392779:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.392780:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:17.0:1619677475.392780:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:24.0:1619677475.392781:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.392782:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.392784:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.392784:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.392786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677475.392786:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.392789:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.392790:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.392790:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.392792:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.392792:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.392793:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.392794:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.392798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.392806:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.392808:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.392809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.392810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.392812:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.392814:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.393012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677475.393018:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.393021:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677475.393022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677475.393025:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.393027:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.393029:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677475.393030:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.393032:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.393033:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.393034:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677475.393038:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.393041:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677475.393044:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.393045:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677475.393046:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677475.393048:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677475.393048:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.393049:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677475.393049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677475.393054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.393062:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.393065:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.393066:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.393067:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.393069:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.393072:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.393123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.393131:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.393134:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.393136:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.393137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.393138:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.393140:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.393141:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.393143:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.393144:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.393145:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.393148:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.393150:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.393153:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.393154:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.393155:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.393156:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.393157:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.393157:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.393158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.393162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.393169:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.393172:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.393173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.393174:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.393176:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.393178:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.393338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.393343:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.393346:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.393347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.393349:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.393351:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.393352:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.393353:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.393354:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.393355:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.393356:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.393360:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.393362:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.393364:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.393364:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.393365:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.393367:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.393367:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.393368:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.393369:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.393370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.393374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.393375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:4.2:1619677475.393377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:22.0:1619677475.393380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.393382:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.393382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.393383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.393383:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.393385:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.393385:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.393387:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:21.0:1619677475.393387:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.393388:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.393389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:26.0:1619677475.393391:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.393393:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.393396:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.393399:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.393402:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.393404:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.393406:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.393408:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.393410:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.393415:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.393418:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.393422:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.393423:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.393424:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.393426:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.393427:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.393428:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.393429:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.393432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.393440:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.393442:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.393443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.393444:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.393446:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.393449:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.393483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.393491:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.393494:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.393496:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.393498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.393499:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.393501:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.393502:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.393503:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.393504:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.393505:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.393509:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.393510:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.393512:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.393513:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.393513:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.393515:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.393515:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.393517:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.393518:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.393519:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.393520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.393524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.393529:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.393530:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.393531:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.393532:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677475.393533:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.393533:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.393534:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.393535:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.393536:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.393537:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.393646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.393653:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.393656:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.393657:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.393658:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.393662:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.393663:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.393664:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.393665:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.393666:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.393667:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.393670:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.393672:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.393675:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.393676:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.393676:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.393678:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.393678:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.393679:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.393679:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.393684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.393694:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.393698:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.393700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.393700:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.393703:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.393707:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.393929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.393936:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.393939:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.393939:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677475.393941:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.393944:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.393945:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.393947:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.393948:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.393949:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.393950:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.393953:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.393954:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.393957:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.393958:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.393958:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.393960:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.393960:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.393961:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.393962:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.393966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.393974:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.393977:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.393977:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.393978:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.393980:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.393982:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.393985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.393992:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.393995:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.393996:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.393996:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.393997:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.393998:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.394271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.394279:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.394282:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.394284:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.394286:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.394286:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.394287:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.394289:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.394290:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.394291:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.394292:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.394301:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.394303:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.394305:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.394305:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.394306:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.394307:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.394308:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.394309:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.394309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.394314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.394322:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.394324:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.394325:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.394327:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.394329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.394329:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.394823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.394831:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.394833:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.394835:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.394837:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.394838:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677475.394838:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.394839:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.394840:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.394841:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.394842:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.394845:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.394847:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.394849:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.394850:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.394851:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.394852:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.394853:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.394854:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.394854:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.394859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.394866:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.394869:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.394870:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.394870:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.394873:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.394875:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.395112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.395119:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.395121:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.395122:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.395124:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.395127:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.395128:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.395130:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.395131:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677475.395131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:18.0:1619677475.395132:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.395133:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.395136:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.395138:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677475.395138:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.395141:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.395142:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:18.0:1619677475.395142:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:8.0:1619677475.395142:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:18.0:1619677475.395143:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:24.0:1619677475.395145:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.395145:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.395145:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.395147:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000400:00000200:24.0:1619677475.395148:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.395148:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677475.395148:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:24.0:1619677475.395149:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.395150:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.395151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:24.0:1619677475.395151:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.395152:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.395153:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677475.395153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:24.0:1619677475.395156:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.395158:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:12.0:1619677475.395158:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677475.395159:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:24.0:1619677475.395161:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.395161:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:2.0:1619677475.395161:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:24.0:1619677475.395162:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.395162:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:12.0:1619677475.395162:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.395162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.395163:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:12.0:1619677475.395163:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.395164:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.395164:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.395165:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:13.0:1619677475.395165:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:24.0:1619677475.395166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:12.0:1619677475.395166:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.2:1619677475.395170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:25.2:1619677475.395171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:17.0:1619677475.395178:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:13.0:1619677475.395178:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677475.395181:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:13.0:1619677475.395181:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.395181:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.395182:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:1.0:1619677475.395182:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.395183:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.395183:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.395184:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:17.0:1619677475.395185:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.395188:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.395260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.395263:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.395266:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.395268:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.395269:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.395271:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.395272:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.395273:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.395275:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.395276:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.395277:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.395280:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.395282:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.395284:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.395285:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.395286:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.395287:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.395288:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.395288:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.395289:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.395293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.395303:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.395305:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.395306:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.395307:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.395308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.395310:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.395344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.395349:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.395352:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.395353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.395355:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.395356:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.395358:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.395359:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.395360:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.395361:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.395362:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.395365:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.395367:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.395369:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.395370:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.395371:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.395372:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.395373:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.395374:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.395374:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.395379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.395386:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.395389:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.395390:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.395391:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.395392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.395393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.395546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.395553:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.395555:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.395557:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.395558:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.395561:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.395562:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.395564:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.395565:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.395566:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.395567:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.395570:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.395572:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.395575:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.395576:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.395576:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.395578:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.395579:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.395580:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.395580:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.395584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.395592:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.395594:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.395595:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.395597:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.395599:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.395600:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.395779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.395787:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.395790:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.395793:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.395795:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.395795:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.395797:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.395798:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.395799:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.395800:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.395801:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.395804:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.395806:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.395808:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.395809:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.395810:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.395812:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.395812:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.395813:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.395814:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.395817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.395824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.395827:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.395828:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.395828:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.395830:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.395832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.395863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677475.395870:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.395873:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677475.395874:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677475.395875:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.395878:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.395879:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677475.395880:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.395882:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.395882:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.395883:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677475.395886:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.395888:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677475.395890:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.395891:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677475.395891:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677475.395893:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677475.395893:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.395894:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677475.395894:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.2:1619677475.395899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677475.395901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:31.0:1619677475.395906:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:1.0:1619677475.395907:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:31.0:1619677475.395908:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.395909:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.395911:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.395911:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.395911:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:30.0:1619677475.395913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.395914:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:1.0:1619677475.395914:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.395916:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.395917:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.395919:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.395920:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.395921:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.395922:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.395925:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.395927:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.395929:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.395930:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.395931:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.395933:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.395933:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.395934:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.395935:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.395939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.395948:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.395950:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.395951:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.395952:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.395953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.395954:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.396081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677475.396089:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.396092:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677475.396094:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.396096:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677475.396097:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.396098:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677475.396100:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.396101:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.396102:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.396103:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.396106:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.396108:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677475.396110:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.396111:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:30.0:1619677475.396112:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.396113:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677475.396114:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.396115:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677475.396116:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677475.396120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.396128:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.396131:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.396131:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.396132:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.396134:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.396137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677475.396166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.396169:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.396172:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.396174:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.396175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.396176:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.396178:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.396179:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.396180:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.396181:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.396182:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.396186:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.396188:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.396190:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.396191:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.396191:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.396193:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.396194:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.396194:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.396195:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.396199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.396207:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.396210:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.396211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.396211:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.396213:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.396215:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.396508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.396515:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.396518:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.396519:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.396521:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.396524:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.396525:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677475.396526:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.396528:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.396529:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.396530:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677475.396533:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.396535:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.396537:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.396537:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.396538:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.396539:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.396540:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.396540:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.396541:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.396546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.396552:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.396554:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.396554:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.396555:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.396556:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.396558:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677475.396565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.396571:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.396573:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.396574:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.396574:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.396575:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.396575:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.396924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.396933:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.396935:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.396938:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.396940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.396941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.396942:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.396943:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.396944:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.396945:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.396946:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.396949:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.396951:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.396954:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.396954:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.396955:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.396956:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.396957:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.396958:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.396958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.396972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.396980:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.396982:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.396983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.396984:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.396986:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.396989:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.397288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.397293:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.397299:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.397301:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.397303:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677475.397306:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.397308:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.397309:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.397310:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.397311:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.397312:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.397315:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.397316:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.397318:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.397319:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.397320:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.397322:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.397322:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.397323:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.397324:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.397328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.397331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.397337:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.397338:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.397338:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.397340:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.397340:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.397342:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:22.0:1619677475.397343:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.397345:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.397345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.397347:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:6.0:1619677475.397347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:22.0:1619677475.397348:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.397348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.397350:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.397351:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.397352:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.397355:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.397357:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.397359:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.397360:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.397360:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.397362:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.397363:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.397363:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.397364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.397368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.397376:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.397378:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.397379:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.397380:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.397381:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.397382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.397807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.397815:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.397818:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.397821:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.397822:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.397824:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.397825:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.397827:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.397828:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.397829:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.397830:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.397833:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.397835:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.397837:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.397838:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.397839:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.397840:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.397841:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.397842:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.397843:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.397846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.397853:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.397855:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.397856:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.397858:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.397860:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.397861:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.398153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.398161:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.398164:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.398165:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.398166:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.398168:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.398170:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.398171:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.398172:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.398173:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.398174:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.398177:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.398178:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.398181:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.398182:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.398183:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.398184:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.398185:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.398186:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.398187:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.398188:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.398191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.398193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.398198:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677475.398200:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.398200:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.398201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.398202:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677475.398202:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.398203:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.398204:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.398204:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.398207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.398303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677475.398308:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.398310:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677475.398311:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677475.398313:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.398316:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.398317:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677475.398318:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.398319:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.398321:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.398321:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677475.398325:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.398327:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677475.398329:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.398330:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677475.398330:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677475.398332:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677475.398332:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.398333:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677475.398333:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677475.398339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.398347:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.398350:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.398351:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.398351:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.398353:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.398356:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.398858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.398864:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.398867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.398867:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.398869:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.398872:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.398873:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.398874:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.398875:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.398877:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.398877:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.398881:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.398882:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.398884:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.398885:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.398885:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.398887:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.398887:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.398888:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.398889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.398893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.398901:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.398903:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.398904:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.398906:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.398907:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.398908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.398911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.398918:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.398921:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.398922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.398923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000400:00000200:8.0:1619677475.398924:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.398926:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.398928:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.398929:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.398930:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.398931:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677475.398931:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.398932:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.398933:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.398934:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.398935:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.398935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677475.398936:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.398937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677475.398937:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.398940:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.398941:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.398942:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.398943:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.398944:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.398944:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.398945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.398949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.398956:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.398959:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.398960:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.398960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.398963:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.398965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.398969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:21.2:1619677475.398976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:6.0:1619677475.398976:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.398979:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.398980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.398980:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.398981:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.398982:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677475.398984:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.398987:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.398989:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.398991:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.398992:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.398993:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.398994:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.398995:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.398996:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.398997:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.399000:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.399002:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.399005:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.399006:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.399006:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.399008:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.399008:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.399010:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.399011:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.399012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.399013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.399018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.399021:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.399023:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.399024:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.399025:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.399025:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.399027:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.399028:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.399029:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677475.399030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.399031:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677475.399162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.399167:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.399170:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.399172:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.399173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.399174:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.399176:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.399177:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.399178:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.399179:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.399180:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.399183:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.399185:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.399187:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.399188:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.399189:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.399190:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.399191:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.399192:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.399193:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.399193:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.399196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.399199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.399205:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.399206:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.399207:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.399207:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677475.399208:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.399208:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.399209:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.399210:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.399211:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677475.399212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677475.399267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.399274:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.399277:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.399277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.399280:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.399282:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.399283:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.399284:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.399286:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.399287:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.399288:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.399291:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.399293:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.399302:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.399302:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.399303:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.399305:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.399305:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.399306:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.399306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.399322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.399330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.399332:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.399333:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.399335:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.399336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.399337:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.399340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.399348:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.399350:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.399351:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.399352:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.399353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.399354:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.399465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.399473:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.399476:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.399477:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.399478:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.399481:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.399483:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.399484:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.399486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677475.399486:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.399486:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.399488:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.399491:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.399493:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.399493:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.399496:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677475.399496:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.399497:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:30.0:1619677475.399498:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.399498:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.399499:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.399500:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:31.0:1619677475.399501:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.399501:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.399501:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:30.0:1619677475.399502:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:10.0:1619677475.399502:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000400:00000200:30.0:1619677475.399503:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.399503:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:30.0:1619677475.399504:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.399505:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.399506:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677475.399508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:30.0:1619677475.399509:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.399511:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.399513:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.399513:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.399514:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.399515:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.399516:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677475.399516:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:25.2:1619677475.399516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.399517:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.399518:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.399518:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677475.399518:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.399518:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.399519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.399521:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.399524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677475.399524:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.399527:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.399527:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.399528:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:6.0:1619677475.399528:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.399530:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:6.0:1619677475.399530:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.399531:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.399532:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677475.399533:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.399533:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:6.0:1619677475.399533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.399534:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.399535:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.399536:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.399537:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.399539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.399545:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.399548:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.399549:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.399549:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.399550:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.399551:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.399869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677475.399875:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.399878:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677475.399879:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677475.399880:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677475.399883:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.399884:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677475.399886:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.399887:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.399888:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.399889:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677475.399892:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677475.399893:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.399896:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.399897:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.399897:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.399899:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.399899:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.399900:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677475.399900:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:25.2:1619677475.399905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.399912:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.399915:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.399915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.399916:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.399917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:22.0:1619677475.399918:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.399920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677475.399922:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.399925:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.399927:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.399927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.399929:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.399930:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.399932:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.399933:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.399934:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.399935:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.399938:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.399940:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.399943:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.399943:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.399944:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.399945:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.399946:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.399947:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.399947:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.399960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.399967:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.399969:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.399970:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.2:1619677475.399970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:6.0:1619677475.399970:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.399972:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.399975:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677475.399976:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.399980:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.399981:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.399982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000400:00000200:2.0:1619677475.399983:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.399986:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.399987:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.399988:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.399989:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:2.0:1619677475.399990:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.399991:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.399992:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677475.399992:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677475.399993:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.399993:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.399994:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.399994:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:2.0:1619677475.399996:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.399998:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.400001:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.400002:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.400003:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.400004:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.400005:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.400006:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.400006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.400010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.400019:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.400021:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.400022:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.400022:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.400024:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.400027:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.400032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.400039:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.400042:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.400043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.400043:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.400043:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.400044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.400062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.400069:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.400072:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.400073:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.400075:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.400078:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.400080:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.400081:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.400082:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.400083:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.400084:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.400087:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.400089:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.400091:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.400092:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.400093:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.400094:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.400095:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.400096:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.400097:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.400097:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.400100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.400103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677475.400108:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677475.400110:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677475.400110:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.400111:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.400112:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:12.0:1619677475.400112:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.400113:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.400114:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677475.400114:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.400115:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677475.400784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.400791:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.400793:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.400794:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.400796:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.400799:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.400800:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.400801:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.400802:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.400803:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.400804:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.400807:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.400809:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.400810:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.400811:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.400812:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.400813:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.400814:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.400815:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.400815:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.400820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.400828:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.400829:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.400830:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.400831:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.400832:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.400833:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.400846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.400852:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.400855:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.400855:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.400857:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.400859:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.400861:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.400862:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.400863:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.400864:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.400865:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.400869:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.400870:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.400873:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.400874:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.400875:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.400877:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.400877:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.400878:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.400879:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.400880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.400883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.400885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.400890:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.400890:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.400891:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.400892:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.400892:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.400893:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.400893:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.400895:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.400896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.400897:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.401577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.401586:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.401589:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.401591:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.401593:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.401593:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.401594:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.401596:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.401597:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.401598:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.401599:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.401603:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.401605:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.401607:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.401608:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.401608:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.401610:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.401611:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.401612:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.401612:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.401617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.401624:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.401626:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.401627:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.401628:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.401630:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.401631:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.401842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.401849:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.401852:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.401854:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.401856:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.401856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.401857:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.401858:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.401859:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.401860:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.401861:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.401864:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.401866:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.401869:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.401869:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.401870:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.401872:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.401872:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.401873:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.401873:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.401890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.401898:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.401901:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.401901:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.401902:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.401904:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.401907:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.401997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.402002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.402005:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.402005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.402008:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.402010:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.402011:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677475.402013:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.402014:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.402015:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.402016:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677475.402019:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.402021:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677475.402024:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.402025:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:22.0:1619677475.402025:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677475.402027:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:22.0:1619677475.402028:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.402029:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.402029:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677475.402033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.402041:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.402044:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677475.402044:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.402045:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.402047:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.402050:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677475.402170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.402176:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.402178:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.402181:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.402181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.402183:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.402184:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.402186:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.402187:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.402188:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.402189:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.402192:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.402194:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.402197:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.402197:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.402198:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.402200:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.402200:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.402201:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.402203:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.402203:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.402205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.402208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.402212:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677475.402213:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.402214:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.402215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677475.402215:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.402216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.402216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677475.402216:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.402219:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.402221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.402306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.402313:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.402316:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.402316:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.402318:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.402321:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.402322:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677475.402324:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.402325:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.402326:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.402327:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677475.402330:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.402332:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.402334:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.402335:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677475.402335:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.402337:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.402338:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.402338:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.402339:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677475.402343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.402351:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.402353:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.402355:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.402355:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.402357:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.402359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.402466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.402474:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.402477:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.402478:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.402480:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.402483:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.402484:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.402486:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.402487:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.402488:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.402489:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.402492:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.402494:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.402497:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.402498:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.402499:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.402500:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.402501:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.402502:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.402504:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.402504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.2:1619677475.402506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:8.2:1619677475.402506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.0:1619677475.402509:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677475.402510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.402510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.0:1619677475.402513:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:1.0:1619677475.402514:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:20.0:1619677475.402515:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.402517:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:1.0:1619677475.402517:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:30.0:1619677475.402518:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677475.402518:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.402518:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.402518:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:1.0:1619677475.402518:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.402520:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:8.0:1619677475.402520:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.402521:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677475.402521:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.402522:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677475.402522:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677475.402522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677475.402524:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.402524:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:20.0:1619677475.402525:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.402526:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677475.402526:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.402527:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.402530:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.402530:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.402532:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677475.402532:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.402533:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.402534:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.402535:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.402536:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.402536:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677475.402536:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.402537:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.402538:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.402539:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677475.402540:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.402540:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.402540:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677475.402542:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.402544:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.402545:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.2:1619677475.402545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.402546:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.402547:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.402548:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.402549:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.402549:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677475.402552:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:25.2:1619677475.402553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677475.402555:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.402556:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.402556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.402558:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.402560:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:17.0:1619677475.402560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677475.402562:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.402563:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.402564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:31.0:1619677475.402565:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.402566:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.402567:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.402568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:1.0:1619677475.402572:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.402573:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:1.0:1619677475.402575:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.402576:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:17.0:1619677475.402576:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.402577:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.402577:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.402578:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:1.0:1619677475.402578:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:21.0:1619677475.402579:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.402581:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.402583:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677475.402584:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.402586:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.402587:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.402588:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677475.402591:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.402593:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677475.402596:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.402597:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677475.402597:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677475.402599:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677475.402600:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.402601:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677475.402602:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677475.402603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:9.2:1619677475.402605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677475.402608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.402611:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:26.0:1619677475.402612:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:20.0:1619677475.402613:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:20.0:1619677475.402614:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:26.0:1619677475.402615:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:20.0:1619677475.402615:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:10.0:1619677475.402615:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.402616:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.402618:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.402621:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.402817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.402822:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.402825:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.402826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.402828:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.402830:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.402831:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.402832:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.402834:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.402835:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.402836:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.402838:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.402840:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.402842:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.402843:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.402844:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.402845:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.402846:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.402847:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.402847:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.402852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.402860:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.402862:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.402863:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.402864:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.402865:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.402867:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.402986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.402993:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.402996:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.402996:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677475.402999:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.403001:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.403002:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.403004:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.403005:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.403006:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.403007:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.403011:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.403012:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.403015:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.403016:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.403016:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.403018:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.403018:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.403019:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.403020:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.403024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.403031:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.403034:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.403035:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.403036:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.403038:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.403041:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.403045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.403051:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.403054:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.403055:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.403055:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.403056:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.403057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.403536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.403542:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.403545:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.403545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.403547:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.403549:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.403551:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.403552:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.403553:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.403554:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.403555:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.403559:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.403561:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.403563:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.403564:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.403565:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.403566:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.403567:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.403567:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.403568:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.403573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.403581:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.403583:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:20.2:1619677475.403584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:6.0:1619677475.403584:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.403585:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.403587:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.403589:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.0:1619677475.403590:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.403593:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677475.403595:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.403595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677475.403598:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.403599:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677475.403601:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.403602:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.403603:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.403604:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677475.403607:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.403609:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677475.403612:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.403613:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677475.403614:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677475.403616:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677475.403616:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.403617:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677475.403618:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:9.2:1619677475.403622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.403629:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.403632:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677475.403632:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.403633:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.403635:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.403638:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.403674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.403682:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.403685:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.403687:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.403689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.403689:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.403690:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.403691:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.403693:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.403694:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.403695:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.403698:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.403700:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.403703:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.403703:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.403704:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.403706:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.403706:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.403707:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.403708:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.403712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.403719:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.403722:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.403723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.403723:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.403725:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.403728:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.403732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.403739:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.403741:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.403742:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.403742:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.403743:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.403744:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.403958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677475.403964:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.403967:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677475.403968:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677475.403969:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677475.403972:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.403974:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677475.403975:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.403976:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.403977:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.403978:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677475.403982:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677475.403983:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677475.403986:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.403987:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677475.403987:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.403989:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.403990:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.403990:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677475.403991:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:9.2:1619677475.404005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.404012:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.404015:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.404016:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.404016:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.404018:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.404021:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.404326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.404330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.404333:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.404335:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.404336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.404338:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.404339:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.404341:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.404342:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.404343:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.404344:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.404348:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.404349:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.404352:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.404353:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.404354:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.404356:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.404356:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.404357:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.404358:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.404362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.404369:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.404372:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.404372:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.404373:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.404375:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.404378:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.404392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.404397:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.2:1619677475.404399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677475.404400:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.404401:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.404402:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.404405:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.404406:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677475.404407:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677475.404407:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.404408:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.404409:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677475.404410:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.404411:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.404413:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.404414:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677475.404415:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.404416:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.404417:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.404418:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.404419:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.404420:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.404421:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.404421:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.404421:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.404422:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.404423:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677475.404423:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.404423:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.404424:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.404425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:31.0:1619677475.404426:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.404428:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:25.2:1619677475.404429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.404430:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.404431:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.404431:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.404433:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.404433:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.404435:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.404435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677475.404437:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:25.2:1619677475.404438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677475.404439:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.404440:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.404441:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.404442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.404444:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.404445:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.404447:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.404448:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.404450:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.404451:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.404452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.404607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.404615:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.404618:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.404619:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.404620:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.404623:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.404624:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.404625:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.404627:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.404628:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.404629:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.404632:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.404634:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.404637:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.404637:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.404638:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.404639:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.404640:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.404641:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.404641:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.404646:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.404653:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.404655:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.404656:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.404657:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.404659:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.404662:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.404887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.404894:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.404897:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.404898:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.404899:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.404902:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.404904:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.404905:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.404907:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.404908:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.404908:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.404912:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.404914:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.404916:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.404917:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.404918:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.404919:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.404920:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.404921:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.404921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.404925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.404933:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.404935:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.404936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.404937:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.404938:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.404941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.404999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.405002:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.405005:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.405007:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.405008:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.405009:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.405011:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.405012:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.405014:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.405015:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.405016:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.405019:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.405021:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.405023:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.405024:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.405024:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.405026:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.405026:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.405027:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.405028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.405032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.405040:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.405042:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.405043:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.405045:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.405045:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.405047:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.405379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.405383:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.405385:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.405387:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.405388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.405389:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.405391:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.405392:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.405393:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.405394:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.405395:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.405398:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.405399:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.405402:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.405403:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.405403:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.405405:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.405406:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.405406:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.405407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.405412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.405419:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.405422:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.405422:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.405423:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.405425:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.405427:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.405431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.405437:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.405440:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.405441:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.405441:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.405442:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.405442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.405784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.405792:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.405795:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.405797:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.405799:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.405799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.405801:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.405802:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.405803:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.405804:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.405805:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.405809:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.405811:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.405814:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.405814:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.405815:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.405817:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.405817:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.405818:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.405819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.405823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.2:1619677475.405827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:22.0:1619677475.405831:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677475.405832:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:22.0:1619677475.405834:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.405835:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.405835:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000800:00000200:6.0:1619677475.405835:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.405837:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:22.0:1619677475.405838:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.405839:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.405841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:21.0:1619677475.405842:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.405844:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.405845:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.405846:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.405847:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.405848:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.405852:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.405854:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677475.405855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fd9e00] (20)++
00000800:00000200:21.0:1619677475.405856:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.405857:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.405858:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.405858:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fd9e00] (21)++
00000800:00000200:21.0:1619677475.405859:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.405860:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.405860:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.46@o2ib3
00000800:00000200:21.0:1619677475.405861:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677475.405861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:21.0:1619677475.405862:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd9e00] (22)--
00000400:00000200:20.0:1619677475.405862:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.46@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.405865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677475.405865:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.405866:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:20.0:1619677475.405867:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.405869:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.405869:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.405870:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677475.405873:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.46@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.405873:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.405875:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.405875:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.405876:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.405877:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.405878:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.405879:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.405879:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:6.0:1619677475.405879:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.405881:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.405881:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fd9e00] (21)++
00000800:00000200:6.0:1619677475.405881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.405882:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fd9e00] (22)--
00000800:00000200:20.0:1619677475.405883:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fd9e00] (21)--
00000800:00000200:25.2:1619677475.405887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677475.405892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.405894:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.405896:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.405897:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.405899:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677475.405899:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.405899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.405901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677475.405902:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.405903:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.405903:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:25.2:1619677475.405904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.405904:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.405905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.405911:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.405914:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.405915:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.405915:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.405916:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.405917:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.406366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.406372:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.406375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.406375:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.406377:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.406379:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.406380:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.406382:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.406383:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.406384:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.406385:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.406388:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.406389:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.406392:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.406392:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.406393:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.406395:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.406395:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.406396:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.406396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.406401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.406410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.406412:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.406413:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.406414:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.406416:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.406416:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.2:1619677475.406711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.406719:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.406722:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.406725:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.406727:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.406728:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.406729:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.406730:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.406731:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.406733:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.406733:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.406737:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.406738:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.406740:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.406741:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.406742:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.406743:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.406744:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.406745:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.406747:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.406747:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.406749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.406753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.406757:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.406759:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.406759:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.406761:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.406761:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.406762:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.406762:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.406763:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.406765:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.406765:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.406768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.406776:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.406778:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.406779:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.406779:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.406780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677475.406780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:31.0:1619677475.406781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.0:1619677475.406784:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.406787:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.406789:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.406789:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.406791:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.406793:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.406794:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.406796:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.406797:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.406798:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.406801:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.406803:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.406805:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.406806:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.406807:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.406809:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.406809:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.406810:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.406811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.406814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.406822:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.406824:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.406825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.406826:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.406828:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.406830:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.406858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.406865:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.406868:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.406869:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.406871:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.406873:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.406875:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.406876:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.406877:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.406878:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.406879:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.406882:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.406884:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.406887:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.406888:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.406889:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.406891:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.406891:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.406892:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.406893:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.406896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.406903:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.406906:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.406907:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.406909:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.406910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.406911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.407358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.407366:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.407369:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.407369:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.407372:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.407374:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.407375:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677475.407377:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.407378:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.407379:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.407380:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677475.407383:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.407384:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.407386:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.407387:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.407388:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.407389:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.407390:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.407390:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.407391:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.407395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.407402:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.407403:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.407404:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.407405:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.407405:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.407407:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677475.407414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.407420:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.407422:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.407423:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.407423:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.407424:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.407425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.407567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.407576:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.407578:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.407580:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.407583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.407583:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.407584:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.407585:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.407586:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.407587:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.407588:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.407591:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.407593:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.407596:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.407596:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.407597:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.407599:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.407599:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.407600:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.407601:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.407602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.407605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.407608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.407613:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.407615:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.407616:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677475.407616:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.407617:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.407618:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.407619:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677475.407619:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.407620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.407622:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.407714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.407722:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.407725:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.407726:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.407728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.407729:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.407731:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.407732:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.407733:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.407734:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.407735:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.407737:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.407739:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.407741:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.407742:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.407743:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.407744:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.407745:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.407746:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.407746:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.407750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.407757:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.407759:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.407760:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.407760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.407762:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.407764:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.407850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.407857:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.407860:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.407861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.407863:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.407866:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.407867:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.407868:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.407869:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.407870:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.407871:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.407875:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.407877:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.407879:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.407880:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.407881:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.407882:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.407883:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.407884:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.407885:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.407886:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.407888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.407892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.407897:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.407899:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677475.407900:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.407900:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.407901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677475.407902:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.407903:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.407905:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.407905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.407908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.407994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.408002:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.408005:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.408006:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.408008:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.408010:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.408011:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.408012:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.408014:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.408015:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.408016:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.408019:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.408021:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.408024:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.408024:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.408025:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.408027:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.408027:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.408028:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.408028:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.408033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.408040:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.408043:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.408044:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.408044:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.408047:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.408049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.408419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.408425:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.408428:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.408428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.408430:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.408433:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.408434:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.408435:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.408436:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.408437:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.408438:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.408441:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.408443:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.408445:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.408446:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.408447:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.408448:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.408449:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.408450:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.408451:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.408452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.408455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.408457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.408462:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.408463:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.408464:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.408465:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.408465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.408466:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.408466:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.408468:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.408469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.408470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.408817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.408825:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.408827:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.408828:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.408830:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.408833:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.408834:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.408836:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.408837:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.408838:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.408839:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.408842:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.408844:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.408846:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.408847:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.408848:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.408850:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.408850:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.408851:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.408851:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.408856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.408863:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.408866:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.408867:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.408867:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.408869:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.408872:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.408921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.408927:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.408929:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.408932:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.408932:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.408934:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.408936:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.408937:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.408938:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.408939:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.408940:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.408943:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.408944:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.408947:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.408948:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.408948:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.408950:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.408951:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.408952:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.408952:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.408957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.408965:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.408968:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.408969:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.408969:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.408971:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.408974:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.409247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.409255:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.409259:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.409261:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.409263:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.409264:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.409265:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.409266:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.409267:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677475.409267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:30.0:1619677475.409269:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.409270:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.409273:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.409274:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.409275:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.409276:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000800:00000200:30.0:1619677475.409277:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.409278:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.409278:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.409278:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677475.409280:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.409280:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.409280:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.409281:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.409282:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677475.409282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:31.0:1619677475.409283:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.409283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677475.409284:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.409285:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.409285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677475.409286:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677475.409289:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.409291:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677475.409292:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.409297:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.409298:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.409300:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.409300:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:30.0:1619677475.409300:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.409301:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677475.409302:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.409303:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677475.409303:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.409303:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.409304:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677475.409305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677475.409309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.409317:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.409320:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.409322:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.409322:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.409324:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.409327:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.409472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.409480:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.409483:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.409484:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.409485:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.409488:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.409490:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.409491:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.409492:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.409493:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.409494:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.409497:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.409498:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.409501:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.409502:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.409503:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.409504:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.409505:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.409505:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.409506:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.409510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.409518:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.409521:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.409522:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.409522:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.409524:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.409526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.409772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.409779:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.409782:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.409783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.409785:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.409788:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.409789:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.409791:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.409792:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.409793:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.409794:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.409797:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.409798:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.409801:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.409802:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.409802:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.409804:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.409804:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.409805:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.409806:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.409810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.409817:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.409820:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.409820:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.409821:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.409823:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.409826:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.409826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:25.2:1619677475.409827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677475.409832:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677475.409834:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677475.409835:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.409835:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:6.0:1619677475.409837:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.409838:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:18.0:1619677475.409838:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.409839:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000400:00000200:18.0:1619677475.409840:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.409840:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.409840:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:18.0:1619677475.409842:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.409843:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.409844:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.409845:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.409846:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.409849:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.409851:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.409853:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.409854:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.409854:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.409855:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.409856:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.409857:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.409858:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.409862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.409869:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.409871:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.409872:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.409872:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.409873:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.409875:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.409884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.409891:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.409893:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.409894:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.409894:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.409895:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.409895:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.409940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677475.409947:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677475.409949:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677475.409950:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677475.409952:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.409955:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.409957:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677475.409958:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.409959:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.409961:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.409961:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:20.2:1619677475.409964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:10.0:1619677475.409965:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.409967:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677475.409969:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.409970:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677475.409970:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:2.0:1619677475.409970:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:10.0:1619677475.409972:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677475.409972:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:2.0:1619677475.409972:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.409973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:10.0:1619677475.409973:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677475.409974:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677475.409975:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:2.0:1619677475.409975:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.409977:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.2:1619677475.409978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000400:00000200:2.0:1619677475.409979:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:4.2:1619677475.409980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000400:00000200:2.0:1619677475.409980:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.409981:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.409982:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.409983:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.409984:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677475.409986:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:2.0:1619677475.409986:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.409987:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:24.0:1619677475.409987:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.409988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:2.0:1619677475.409988:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677475.409989:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000400:00000200:24.0:1619677475.409989:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.409990:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677475.409991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:2.0:1619677475.409991:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.409992:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:2.0:1619677475.409992:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.409993:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.409994:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.409995:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.409996:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.409997:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677475.410001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.410009:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.410011:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.410013:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.410013:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.410014:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.410017:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.410023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.410031:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.410034:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.2:1619677475.410036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:30.0:1619677475.410037:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.410038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.410039:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.410041:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677475.410041:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677475.410042:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.410043:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.410044:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:30.0:1619677475.410045:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.410045:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:30.0:1619677475.410046:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677475.410047:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.410050:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.410052:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677475.410053:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.410055:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.410055:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:30.0:1619677475.410056:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:22.0:1619677475.410056:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96500047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.410057:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677475.410058:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.410059:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.410059:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:22.0:1619677475.410059:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.410060:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677475.410060:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.410061:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:22.0:1619677475.410063:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.410064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.410065:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.410068:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.410069:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.410069:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.410072:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.410072:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.410073:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.410073:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.410074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:6.0:1619677475.410074:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.410075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.410075:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.410077:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.410078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.410080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.410085:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.410087:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.410088:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.410090:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.410091:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.410092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.410747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.410752:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.410755:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.410757:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.410758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.410759:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.410761:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.410762:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.410763:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.410765:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.410765:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.410769:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.410771:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.410773:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.410774:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.410774:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.410776:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.410777:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.410778:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.410779:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.410780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.410783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.410786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.410792:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.410793:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.410794:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.410794:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677475.410795:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.410795:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.410796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.410797:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.410798:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677475.410799:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677475.410872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.410880:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.410882:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.410885:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.410887:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.410887:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.410889:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.410890:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.410891:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.410892:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.410893:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.410896:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.410898:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.410900:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.410901:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.410902:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.410903:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.410904:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.410905:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.410905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.410910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.410916:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.410918:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.410919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.410920:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.410921:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.410924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.410927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.410932:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.410935:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.410936:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.410936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.410937:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.410937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.410969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677475.410976:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.410979:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677475.410979:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.410981:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.410983:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.410985:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677475.410986:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.410988:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.410988:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.410990:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677475.410993:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.410995:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677475.410997:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.410998:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677475.410998:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677475.411000:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677475.411000:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.411001:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677475.411002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677475.411006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.411014:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.411017:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.411018:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.411018:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.411020:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.411022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.411760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.411768:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.411771:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.411773:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.411775:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.411776:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.411777:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.411779:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.411780:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.411781:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.411782:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.411785:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.411787:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.411790:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.411790:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.411791:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.411793:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.411793:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.411794:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.411795:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.411800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.411808:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.411811:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.411812:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.411812:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.411814:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.411817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.411897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677475.411903:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.411905:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677475.411907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677475.411908:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.411910:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.411912:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.411913:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.411914:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.411915:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.411917:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.411920:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.411922:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.411924:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.411925:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.411926:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.411928:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.411928:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.411929:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677475.411930:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:25.2:1619677475.411934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.411941:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.411944:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.411945:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.411946:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.411947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.411950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.411953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:25.2:1619677475.411956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.411961:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.411963:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.411963:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677475.411965:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.411966:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:30.0:1619677475.411966:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.411967:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.411967:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.411969:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.411969:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:30.0:1619677475.411969:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.411970:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.411971:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.411972:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.411974:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.411975:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.411977:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.411979:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.411981:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.411982:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.411983:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.411984:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.411985:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.411986:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.411988:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.411988:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.411990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.411994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.411999:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.412000:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.412000:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.412002:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.412002:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.412003:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.412004:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.2:1619677475.412004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.412005:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.412006:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.412006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677475.412009:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.412012:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.412013:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.412015:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.412017:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.412018:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.412019:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.412020:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.412022:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.412022:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.412025:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.412027:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.412029:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.412030:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.412031:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.412032:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.412033:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.412033:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.412034:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.412038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.412046:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.412048:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.412049:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.412050:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.412052:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.412052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.412061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.412068:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.412071:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.412072:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.412072:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.412073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.412074:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.412514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677475.412522:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677475.412525:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677475.412526:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677475.412527:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.412530:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.412532:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677475.412533:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.412534:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.412535:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.412536:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677475.412540:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.412542:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677475.412544:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.412545:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677475.412545:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677475.412547:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677475.412547:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677475.412549:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:8.0:1619677475.412550:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677475.412550:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:9.2:1619677475.412553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677475.412557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000800:00000200:10.0:1619677475.412560:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.412563:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.412563:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.412564:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677475.412564:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000400:00000200:10.0:1619677475.412565:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.412566:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000800:00000200:24.0:1619677475.412566:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677475.412568:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:10.0:1619677475.412568:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677475.412769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.412776:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.412779:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.412779:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.412782:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.412785:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.412786:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.412787:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.412789:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.412789:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.412791:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.412794:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.412795:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.412798:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.412798:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.412799:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.412800:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.412801:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.412803:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.412804:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.412804:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677475.412807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677475.412809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.412815:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677475.412815:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677475.412817:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.412817:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.412818:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:1.0:1619677475.412818:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.412819:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.412819:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.412821:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.412823:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677475.412948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.412954:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.412956:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.412957:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.412959:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.412961:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.412962:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.412963:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.412965:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.412965:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.412966:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:17.0:1619677475.412970:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.412972:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.412974:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.412975:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.412976:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.412977:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.412978:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.412979:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.412980:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.412980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.412984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:5.2:1619677475.412985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677475.412991:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:1.0:1619677475.412992:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677475.412993:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:1.0:1619677475.412994:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:22.0:1619677475.412995:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.412995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:1.0:1619677475.412995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.412996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677475.412997:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.412999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.413298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.413307:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.413309:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.413311:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.413313:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.413314:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.413315:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.413316:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.413317:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.413318:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.413320:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.413323:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.413325:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.413327:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.413328:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.413329:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.413330:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.413331:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.413332:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.413333:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.413337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.413343:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.413346:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.413347:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.413347:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.413349:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.413351:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.413906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.413912:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.413915:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.413915:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.413917:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.413919:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.413921:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.413922:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.413924:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.413924:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.413926:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.413929:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.413931:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.413933:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.413934:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.413935:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.413936:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.413937:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.413937:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.413938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.413943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.413950:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.413952:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.413953:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.413955:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.413956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.413957:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.413995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.414002:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.414006:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.414008:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.414009:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.414010:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.414012:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.414013:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.414015:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.414016:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.414017:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.414020:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.414021:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.414024:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.414025:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.414026:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.414027:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.414028:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.414029:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.414030:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.414033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.414041:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.414043:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.414044:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.414045:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.414047:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.414049:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.414168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.414176:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.414178:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.414179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.414181:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.414183:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.414185:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677475.414186:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.414187:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.414188:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.414189:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677475.414193:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.414194:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677475.414197:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.414198:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677475.414198:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677475.414200:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677475.414201:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.414201:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.414202:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.414208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.414217:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.414220:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.414221:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.414221:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.414223:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.414226:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.414229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.414236:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.414239:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.414240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.414241:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.414242:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.414242:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.414269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.414277:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.414280:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.414282:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.414283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.414285:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.414286:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.414287:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.414288:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.414290:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.414290:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.414297:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.414299:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.414301:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.414301:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.414302:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.414303:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.414304:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.414305:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.414305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.414310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.414317:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.414320:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.414320:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.414322:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.414324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.414325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.414416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.414424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.414426:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.414427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.414428:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.414432:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.414433:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.414434:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.414436:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.414437:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.414438:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.414441:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.414442:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.414445:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.414446:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.414446:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.414448:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.414448:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.414450:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.414451:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.414451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.414454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.414457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.414462:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.414464:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.414465:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.414465:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677475.414466:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:6.0:1619677475.414466:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.414467:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.414468:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677475.414468:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.414471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.414472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.414479:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.414482:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.414482:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.414483:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.414484:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.414484:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.414714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.414718:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.414720:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.414722:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.414723:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.414725:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.414726:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.414727:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.414729:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.414730:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.414731:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.414734:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.414735:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.414738:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.414739:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.414739:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.414741:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.414742:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.414742:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.414743:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.414748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.414756:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.414759:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.414760:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.414760:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.414762:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.414764:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.414801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.414809:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.414811:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.414812:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.414814:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.414817:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.414819:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.414820:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.414821:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.414822:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.414823:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.414827:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.414828:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.414831:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.414832:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.414833:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.414834:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.414835:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.414835:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.414836:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.414840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.414847:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.414850:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.414851:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.414851:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.414853:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.414856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.415388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.415393:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.415396:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.415397:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.415398:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.415401:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.415402:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.415403:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.415404:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.415405:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.415406:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.415410:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.415412:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.415414:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.415415:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.415415:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.415417:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.415418:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.415419:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.415420:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.415420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.415424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.415425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.415430:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.415432:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.415432:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.415432:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.415433:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.415435:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.415436:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.415437:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.415438:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.415439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.415443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.415451:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.415453:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.415454:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.415456:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.415459:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.415461:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.415462:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.415464:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.415465:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.415466:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.415469:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.415471:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.415474:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.415475:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.415476:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.415477:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.415478:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.415478:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.415479:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.415483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.415491:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.415494:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.415495:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.415495:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.415498:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.415500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.415725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.415732:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.415735:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.415736:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.415738:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.415741:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.415742:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677475.415743:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.415745:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.415746:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.415747:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677475.415749:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.415749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:26.0:1619677475.415751:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.415752:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.415753:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.415753:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.415755:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.415755:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677475.415757:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:26.0:1619677475.415757:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.415758:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.415758:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:31.0:1619677475.415760:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:9.2:1619677475.415760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:31.0:1619677475.415762:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.415764:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.415764:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:4.2:1619677475.415764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000400:00000200:31.0:1619677475.415765:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.415767:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.415768:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.415769:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677475.415769:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000400:00000200:31.0:1619677475.415770:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.415770:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.415771:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.415772:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:31.0:1619677475.415773:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.415773:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:10.0:1619677475.415773:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.415774:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.415774:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677475.415775:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.415775:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:10.0:1619677475.415775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677475.415778:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.415779:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.415779:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.415781:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.415782:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.415783:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.415785:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.415785:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.415787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.415791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.415795:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.415798:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.415799:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.415799:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.415799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.415801:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677475.415801:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.415802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.415803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.415804:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.415857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.415860:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.415863:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.415865:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.415866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.415867:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.415869:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677475.415870:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.415871:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.415872:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.415873:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677475.415877:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.415879:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677475.415882:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.415883:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677475.415883:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677475.415885:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677475.415886:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.415886:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.415887:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677475.415891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.415898:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.415901:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.415901:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.415903:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.415905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.415906:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.415926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.415933:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.415936:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.415938:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.415940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.415941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.415942:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.415944:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.415945:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.415946:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.415947:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.415950:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.415952:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.415954:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.415955:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.415956:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.415957:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.415958:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.415958:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.415959:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.415963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.415969:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.415971:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.415972:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.415974:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.415975:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.415976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.415984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.415991:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.415994:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.415995:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.415995:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.415996:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.415997:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.416149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.416156:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.416159:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.416160:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.416161:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.416163:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.416164:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.416166:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.416167:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.416168:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.416169:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.416172:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.416173:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.416176:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.416177:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.416178:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.416179:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.416180:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.416181:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.416181:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.416185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.416193:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.416195:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.416196:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.416197:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.416199:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.416202:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.416209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.416216:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.416218:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.416219:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.416219:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.416220:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.416221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.416390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.416396:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.416398:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.416400:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.416401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.416403:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.416404:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.416405:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.416407:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.416408:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.416408:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.416411:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.416413:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.416414:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.416415:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.416416:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.416417:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.416418:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.416419:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.416419:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.416424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.416432:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.416434:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.416435:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.416437:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.416437:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.416439:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.416444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.416452:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.416455:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.416456:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.416456:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.416457:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.416458:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.416505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06fbb6400] (20)++
00000800:00000200:10.0:1619677475.416513:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb6400] (21)++
00000800:00000200:26.0:1619677475.416516:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb6400] (22)--
00000800:00000200:10.0:1619677475.416516:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.58@o2ib3
00000400:00000200:10.0:1619677475.416519:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.4.58@o2ib3 : GET - for me
00000400:00000200:10.0:1619677475.416521:0:19940:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.4.58@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:10.0:1619677475.416524:0:19940:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.4.58@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:10.0:1619677475.416525:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06fbb6400] (21)++
00000800:00000200:10.0:1619677475.416526:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06fbb6400] (22)--
00000400:00000200:10.0:1619677475.416528:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.4.58@o2ib3 local destination
00000400:00000200:10.0:1619677475.416531:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.4.58@o2ib3(10.51.4.58@o2ib3:10.51.4.58@o2ib3) : REPLY try# 0
00000800:00000200:10.0:1619677475.416533:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.4.58@o2ib3
00000800:00000200:10.0:1619677475.416535:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3c4599900] -> 10.51.4.58@o2ib3 (2) version: 12
00000800:00000200:10.0:1619677475.416536:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a06fbb6400] (21)++
00000800:00000200:10.0:1619677475.416536:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06fbb6400] (22)++
00000800:00000200:10.0:1619677475.416538:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a06fbb6400] (23)--
00000800:00000200:10.0:1619677475.416538:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb6400] (22)--
00000800:00000200:4.2:1619677475.416544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06fbb6400] (21)++
00000800:00000200:26.0:1619677475.416551:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06fbb6400] (22)++
00000800:00000200:26.0:1619677475.416553:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06fbb6400] (23)--
00000400:00000200:26.0:1619677475.416554:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.416554:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb6400] (22)--
00000400:00000200:26.0:1619677475.416555:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.4.58@o2ib3: REPLY: OK
00000800:00000200:26.0:1619677475.416557:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06fbb6400] (21)--
00000800:00000200:20.2:1619677475.416637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.416644:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.416647:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.416647:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.416650:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.416652:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.416653:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.416655:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.416656:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.416657:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.416659:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.416662:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.416663:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.416665:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.416666:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.416667:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.416668:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.416669:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.416669:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.416670:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.416671:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.416675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.416678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.416683:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.416685:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.416685:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.416685:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.416686:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.416686:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.416687:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.416687:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.416688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677475.416689:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.416803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.416810:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.416813:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.416814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.416816:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.416818:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.416819:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.416820:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.416822:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.416823:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.416824:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.416827:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.416829:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.416832:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.416833:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.416833:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.416835:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.416835:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.416836:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.416837:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.416841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.416848:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.416851:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.416851:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.416853:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.416855:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.416856:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.417172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.417179:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.417182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.417182:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.417184:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.417186:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.417188:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.417189:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.417190:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.417191:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.417192:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.417196:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.417198:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.417200:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.417201:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.417202:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.417203:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.417204:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.417204:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.417205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.417210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.417216:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.417219:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.417220:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.417221:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.417222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.417223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.417522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677475.417529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.417533:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.417535:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.417537:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.417538:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.417539:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.417539:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.417541:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.417542:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.417543:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.417544:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.417545:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.417546:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.417549:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.417550:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.417550:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.417551:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.417552:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.417553:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.417554:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.417554:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.417558:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.417558:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.417558:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.417559:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.417560:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.417561:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.417561:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.417562:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.417562:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.417563:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.417563:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.417563:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.417564:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677475.417565:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.417565:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.417565:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.417567:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:25.2:1619677475.417568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.417569:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.417569:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.417571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.417575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.417576:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.417579:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677475.417579:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677475.417579:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.417580:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.417581:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.417582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:22.0:1619677475.417582:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.417582:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.417583:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:22.0:1619677475.417585:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:8.0:1619677475.417585:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.417586:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.417587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.417588:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677475.417588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.417842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.417849:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.417852:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.417853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.417855:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.417857:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.417858:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.417860:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.417861:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.417862:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.417863:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.417867:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.417869:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.417872:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.417873:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.417873:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.417875:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.417876:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.417877:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.417877:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.417881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.417889:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.417892:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.417893:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.417895:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.417896:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.417897:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.417901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677475.417907:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.417910:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.417911:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.417912:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.417912:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.417914:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.418022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4fe00] (20)++
00000800:00000200:6.0:1619677475.418027:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4fe00] (21)++
00000800:00000200:22.0:1619677475.418030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4fe00] (22)--
00000800:00000200:6.0:1619677475.418030:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.42@o2ib3
00000400:00000200:6.0:1619677475.418033:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.42@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.418035:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.418037:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:6.0:1619677475.418038:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.418039:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.418041:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.418042:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:6.0:1619677475.418045:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.42@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.418047:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:6.0:1619677475.418050:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.418051:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677475.418052:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677475.418054:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:6.0:1619677475.418054:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4fe00] (21)++
00000800:00000200:6.0:1619677475.418055:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4fe00] (22)++
00000800:00000200:6.0:1619677475.418056:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4fe00] (23)--
00000800:00000200:6.0:1619677475.418056:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4fe00] (22)--
00000800:00000200:9.2:1619677475.418060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.418063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4fe00] (21)++
00000800:00000200:24.0:1619677475.418066:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.418068:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:22.0:1619677475.418068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4fe00] (22)++
00000400:00000200:24.0:1619677475.418070:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.418070:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4fe00] (23)--
00000800:00000200:8.0:1619677475.418070:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677475.418071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4fe00] (22)--
00000800:00000200:22.0:1619677475.418071:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4fe00] (21)++
00000400:00000200:24.0:1619677475.418072:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.418073:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.42@o2ib3
00000800:00000200:6.0:1619677475.418073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4fe00] (22)--
00000800:00000200:24.0:1619677475.418075:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:22.0:1619677475.418077:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.42@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.418079:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.418081:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677475.418082:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.418084:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.418085:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.418085:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677475.418089:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.42@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.418090:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677475.418093:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.418093:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677475.418094:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677475.418096:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677475.418096:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4fe00] (21)++
00000800:00000200:22.0:1619677475.418097:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4fe00] (22)--
00000800:00000200:22.0:1619677475.418097:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4fe00] (21)--
00000800:00000200:8.2:1619677475.418102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.418110:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.418113:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.418114:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.418114:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.418117:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.418119:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.418486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4fe00] (20)++
00000800:00000200:6.0:1619677475.418492:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4fe00] (21)++
00000800:00000200:6.0:1619677475.418495:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.42@o2ib3
00000800:00000200:22.0:1619677475.418496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4fe00] (22)--
00000400:00000200:6.0:1619677475.418498:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.42@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.418501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.418502:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:6.0:1619677475.418503:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.418505:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.418506:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.418507:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:6.0:1619677475.418509:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.42@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.418511:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677475.418512:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.418513:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:6.0:1619677475.418514:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677475.418515:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:6.0:1619677475.418516:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4fe00] (21)++
00000800:00000200:6.0:1619677475.418516:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4fe00] (22)--
00000800:00000200:6.0:1619677475.418517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4fe00] (21)--
00000800:00000200:8.2:1619677475.418522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.418531:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.418533:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.418534:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.418536:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.418537:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.418538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.418604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.418613:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.418615:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.418618:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.418620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.418621:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.418622:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.418623:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.418624:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.418626:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.418627:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.418630:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.418632:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:4.2:1619677475.418633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:30.0:1619677475.418634:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.418635:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.418635:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.418637:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.418637:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.418638:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.418639:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677475.418641:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.418643:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:25.2:1619677475.418643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677475.418644:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.418646:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.418648:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.418650:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:26.0:1619677475.418650:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:26.0:1619677475.418651:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.418652:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.418653:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:26.0:1619677475.418653:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.418654:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.418654:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677475.418655:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.418656:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:26.0:1619677475.418657:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.418658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:26.0:1619677475.418659:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:26.0:1619677475.418662:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.418663:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:26.0:1619677475.418663:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:26.0:1619677475.418665:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677475.418665:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.418666:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.418666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.418671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.418679:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.418682:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.418683:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.418683:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.418685:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.418689:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.418774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4fe00] (20)++
00000800:00000200:22.0:1619677475.418780:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4fe00] (21)++
00000800:00000200:22.0:1619677475.418782:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.42@o2ib3
00000800:00000200:6.0:1619677475.418783:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4fe00] (22)--
00000400:00000200:22.0:1619677475.418785:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.42@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.418788:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.418790:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:22.0:1619677475.418791:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.418792:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.418793:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.418794:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:22.0:1619677475.418797:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.42@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.418798:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677475.418802:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.418802:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:22.0:1619677475.418803:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:22.0:1619677475.418805:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:22.0:1619677475.418806:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4fe00] (21)++
00000800:00000200:22.0:1619677475.418806:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4fe00] (22)--
00000800:00000200:22.0:1619677475.418807:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4fe00] (21)--
00000800:00000200:25.2:1619677475.418811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677475.418819:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677475.418822:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677475.418822:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.418824:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.418826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677475.418827:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677475.419149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.419154:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.419157:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.419158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.419160:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.419163:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.419164:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.419165:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.419166:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.419167:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.419168:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.419171:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.419172:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.419174:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.419175:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.419175:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.419177:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.419177:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.419178:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.419178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.419183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.419190:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.419193:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.419193:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.419195:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.419197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.419197:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.419205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.419213:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.419215:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.419216:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.419217:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.419218:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.2:1619677475.419219:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.0:1619677475.419222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.419226:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.419229:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.419231:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.419233:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.419234:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.419235:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.419236:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.419238:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.419239:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.419240:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.419243:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.419245:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.419248:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.419248:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.419249:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.419251:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.419251:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.419252:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.419253:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.419257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.419264:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.419267:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.419267:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.419268:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.419270:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.419273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.419276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.419284:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.419286:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.419287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.419287:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.419288:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.419289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.419599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.419605:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.419608:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.419609:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.419611:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677475.419614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:1.0:1619677475.419614:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.419615:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.419616:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.419618:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.419619:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.419620:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677475.419622:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677475.419622:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.419624:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:10.0:1619677475.419625:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.419626:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677475.419627:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.419627:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:10.0:1619677475.419628:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.419628:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.419630:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.419630:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:10.0:1619677475.419631:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.419631:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:10.0:1619677475.419632:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:1.0:1619677475.419632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:10.0:1619677475.419633:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96500005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.419634:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.419636:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677475.419636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:10.0:1619677475.419637:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.419639:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.419641:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.419644:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:10.0:1619677475.419644:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.419645:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.419645:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.419647:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:10.0:1619677475.419647:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:20.0:1619677475.419648:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.419648:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.419648:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000400:00000200:20.0:1619677475.419649:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.419649:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.419650:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677475.419651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.419653:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:25.2:1619677475.419654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.419656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.419662:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.419663:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.419665:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677475.419665:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.419665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.419666:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677475.419666:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.419667:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.419668:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.419671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.419724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.419732:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.419734:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.419735:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.419737:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.419740:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.419741:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.419743:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96500004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.419744:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.419745:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.419746:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.419749:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.419751:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.419753:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.419754:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.419755:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.419756:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.419757:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.419758:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.419758:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.419762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.419770:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.419772:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.419773:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.419773:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.419776:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.419779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.420166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.420175:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.420177:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.420180:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.420182:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.420183:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.420184:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.420185:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.420186:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.420188:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.420188:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.420192:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.420193:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.420196:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.420197:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.420197:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.420199:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.420199:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.420200:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.420201:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.420204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.420210:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.420213:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.420214:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.420214:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.420216:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.420218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.420726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.420732:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.420734:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.420735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.420737:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.420740:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.420741:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.420742:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96500002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.420743:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.420744:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.420745:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.420749:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.420750:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.420753:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.420754:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.420754:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.420756:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.420756:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.420757:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.420757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.420763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.420771:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.420773:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.420774:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.420776:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.420776:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.420778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.421015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.421024:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.421026:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677475.421027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:30.0:1619677475.421029:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.421031:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.421031:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.421032:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.421034:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96500001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.421035:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.421035:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677475.421036:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.421037:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677475.421037:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.421038:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.421040:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.421041:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.421043:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677475.421043:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.421044:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677475.421045:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.421046:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.421046:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:8.0:1619677475.421046:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96500000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.421047:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.421048:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:8.0:1619677475.421048:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.421048:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677475.421049:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.421050:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.421050:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:8.0:1619677475.421052:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.421053:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:4.2:1619677475.421053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677475.421054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677475.421056:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.421057:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.421058:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.421059:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677475.421060:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:8.0:1619677475.421060:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.421060:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677475.421061:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:8.0:1619677475.421061:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677475.421063:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:10.0:1619677475.421063:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:31.0:1619677475.421064:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.421064:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677475.421065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:31.0:1619677475.421066:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.421067:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:10.0:1619677475.421067:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.421069:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:10.0:1619677475.421070:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.421071:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.421073:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.421074:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:10.0:1619677475.421074:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.421075:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.421076:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.421077:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.421077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.421078:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.421080:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.421080:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.421082:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.421083:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:10.0:1619677475.421085:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.421086:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.421086:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.421088:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.421088:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.421090:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.421091:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.421091:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.421094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.421097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.421101:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677475.421104:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.421104:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.421105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.421105:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.421106:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.421106:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.421107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.421107:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.421110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.421122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.421125:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.421128:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.421130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.421130:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.421132:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.421134:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.421135:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.421136:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.421137:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.421138:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.421142:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.421143:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.421146:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.421146:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.421147:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.421149:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.421149:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.421150:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.421151:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.421155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.421163:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.421165:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.421166:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.421167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.421169:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.421171:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.421175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.421183:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.421186:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.421186:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.421187:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.421187:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.421188:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.421500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.421508:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.421511:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.421513:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.421515:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.421516:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.421517:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.421518:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.421520:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.421521:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.421522:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.421525:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.421527:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.421530:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.421531:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.421531:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.421533:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.421534:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.421535:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.421535:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.421540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.421548:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.421551:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.421551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.421552:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.421554:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.421557:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.422330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.422335:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.422338:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.422340:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.422341:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.422343:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.422344:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.422346:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.422347:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.422348:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.422349:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.422353:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.422355:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.422357:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.422358:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.422359:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.422360:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.422361:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.422362:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.422363:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.422364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.422367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.422369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.422372:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.422374:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.422375:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.422376:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677475.422376:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.422378:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.422379:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.422380:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.422381:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.422384:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.422678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.422684:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.422686:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.422687:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.422689:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.422691:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.422692:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.422693:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.422695:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.422696:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.422697:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.422700:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.422702:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.422704:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.422705:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.422706:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.422707:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.422708:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.422709:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.422710:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.422710:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.422714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.422715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.422720:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.422721:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.422722:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.422722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.422723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.422723:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.422724:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.422726:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.422727:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.422728:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.422752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677475.422759:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.422761:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677475.422762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677475.422764:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.422764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:24.0:1619677475.422767:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.422769:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677475.422770:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.422771:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.422772:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677475.422772:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.422773:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:31.0:1619677475.422774:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.422777:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.422778:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.422779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.422780:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.422780:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:31.0:1619677475.422781:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677475.422782:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.422783:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.422783:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:31.0:1619677475.422784:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677475.422784:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:31.0:1619677475.422785:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.422785:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:31.0:1619677475.422786:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677475.422786:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.422787:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:24.0:1619677475.422787:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:31.0:1619677475.422789:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.422791:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:9.2:1619677475.422791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:31.0:1619677475.422793:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.422794:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.422794:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.422796:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.422796:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.422798:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677475.422798:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:31.0:1619677475.422799:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.422800:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677475.422800:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.422801:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:25.2:1619677475.422801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:10.0:1619677475.422801:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.422803:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:21.2:1619677475.422805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.422806:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:30.0:1619677475.422810:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.422811:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.422812:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.422812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:30.0:1619677475.422813:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.422814:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.422815:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.422815:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.422816:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.422817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.422818:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677475.422818:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.422821:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.422822:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.422822:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.422823:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.422823:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.423213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.423220:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.423223:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.423225:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.423227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.423227:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.423228:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.423230:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.423231:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.423232:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.423233:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.423236:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.423238:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.423241:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.423242:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.423242:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.423244:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.423244:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.423246:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.423247:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.423248:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.423250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.423253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.423257:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.423260:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.423261:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.423261:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.423261:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.423263:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.423264:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.423265:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.423266:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.423267:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.423269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.423277:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.423279:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.423280:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.423280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.423281:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.423282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.423726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.423734:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.423737:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.423739:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.423742:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.423742:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.423743:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.423745:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.423746:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.423747:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.423748:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.423751:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.423752:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.423754:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.423755:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.423756:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.423757:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.423758:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.423759:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.423759:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.423764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:5.2:1619677475.423770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677475.423771:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.423774:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.423774:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.423775:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.423776:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:8.0:1619677475.423776:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.423779:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:8.0:1619677475.423779:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677475.423780:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.423782:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677475.423783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:17.0:1619677475.423784:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.423785:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:17.0:1619677475.423786:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.423788:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677475.423789:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677475.423789:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.423790:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677475.423792:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.423793:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.423793:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.423793:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.423794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:17.0:1619677475.423794:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.423796:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.423798:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.423799:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.423800:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.423801:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:17.0:1619677475.423802:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.423802:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.423803:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.423807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.423815:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.423817:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.423818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.423818:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.423821:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.423823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.424434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677475.424441:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.424444:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677475.424445:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677475.424447:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.424449:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.424450:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677475.424452:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.424453:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.424454:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.424455:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677475.424458:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.424460:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677475.424463:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.424463:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677475.424464:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677475.424465:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677475.424466:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.424467:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677475.424468:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:8.0:1619677475.424469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:21.2:1619677475.424471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:25.2:1619677475.424472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.424475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:1.0:1619677475.424477:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:30.0:1619677475.424479:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:1.0:1619677475.424480:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.424481:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:30.0:1619677475.424482:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:24.0:1619677475.424482:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000400:00000200:30.0:1619677475.424483:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.424483:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.424484:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000400:00000200:30.0:1619677475.424485:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.424485:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:1.0:1619677475.424485:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.424486:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:31.0:1619677475.424487:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.424487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:1.0:1619677475.424487:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677475.424488:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.424489:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.424491:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.424491:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677475.424494:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.424496:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.424498:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.424499:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.424500:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.424502:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.424502:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.424503:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.424503:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677475.424508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.424516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.424518:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.424519:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.424519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.424522:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.424524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.424586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.424594:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.424599:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.424601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.424604:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.424607:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.424609:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.424611:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.424613:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.424615:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.424617:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.424622:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.424625:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.424628:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.424629:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.424630:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.424632:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.424633:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.424635:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.424636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.424639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.424648:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.424651:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.424652:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.424653:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.424656:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.424658:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.424727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.424733:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.424736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.424736:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.424738:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.424741:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.424742:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.424743:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.424745:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.424746:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.424747:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.424750:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.424752:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.424755:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.424756:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.424757:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.424758:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.424759:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.424760:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.424760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.424765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.424772:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.424775:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.424776:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.424777:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.424778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.424780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.424914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:20.2:1619677475.424922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677475.424922:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.424926:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.424926:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677475.424928:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.424930:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:10.0:1619677475.424931:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.424932:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.424933:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.424935:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:10.0:1619677475.424935:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.424936:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677475.424936:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:10.0:1619677475.424938:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677475.424940:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.424941:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.424943:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.424943:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:18.0:1619677475.424945:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:10.0:1619677475.424946:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677475.424947:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.424947:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.424948:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.424949:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:18.0:1619677475.424950:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.424950:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.424950:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000400:00000200:18.0:1619677475.424951:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677475.424951:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:18.0:1619677475.424953:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:25.2:1619677475.424955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:18.0:1619677475.424958:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.424961:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677475.424962:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.424965:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:18.0:1619677475.424965:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.424966:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.424966:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677475.424967:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.424968:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:22.0:1619677475.424969:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.424970:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677475.424971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677475.424971:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.424973:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.424974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.424976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.424983:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.424986:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.424987:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.424989:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.424990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.424993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.425096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.425104:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.425106:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.425109:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.425111:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.425111:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.425112:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.425113:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.425114:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.425116:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.425117:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.425119:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.425121:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.425124:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.425124:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.425125:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.425127:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.425127:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.425128:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.425129:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.425132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.425139:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.425141:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.425142:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.425142:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.425144:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.425146:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.425243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.425250:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.425253:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.425255:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.425257:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.425257:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.425259:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.425260:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.425261:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.425262:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.425263:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.425266:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.425267:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.425270:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.425271:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.425272:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.425273:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.425274:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.425275:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.425275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.425279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.425287:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.425289:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.425290:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.425291:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.425297:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.425300:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.425377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.425385:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.425387:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.425388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.425390:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.425392:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.425394:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.425395:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.425396:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.425397:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.425398:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.425401:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.425403:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.425406:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.425406:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.425407:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.425409:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.425409:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.425410:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.425411:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.425412:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.425415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.425417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.425422:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.425424:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.425425:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.425426:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.425426:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677475.425426:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.425427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.425428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.425429:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.425431:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.425433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.425441:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.425443:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.425444:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.425444:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.425445:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.425446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.425820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.425827:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.425830:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.425831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.425832:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.425835:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.425837:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.425838:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.425839:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.425840:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.425841:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.425844:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.425846:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.425849:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.425850:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.425850:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.425852:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.425852:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.425853:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.425853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.425858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.425865:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.425868:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.425869:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.425869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.425871:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.425874:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.425993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.426001:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.426004:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.426006:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.426008:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.426009:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.426010:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.426011:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.426012:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.426013:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.426014:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.426017:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.426019:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.426021:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.426022:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.426022:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.426024:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.426024:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.426025:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.426026:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.426030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.426036:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.426039:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.426040:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.426040:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.426041:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.426044:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.426305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.426310:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.426313:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.426313:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.426315:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.426317:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.426319:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.426320:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.426321:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.426322:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.426323:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.426327:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.426328:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.426331:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.426332:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.426333:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.426334:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.426335:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.426335:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.426336:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.426341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.426349:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.426351:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.426352:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.426354:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.426355:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.426356:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.426356:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.426357:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.426358:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.426358:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.426359:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.426711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.426714:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.426716:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.426719:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.426719:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.426721:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.426722:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.426724:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.426725:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.426726:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.426727:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.426731:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.426732:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.426735:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.426736:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.426736:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.426738:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.426738:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.426739:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.426740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.426745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.426754:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.426756:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.426757:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.426757:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.426759:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.426762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.426986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.426994:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.426996:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.426999:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.427001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.427002:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.427003:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.427004:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.427006:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.427007:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.427008:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.427011:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.427013:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.427016:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.427017:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.427017:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.427019:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.427020:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.427021:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.427021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.427025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.427033:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.427036:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.427037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.427038:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.427040:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.427042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.427076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.427082:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.427084:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.427086:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.427087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.427089:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.427090:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.427092:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.427093:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.427094:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.427095:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.427098:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.427100:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.427103:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.427103:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.427104:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.427106:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.427106:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.427107:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.427109:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.427109:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.2:1619677475.427112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:8.2:1619677475.427112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.427116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:1.0:1619677475.427118:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677475.427121:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.427121:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.427122:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677475.427123:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.427123:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.427124:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:20.0:1619677475.427124:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.427124:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.427125:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:1.0:1619677475.427125:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.427126:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.427126:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.427127:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:1.0:1619677475.427127:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677475.427128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:1.0:1619677475.427129:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.427130:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.427131:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.427132:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.427135:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.427137:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.427140:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.427140:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.427141:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.427143:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.427143:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.427144:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.427144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.427148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.427157:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.427159:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.427160:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.427162:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.427163:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.427164:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.427165:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.427166:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.427167:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:20.0:1619677475.427167:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.427168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.427169:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.427170:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.427681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.427688:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.427691:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.427694:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.427696:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.427697:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.427698:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.427699:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.427701:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.427702:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.427703:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.427707:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.427709:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.427711:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.427712:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.427713:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.427714:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.427715:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.427716:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.427718:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.427718:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.427720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.427725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.427729:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.427731:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.427731:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.427733:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.427733:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.427734:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.427734:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.427735:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.427736:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.427737:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.427740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.427746:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.427749:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.427749:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.427750:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.427751:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.427751:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.427930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.427933:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.427935:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.427937:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.427938:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.427939:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.427941:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.427942:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.427944:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.427945:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.427946:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.427949:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.427951:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.427954:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.427954:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.427955:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.427956:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.427957:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.427958:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.427958:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.427963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.427972:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.427974:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.427975:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.427975:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.427977:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.427979:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.428141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.428148:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.428151:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.428152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.428154:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.428156:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.428158:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.428159:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.428161:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.428162:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.428163:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.428165:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.428167:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.428170:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.428171:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.428171:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.428173:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.428173:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.428174:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.428174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.428179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.428187:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.428189:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.428190:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.428191:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.428193:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.428196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.428219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.428225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.428228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.428228:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.428230:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.428232:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.428234:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.428235:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.428236:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.428237:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.428238:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.428242:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.428243:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.428245:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.428246:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.428247:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.428249:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.428249:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.428249:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.428250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.428255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.428262:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.428264:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.428265:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.428267:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.428267:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.428269:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.428363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677475.428371:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.428374:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677475.428376:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.428378:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677475.428379:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.428380:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.428381:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.428383:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.428384:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.428385:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.428388:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.428389:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.428392:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.428393:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.428393:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.428395:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.428395:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.428396:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677475.428397:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677475.428401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.428408:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.428410:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.428411:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.428411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.428413:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.428416:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677475.428696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.428702:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.428705:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.428706:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.428708:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.428709:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.428711:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677475.428712:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.428713:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.428715:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.428716:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677475.428719:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.428720:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677475.428723:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.428723:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677475.428724:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677475.428726:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677475.428727:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.428727:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.428728:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.428732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.428739:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.428742:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.428742:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.428744:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.428746:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.428747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.428935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.428943:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.428946:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.428948:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.428949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.428950:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.428952:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.428953:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.428954:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.428955:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.428956:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.428960:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.428961:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.428963:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.428964:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.428965:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.428966:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.428967:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.428968:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.428969:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.428972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677475.428976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:30.0:1619677475.428978:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.428981:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.428981:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.428983:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.428983:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:31.0:1619677475.428985:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:8.0:1619677475.428985:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:30.0:1619677475.428986:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:24.0:1619677475.428986:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677475.428988:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.428990:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.428992:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.428993:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.428994:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.428995:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.428996:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.428999:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.429000:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.429003:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.429004:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.429004:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.429006:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.429007:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.429008:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677475.429009:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677475.429010:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:25.2:1619677475.429012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.429014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:4.2:1619677475.429015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:22.0:1619677475.429019:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677475.429021:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.429022:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:22.0:1619677475.429022:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.429023:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.429023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.429024:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:24.0:1619677475.429024:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677475.429024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677475.429025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:22.0:1619677475.429026:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.429027:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.429028:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.429030:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.429030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677475.429032:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.429033:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.429034:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.429035:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.429036:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.429039:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.429040:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.429043:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.429044:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.429044:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.429046:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.429047:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.429048:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.429048:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.429052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.429059:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.429062:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.429063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.429063:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.429066:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.429068:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.429764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.429772:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.429775:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.429775:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677475.429778:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.429780:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.429782:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.429783:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.429784:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.429785:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.429786:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.429790:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.429791:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.429794:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.429795:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.429795:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.429797:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.429798:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.429798:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.429799:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.429803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.429812:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.429814:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.429815:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.429815:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.429818:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.429820:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.429827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.429834:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.429836:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.429837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.429838:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.429839:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.429839:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.430078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.430084:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.430087:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.430088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.430090:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.430093:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.430094:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.430095:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.430097:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.430098:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.430099:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.430102:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.430103:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.2:1619677475.430104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677475.430106:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.430107:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.430107:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.430109:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.430109:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.430110:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.430110:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677475.430112:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.430115:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677475.430116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:30.0:1619677475.430117:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.430120:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.430120:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.430122:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.430123:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.430124:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.430124:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:30.0:1619677475.430125:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.430126:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677475.430127:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.430128:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.430128:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.430130:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:30.0:1619677475.430131:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.430133:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:1.0:1619677475.430133:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677475.430135:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677475.430135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:4.2:1619677475.430135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (20)++
00000800:00000200:30.0:1619677475.430136:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.430137:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.430138:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.430139:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.430140:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.430141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677475.430142:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.430142:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (21)++
00000800:00000200:9.2:1619677475.430144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677475.430146:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.430146:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.3.3@o2ib3
00000800:00000200:1.0:1619677475.430146:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.430148:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:17.0:1619677475.430148:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.430149:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.430149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:8.0:1619677475.430149:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : REPLY - routed
00000800:00000200:24.0:1619677475.430150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.430151:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.430152:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.430152:0:19938:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000400:00000200:24.0:1619677475.430154:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.430157:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.430157:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677475.430159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677475.430159:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (22)++
00000800:00000200:8.0:1619677475.430160:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (23)--
00000800:00000200:8.0:1619677475.430160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:9.2:1619677475.430162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:4.2:1619677475.430166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677475.430171:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.430173:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.430173:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.430174:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.430175:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.430175:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:24.0:1619677475.430176:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677475.430176:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:8.0:1619677475.430177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:4.2:1619677475.430181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000800:00000200:24.0:1619677475.430187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:24.0:1619677475.430190:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.3.3@o2ib3
00000800:00000200:8.0:1619677475.430191:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (23)++
00000800:00000200:8.0:1619677475.430192:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[0] from 10.51.3.3@o2ib3
00000400:00000200:24.0:1619677475.430193:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : REPLY - routed
00000400:00000200:8.0:1619677475.430194:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.3@o2ib3 : REPLY - routed
00000800:00000200:24.0:1619677475.430195:0:19939:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:8.0:1619677475.430196:0:19938:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:24.0:1619677475.430200:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39c600] (24)++
00000800:00000200:24.0:1619677475.430201:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (25)++
00000800:00000200:24.0:1619677475.430202:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (26)--
00000800:00000200:24.0:1619677475.430202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (25)--
00000800:00000200:8.0:1619677475.430203:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39c600] (24)++
00000800:00000200:24.0:1619677475.430204:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (24)--
00000800:00000200:8.0:1619677475.430204:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (24)++
00000800:00000200:8.0:1619677475.430205:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (25)--
00000800:00000200:8.0:1619677475.430206:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (24)--
00000800:00000200:4.2:1619677475.430208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (23)++
00000800:00000200:24.0:1619677475.430215:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (24)++
00000800:00000200:24.0:1619677475.430217:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (25)--
00000800:00000200:24.0:1619677475.430218:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (24)++
00000800:00000200:24.0:1619677475.430218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (25)--
00000800:00000200:8.0:1619677475.430219:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (24)--
00000800:00000200:21.2:1619677475.430249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.430254:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.430257:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.430258:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.430260:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.430262:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.430264:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.430265:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.430267:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.430268:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.430269:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.430272:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.430274:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.430277:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.430277:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.430278:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.430280:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.430281:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.430282:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.430283:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.430283:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.430286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.430289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.430294:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.430297:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.430299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.430299:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.430300:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.430300:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.430301:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.430302:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.430303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.430304:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.430627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (23)++
00000800:00000200:24.0:1619677475.430636:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (24)++
00000800:00000200:24.0:1619677475.430639:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.3.3@o2ib3
00000800:00000200:8.0:1619677475.430640:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (25)--
00000800:00000200:24.0:1619677475.430641:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39c600] (24)--
00000400:00000200:24.0:1619677475.430642:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.430643:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677475.430645:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.430646:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.430647:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.430648:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677475.430652:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:24.0:1619677475.430653:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677475.430655:0:19939:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:24.0:1619677475.430664:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.430665:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:24.0:1619677475.430665:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677475.430667:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677475.430667:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (23)++
00000800:00000200:24.0:1619677475.430668:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (24)--
00000800:00000200:24.0:1619677475.430668:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:25.2:1619677475.430673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.430680:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.430683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.430684:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:25.2:1619677475.430703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.430710:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.430712:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[3] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.430716:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.430717:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.430730:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.430731:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:4.2:1619677475.430791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.430798:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.430801:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.430802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.430803:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.430806:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.430808:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.430809:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.430810:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.430811:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.430812:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.430814:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.430816:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.430819:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.430819:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.430820:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.430821:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.430822:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.430823:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.430823:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.430890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.430897:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.430900:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.430901:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.430901:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.430903:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.430906:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.430963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (22)++
00000800:00000200:21.2:1619677475.430969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677475.430970:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (23)++
00000800:00000200:24.0:1619677475.430973:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.3.3@o2ib3
00000800:00000200:8.0:1619677475.430974:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (24)--
00000800:00000200:24.0:1619677475.430975:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39c600] (24)--
00000400:00000200:24.0:1619677475.430976:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.430977:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:24.0:1619677475.430977:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:24.0:1619677475.430979:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.430980:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:24.0:1619677475.430980:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.430982:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.430982:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.430983:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.430984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:24.0:1619677475.430985:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000400:00000200:30.0:1619677475.430987:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.430987:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:24.0:1619677475.430988:0:19939:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000400:00000200:30.0:1619677475.430989:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.430990:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96499957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.430991:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.430992:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.430993:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.430996:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.430997:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:24.0:1619677475.430997:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.430997:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677475.430998:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:24.0:1619677475.430999:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.431000:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.431000:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (22)++
00000800:00000200:24.0:1619677475.431000:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (23)--
00000800:00000200:30.0:1619677475.431001:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.431001:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:24.0:1619677475.431001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (22)--
00000800:00000200:30.0:1619677475.431002:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.431003:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.431004:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.431004:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.431066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39c600] (21)++
00000800:00000200:25.2:1619677475.431071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677475.431073:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39c600] (22)++
00000800:00000200:8.0:1619677475.431076:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.3.3@o2ib3
00000800:00000200:24.0:1619677475.431077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (23)--
00000800:00000200:31.0:1619677475.431078:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:8.0:1619677475.431078:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39c600] (22)--
00000400:00000200:8.0:1619677475.431079:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.431080:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:8.0:1619677475.431080:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677475.431081:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.431081:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:10, s:-96499956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.431082:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.431082:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.431083:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.431084:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:8.0:1619677475.431084:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.431085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:8.0:1619677475.431087:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.3@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:8.0:1619677475.431089:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 1048576 bytes in 256 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677475.431090:0:19938:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 1048576
00000800:00000200:8.0:1619677475.431099:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.431100:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:8.0:1619677475.431101:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:8.0:1619677475.431102:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (24)--
00000800:00000200:8.0:1619677475.431102:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39c600] (21)++
00000800:00000200:8.0:1619677475.431103:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39c600] (22)--
00000800:00000200:8.0:1619677475.431103:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39c600] (21)--
00000800:00000200:21.2:1619677475.431204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.431212:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.431215:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.431216:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.431218:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.431218:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.431219:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.431220:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:509, d:12, s:-96499955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.431222:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.431222:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.431223:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.431226:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.431228:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.431231:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.431232:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.431232:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.431234:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.431234:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.431236:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.431237:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.431238:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.431240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.431244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.431247:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.431250:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.431251:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.431251:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.431251:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.431254:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677475.431254:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.431255:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.431255:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.431257:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.431322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:30.0:1619677475.431330:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (24)++
00000800:00000200:30.0:1619677475.431334:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (25)--
00000400:00000200:30.0:1619677475.431335:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.431337:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: REPLY: OK
00000800:00000200:31.0:1619677475.431338:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (24)++
00000800:00000200:31.0:1619677475.431339:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (25)--
00000800:00000200:30.0:1619677475.431339:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (24)--
00000800:00000200:31.0:1619677475.431340:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:31.0:1619677475.431342:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (24)--
00000800:00000200:31.0:1619677475.431343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:25.2:1619677475.431361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.431368:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:31.0:1619677475.431370:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[1] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.431374:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (24)++
00000800:00000200:31.0:1619677475.431375:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (25)++
00000800:00000200:30.0:1619677475.431376:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.431388:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (26)--
00000800:00000200:31.0:1619677475.431388:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (25)--
00000800:00000200:31.0:1619677475.431390:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (24)--
00000800:00000200:30.0:1619677475.431392:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:30.0:1619677475.431411:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (24)--
00000800:00000200:30.0:1619677475.431412:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:4.2:1619677475.431501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677475.431509:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.431511:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677475.431513:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.431515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677475.431516:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.431518:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.431518:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96499954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.431520:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.431521:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.431522:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.431524:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.431526:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.431528:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.431529:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.431529:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.431531:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.431531:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.431532:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677475.431533:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.431607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.431614:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.431616:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.431617:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.431618:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.431619:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.431622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.431736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.431742:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.431745:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.431746:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.431747:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.431750:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.431751:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677475.431752:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96499953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.431754:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.431755:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.431756:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677475.431759:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.431761:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677475.431763:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.431764:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677475.431765:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677475.431766:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677475.431767:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.431767:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.431768:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677475.431773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.431781:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.431784:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677475.431784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.431785:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.431787:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.431790:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:25.2:1619677475.431807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:21.2:1619677475.431812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.431815:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)++
00000800:00000200:30.0:1619677475.431818:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (24)--
00000800:00000200:22.0:1619677475.431818:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677475.431819:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.431820:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: REPLY: OK
00000800:00000200:22.0:1619677475.431821:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:31.0:1619677475.431822:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:6.0:1619677475.431822:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677475.431823:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:22.0:1619677475.431824:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.431826:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.431827:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.431829:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.431830:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.431831:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.431832:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.431836:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.431838:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.431840:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.431841:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.431841:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.431843:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.431844:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.431844:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.431845:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.431856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.431863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.431866:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.431867:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.431868:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.431869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.431871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.431886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.431893:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677475.431895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.431895:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.431896:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.431897:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.431898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.431898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.431903:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.431906:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.431907:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.431909:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: REPLY: OK
00000800:00000200:30.0:1619677475.431909:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.431912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.432128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.432135:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.432138:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.432139:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.432141:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.432144:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.432145:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677475.432147:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.432148:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.432149:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.432150:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677475.432154:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.432155:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.432158:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.432159:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677475.432160:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677475.432162:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677475.432162:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.432163:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.432163:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.432168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.432176:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.432179:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.432180:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.432180:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.432183:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.432185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.432187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.432194:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.432196:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.432197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.432198:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.432199:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.432199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.432228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677475.432233:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.432236:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677475.432237:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.432238:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.432241:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.432242:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.432243:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.432244:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.432245:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.432246:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.432248:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.432250:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.432252:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.432252:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.432253:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.432255:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.432255:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.432256:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677475.432256:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677475.432260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.432267:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.432269:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.432269:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.432270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.432271:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.432272:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.432689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.432697:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.432699:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.432700:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.432702:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.432704:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.432706:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.432707:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.432708:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.432709:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.432710:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.432714:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.432716:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.432719:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.432720:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.432720:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.432722:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.432722:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.432723:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.432724:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.432729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.432737:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.432740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.432740:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.432741:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.432743:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.432745:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.432751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.432758:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.432760:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.432761:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.432762:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.432762:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.432763:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.432772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677475.432780:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.432782:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677475.432783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677475.432785:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.432788:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.432789:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677475.432790:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.432792:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.432793:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.432794:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677475.432797:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.432798:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677475.432801:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.432802:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677475.432802:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677475.432804:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677475.432804:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.432805:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677475.432805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677475.432810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.432818:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.432821:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.432822:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.432822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.432824:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.432827:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677475.432981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.432987:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.432989:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.432992:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.432992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.432994:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.432996:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.432997:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.432998:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.432999:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.433000:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.433004:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.433005:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.433008:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.433008:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.433009:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.433011:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.433011:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.433012:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.433013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.433017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.433025:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.433027:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.433028:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.433028:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.433030:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.433032:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.433091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.433098:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.433101:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.433101:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.433104:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.433107:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.433108:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.433109:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.433111:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.433111:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.433112:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.433115:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.433117:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.433120:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.433121:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.433122:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.433123:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.433124:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.433124:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.433125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.433129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.433138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.433141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.433142:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.433145:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.433148:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.433149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.433397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677475.433403:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.433406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.433406:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000400:00000200:6.0:1619677475.433409:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.433411:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.433413:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677475.433414:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.433415:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.433417:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.433417:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:6.0:1619677475.433421:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.433423:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677475.433426:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.433427:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677475.433427:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677475.433430:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677475.433430:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.433431:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.433431:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677475.433436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.433445:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.433448:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.433448:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.433450:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.433452:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.433454:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677475.433550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.433556:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.433559:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.433560:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.433563:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.433565:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.433566:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.433567:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.433568:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.433570:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.433571:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.433574:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.433575:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.433577:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.433578:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.433579:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.433580:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.433581:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.433582:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.433582:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.433587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.433594:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.433597:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.433597:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.433598:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.433599:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.433601:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.433609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.433616:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.433618:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.433619:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.433620:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.433620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677475.433620:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.2:1619677475.433687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.433695:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.433698:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.433701:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.433702:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.433703:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.433705:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.433706:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.433707:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.433708:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.433709:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.433712:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.433714:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.433717:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.433718:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.433718:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.433720:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.433721:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.433722:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.433722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.433726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.433734:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.433737:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.433737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.433738:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.433740:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.433743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.433798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.433803:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.433806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.433806:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.433808:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.433811:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.433812:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.433813:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.433814:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.433815:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.433816:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.433820:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.433822:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.433824:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.433825:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.433826:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.433827:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.433828:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.433828:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.433829:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.433833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.433841:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.433843:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.433844:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.433846:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.433847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.433848:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.434002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907cbe3400] (20)++
00000800:00000200:2.0:1619677475.434008:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677475.434011:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.18@o2ib3
00000800:00000200:18.0:1619677475.434012:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (22)--
00000400:00000200:2.0:1619677475.434013:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.18@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.434016:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.434018:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677475.434019:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.434020:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.434021:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.434022:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677475.434025:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.18@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.434027:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677475.434030:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.434031:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677475.434032:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677475.434034:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677475.434034:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907cbe3400] (21)++
00000800:00000200:2.0:1619677475.434035:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907cbe3400] (22)--
00000800:00000200:2.0:1619677475.434036:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907cbe3400] (21)--
00000800:00000200:25.2:1619677475.434050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.434056:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.434059:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.434060:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.434062:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.434063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.434065:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.434211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.434216:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.434219:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.434221:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.434221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.434223:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.434225:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.434226:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.434227:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.434228:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.434229:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.434232:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.434234:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.434236:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.434237:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.434238:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.434239:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.434240:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.434241:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.434242:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.434243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.434246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.434247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.434250:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.434252:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.434253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.434254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.434254:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.434257:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.434258:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.434258:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.434260:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.434262:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.434265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.434272:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.434275:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.434276:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.434276:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.434277:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.434277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.434296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.434304:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.434306:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.434308:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.434309:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.434312:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.434314:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.434315:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.434316:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.434317:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.434318:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.434321:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.434322:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.434325:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.434326:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.434327:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.434328:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.434329:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.434330:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.434330:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.434334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.434343:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.434345:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.434346:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.434347:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.434349:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.434352:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.434418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.434426:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.434429:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.434431:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.434433:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.434434:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.434435:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.434437:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.434438:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.434439:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.434439:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.434443:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.434444:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.434446:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.434447:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.434448:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.434449:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.434450:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.434451:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.434453:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.434454:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.2:1619677475.434459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677475.434466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.434469:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.434471:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.434472:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.434473:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.434475:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.434475:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.434475:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.434475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:31.0:1619677475.434477:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677475.434477:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.434479:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.0:1619677475.434483:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:25.2:1619677475.434485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677475.434485:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677475.434486:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677475.434488:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677475.434490:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.434492:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:8.0:1619677475.434492:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677475.434493:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.434494:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.434495:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.434495:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000400:00000200:8.0:1619677475.434495:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.434496:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000400:00000200:8.0:1619677475.434496:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.434497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.434498:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:8.0:1619677475.434498:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677475.434500:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.434501:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.434502:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.434502:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.434504:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.434504:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.434505:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677475.434506:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677475.434509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.434515:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.434517:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.434518:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.434519:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.434520:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.434522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.434689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677475.434697:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.434699:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677475.434701:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.434703:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.434703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677475.434705:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677475.434706:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.434707:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.434708:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.434709:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677475.434711:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.434713:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.434715:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.434715:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.434716:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.434718:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.434718:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.434719:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677475.434719:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:25.2:1619677475.434723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.434729:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.434732:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.434733:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.434734:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.434736:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.434737:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.434742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.434748:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.434750:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.434751:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.434752:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.434752:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.434753:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.434840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.434847:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.434850:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.434851:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.434853:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.434855:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.434856:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.434857:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.434858:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.434859:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.434860:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.434863:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.434865:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.434868:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.434869:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.434869:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.434871:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.434871:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.434872:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.434872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.434877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.434884:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.434887:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.434888:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.434888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.434891:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.434894:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.434896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.434903:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.434906:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.434907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.434907:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.434908:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.434909:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.435444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.435452:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.435455:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.435457:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.435459:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.435460:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.435460:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.435461:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.435463:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.435464:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.435465:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.435468:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.435470:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.435471:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.435472:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.435473:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.435474:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.435475:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.435476:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.435476:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.435480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.435487:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.435490:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.435491:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.435492:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.435494:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.435494:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.435796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.435802:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.435805:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.435806:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.435807:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.435810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.435811:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.435812:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.435814:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.435815:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.435816:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:5.2:1619677475.435817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:22.0:1619677475.435819:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.435820:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.435822:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.435823:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:1.0:1619677475.435823:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677475.435824:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677475.435825:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:22.0:1619677475.435826:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.435826:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677475.435826:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:22.0:1619677475.435827:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.435828:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:1.0:1619677475.435828:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.435830:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.435832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:1.0:1619677475.435832:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:1.0:1619677475.435833:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.435834:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.435835:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.435836:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:20.0:1619677475.435839:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:1.0:1619677475.435840:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.435841:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.435842:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.435842:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677475.435844:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.435845:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:1.0:1619677475.435845:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.435846:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677475.435846:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:1.0:1619677475.435846:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677475.435849:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:1.0:1619677475.435849:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.435850:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.435850:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.435854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.435862:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.435864:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677475.435866:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.435867:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.435869:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677475.435870:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677475.435927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.435934:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.435937:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.435940:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.435941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.435942:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.435943:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677475.435944:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.435946:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.435947:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.435948:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677475.435951:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.435953:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.435956:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.435957:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677475.435957:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677475.435959:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677475.435960:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.435961:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.435961:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.435965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.435974:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.435977:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.435977:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.435978:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.435980:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.435984:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.436054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.436059:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.436062:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.436063:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.436064:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.436066:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.436068:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.436069:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.436070:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.436071:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.436073:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.436076:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.436078:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.436081:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.436082:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.436083:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.436084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:2.0:1619677475.436085:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.436085:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.436087:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.436088:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.436088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.436091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677475.436091:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.2:1619677475.436093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677475.436094:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.436095:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.436097:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.436099:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:8.0:1619677475.436099:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.436100:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.436101:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:8.0:1619677475.436101:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677475.436102:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:13.0:1619677475.436102:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:8.0:1619677475.436102:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.436103:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:8.0:1619677475.436103:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.436104:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.436104:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677475.436106:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:8.0:1619677475.436106:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:13.0:1619677475.436107:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:8.0:1619677475.436109:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.436110:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:8.0:1619677475.436111:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.436114:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.436115:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.436115:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.436117:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.436118:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.436119:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.436120:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.436121:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.436123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.436126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.436131:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.436133:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.436133:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.436134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.436135:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:6.0:1619677475.436135:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.436136:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.436137:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677475.436137:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.436140:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.436502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.436508:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.436511:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.436511:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.436514:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.436516:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.436517:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677475.436519:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499928] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.436520:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.436521:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.436522:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677475.436525:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.436527:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677475.436529:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.436530:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677475.436531:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677475.436532:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677475.436533:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.436534:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.436535:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.436536:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677475.436539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:5.2:1619677475.436541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:2.0:1619677475.436547:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677475.436547:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:2.0:1619677475.436549:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:1.0:1619677475.436549:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.436550:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:18.0:1619677475.436551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.436551:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.436551:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:2.0:1619677475.436553:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.436556:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677475.436560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677475.436568:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.436570:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677475.436571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.436572:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.436573:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.436573:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677475.436816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.436823:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.436826:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.436827:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.436827:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.436828:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.436829:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.437341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.437347:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.437350:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.437350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.437352:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.437355:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.437356:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.437357:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.437359:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.437360:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.437361:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.437364:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.437366:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.437368:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.437369:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.437370:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.437371:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.437372:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.437373:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.437374:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.437374:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.437378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.437380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.437385:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.437387:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.437387:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.437388:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.437389:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.437389:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.437390:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.437392:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.437393:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.437395:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.437602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.437610:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.437613:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.437613:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677475.437616:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.437618:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.437620:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.437621:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.437622:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677475.437623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:10.0:1619677475.437623:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.437624:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.437628:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.437629:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677475.437631:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:10.0:1619677475.437632:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.437633:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:30.0:1619677475.437634:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:10.0:1619677475.437634:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:30.0:1619677475.437636:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.437636:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.437636:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.437637:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.437637:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677475.437638:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.437639:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.437641:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.437642:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.437642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:30.0:1619677475.437643:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.437644:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.437645:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.437648:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.437649:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:30.0:1619677475.437650:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.437652:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.437652:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:30.0:1619677475.437653:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:22.0:1619677475.437653:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.437653:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:30.0:1619677475.437654:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677475.437655:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.437656:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.437656:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.437657:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.437658:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.437658:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.437662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.437669:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.437671:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.437672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.437672:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.437674:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.437677:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.437981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.437989:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.437992:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.437994:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.437997:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.437997:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.437998:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.437999:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.438001:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.438002:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.438003:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.438006:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.438008:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.438010:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.438011:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.438011:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.438013:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.438013:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.438014:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.438015:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.438018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.438025:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.438028:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.438029:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.438029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.438031:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.438033:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.438106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677475.438113:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.438115:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677475.438116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677475.438118:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677475.438120:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.438121:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677475.438122:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.438123:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.438124:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.438125:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677475.438128:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677475.438130:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.438131:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.438132:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.438132:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.438133:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.438133:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.438134:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677475.438135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677475.438138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.438144:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.438146:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.438147:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.438148:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.438148:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.438150:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.438633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.438639:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.438642:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.438643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.438645:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.438647:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.438649:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.438650:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.438651:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.438652:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.438654:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.438657:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.438659:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.438661:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.438662:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.438662:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.438664:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.438665:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.438665:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.438666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.438672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.438680:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.438682:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.438683:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.438685:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.438686:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.438688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677475.438855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.438860:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.438863:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.438864:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.438865:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.438868:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.438869:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.438870:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.438871:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.438872:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.438873:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.438876:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.438877:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.438878:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.438879:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.438879:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.438881:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.438881:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.438882:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.438883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.438887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.438895:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.438897:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.438898:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.438898:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.438899:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.438901:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677475.438918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.438924:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.438926:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.438929:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.438929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.438931:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.438933:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.438934:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.438935:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.438936:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.438937:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.438941:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.438943:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.438945:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.438946:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.438946:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.438948:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.438949:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.438950:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.438950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.438954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.438962:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.438965:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.438965:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.438966:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.438967:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.438970:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.439185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.439193:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.439196:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.439198:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.439200:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.439201:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.439201:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.439203:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.439204:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.439205:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.439206:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.439209:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.439211:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.439213:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.439214:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.439214:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.439216:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.439217:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.439218:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.439218:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.439222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.439229:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.439232:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.439233:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.439234:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.439236:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.439237:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.439274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.439280:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.439283:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.439284:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.439286:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.439288:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.439290:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.439291:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.439293:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.439297:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.439299:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.439302:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.439303:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.439306:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.439306:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.439307:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.439309:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.439309:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.439310:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.439310:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.439315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.439322:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.439325:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.439325:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.439327:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.439328:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.439329:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.439334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.439341:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.439343:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.439344:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.439345:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.439346:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.439347:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.439458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.439463:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.439465:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.439467:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.439468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.439469:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.439471:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.439472:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.439473:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.439474:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.439475:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.439477:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.439479:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.439481:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.439482:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.439482:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.439484:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.439484:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.439485:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.439486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.439490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.439498:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.439500:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.439501:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.439502:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.439503:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.439505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.439683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.439691:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.439694:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.439696:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.439698:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.439698:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.439700:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.439701:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.439702:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.439703:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.439704:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.439708:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.439709:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.439712:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.439713:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.439714:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.439715:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.439716:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.439717:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.439719:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.439719:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.439721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.439727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.439729:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.439732:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.439732:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.439733:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.439735:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677475.439735:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.439737:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.439738:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677475.439738:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677475.439739:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677475.439960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.439966:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.439969:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.439970:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.439971:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.439975:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.439976:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.439977:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.439978:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.439980:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.439980:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.439984:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.439985:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.439987:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.439988:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.439989:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.439990:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.439991:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.439992:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.439992:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.439996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.440003:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.440006:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.440006:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.440007:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.440008:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.440009:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.440112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.440118:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.440120:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.440122:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.440123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.440125:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.440126:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.440128:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.440129:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.440130:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.440131:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.440134:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.440136:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.440138:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.440139:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.440139:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.440141:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.440141:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.440142:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.440143:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.440147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.440155:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.440158:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.440159:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.440159:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.440161:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.440163:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.440256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.440264:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.440267:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.440268:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.440270:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.440273:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.440274:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.440275:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.440276:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.440277:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.440278:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.440281:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.440283:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.440286:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.440287:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.440287:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.440289:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.440289:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.440290:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.440292:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.440292:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.440297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.440298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.440306:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.440306:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.440308:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677475.440309:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677475.440309:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.440310:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.440310:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.440310:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.440312:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.440313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.440316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.440318:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.440320:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.440321:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.440321:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.440322:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.440322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.440475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.440483:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.440485:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.440488:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.440490:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.440491:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.440492:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.440493:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.440494:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.440495:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.440496:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.440499:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.440501:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.440502:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.440503:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.440504:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.440505:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.440506:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.440507:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.440509:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.440510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.440511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.440515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.440519:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.440521:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.440522:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.440524:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.440524:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.440525:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.440525:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.440526:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.440527:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.440528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.440697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.440705:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.440707:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.440709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.440710:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.440713:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.440714:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.440715:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.440716:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.440717:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.440718:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.440721:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.440723:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.440726:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.440726:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.440727:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.440728:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.440729:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.440730:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.440730:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.440734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.440743:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.440746:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.440747:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.440747:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.440749:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.440752:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.441261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.441266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.441269:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.441270:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.441272:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.441274:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.441275:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.441276:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.441277:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.441278:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.441279:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.441283:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.441285:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.441287:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.441288:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.441289:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.441290:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.441291:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.441292:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.441292:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.441297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.441305:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.441307:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.441308:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.441310:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.441311:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.441312:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.441347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.441355:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.441357:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.441360:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.441362:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.441362:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.441363:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.2:1619677475.441363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000400:00000200:31.0:1619677475.441365:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.441366:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.441367:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.441368:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.441368:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.441371:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:31.0:1619677475.441372:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.441373:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.441373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:31.0:1619677475.441374:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.441376:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.441377:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:31.0:1619677475.441378:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.441379:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.441379:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677475.441379:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.441380:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.441381:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:21.0:1619677475.441381:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.441382:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:21.0:1619677475.441382:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.441383:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.441383:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:21.0:1619677475.441385:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.441387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.441387:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.441389:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.441390:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.441391:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.441392:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.441393:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:22.0:1619677475.441394:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677475.441394:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:21.0:1619677475.441395:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677475.441396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:22.0:1619677475.441397:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.441398:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.441398:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.2:1619677475.441399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677475.441400:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.2:1619677475.441402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:22.0:1619677475.441403:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677475.441409:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.441410:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.441410:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:21.0:1619677475.441411:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.441411:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677475.441412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677475.441413:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.441413:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.441414:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:21.0:1619677475.441415:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.441986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.441993:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.441997:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.441998:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.441999:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.442002:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.442004:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.442005:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.442006:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.442007:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.442008:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.442012:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.442014:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.442017:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.442017:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.442018:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.442020:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.442020:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.442021:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.442021:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.442026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.442033:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.442036:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.442037:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.442037:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.442039:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.442042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.442327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.442332:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.442335:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.442335:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.442337:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.442340:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.442342:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677475.442343:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.442344:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.442345:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.442346:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677475.442349:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.442351:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677475.442354:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.442355:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677475.442356:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677475.442358:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677475.442358:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.442359:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.442359:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677475.442365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.442375:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.442378:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:4.2:1619677475.442378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:12.0:1619677475.442379:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.442382:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.442385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.442386:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677475.442386:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.442388:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.442389:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.442391:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.442394:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.442395:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.442396:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.442398:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.442399:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.442400:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.442403:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.442405:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.442408:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.442409:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.442410:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.442411:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.442412:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.442413:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.442414:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.442414:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.442417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.442420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.442424:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677475.442427:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.442427:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.442428:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.442428:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.442429:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.442430:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677475.442430:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.442431:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677475.442434:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.442818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.442823:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.442826:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.442826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.442828:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.442831:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.442833:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.442833:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.442835:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.442836:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.442837:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.442840:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.442842:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.442844:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.442845:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.442846:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.442848:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.442848:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.442848:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.442849:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.442854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.442861:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.442863:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.442864:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.442866:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.442867:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.442868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.443065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.443072:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.443074:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.443076:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.443077:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.443079:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.443081:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.443082:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.443083:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.443084:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.443085:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.443089:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.443091:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.443094:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.443095:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.443095:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.443097:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.443098:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.443099:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.443099:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.443103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.443111:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.443114:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.443115:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.443116:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.443118:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.443119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.443279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.443286:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.443289:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.443291:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.443293:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.443300:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.443302:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.443303:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.443305:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.443305:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.443307:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.443310:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.443312:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.443315:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.443316:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.443316:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.443318:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.443319:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.443320:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.443320:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.443325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.443333:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.443336:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.443337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.443337:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.443339:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.443342:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.443412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.443420:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.443422:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.443424:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.443426:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.443427:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.443429:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.443430:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.443431:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.443432:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.443433:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.443436:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.443438:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.443440:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.443441:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.443442:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.443443:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.443444:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.443445:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.443445:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.443449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.443456:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.443458:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.443459:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.443459:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.443461:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.443464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.443703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.443709:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.443712:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.443713:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.443715:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.443717:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.443719:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.443720:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.443722:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.443722:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.443724:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.443727:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.443729:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.443731:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.443732:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.443732:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.443734:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.443735:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.443736:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.443736:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677475.443740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.443748:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.443751:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.443752:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.443752:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.443754:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.443756:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.444319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.444322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:31.0:1619677475.444327:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.444330:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677475.444331:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:31.0:1619677475.444332:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.444333:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.444334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:31.0:1619677475.444335:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.444335:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.444336:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.444336:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.444337:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.444339:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.444339:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.444339:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.444340:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.444340:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.444341:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.444343:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.444344:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.444345:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.444346:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.444348:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.444349:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.444350:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.444351:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.444351:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.444352:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.444353:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.444354:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.444354:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.444355:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.444355:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677475.444355:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.444356:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.444357:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.444358:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.444358:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.444359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.444360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.2:1619677475.444365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677475.444367:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.444370:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.444370:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.444371:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.444372:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.444373:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:24.0:1619677475.444375:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677475.444376:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.444377:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.444378:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.444379:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.444382:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.2:1619677475.444382:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:21.0:1619677475.444383:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.444386:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.444387:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.444387:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.444388:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:30.0:1619677475.444390:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.444393:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677475.444395:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.444396:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677475.444397:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.444399:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:30.0:1619677475.444400:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.444401:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.444402:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.444403:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677475.444406:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.444408:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:30.0:1619677475.444411:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.444411:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:30.0:1619677475.444412:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:30.0:1619677475.444413:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:30.0:1619677475.444414:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.444415:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677475.444416:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:25.2:1619677475.444419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677475.444426:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677475.444429:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677475.444430:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.444431:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.444433:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677475.444434:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:25.2:1619677475.444441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.2:1619677475.444441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:13.0:1619677475.444448:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:1.0:1619677475.444448:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:13.0:1619677475.444450:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677475.444451:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:1.0:1619677475.444451:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.444452:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:13.0:1619677475.444452:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677475.444452:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677475.444454:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:1.0:1619677475.444454:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.444455:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.444457:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:1.0:1619677475.444458:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.444460:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.444461:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.444462:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:1.0:1619677475.444465:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.444467:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677475.444470:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.444470:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:1.0:1619677475.444471:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:1.0:1619677475.444473:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:1.0:1619677475.444473:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.444474:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.444474:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677475.444478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.444487:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.444489:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.444490:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.444492:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.444493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.444494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.444742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.444748:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.444751:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.444751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.444754:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.444756:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.444757:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.444758:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.444760:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.444761:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.444762:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.444765:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.444767:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.444769:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.444770:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.444771:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.444772:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.444773:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.444774:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.444775:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.444775:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.444778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.444780:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.444785:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.444785:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.444787:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.444788:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.444788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.444789:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.444789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.444790:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.444792:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.444792:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.444812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.444817:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.444819:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.444821:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.444822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.444823:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.444824:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.444826:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.444827:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.444828:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.444829:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.444832:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.444834:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.444836:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.444837:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.444838:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.444840:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.444840:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.444841:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.444841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.444846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.444854:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.444857:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.444857:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.444858:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.444860:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.444862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.444865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.444872:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.444875:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.444875:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.444876:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.444877:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.444877:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.444956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.444964:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.444966:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.444969:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.444970:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.444971:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.444972:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.444974:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.444975:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.444976:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.444977:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.444981:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.444982:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.444986:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.444986:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.444987:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.444989:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.444989:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.444990:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.444991:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.444995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.445002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.445005:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.445006:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.445006:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.445008:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.445011:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.445014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.445021:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.445024:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.445025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.445025:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.445026:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.445027:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.445081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.445089:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.445092:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.445093:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.445095:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.445097:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.445099:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.445100:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.445101:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.445102:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.445103:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.445106:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.445107:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.445110:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.445111:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.445112:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.445113:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.445114:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.445114:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.445115:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.445119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.445126:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.445129:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.445129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.445130:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.445132:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.445134:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.445234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.445241:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:4.2:1619677475.445241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677475.445243:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.445244:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.445246:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.445248:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.445248:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:17.0:1619677475.445250:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677475.445251:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:17.0:1619677475.445251:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.445251:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:17.0:1619677475.445252:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.445253:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.445255:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677475.445255:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677475.445258:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.445258:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.445259:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677475.445260:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:10.0:1619677475.445260:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.445262:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677475.445262:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.445263:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.445263:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:10.0:1619677475.445263:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.445264:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677475.445265:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.445266:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.445267:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677475.445268:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:10.0:1619677475.445268:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.445269:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:10.0:1619677475.445269:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.2:1619677475.445271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677475.445272:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.445272:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.445273:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.2:1619677475.445275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:10.0:1619677475.445275:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.445275:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.445276:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.445276:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.445280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677475.445280:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.445281:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.445282:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:1.0:1619677475.445282:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.445283:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677475.445283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.445284:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.445284:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.445286:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:12.0:1619677475.445286:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:1.0:1619677475.445286:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677475.445289:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.445290:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.445292:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.445293:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.2:1619677475.445298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:25.2:1619677475.445303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.445305:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677475.445307:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:12.0:1619677475.445307:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677475.445310:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:13.0:1619677475.445310:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.445310:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.445311:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677475.445311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.445312:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:12.0:1619677475.445312:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.445312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:17.0:1619677475.445313:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.445313:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.445621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.445630:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.445632:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.445635:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.445637:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.445638:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.445639:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.445640:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.445642:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.445643:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.445643:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.445647:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.445648:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.445650:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.445651:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.445652:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.445653:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.445654:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.445655:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.445655:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.445659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.445665:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.445668:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.445668:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.445670:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.445672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.445673:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.445676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.445682:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.445685:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.445686:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.445686:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.445687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.445688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.446006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.446009:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.446011:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.446014:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.446014:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.446016:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.446018:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.446019:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.446021:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.446022:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.446023:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.446026:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.446028:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.446031:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.446031:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.446032:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.446034:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.446034:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.446036:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.446037:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.446038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.446040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.446043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677475.446047:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.446048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.446050:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677475.446050:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677475.446051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677475.446051:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.446052:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.446052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.446054:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.446056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.446304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.446312:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.446314:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.446315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.446317:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.446320:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.446322:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.446323:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.446324:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.446325:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.446326:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.446330:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.446331:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.446334:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.446335:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.446336:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.446338:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.446338:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.446339:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.446339:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.446345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.446353:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.446356:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.446357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.446357:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.446359:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.446362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.446375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393400] (20)++
00000800:00000200:26.0:1619677475.446383:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393400] (21)++
00000800:00000200:26.0:1619677475.446386:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.43@o2ib3
00000800:00000200:10.0:1619677475.446386:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393400] (22)--
00000400:00000200:26.0:1619677475.446389:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.5.43@o2ib3 : GET - for me
00000400:00000200:26.0:1619677475.446392:0:19941:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.5.43@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:26.0:1619677475.446393:0:19941:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.5.43@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:26.0:1619677475.446395:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393400] (21)++
00000800:00000200:26.0:1619677475.446395:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393400] (22)--
00000400:00000200:26.0:1619677475.446397:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.5.43@o2ib3 local destination
00000400:00000200:26.0:1619677475.446401:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.5.43@o2ib3(10.51.5.43@o2ib3:10.51.5.43@o2ib3) : REPLY try# 0
00000800:00000200:26.0:1619677475.446402:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.5.43@o2ib3
00000800:00000200:26.0:1619677475.446405:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a074f81e80] -> 10.51.5.43@o2ib3 (2) version: 12
00000800:00000200:26.0:1619677475.446406:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a06f393400] (21)++
00000800:00000200:26.0:1619677475.446406:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393400] (22)++
00000800:00000200:26.0:1619677475.446408:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a06f393400] (23)--
00000800:00000200:26.0:1619677475.446408:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393400] (22)--
00000800:00000200:4.2:1619677475.446415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393400] (21)++
00000800:00000200:10.0:1619677475.446422:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393400] (22)++
00000800:00000200:10.0:1619677475.446424:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393400] (23)--
00000800:00000200:26.0:1619677475.446425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393400] (22)--
00000400:00000200:10.0:1619677475.446425:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.446427:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.5.43@o2ib3: REPLY: OK
00000800:00000200:10.0:1619677475.446428:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393400] (21)--
00000800:00000200:21.2:1619677475.446488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.446496:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.446499:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.446500:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.446502:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.446504:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677475.446504:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.446505:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.446506:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.446507:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.446508:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.446511:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.446513:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.446515:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.446516:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.446516:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.446518:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.446518:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.446520:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.446521:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.446522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.446524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.446528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.446530:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.446532:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.446533:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.446533:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.446535:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.446535:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.446537:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.446538:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.446538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.446539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.446545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.446551:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.446553:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.446554:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.446554:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.446555:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.446556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.446754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.446760:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.446763:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.446763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.446765:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.446768:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.446769:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.446770:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.446772:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.446773:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.446774:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.446777:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.446779:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.446782:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.446782:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.446783:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.446785:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.446785:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.446786:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.446786:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.446791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.446800:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.446802:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.446803:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.446804:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.446805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.446807:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.446811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.446818:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.446821:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.446821:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.446822:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.446823:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.446824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.446959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.446967:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.446970:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.446972:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.446974:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.446974:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.446976:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.446977:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.446979:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.446980:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.446981:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.446984:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.446985:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.446989:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.446989:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.446990:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.446992:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.446992:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.446993:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.446995:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.446995:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.446998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.447001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.447005:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.447008:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.447009:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.447009:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.447009:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.447011:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.447012:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.447013:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:4.2:1619677475.447013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677475.447014:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.447014:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.447017:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.447020:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.447022:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.447023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.447024:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.447026:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.447027:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.447029:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.447030:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.447031:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.447033:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.447035:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.447037:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.447038:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.447039:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.447040:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.447041:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.447042:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.447042:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.447046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.447054:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.447056:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.447057:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.447059:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.447059:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.447060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.447241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.447248:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.447252:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.447252:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.447254:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.447256:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.447258:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.447259:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.447260:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.447261:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.447262:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.447265:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.447267:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.447270:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.447270:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.447271:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.447273:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.447273:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.447274:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.447274:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.447279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.447286:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:4.2:1619677475.447288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:22.0:1619677475.447289:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.447290:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.447290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.447292:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.447296:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:22.0:1619677475.447300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677475.447301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.447305:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677475.447307:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.447309:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.447310:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677475.447312:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.447313:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.447314:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.447315:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677475.447318:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.447319:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677475.447321:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.447321:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677475.447322:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677475.447323:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677475.447324:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.447325:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.447326:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.447327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.447330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677475.447333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677475.447338:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.447340:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:26.0:1619677475.447340:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:31.0:1619677475.447341:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.447342:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.447342:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.447343:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.447344:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:30.0:1619677475.447345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.447346:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.447755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677475.447762:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677475.447765:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677475.447766:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677475.447767:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.447771:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.447772:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:24.0:1619677475.447773:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.447774:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.447775:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.447776:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.2:1619677475.447776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:24.0:1619677475.447778:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.447780:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.447782:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.447782:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:2.0:1619677475.447782:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677475.447783:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.447784:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.447784:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677475.447785:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:2.0:1619677475.447785:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:24.0:1619677475.447786:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:18.0:1619677475.447787:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.447788:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:9.2:1619677475.447790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:2.0:1619677475.447791:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.447793:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.447794:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.447795:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.447796:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:2.0:1619677475.447796:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.447797:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.447798:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.447799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.447799:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.447801:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:2.0:1619677475.447801:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677475.447803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:2.0:1619677475.447803:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.447806:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.447806:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.447807:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.447809:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.447809:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.447810:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.447811:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.447815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.447823:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.447826:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.447827:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.447827:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.447829:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.447831:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.448013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.448021:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.448024:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.448026:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.448027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.448028:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.448030:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.448031:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.448033:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.448034:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.448035:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.448038:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.448039:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.448042:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.448042:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.448043:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.448045:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.448045:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.448046:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.448047:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.448050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.448056:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.448058:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.448059:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.448060:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.448061:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.448064:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.448304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.448312:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.448314:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.448315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.448316:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.448318:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.448320:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.448321:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.448322:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.448323:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.448324:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.448327:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.448328:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.448331:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.448332:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.448332:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.448334:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.448335:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.448336:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.448337:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.448338:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.448340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.448343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.448348:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.448350:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.448351:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.448352:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677475.448352:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.448352:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.448353:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.448354:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677475.448354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:1.0:1619677475.448354:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.448358:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677475.448360:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.448363:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.448364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.448365:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.448368:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.448369:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.448370:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.448372:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.448373:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.448374:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:20.2:1619677475.448377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:6.0:1619677475.448377:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.448379:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.448382:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.448382:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.448383:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.448384:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:6.0:1619677475.448385:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.448385:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:18.0:1619677475.448386:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:6.0:1619677475.448386:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.448386:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:2.0:1619677475.448387:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.448389:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.448391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677475.448391:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.448393:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.448394:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.448395:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.448396:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.448397:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.448399:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:18.0:1619677475.448400:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.448401:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.448402:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.448402:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.448404:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.448404:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.448405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677475.448405:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.448406:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677475.448406:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.448407:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.448408:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.448409:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.448410:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.448411:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.448414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.448416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.448423:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.448423:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.448424:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.448424:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.448425:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.448425:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.448426:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.448427:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:2.0:1619677475.448427:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.448428:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.448599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.448607:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.448610:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.448612:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.448615:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.448615:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.448616:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.448618:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.448619:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.448620:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.448621:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.448625:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.448627:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.448630:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.448630:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.448631:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.448633:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.448633:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.448634:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.448635:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.448639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.448647:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.448650:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.448650:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.448651:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.448653:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.448655:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.448657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.448664:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.448667:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.448668:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.448668:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.448669:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.448670:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.448713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.448719:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.448722:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.448723:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.448725:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.448727:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.448728:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.448729:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.448731:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.448732:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.448733:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.448736:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.448738:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.448741:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.448741:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.448742:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.448744:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.448744:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.448746:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.448747:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.448747:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.448750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.448753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.448758:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677475.448760:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.448760:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.448761:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.448761:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677475.448762:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.448763:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.448765:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.448766:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.448768:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.448770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.448778:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.448780:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.448781:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.448782:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.448783:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.448784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.448876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.448884:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.448886:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.448888:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.448890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.448891:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.448893:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.448894:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.448895:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.448896:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.448898:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.448901:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.448903:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.448905:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.448906:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.448907:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.448908:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.448909:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.448910:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.448910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.448914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.448920:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.448923:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.448923:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.448924:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.448926:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.448928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.449229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677475.449235:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.449238:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677475.449238:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.449241:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.449243:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.449244:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677475.449245:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.449247:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.449248:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.449249:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677475.449252:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.449253:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677475.449256:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.449256:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677475.449257:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677475.449259:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677475.449259:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.449260:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677475.449261:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:17.0:1619677475.449261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:9.2:1619677475.449265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.2:1619677475.449266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.449272:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:18.0:1619677475.449273:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677475.449273:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677475.449274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:18.0:1619677475.449275:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:1.0:1619677475.449275:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000400:00000200:18.0:1619677475.449276:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.449277:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.449278:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.449281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.449852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.449860:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.449863:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.449864:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.449865:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.449868:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.449869:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.449870:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.449872:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.449873:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.449874:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.449877:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.449879:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.449882:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.449883:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.449883:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.449885:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.449886:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.449887:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.449887:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.449892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.449901:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.449904:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.449905:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.449905:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.449907:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.449910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.450185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.450193:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.450196:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.450198:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.450201:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.450201:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.450202:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.450203:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.450204:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.450205:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.450206:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.450209:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.450211:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.450213:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.450214:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.450214:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.450216:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.450217:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.450218:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.450218:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.450222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.450228:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.450230:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.450231:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.450231:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.450233:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.450235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.450242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.450248:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.450251:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.450251:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.450252:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.450253:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.450253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.450328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.450333:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.450336:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.450337:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.450339:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.450341:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.450342:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.450343:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.450345:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.450346:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.450347:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.450350:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.450352:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.450354:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.450355:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.450356:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.450357:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.450358:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.450358:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.450359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.450364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.450372:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.450374:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.450375:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.450377:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.450378:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.450379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.450448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.2:1619677475.450451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.450454:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.450455:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.450455:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.450456:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.450458:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.450459:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.450461:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.450463:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.450464:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.450465:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.450466:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.450467:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.450469:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.450470:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.450472:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.450472:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.450473:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.450474:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.450474:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.450475:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.450479:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.450481:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.450483:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.450483:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.450483:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.450484:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.450484:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.450485:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.450486:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.450486:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.450487:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.450487:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.450488:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.450489:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.450489:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (23)++
00000800:00000200:21.0:1619677475.450490:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (24)--
00000800:00000200:20.0:1619677475.450490:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (24)--
00000800:00000200:21.0:1619677475.450491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677475.450491:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.450493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677475.450493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.450497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677475.450500:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.450501:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.450502:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677475.450503:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677475.450503:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.450503:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.450504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.450504:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.450504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.450505:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.450506:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.450506:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.450507:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:24.0:1619677475.450508:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.450508:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677475.450511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.450629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677475.450637:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.450639:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677475.450640:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677475.450642:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.450645:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.450646:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677475.450647:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.450648:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.450650:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.450650:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677475.450654:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.450656:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677475.450658:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.450659:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677475.450660:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677475.450661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.0:1619677475.450661:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677475.450662:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.450662:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677475.450663:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:25.2:1619677475.450667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.450671:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.450673:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.450675:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.450676:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.450678:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.450678:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.450678:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.450680:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.450680:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677475.450681:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.450682:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.450683:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.450684:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.450686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:30.0:1619677475.450687:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.450688:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.450689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677475.450689:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.450692:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.450693:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677475.450693:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.450695:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677475.450696:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.450697:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.450697:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.450701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.450710:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.450713:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.450714:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.450714:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.450716:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.450720:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.450725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.450732:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.450735:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.450736:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.450736:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.450737:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.450737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.451126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.451133:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.451136:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.451138:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.451140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.451141:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.451143:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.451144:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.451145:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.451146:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.451147:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.451150:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.451151:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.451154:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.451154:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.451155:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.451156:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.451157:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.451158:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.451160:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.451160:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.451162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.451166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677475.451168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.451171:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.451173:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:30.0:1619677475.451174:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.451174:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.451176:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:24.0:1619677475.451176:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.451176:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677475.451177:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.451178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677475.451178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.451178:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.451180:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:8.0:1619677475.451180:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.451181:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.451183:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.451184:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.451185:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.451186:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.451187:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.451188:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.451191:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.451193:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.451196:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.451196:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.451197:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.451199:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.451199:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.451200:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.451201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.451205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.451213:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.451215:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.451216:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.451216:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.451218:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.451222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.451567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.451576:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.451578:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.451580:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.451583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.451583:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.451584:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677475.451585:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.451587:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.451588:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.451589:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.451592:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.451593:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.451596:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.451597:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677475.451598:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.451599:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677475.451600:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.451601:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.451601:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.451606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.451616:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.451618:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.451620:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.451620:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.451622:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.451625:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.452087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:20.2:1619677475.452088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677475.452095:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677475.452096:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.452099:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:26.0:1619677475.452100:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.452100:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:20.0:1619677475.452102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:26.0:1619677475.452103:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.452103:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.452106:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.452106:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.452108:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677475.452109:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:21.0:1619677475.452110:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.452111:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.452113:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.452113:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.452115:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.452115:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.452117:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:21.0:1619677475.452117:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.452123:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.452124:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.452126:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677475.452127:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.0:1619677475.452129:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.452130:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.452131:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:21.0:1619677475.452131:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677475.452132:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677475.452132:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.452134:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:21.0:1619677475.452134:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.452135:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677475.452135:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.452136:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:26.0:1619677475.452137:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.0:1619677475.452137:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:26.0:1619677475.452138:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.452141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:8.2:1619677475.452141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677475.452150:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:17.0:1619677475.452152:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:6.0:1619677475.452153:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.452154:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.452155:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.452157:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:6.0:1619677475.452157:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.452159:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.452160:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:1.0:1619677475.452160:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:25.2:1619677475.452161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000400:00000200:17.0:1619677475.452163:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.452167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677475.452169:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.452172:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.452173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.452174:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.452175:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.452176:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.452215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.452219:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.452222:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677475.452225:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.452225:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.452228:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.452230:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677475.452231:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.452233:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.452234:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.452235:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.452238:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.452240:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.452243:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.452244:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.452244:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.452246:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677475.452247:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.452248:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.452248:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.452253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.452261:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.452263:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.452264:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.452265:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.452267:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.452269:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.452328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.452337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.452340:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.452342:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.452344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.452345:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.452346:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.452348:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.452348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:31.0:1619677475.452349:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.452350:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.452351:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.452354:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.452354:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.452356:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677475.452357:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.452358:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.452359:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.452360:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.452360:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.452361:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:6.0:1619677475.452362:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.452363:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:6.0:1619677475.452363:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677475.452364:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.452365:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.452365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677475.452365:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.452366:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.452367:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.452368:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.2:1619677475.452370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:6.0:1619677475.452372:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.452374:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.452376:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.452377:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.452378:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:6.0:1619677475.452378:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.452380:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:6.0:1619677475.452380:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.452380:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677475.452381:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.452381:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.452382:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:21.0:1619677475.452383:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.452383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.452384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:25.2:1619677475.452386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.452389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.452391:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.452394:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.452396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.452396:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.452396:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.452398:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:21.0:1619677475.452398:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.452399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.452400:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.452400:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.452401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.453095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.453102:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.453105:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.453107:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.453109:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.453110:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.453112:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.453113:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.453114:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.453115:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.453116:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.453120:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.453121:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.453125:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.453125:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.453126:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.453128:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.453128:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.453130:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.453131:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.453132:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.453134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.453138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.453142:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.453145:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.453146:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.453146:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.453146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.453148:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.453149:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677475.453149:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.453150:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.453152:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.453363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.453370:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.453373:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.453375:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.453378:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.453379:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.453380:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.453381:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.453382:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.453384:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.453384:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.453388:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.453389:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.453391:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.453392:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.453392:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.453393:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.453394:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.453395:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.453395:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.453400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.453407:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.453409:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.453410:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.453411:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.453413:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.453414:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.453913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:4.2:1619677475.453917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:6.0:1619677475.453919:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.453921:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.453922:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.453924:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.453925:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:6.0:1619677475.453926:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.453927:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677475.453928:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.453929:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:6.0:1619677475.453929:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.453931:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.453932:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:6.0:1619677475.453932:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.453933:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:10.0:1619677475.453935:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.453936:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.453937:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.453938:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.453938:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:10.0:1619677475.453940:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.453941:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.453941:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677475.453942:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:6.0:1619677475.453942:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.453942:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.453944:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.453944:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.453945:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677475.453946:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677475.453946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:10.0:1619677475.453947:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:25.2:1619677475.453950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677475.453950:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.453951:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.453951:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.453953:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.453953:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.453954:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.453955:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.453956:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.453958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.0:1619677475.453958:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.453960:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.453961:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.453962:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:4.2:1619677475.453962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677475.453964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.453965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:13.0:1619677475.453966:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.453968:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:26.0:1619677475.453969:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:13.0:1619677475.453969:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.453971:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677475.453972:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.453972:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.453973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:12.0:1619677475.453973:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.453974:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677475.453974:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.0:1619677475.453980:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.453982:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.453983:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.453984:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.453985:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.453986:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.454008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.454014:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.454017:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.454018:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.454019:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.454022:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.454023:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677475.454024:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.454026:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.454027:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.454028:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677475.454031:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.454033:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677475.454035:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.454036:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677475.454037:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677475.454038:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677475.454039:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.454039:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.454040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677475.454045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.454053:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.454055:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677475.454056:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.454057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.454058:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.454061:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677475.454185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:4.2:1619677475.454186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:18.0:1619677475.454191:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677475.454194:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677475.454194:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.454194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.454196:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:24.0:1619677475.454197:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:18.0:1619677475.454198:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.454198:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677475.454200:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:24.0:1619677475.454201:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.454202:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.454203:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.454204:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.454204:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.454205:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:24.0:1619677475.454206:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.454207:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.454209:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.454209:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:24.0:1619677475.454211:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.454212:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.454213:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.454216:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677475.454217:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.454217:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.454217:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677475.454219:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.454219:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.454220:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.454221:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.454221:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677475.454222:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.454222:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.454223:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:25.2:1619677475.454225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677475.454225:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.454225:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.454226:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.454226:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.454231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:12.0:1619677475.454232:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:4.2:1619677475.454233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:12.0:1619677475.454235:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.454235:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.454237:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.454238:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:17.0:1619677475.454239:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:12.0:1619677475.454239:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677475.454240:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:17.0:1619677475.454241:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.454243:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.454243:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:1.0:1619677475.454243:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677475.454244:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:17.0:1619677475.454245:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.454247:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.454248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:10.0:1619677475.454249:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.454250:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677475.454251:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.454253:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.454254:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.454255:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677475.454257:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.454259:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677475.454260:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.454261:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677475.454261:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.454263:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.454263:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.454264:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.454264:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.454268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.454275:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.454276:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.454277:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.454278:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.454279:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.454280:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677475.454314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.454319:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.454322:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.454324:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.454325:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.454326:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.454328:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.454329:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.454331:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.454332:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.454333:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.454336:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.454338:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.454340:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.454341:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.454341:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.454343:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.454344:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.454344:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.454345:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.454350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.454358:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.454360:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.454361:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.454362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.454363:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.454365:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.454370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.454376:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.454379:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.454380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.454380:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.454381:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.454381:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.454579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.454587:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.454590:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.454592:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.454594:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.454594:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.454595:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.454597:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.454598:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.454599:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.454600:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.454603:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.454605:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.454607:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.454608:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.454608:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.454610:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.454610:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.454611:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.454612:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.454615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.454622:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.454625:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.454626:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.454627:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.454629:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.454630:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.454728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.454736:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.454738:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.454739:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.454741:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.454743:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.454744:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.454745:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.454746:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.454747:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.454748:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.454751:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.454753:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.454756:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.454756:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.454757:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.454759:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.454759:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.454760:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.454760:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.454765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.454772:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.454775:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.454776:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.454776:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.454778:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.454781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.454870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677475.454875:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.454877:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677475.454878:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677475.454880:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.454882:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.454883:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677475.454884:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.454886:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.454887:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.454887:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677475.454890:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.454892:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.454894:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.454895:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677475.454896:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677475.454898:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677475.454898:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.454899:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677475.454899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677475.454904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.454913:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.454915:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.454917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.454917:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.454919:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.454921:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.455364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.455372:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.455375:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.455377:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.455379:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.455380:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.455381:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.455383:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.455384:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.455385:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.455386:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.455389:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.455391:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.455395:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.455395:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.455396:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.455398:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.455398:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.455399:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.455400:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.455405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.455413:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.455416:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.455417:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.455417:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.455420:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.455422:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.455515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.455523:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.455526:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.455528:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.455529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.455531:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.455532:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.455534:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.455535:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.455536:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.455537:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.455540:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.455542:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.455544:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.455544:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.455545:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.455546:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.455547:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.455548:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.455550:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.455550:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.455552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.455556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.455560:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.455562:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.455562:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.455564:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.455564:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.455565:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.455566:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.455567:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.455568:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.455569:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.455569:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677475.455570:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.455570:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (23)++
00000800:00000200:31.0:1619677475.455571:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.455571:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (24)--
00000800:00000200:31.0:1619677475.455572:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.455573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.455813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.455818:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.455821:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.455822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.455824:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.455826:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.455827:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.455828:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.455830:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.455831:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.455832:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.455835:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.455837:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.455839:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.455840:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.455841:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.455842:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.455843:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.455843:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.455844:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.455848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.455856:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.455858:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.455859:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.455861:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.455863:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.455863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.455889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.455894:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.455897:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.455899:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.455899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.455901:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.455903:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.455904:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.455906:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.455907:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.455908:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.455911:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.455913:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.455916:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.455916:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.455917:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.455919:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.455919:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.455920:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.455921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.455925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.455932:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.455935:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.455936:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.455936:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.455938:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.455941:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.456501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.456508:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.456511:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.456512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.456513:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.456516:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.456517:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.456518:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.456520:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.456520:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.456521:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.456524:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.456526:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.456529:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.456530:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.456530:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.456532:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.456532:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.456533:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.456533:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.456539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.456547:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.456550:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.456551:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.456551:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.456553:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.456556:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.456766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.2:1619677475.456766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.456769:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.456772:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677475.456774:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:20.0:1619677475.456774:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.456775:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677475.456777:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:20.0:1619677475.456777:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.456778:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:31.0:1619677475.456780:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.456780:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.456781:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:20.0:1619677475.456781:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.456782:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.456782:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.456783:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677475.456784:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.456786:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.456787:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.456788:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.456789:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.456789:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677475.456790:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.456792:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.456793:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.456793:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:31.0:1619677475.456794:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.456795:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.456795:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677475.456796:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.456796:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.456797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677475.456800:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.456800:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.456801:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.2:1619677475.456801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:31.0:1619677475.456803:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.456804:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.456805:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.456805:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.456809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.456809:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.456811:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.456812:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.456814:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.456815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.456816:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677475.456817:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.456820:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.456821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.456821:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.456823:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.456826:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.456836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.456843:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.456846:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.456846:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.456847:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.456848:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.456848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.456951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.456958:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.456961:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.456963:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.456965:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.456966:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.456968:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.456969:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.456970:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.456971:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.456972:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.456975:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.456977:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.456979:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.456980:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.456980:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.456982:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.456983:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.456984:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.456984:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.456987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.456994:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.456996:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.456998:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.456998:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.457000:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.457002:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.457701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.457707:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.457710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.457710:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.457712:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.457714:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.457716:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.457717:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.457718:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.457720:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.457720:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.457724:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.457726:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.457728:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.457729:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.457730:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.457732:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.457732:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.457732:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.457733:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.457738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.457747:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.457749:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.457750:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.457751:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.457753:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.457753:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.457972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.457980:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.457983:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.457985:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.457987:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.457987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.457989:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.457990:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.457991:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.457992:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.457993:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.457997:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.457998:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.458001:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.458001:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.458002:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.458004:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.458005:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.458006:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.458006:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.458010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.458016:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.458019:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.458020:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.458020:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.458022:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.458024:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.458574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.458579:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.458582:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.458584:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.458585:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.458586:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.458588:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.458589:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.458590:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.458591:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.458592:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.458596:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.458597:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.458600:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.458600:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.458601:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.458602:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.458603:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.458604:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.458605:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.458606:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.458609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.458611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.458614:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.458616:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.458616:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.458617:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.458618:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.458621:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.458622:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.458622:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.458624:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.458626:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.458628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.458635:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.458638:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.458638:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.458639:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.458640:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.458640:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.458764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.458772:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.458775:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.458777:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.458779:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.458780:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.458781:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.458783:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.458784:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.458785:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.458786:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.458789:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.458791:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.458794:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.458795:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.458796:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.458797:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.458798:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.458799:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.458799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.458803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.458812:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.458814:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.458815:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.458817:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.458818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.458819:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.458841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.458849:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.458852:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.458854:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.458855:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.458857:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.458858:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.458859:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.458860:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.458861:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.458862:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.458865:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.458867:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.458870:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.458870:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.458871:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.458873:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.458873:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.458874:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.458875:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.458879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.458888:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.458891:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.458891:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.458892:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.458894:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.458897:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.458910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.458916:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.458919:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.458920:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.458922:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.458926:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.458927:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.458928:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.458930:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.458931:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.458931:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.458934:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.458936:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.458937:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.458939:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.458939:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.458941:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.458941:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.458942:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.458942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.458947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.458954:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.458956:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.458957:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.458957:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.458958:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.458959:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.459049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.459054:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.459057:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.459059:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.459059:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.459062:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.459063:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.459064:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.459066:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.459067:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.459068:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.459070:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.459072:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.459074:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.459075:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.459075:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.459077:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.459077:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.459079:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.459080:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.459081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677475.459082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.459086:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.459091:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.459092:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.459092:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.459093:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:20.0:1619677475.459093:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.459094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.459095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.459095:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.459096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677475.459097:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.459163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.459169:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.459172:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.459172:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.459174:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.459176:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.459178:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.459179:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.459180:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.459181:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.459182:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.459186:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.459187:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.459190:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.459190:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.459191:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.459193:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.459193:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.459194:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.459195:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.459195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.459199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.459201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.459206:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.459208:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.459208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.459208:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.459209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.459210:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.459211:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.459213:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.459213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.459215:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.459957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.459965:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.459968:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:21.2:1619677475.459968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677475.459969:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.459971:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.459974:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.459976:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677475.459976:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677475.459977:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.459978:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677475.459979:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.459980:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.459981:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.459982:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.459983:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.459985:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.459985:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.459986:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:26.0:1619677475.459986:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.459987:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.459988:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.459989:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.459990:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677475.459990:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.459991:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677475.459991:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:30.0:1619677475.459993:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.459993:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.459993:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.459994:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:30.0:1619677475.459995:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677475.459995:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:30.0:1619677475.459997:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.459997:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.459998:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.460000:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.460000:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677475.460000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.460002:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.460003:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.460004:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.460006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.460009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677475.460009:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.460011:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.460012:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.460013:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.460014:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.460015:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:17.0:1619677475.460015:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.460016:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.460017:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:17.0:1619677475.460017:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:31.0:1619677475.460018:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.460019:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.460020:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.460021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.460022:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.460023:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.460043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.460051:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.460053:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.460054:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.460056:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.460059:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.460060:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.460061:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.460062:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.460063:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.460064:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.460067:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.460069:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.460072:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.460073:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.460073:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.460075:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.460075:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.460076:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.460077:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.460078:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.460081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.460083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.460088:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.460090:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.460091:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677475.460092:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:22.0:1619677475.460092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.460092:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.460093:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.460094:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.460094:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:4.2:1619677475.460095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:6.0:1619677475.460097:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677475.460102:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.460105:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.460106:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.460107:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.460110:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.460111:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.460113:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.460114:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.460115:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.460116:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.460119:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.460120:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.460123:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.460124:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.460124:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.460126:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.460126:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.460127:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.460127:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.460133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.460140:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:4.2:1619677475.460142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:22.0:1619677475.460143:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.460144:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.460144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677475.460145:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:22.0:1619677475.460146:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.460148:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:22.0:1619677475.460149:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:20.0:1619677475.460150:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.460151:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.460152:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.460154:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.460155:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.460156:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.460157:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.460158:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.460162:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.460163:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.460166:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.460167:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.460168:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.460169:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.460170:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.460171:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.460171:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.460175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.460182:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.2:1619677475.460183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.0:1619677475.460184:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.460186:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.460186:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.460188:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.460190:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.460190:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.460191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.460193:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.460195:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.460197:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.460197:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.460197:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:30.0:1619677475.460199:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:24.0:1619677475.460199:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.460200:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.460200:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000400:00000200:30.0:1619677475.460201:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677475.460201:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.460201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:30.0:1619677475.460202:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.460202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:30.0:1619677475.460203:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.460206:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.460207:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.460211:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.460212:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.460212:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.460214:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.460215:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.460216:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.460217:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.460218:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.460220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.460224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.460228:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.460231:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.460232:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.460232:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.460232:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.460234:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:6.0:1619677475.460234:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.460235:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.460236:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.460237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.460705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.460713:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.460716:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.460717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.460718:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.460722:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.460723:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.460724:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.460726:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.460727:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.460728:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.460731:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.460733:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.460736:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.460736:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.460737:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.460739:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.460739:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.460740:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.460742:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.460742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.460746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.460748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.460754:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.460756:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.460756:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.460757:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.460758:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677475.460758:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.460759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.460760:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.460760:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.460762:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.460765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.460772:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.460774:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.460775:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.460775:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.460776:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.460777:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.460802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.460809:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.460811:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.460813:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.460814:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.460817:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.460818:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.460820:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.460821:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.460822:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.460823:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.460827:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.460828:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.460831:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.460832:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.460833:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.460835:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.460835:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.460836:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.460837:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.460838:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.460841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677475.460843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677475.460849:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677475.460849:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677475.460851:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:18.0:1619677475.460852:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.460852:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:31.0:1619677475.460853:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.460853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677475.460854:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.460856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.460858:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.460923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.460929:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.460932:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.460933:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.460935:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.460937:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.460939:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677475.460940:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.460942:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.460943:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.460944:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677475.460947:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.460949:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677475.460952:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.460953:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677475.460954:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677475.460955:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677475.460956:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.460956:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.460957:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677475.460961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.460969:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.460972:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677475.460972:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.460973:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.460975:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.460978:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.461041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.461046:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.461050:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.461050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.461052:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.461054:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.461056:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.461057:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.461058:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.461059:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.461060:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.2:1619677475.461061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:22.0:1619677475.461064:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.461066:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.461068:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.461069:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.461069:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.461070:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.461071:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.461072:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677475.461072:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.461073:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.461073:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:30.0:1619677475.461074:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.461076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.461077:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.461077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.461078:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.461080:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.461081:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.461082:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.461083:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:21.0:1619677475.461085:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:30.0:1619677475.461087:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.461087:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.461088:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.461089:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:21.0:1619677475.461090:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.461090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.461092:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.461092:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:21.0:1619677475.461092:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.461093:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.461095:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.461095:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.461096:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.461097:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.461097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:9.2:1619677475.461101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677475.461104:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.461107:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677475.461107:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.0:1619677475.461108:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.461108:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.461109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677475.461110:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.461111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677475.461111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677475.461111:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.461114:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.461116:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.461121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.461127:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.461130:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.461130:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.461131:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.461131:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.461132:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.461217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.461224:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.461227:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.461229:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.461231:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.461232:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.461233:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.461234:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.461236:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.461237:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.461238:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.461241:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.461243:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.461246:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.461247:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.461247:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.461249:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.461250:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.461251:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.461251:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.461255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.461263:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.461266:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.461267:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.461267:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.461269:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.461272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.461354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.461358:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.461360:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.461362:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.461363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.461365:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.461366:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.461367:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.461369:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.461370:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.461371:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.461374:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.461376:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.461378:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.461379:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.461379:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.461381:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.461382:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.461383:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.461384:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.461385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.461387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.461390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.461395:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677475.461396:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677475.461397:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:22.0:1619677475.461398:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677475.461398:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.461398:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:22.0:1619677475.461399:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.461399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.461401:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.461404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.461750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.461757:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.461760:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.461761:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.461763:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.461766:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.461767:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.461768:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.461770:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.461771:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.461772:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.461775:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.461777:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.461780:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.461781:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.461782:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.461783:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.461784:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.461785:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.461785:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.461789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.461797:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.461799:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.461800:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.461801:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.461802:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.461805:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.462353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.462361:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.462364:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.462366:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.462369:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.462369:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.462370:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.462371:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.462373:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.462374:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.462375:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.462378:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.462379:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.462381:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.462381:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.462382:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.462383:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.462384:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.462385:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.462385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.462390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.462397:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.462399:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.462400:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.462402:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.462403:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.462404:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.462686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.462692:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:4.2:1619677475.462693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677475.462695:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.462696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.462698:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.462701:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.462701:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:17.0:1619677475.462702:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.462704:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.462704:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.462705:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:17.0:1619677475.462705:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.462706:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.462707:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677475.462708:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.462710:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.462711:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.462712:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:17.0:1619677475.462713:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677475.462713:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.462715:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.462716:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.462717:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.462717:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:10.0:1619677475.462717:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.462718:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:17.0:1619677475.462719:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.462720:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.462720:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.462721:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:10.0:1619677475.462721:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.462723:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.462726:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677475.462726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:10.0:1619677475.462727:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.462728:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.462729:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.462730:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.462731:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.462731:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677475.462734:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.2:1619677475.462736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.462737:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.462738:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.462739:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.462741:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.462742:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.462743:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.2:1619677475.462745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.462746:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.462747:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.462748:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.462750:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.462751:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.462752:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.0:1619677475.462753:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.462753:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.462754:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.462755:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.462755:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.462757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.462765:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.462768:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.462769:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.462769:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.462770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.462771:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.462823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.462828:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.462831:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.462833:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.462833:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.462835:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.462837:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.462838:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.462839:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.462840:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.462841:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.462845:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.462846:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.462849:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.462850:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.462850:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.462852:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.462852:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.462853:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.462854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.462857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.462865:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.462867:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.462867:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.462869:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.462870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.462871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.463091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.463097:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.2:1619677475.463099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.463100:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.463101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.463103:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.463105:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.463107:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677475.463107:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.463108:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.463109:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677475.463109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.463110:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:22.0:1619677475.463110:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.463112:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.463114:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.463115:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.463115:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.463116:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:22.0:1619677475.463116:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.463117:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677475.463117:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.463118:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.463118:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.463120:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.463121:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.463121:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.463122:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.463122:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.463122:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.463124:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.463124:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.463125:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677475.463126:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.463126:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677475.463128:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.463129:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.463130:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677475.463130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.463131:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:30.0:1619677475.463131:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.463132:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.463133:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677475.463133:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.463134:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96499807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.463135:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.463136:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.463137:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.463138:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.463138:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677475.463139:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.463139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.463140:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677475.463140:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677475.463141:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.463141:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.463141:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.463143:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.463143:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.463143:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.463143:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.463143:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677475.463144:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.463145:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.463146:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.463146:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.463147:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.463147:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.463148:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.463149:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.463149:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.463150:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.463150:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.463151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.463155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.463163:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.463165:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.463166:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.463167:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.463169:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.463172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.463174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.463181:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.463184:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.463185:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.463185:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.463186:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.463187:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.463882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:26.0:1619677475.463890:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677475.463893:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:10.0:1619677475.463894:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:26.0:1619677475.463896:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.463898:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.463900:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677475.463901:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.463903:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.463904:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.463904:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677475.463908:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.463909:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677475.463911:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.463912:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677475.463912:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677475.463914:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677475.463915:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677475.463916:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:26.0:1619677475.463916:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:25.2:1619677475.463921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.463928:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.463931:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.463931:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.463933:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.463935:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.463936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.463939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.463946:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.463948:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.463949:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.463950:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.463950:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.463951:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.464469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.464476:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.464479:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.464480:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.464481:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.464483:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.464484:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.464486:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.464487:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.464488:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.464488:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:21.2:1619677475.464490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:10.0:1619677475.464491:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.464493:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.464495:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.464496:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.464497:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677475.464498:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.464498:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.464499:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:30.0:1619677475.464500:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677475.464500:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.464501:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:30.0:1619677475.464502:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.464502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:30.0:1619677475.464505:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.464505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677475.464506:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.464507:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.464508:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499804] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677475.464508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677475.464509:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.464510:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.464510:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677475.464513:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:30.0:1619677475.464514:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.464515:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677475.464515:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.464516:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.464516:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:30.0:1619677475.464517:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.464517:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.464518:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.464518:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677475.464518:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.464518:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.464519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.464519:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.464520:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.464520:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.464522:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677475.464522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677475.464523:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.464524:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.464525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.464530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.464534:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.464536:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.464536:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.464538:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.464538:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.464539:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.464539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.464540:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.464541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.464542:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.464548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.464554:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.464557:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.464558:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.464559:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.464559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.464560:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.464682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.464687:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.464690:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.464690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.464692:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.464695:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.464696:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.464697:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.464699:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.464700:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.464701:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.464704:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.464706:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.464708:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.464709:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.464710:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.464711:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.464712:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.464713:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.464713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.464717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.464725:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.464727:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.464728:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.464730:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.464731:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.464732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.464857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.2:1619677475.464862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677475.464864:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.464867:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.0:1619677475.464867:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677475.464870:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.464870:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677475.464871:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:20.0:1619677475.464872:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677475.464873:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.464873:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.464874:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.464875:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.464876:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.464876:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:30.0:1619677475.464879:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.464879:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.464881:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.464881:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.464882:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.464884:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.464884:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.464890:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.464890:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.464893:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.464893:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677475.464896:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.464896:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.464897:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.464897:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:30.0:1619677475.464898:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.464898:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.464900:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.464900:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677475.464901:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677475.464901:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677475.464902:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677475.464902:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677475.464903:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677475.464903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677475.464906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.2:1619677475.464907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677475.464912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677475.464913:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.464916:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677475.464916:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677475.464917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.464917:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.464918:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.464919:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.464920:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677475.464920:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:6.0:1619677475.464920:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.464922:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:1.0:1619677475.464922:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.464923:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.464923:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:1.0:1619677475.464925:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:10.0:1619677475.464926:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.464928:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.464930:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.464931:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.464932:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.464933:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.464934:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.464938:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.464940:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.464943:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.464943:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.464944:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.464946:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.464946:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.464947:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.464948:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.464952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.464959:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.464962:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.464963:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.464963:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.464966:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.464968:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.465455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.465463:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.465466:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.465468:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.465470:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.465471:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.465472:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.465473:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.465475:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.465476:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.465477:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.465480:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.465482:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.465484:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.465484:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.465485:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.465486:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.465487:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.465488:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.465488:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.465493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.465499:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.465502:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.465503:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.465504:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.465506:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.465507:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.465818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.465825:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.465828:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.465828:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.465831:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.465834:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.465835:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.465837:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.465838:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.465839:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.465840:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.465843:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.465845:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.465847:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.465848:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.465849:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.465850:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.465851:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.465852:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.465852:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.465857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.465865:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.465868:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.465869:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.465869:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.465871:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.465873:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.465877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.465884:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.465886:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.465887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.465887:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.465888:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.465889:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.466536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.466544:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.466547:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.466547:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.466548:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677475.466549:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.466550:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.466553:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.466553:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.466555:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.466556:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.466558:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.466559:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.466560:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.466562:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.466563:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.466564:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.466565:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.466565:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.466566:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.466566:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.466570:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677475.466570:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.466571:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.466572:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.466572:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.466574:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.466575:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (23)++
00000800:00000200:8.0:1619677475.466575:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.466576:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (24)--
00000800:00000200:8.0:1619677475.466576:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.466577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.466577:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.466578:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.466579:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.466580:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.466580:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.466581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.466581:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677475.466584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.2:1619677475.466585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.466590:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.466591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.466592:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.466593:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:17.0:1619677475.466593:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:6.0:1619677475.466593:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.466595:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.466595:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.466596:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.466596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.466597:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.466599:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.466599:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.466599:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.466600:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:22.0:1619677475.466601:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:6.0:1619677475.466602:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.466603:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.466603:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:22.0:1619677475.466604:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.466605:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.466607:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:4.2:1619677475.466608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:22.0:1619677475.466610:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.466612:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.466612:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:22.0:1619677475.466614:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.466614:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:22.0:1619677475.466615:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.466616:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677475.466616:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.466617:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.466617:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:22.0:1619677475.466618:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.466619:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:20.0:1619677475.466619:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.466620:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677475.466620:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:22.0:1619677475.466621:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.466622:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.466623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677475.466623:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.466624:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.466625:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677475.466626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:20.0:1619677475.466627:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.466629:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.466631:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.466631:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.466632:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.466633:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.466633:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677475.466633:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.466634:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.466634:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:6.0:1619677475.466634:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677475.466635:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.466635:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:6.0:1619677475.466635:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.466636:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.466636:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:21.0:1619677475.466637:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.466638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.466639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677475.466640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.466647:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.466650:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.466650:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.466652:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.466653:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.466654:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.466852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.466860:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.466863:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.466865:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.466867:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.466868:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.466868:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.466869:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.466871:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.466872:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.466873:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.466876:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.466877:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.466880:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.466881:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.466881:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.466883:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.466883:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.466884:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.466885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.466889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.466895:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.466897:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.466898:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.466899:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.466900:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.466903:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.467416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.467424:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.467427:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.467429:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.467432:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.467432:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.467433:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.467434:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.467436:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.467436:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.467437:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.467440:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.467442:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.467445:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.467446:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.467446:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.467448:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.467448:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.467450:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.467451:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.467452:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.467455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.467458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.467463:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.467466:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.467466:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.467467:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.467467:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.467469:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677475.467469:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.467470:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.467471:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.467472:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.467474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.467481:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.467484:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.467485:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.467485:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.467486:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.467486:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.467638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.467647:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.467649:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.467650:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.467652:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.467655:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.467656:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.467657:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.467658:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.467659:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.467660:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.467663:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.467665:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.467667:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.467668:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.467669:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.467670:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.467671:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.467672:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.467673:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.467673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.467676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.467679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.467684:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.467686:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.467687:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.467688:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677475.467688:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.467688:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.467689:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.467690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677475.467691:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.467694:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.467707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.467715:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.467717:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.467719:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.467721:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.467722:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.467724:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.467725:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.467726:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.467727:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.467728:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.467731:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.467733:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.467735:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.467735:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.467736:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.467737:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.467738:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.467739:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.467739:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.467743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.467749:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.467752:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.467753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.467753:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.467755:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.467757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.468056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.468062:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.468064:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.468067:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.468067:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.468069:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.468071:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.468072:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.468073:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.468074:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.468075:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.468079:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.468080:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.468082:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.468083:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.468084:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.468085:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.468086:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.468086:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.468087:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.468091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.468099:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.468101:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.468102:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.468103:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.468104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.468106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.468277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.468284:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.468289:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.468291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.468302:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.468305:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.468307:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.468310:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.468312:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.468314:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.468315:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.468320:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.468323:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.468326:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.468327:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.468328:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.468330:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.468331:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.468332:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.468333:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.468337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.468346:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.468350:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.468351:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.468352:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.468354:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677475.468356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.468360:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.468362:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.468365:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.468366:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.468366:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.468368:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.468369:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.468473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.468482:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.468485:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.468487:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.468489:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.468490:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.468491:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.468492:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.468494:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.468495:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.468496:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.468499:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.468501:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.468503:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.468504:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.468504:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.468506:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.468507:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.468508:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.468510:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.468510:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.468512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.468516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.468520:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.468522:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.468522:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.468524:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.468524:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.468525:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.468525:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.468526:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.468527:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.468528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.468538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.468544:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.468546:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.468548:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.468549:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.468552:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.468554:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677475.468555:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.468556:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.468557:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.468559:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:22.0:1619677475.468561:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.468563:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:22.0:1619677475.468566:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.468566:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:22.0:1619677475.468567:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677475.468569:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:22.0:1619677475.468569:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.468570:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677475.468571:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:21.2:1619677475.468571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.468573:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.468574:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.468575:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:8.2:1619677475.468575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.468576:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.468577:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.468577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000400:00000200:22.0:1619677475.468580:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.468582:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.468583:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.468584:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677475.468584:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000400:00000200:22.0:1619677475.468585:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.468585:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.468585:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000800:00000200:6.0:1619677475.468585:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.468586:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.468586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:17.0:1619677475.468587:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.468588:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.468588:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.468590:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.468591:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.468593:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677475.468594:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.468596:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.468597:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.468597:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.468599:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.468599:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.468600:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.468601:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.468605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.468613:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.468615:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.468616:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.468618:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.468619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.468620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.468622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.468629:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.468632:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.468633:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.468633:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.468634:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.468635:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.468659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.468667:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.468670:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.468672:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.468673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.468674:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.468676:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.468677:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.468678:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.468679:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.468680:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.468684:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.468686:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.468689:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.468690:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.468690:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.468692:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.468693:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.468694:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.468694:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.468698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.468706:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.468708:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.468709:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.468710:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.468712:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.468715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.469301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.469308:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.469311:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.469312:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.469314:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.469317:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.469318:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.469319:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499783] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.469320:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.469321:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.469322:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.469325:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.469327:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.469330:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.469330:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.469331:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.469333:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.469333:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.469334:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.469334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.469339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.469347:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.469350:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.469351:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.469351:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.469353:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.469356:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.469777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.469777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.469785:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677475.469786:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.469787:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:30.0:1619677475.469788:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677475.469788:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.469791:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.469792:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.469793:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677475.469794:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.469795:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.469795:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.469797:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677475.469797:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.469798:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.469799:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.469800:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.469800:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.469801:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.469802:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677475.469802:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.469806:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.469807:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.469808:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.469809:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.469811:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.469811:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677475.469811:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.469812:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677475.469812:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.469813:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.469814:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.469814:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.469814:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.469815:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677475.469815:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:21.2:1619677475.469816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:10.0:1619677475.469816:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.469816:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677475.469819:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.469820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.469821:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.469823:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:25.2:1619677475.469823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.469824:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.469825:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.469825:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.469826:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.469827:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.469827:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.469829:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.469830:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96499780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.469830:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677475.469832:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.469833:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.469833:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.469833:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.469833:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.469834:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.469835:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.469837:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677475.469837:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.469838:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:30.0:1619677475.469838:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.469840:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.469842:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.469844:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.469845:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.469845:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.469847:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.469847:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.469848:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.469849:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.469853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.469860:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.469862:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.469863:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.469863:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.469866:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.469868:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.470081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.470086:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.470089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.470089:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.470092:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.470093:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.470094:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.470096:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.470097:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.470098:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.470099:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.470102:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.470104:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.470107:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.470108:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.470108:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.470110:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.470111:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.470111:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.470112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.470116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.470123:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.470126:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.470126:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.470128:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.470129:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.470130:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.470689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.470699:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.470703:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.470707:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.470709:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.470711:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.470713:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:13.0:1619677475.470716:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.470718:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.470720:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.470722:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:13.0:1619677475.470727:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.470730:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677475.470734:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.470735:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:13.0:1619677475.470736:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677475.470737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:13.0:1619677475.470738:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677475.470739:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.470740:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677475.470741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:21.0:1619677475.470743:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.470745:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.470748:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.470748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.470750:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.470751:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.470753:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.470754:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.470755:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.470756:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.2:1619677475.470756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:21.0:1619677475.470760:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.470762:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.470764:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.470765:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.470765:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:21.0:1619677475.470766:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.470767:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:21.0:1619677475.470768:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.470768:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677475.470768:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:21.0:1619677475.470769:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:1.0:1619677475.470769:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.470770:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677475.470770:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:21.0:1619677475.470771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677475.470771:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)++
00000400:00000200:1.0:1619677475.470771:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.470772:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (24)--
00000800:00000200:17.0:1619677475.470772:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.470774:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.2:1619677475.470774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.470775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.470776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677475.470776:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.470778:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:20.0:1619677475.470779:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:17.0:1619677475.470779:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.470780:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.470781:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:17.0:1619677475.470781:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.470782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.470782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.470783:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.471054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.471062:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.471065:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.471067:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.471069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.471070:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.471072:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.471073:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.471074:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.471075:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.471076:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.471079:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.471081:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.471084:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.471085:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.471086:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.471087:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.471088:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.471089:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.471090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.471094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.471103:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.471106:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.471106:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.471107:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.471109:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:21.2:1619677475.471111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.471112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.471115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.471119:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.471122:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677475.471123:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.471124:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.471126:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677475.471126:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.471127:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.471127:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.471127:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.471128:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677475.471129:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:6.0:1619677475.471129:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677475.471130:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.471131:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.471132:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.471133:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.471137:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.471138:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.471142:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.471143:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.471143:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.471145:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.471146:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.471146:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.471147:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.471151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.471159:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.471161:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.471162:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.471164:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.471165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.471167:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.471566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.471574:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.471577:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.471577:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.471579:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.471582:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.471584:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.471585:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.471586:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.471587:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.471588:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.471591:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.471593:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.471596:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.471597:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.471597:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.471599:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.471600:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.471601:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.471601:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677475.471606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.471614:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.471617:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.471618:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.471619:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.471621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.471623:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.471803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.471810:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.471812:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.471814:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.471815:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.471818:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.471820:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.471821:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.471822:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.471823:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.471824:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.471827:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.471829:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.471832:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.471833:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.471833:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.471835:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.471835:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.471836:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.471837:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.471842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.471850:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.471853:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.471854:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.471854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.471856:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.471859:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.471991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.471996:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.471998:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.471999:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.472001:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.472003:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.472004:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.472005:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.472006:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.472007:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.472008:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.472011:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.472013:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.472015:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.472016:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.472017:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.472018:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.472019:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.472019:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.472020:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.472024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.472031:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.472034:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.472034:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.472036:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.472037:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.472038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.472367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.472374:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.472376:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.472378:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.472379:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.472382:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.472383:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.472384:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.472385:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.472387:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.472388:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.472391:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.472393:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.472396:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.472397:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.472398:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.472400:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.472400:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.472401:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.472402:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.472406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.472414:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.472416:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.472417:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.472419:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.472421:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.472422:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.472447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.472455:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.472458:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.472460:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.472463:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.472463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.472464:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.472465:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.472467:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.472467:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.472468:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.472471:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.472473:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.472475:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.472476:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.472476:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.472478:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.472478:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.472480:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.472480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.472483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.472490:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.472492:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.472493:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.472495:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.472496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.472497:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.472503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.472510:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.472513:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.472514:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.472514:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.472515:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.472516:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.472783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.472788:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.472791:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.472793:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.472793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.472795:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.472797:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.472798:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.472800:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.472801:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.472802:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.472805:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.472807:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.472809:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.472810:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.472810:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.472812:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.472813:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.472814:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.472814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.472818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.472826:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.472828:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.472829:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.472829:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.472831:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.472834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.473026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.473034:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.473036:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.473038:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.473040:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.473041:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.473043:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677475.473044:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.473045:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.473046:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.473047:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677475.473050:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.473052:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677475.473054:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.473055:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677475.473056:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677475.473058:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677475.473058:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.473059:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677475.473059:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:21.2:1619677475.473062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:25.2:1619677475.473065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.473071:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.473072:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677475.473073:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.473074:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.473074:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.473074:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.473076:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.473076:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.473077:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.473078:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.473080:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.473080:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.473081:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.473084:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:30.0:1619677475.473084:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.473085:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.473086:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677475.473086:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.473089:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.473090:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.473090:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.473092:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.473093:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.473094:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.473096:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.473096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.473098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.473102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.473105:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.473108:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.473109:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.473109:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.473110:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677475.473111:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.473112:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.473113:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.473114:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.473114:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.473156:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.473164:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.473167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.473167:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.473169:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.473172:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.473174:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.473175:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.473176:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.473177:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.473178:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.473182:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.473183:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.473186:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.473187:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.473187:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.473189:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.473189:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.473190:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.2:1619677475.473191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.473191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.473195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677475.473199:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.473201:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.473203:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.473203:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:30.0:1619677475.473205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.473206:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.473206:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:31.0:1619677475.473207:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:17.0:1619677475.473207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.473207:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.473209:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.473210:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.473211:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.473211:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.473212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:31.0:1619677475.473213:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.473216:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.473218:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.473220:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.473221:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.473222:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.473223:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.473224:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.473225:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.473227:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.473227:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.473229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.473233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.473235:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.473238:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.473239:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.473239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.473240:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.473241:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.473243:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677475.473243:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.473244:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.473245:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.473617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.473623:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.473626:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.473626:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.473628:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.473631:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.473632:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.473633:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.473635:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.473636:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.473637:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.473640:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.473642:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.473645:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.473646:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.473646:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.473648:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.473649:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.473650:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.473651:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.473652:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.473655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.473657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.473662:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.473663:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.473665:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.473665:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.473666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.473666:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.473666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.473667:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.473668:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.473670:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.473747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.473752:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.473755:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.473757:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.473757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.473759:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.473761:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.473762:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.473764:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.473765:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.473766:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.473769:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.473771:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.473773:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.473774:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.473774:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.473776:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.473776:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.473777:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.473778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.473783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.473790:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.473792:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.473793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.473795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.473795:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.473797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.473803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.473811:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.473814:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.473814:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.473815:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.473815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.473817:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.474135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.474142:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.474145:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.474146:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.474148:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.474151:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.474152:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.474153:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.474154:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.474156:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.474157:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.474160:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.474161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677475.474162:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.474164:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.474165:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.474166:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.474168:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.474168:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.474169:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.474169:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677475.474170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.474172:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.474174:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.474174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:30.0:1619677475.474176:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.474177:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.474178:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.474179:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499761] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.474180:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.474181:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.474181:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:30.0:1619677475.474182:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677475.474184:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:30.0:1619677475.474185:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.474185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.474185:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.474186:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677475.474187:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.474188:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.474189:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.474190:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677475.474190:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677475.474191:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.474192:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.474193:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.474193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.474197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:25.2:1619677475.474201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:31.0:1619677475.474204:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.474206:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.474207:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.474208:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.474208:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677475.474210:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677475.474210:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:31.0:1619677475.474211:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677475.474211:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.474212:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.474213:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.474214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.474228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.474235:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.474237:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.474238:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.474240:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.474242:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.474244:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:17.0:1619677475.474245:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.474247:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.474248:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.474249:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677475.474252:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.474253:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:17.0:1619677475.474256:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.474257:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:17.0:1619677475.474257:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677475.474259:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677475.474260:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.474260:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.474261:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:25.2:1619677475.474265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.474273:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.474275:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.474276:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.474277:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.474278:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.474280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.474289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.474297:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.474299:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.474300:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.474301:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.474302:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.474303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.474596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.474603:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.474606:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.474607:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.474609:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.474612:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.474613:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.474615:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.474616:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.474617:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.474618:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.474621:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.474623:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.474626:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.474627:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.474627:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.474629:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.474629:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.474630:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.474631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.474635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.474644:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.474646:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.474647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.474648:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.474650:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.474652:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.474655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.474662:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.474664:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.474665:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.474665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.474666:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.474667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.474676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.474681:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.474683:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.474684:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.474686:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.474688:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.474689:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.474690:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.474691:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.474692:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.474694:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.474697:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.474698:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.474701:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.474701:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.474702:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.474704:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.474704:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.474705:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.474705:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.474710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.474717:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.474719:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.474720:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.474722:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.474722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.474724:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.474839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.474847:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.474849:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.474852:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.474853:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.474854:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.474855:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.474857:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.474858:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.474859:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.474860:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.474864:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.474865:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.474869:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.474869:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.474870:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.474872:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.474872:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.474873:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.474874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.474878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.474885:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.474888:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.474888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.474889:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.474891:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.474894:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.474920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.474924:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.474926:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.474928:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.474929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.474930:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.474931:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.474932:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.474934:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.474935:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.474936:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.474939:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.474941:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.474944:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.474945:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:4.2:1619677475.474945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:20.0:1619677475.474946:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.474948:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.474949:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.474950:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.474951:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:12.0:1619677475.474954:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:8.2:1619677475.474954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:12.0:1619677475.474957:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.474960:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.474962:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:13.0:1619677475.474962:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.474962:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.474964:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:12.0:1619677475.474964:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.474965:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.474966:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000400:00000200:12.0:1619677475.474966:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.474967:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:12.0:1619677475.474968:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.474969:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:12.0:1619677475.474970:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.474972:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677475.474977:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.474979:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677475.474983:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.474985:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677475.474986:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677475.474988:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677475.474989:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.474990:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.474991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.474994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.475003:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.475006:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.475006:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.475007:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.475009:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.475013:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.475055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.475063:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.475066:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.475068:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.475070:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.475070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.475072:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.475073:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.475074:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.475075:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.475076:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.475078:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.475080:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.475082:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.475083:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.475083:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.475085:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.475085:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.475086:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.475087:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.475090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.475097:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.475099:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.475100:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.475101:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.475103:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.475104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.475739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.475745:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.475748:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.475749:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.475750:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.475753:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.475754:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.475756:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.475757:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.475758:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.475759:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.475762:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.475764:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.475766:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.475767:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.475767:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.475769:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.475770:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.475771:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.475771:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.475776:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.475784:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.475786:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.475786:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.475787:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.475788:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.475790:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.475918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.475924:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.475927:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.475928:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.475930:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.475932:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.475933:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.475934:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.475936:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.475936:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.475938:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.475941:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.475943:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.475945:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.475946:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.475947:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.475948:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.475949:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.475949:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.475950:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.475954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.475962:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.475964:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.475965:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.475966:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.475968:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.475968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.476459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.476465:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.476468:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.476469:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.476470:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.476473:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.476475:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677475.476476:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.476477:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.476478:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.476479:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677475.476483:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.476485:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677475.476488:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.476489:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677475.476490:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677475.476492:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677475.476492:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.476493:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.476494:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677475.476498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.476506:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.476508:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.476509:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.476510:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.476512:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.476515:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677475.476640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.476645:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.476648:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.476650:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.476650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.476652:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.476654:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.476655:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.476656:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.476657:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.476658:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.476662:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.476664:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.476666:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.476667:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.476667:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.476669:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.476669:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.476670:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.476671:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.476675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.476682:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.476685:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.476686:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.476686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.476688:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.476690:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.476778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.476784:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.476788:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.476788:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:21.2:1619677475.476790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:1.0:1619677475.476790:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.476793:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.476794:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.476796:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.476797:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.476798:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677475.476798:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.476799:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677475.476801:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677475.476803:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.476804:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.476805:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677475.476806:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.476806:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.476808:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677475.476808:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.476809:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.476809:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.476809:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:30.0:1619677475.476810:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.476811:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.476811:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:30.0:1619677475.476812:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677475.476812:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.476813:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.476813:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:30.0:1619677475.476816:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.476818:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.2:1619677475.476818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677475.476821:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.476822:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.476822:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.476824:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.476825:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.476825:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.476826:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.476826:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.476828:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.476829:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.476830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:20.0:1619677475.476831:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.476833:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.476834:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.476834:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677475.476837:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.476837:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.476839:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.476840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.476842:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.476987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.476994:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.476997:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.476999:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.477001:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.477002:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.477004:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.477005:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.477006:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.477007:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.477008:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.477011:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.477013:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.477014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.477016:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.477017:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.477017:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.477019:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.477019:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.477020:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.477020:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.477021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.477022:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.477023:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.477025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677475.477025:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.477027:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.477029:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.477030:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.477033:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:6.0:1619677475.477033:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.477034:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.477035:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.477035:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.477036:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.477038:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.477040:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.477041:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.477042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.477042:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.477045:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.477046:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.477046:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.477048:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.477048:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.477049:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.477049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.2:1619677475.477052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677475.477054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.477058:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.477060:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.477062:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:20.0:1619677475.477062:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.477063:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.477064:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.477064:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.477066:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:20.0:1619677475.477066:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.477068:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.477069:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.477070:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.477071:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.477073:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.477074:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.477074:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.477075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.477076:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.477078:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.477079:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.477079:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.477081:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.477082:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.477083:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.477084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.477088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.477095:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.477098:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.477099:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.477099:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.477101:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.477103:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.477107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.477113:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.477116:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.477117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.477117:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.477118:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.477119:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.477475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.477484:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.477486:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.477487:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.477489:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.477492:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.477493:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.477494:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.477496:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.477497:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.477498:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.477501:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.477503:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.477505:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.477506:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.477507:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.477508:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.477509:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.477510:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.477510:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.477515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.477524:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.477526:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.477527:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.477528:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.477530:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.477532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.477535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.477536:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.477538:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.477539:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.477539:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.477540:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.477540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.477757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.477766:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.477768:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.477771:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.477773:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.477773:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.477775:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.477776:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.477778:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.477779:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.477780:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.477783:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.477785:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.477787:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.477787:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.477788:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.477789:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.477790:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.477791:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.477793:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.477793:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.477795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.477799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.477803:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.477805:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.477806:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.477807:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.477807:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.477808:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.477809:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.477809:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.477810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.477811:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.478020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.478025:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.478028:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.478029:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.478031:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.478033:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.478034:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.478035:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.478037:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.478038:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.478039:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.478042:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.478044:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.478046:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.478047:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.478048:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.478049:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.478050:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.478051:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.478052:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.478052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.478055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.478057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.478062:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.478064:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.478064:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.478065:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.478066:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.478067:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.478067:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.478069:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.478071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.478072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.478075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.478082:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.478085:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.478086:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.478086:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.478087:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.478089:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.478372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.478379:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.478382:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.478384:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.478387:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.478387:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.478388:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.478389:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.478390:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.478392:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.478393:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.478396:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.478397:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.478400:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.478401:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.478401:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.478403:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.478403:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.478404:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.478405:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.478409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.478416:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.478418:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.478419:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.478419:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.478421:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.478423:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.478429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.478435:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.478437:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.478438:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.478438:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.478439:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.478440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.478479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.478486:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.478489:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.478490:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.478492:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.478494:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.478496:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.478497:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.478499:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.478500:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.478501:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.478504:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.478506:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.478509:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.478510:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.478510:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.478512:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.478513:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.478514:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.478514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.478518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.478525:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.478528:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.478529:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.478531:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.478532:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.478534:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.478563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.478570:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.478573:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.478573:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.478576:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.478578:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.478579:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677475.478581:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.478582:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.478583:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.478584:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.478587:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.478589:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.478592:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.478592:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677475.478593:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677475.478595:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677475.478595:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.478596:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.478596:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677475.478602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.478608:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.478611:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.478612:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.478612:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.478614:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.478617:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.478928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.478935:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.478938:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.478939:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.478940:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.478943:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.478944:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.478946:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.478947:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.478948:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.478949:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.478952:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.478954:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.478957:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.478958:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.478959:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.478960:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.478961:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.478961:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.478962:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.478966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.478974:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.478977:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.478978:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.478978:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.478980:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:4.2:1619677475.478982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:22.0:1619677475.478983:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:30.0:1619677475.478990:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.478992:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.478994:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.478997:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.478997:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.478998:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.479000:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.479001:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.479002:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.479003:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.479006:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.479007:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.479010:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.479010:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.479011:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.479012:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.479013:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.479014:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677475.479014:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.479018:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.479025:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.479027:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.479028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.479028:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.479030:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.479032:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.479179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.479187:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.479189:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.479191:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.479193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.479194:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.479195:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.479196:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.479197:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.479198:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.479199:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.479202:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.479204:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.479207:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.479208:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.479208:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.479210:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.479210:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.479211:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.479212:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.479213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:25.2:1619677475.479216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.479220:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.479222:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.479223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677475.479223:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:24.0:1619677475.479225:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.479227:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.479227:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:24.0:1619677475.479228:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.479228:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.479230:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677475.479231:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:24.0:1619677475.479232:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.479233:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.479233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:24.0:1619677475.479234:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.479235:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.479238:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.479239:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.479242:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.479243:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.479244:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.479245:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.479246:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.479247:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.479248:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.479249:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.2:1619677475.479251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.2:1619677475.479251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.479255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.479256:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.479259:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.479260:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:1.0:1619677475.479260:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:22.0:1619677475.479261:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.479262:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:22.0:1619677475.479263:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.479263:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.479264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.479264:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677475.479264:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.479265:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.479265:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.0:1619677475.479265:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677475.479266:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.479267:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.479268:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.479269:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.479270:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677475.479270:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:22.0:1619677475.479274:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.479275:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.479277:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.479278:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.479279:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.479280:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.479281:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.479282:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.479282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.479286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.479300:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.479302:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.479303:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.479305:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.479306:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.479307:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.479463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.479471:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.479474:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.479475:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.479477:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.479479:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.479481:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677475.479482:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677475.479484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677475.479484:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.479485:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.479486:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.479487:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.479490:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:10.0:1619677475.479490:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.479492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677475.479492:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.479493:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.479495:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.479495:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.479496:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:20.0:1619677475.479497:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677475.479497:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:20.0:1619677475.479498:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.479498:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:20.0:1619677475.479499:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.479499:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000400:00000200:20.0:1619677475.479500:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677475.479500:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.479500:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:20.0:1619677475.479501:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.479504:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677475.479505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.479506:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.479508:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.479509:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.479510:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.479511:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.479512:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.479512:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.479513:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.479514:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.479517:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:8.2:1619677475.479517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.479517:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.479518:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.479520:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.479522:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677475.479525:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.479528:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.479528:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.479530:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.479531:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.479532:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.479638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.479646:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.479649:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.479651:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.479653:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.479654:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.479655:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.479656:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.479658:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.479659:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.479660:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.479663:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.479665:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.479667:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.479668:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.479668:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.479670:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.479670:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.479671:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.479672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.479676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.479683:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.479685:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.479686:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.479688:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.479689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.479690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.479693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.479699:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.479702:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.479702:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.479703:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.479704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.479704:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.479915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.479922:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.479925:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.479926:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.479928:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.479930:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.479932:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.479933:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.479934:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.479935:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.479936:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.479938:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.479940:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.479943:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.479944:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.479944:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.479946:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.479946:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.479947:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.479949:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.479949:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.479952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.479955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.479960:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.479962:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.479963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.479963:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677475.479964:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677475.479964:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.479965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677475.479965:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677475.479966:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.479969:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.480039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677475.480044:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.480047:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677475.480048:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677475.480051:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.480054:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.480055:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677475.480056:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.480058:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.480059:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.480060:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677475.480063:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.480064:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677475.480067:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.480068:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677475.480069:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677475.480071:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677475.480071:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.480073:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677475.480074:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677475.480074:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677475.480077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.2:1619677475.480079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:10.0:1619677475.480083:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:18.0:1619677475.480085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:10.0:1619677475.480085:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.480086:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677475.480087:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000400:00000200:10.0:1619677475.480087:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.480088:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:10.0:1619677475.480089:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.480089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:10.0:1619677475.480092:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.480411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.480417:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.480419:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.480420:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.480422:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.480425:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.480426:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.480427:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.480429:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.480430:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.480431:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.480434:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.480436:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.480439:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.480439:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.480440:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.480442:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.480442:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.480442:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.480443:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.480448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.480456:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.480459:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.480460:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.480461:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.480463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.480464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.480465:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.480473:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.480475:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.480478:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.480480:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.480480:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.480481:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.480482:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.480483:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.480485:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.480485:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.480489:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.480490:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:4.2:1619677475.480491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:31.0:1619677475.480492:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.480493:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.480494:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.480495:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.480496:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.480497:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.480497:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677475.480498:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.480501:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:25.2:1619677475.480501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677475.480501:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.480503:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.480506:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.480507:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677475.480508:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:26.0:1619677475.480508:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677475.480508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:26.0:1619677475.480509:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.480510:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:26.0:1619677475.480510:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.480511:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.480511:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.480513:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.480513:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:31.0:1619677475.480514:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.480515:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:26.0:1619677475.480515:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.480515:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:26.0:1619677475.480518:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.480518:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.480519:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:26.0:1619677475.480521:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.480521:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.480522:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.480522:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:21.0:1619677475.480523:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:26.0:1619677475.480524:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.480524:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.480525:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677475.480525:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677475.480526:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:21.0:1619677475.480526:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.480527:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.480528:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.480531:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.2:1619677475.480531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.480533:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.480536:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.480537:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.480538:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.480538:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.480540:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.480540:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.480540:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.480541:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.480541:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:20.0:1619677475.480541:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.480542:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.480543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.480545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677475.480545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677475.480550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677475.480556:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677475.480558:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677475.480558:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.480559:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.480560:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677475.480560:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.480561:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.480562:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000400:00000200:8.0:1619677475.480562:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.480563:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.480563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.0:1619677475.480565:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.481269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.481276:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.481279:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.481282:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.481283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.481284:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.481285:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.481287:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.481288:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.481289:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.481290:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.481298:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.481300:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.481303:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.481303:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.481304:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.481306:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.481306:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.481308:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.481309:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.481310:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.481312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.481316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.481320:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.481323:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.481324:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.481324:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.481324:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.481326:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.481327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:6.0:1619677475.481327:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.481328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.481330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.481459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.481465:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.481468:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.2:1619677475.481469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:17.0:1619677475.481469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.481471:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.481473:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.481474:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:1.0:1619677475.481475:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.481476:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.481477:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.481478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:1.0:1619677475.481478:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.481479:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.481480:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677475.481481:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.481483:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.481483:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.481485:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:1.0:1619677475.481485:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677475.481486:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.481488:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.481488:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.481488:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:22.0:1619677475.481489:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.481489:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:22.0:1619677475.481490:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677475.481490:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.481491:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.481491:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.481492:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677475.481493:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.481495:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.2:1619677475.481497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677475.481498:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.481499:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.481499:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.481501:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.481501:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.481502:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.481502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:17.0:1619677475.481505:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:25.2:1619677475.481507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677475.481507:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.481508:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.481509:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.481510:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.481512:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.0:1619677475.481515:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.481517:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.481518:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.481519:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.481521:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.481522:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.481839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.481847:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.481850:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.481851:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677475.481854:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.481854:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.481856:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677475.481857:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.481858:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.481859:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.481860:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677475.481864:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.481866:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677475.481868:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.481869:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677475.481870:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677475.481871:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677475.481872:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.481873:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677475.481875:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677475.481876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677475.481877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677475.481881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:10.0:1619677475.481884:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.481886:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.481887:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677475.481887:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.481889:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:10.0:1619677475.481889:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.481891:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:10.0:1619677475.481891:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:12.0:1619677475.481892:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.481893:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.481896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677475.481901:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.481904:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.481905:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.481905:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.481906:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.481906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.482107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.482114:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.482117:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.482119:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.482121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.482122:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.482123:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.482124:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.482125:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.482126:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.482127:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.482130:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.482132:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.482134:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.482134:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.482135:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.482136:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.482137:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.482138:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.482140:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.482140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.482142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.482146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.482150:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.482152:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.482153:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.482154:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.482155:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.482155:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.482156:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.482157:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.482158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.482159:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.482696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.482702:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.482704:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.482705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.482707:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.482709:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.482710:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.482712:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.482713:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.482714:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.482715:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.482718:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.482720:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.482722:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.482723:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.482724:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.482725:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.482726:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.482727:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.482728:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.482729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.482732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.482734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.482739:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.482740:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.482742:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.482742:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.482742:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.482743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.482743:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.482745:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.482746:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.482747:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.482751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.482758:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.482760:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.482761:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.482762:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.482763:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.482763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.482830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.482837:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.482840:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.482841:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.482843:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.482846:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.482847:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.482848:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.482850:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.482851:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.482852:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.482855:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.482856:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.482859:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.482860:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.482860:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.482862:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.482862:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.482863:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.482864:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.482868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.482876:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.482878:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.482879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.482880:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.482882:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.482885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.482888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.482895:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.482898:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.482898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.482899:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.482900:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.482900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.483020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.483028:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.483030:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.2:1619677475.483031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:31.0:1619677475.483033:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.483034:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.483036:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677475.483036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.483037:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.483037:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.483038:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.483039:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.483040:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.483041:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.483043:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.483044:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.483045:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.483046:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.483046:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.483048:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.483049:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.483050:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.483051:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.483054:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.483056:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.483058:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.483058:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.483059:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.483059:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.483060:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.483060:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.483061:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.483061:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.483062:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.483062:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.483063:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.483063:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.483064:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:31.0:1619677475.483065:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.483065:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.483067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677475.483071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:6.0:1619677475.483074:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.483077:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:6.0:1619677475.483077:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.483078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.483078:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.483080:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:6.0:1619677475.483080:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.483081:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.483083:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.483083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677475.483084:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.483085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.483259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.483265:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.483268:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.483269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.483271:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.483274:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.483276:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.483277:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.483278:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.483279:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.483280:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.483284:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.483285:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.483288:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.483289:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.483290:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.483292:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.483292:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:25.2:1619677475.483298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.483299:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.483300:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.483301:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:31.0:1619677475.483305:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.2:1619677475.483306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677475.483308:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.483309:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.483311:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.483312:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.483312:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677475.483314:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:2.0:1619677475.483314:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.483315:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.483316:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.2:1619677475.483358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.483364:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.483366:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.483368:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.483369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.483371:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.483373:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:4.2:1619677475.483373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:21.0:1619677475.483374:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.483375:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.483376:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.483376:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:21.0:1619677475.483377:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.483377:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.483381:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677475.483381:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.483383:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.483384:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.483385:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.483386:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.483387:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.483388:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.483389:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.483393:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.483394:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.483396:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.483396:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.483397:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.483397:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.483398:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.483398:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.483400:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.483400:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.483401:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.483401:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.483402:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.483403:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.483403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.483404:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.483404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.483404:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.483407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.483409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677475.483411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.483415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677475.483415:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:20.0:1619677475.483416:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.483417:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.483418:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.483418:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.483419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.483419:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.483420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677475.483420:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.483420:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.483421:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.483422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.483423:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.483424:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677475.483425:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.483427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.483435:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.483437:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.483438:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.483439:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.483439:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.483440:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.483657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.483663:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.483666:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.483667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.483668:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.483671:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.483673:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.483674:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.483675:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.483676:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.483677:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.483680:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.483682:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.483684:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.483685:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.483686:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.483687:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.483688:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.483688:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.483689:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.483694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.483701:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.483703:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.483704:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.483706:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.483707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.483708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.483819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.483827:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.483830:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.483831:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.483833:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.483835:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.483837:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.483838:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.483839:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.483840:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.483841:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.483845:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.483847:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.483850:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.483850:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.483851:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.483853:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.483853:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.483855:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.483856:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.483856:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.483859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.483862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.483866:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677475.483869:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.483869:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.483870:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.483870:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.483872:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.483872:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677475.483872:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.483874:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677475.483875:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.484144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.484150:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.484153:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.484154:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.484156:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.484158:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.484160:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.484161:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.484162:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.484163:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.484164:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.484168:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.484170:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.484171:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.484172:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.484173:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.484174:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.484174:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.484175:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.484176:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.484177:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.484180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.484183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.484189:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.484190:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.484190:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.484190:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.484190:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.484191:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.484192:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.484192:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.484193:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677475.484193:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.484211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.484219:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.484222:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.484224:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.484225:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.484226:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.484227:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.484228:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.484229:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.484230:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.484232:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.484235:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.484237:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.484240:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.484240:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.484241:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.484243:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.484243:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.484244:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.484245:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.484248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.484255:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.484257:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.484258:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.484258:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.484260:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.484263:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.484657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.484663:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.484665:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.484666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.484668:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.484670:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.484671:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.484672:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.484674:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.484675:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.484676:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.484679:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.484681:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.484684:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.484684:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.484685:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.484687:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.484687:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.484688:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.484688:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.484693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.484702:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.484704:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.484705:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.484706:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.484708:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.484709:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.484754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.484762:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.484765:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.484765:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677475.484767:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.484770:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.484772:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.484773:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.484774:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.484775:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.484776:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.484780:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.484782:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.484784:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.484785:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.484786:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.484788:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.484788:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.484789:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.484789:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.484794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.484801:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.484804:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.484805:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.484805:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.484807:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.484810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.484900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.484908:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.484910:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.484911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.484912:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.484916:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.484918:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.484919:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.484920:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.484921:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.484922:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.484925:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.484926:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.484929:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.484930:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.484930:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.484932:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.484932:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.484933:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.484934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.484938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.484947:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.484949:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.484950:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.484951:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.484953:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.484955:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.484985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.484994:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.484997:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.484999:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.485002:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.485002:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.485003:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.485005:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.485006:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.485007:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.485008:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.485012:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.485014:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.485016:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.485016:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.485017:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.485019:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.485019:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.485020:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.485021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.485025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.485031:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.485034:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.485035:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.485036:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.485038:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.485039:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.485047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.485054:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.485056:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.485057:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.485058:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.485058:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.485059:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.485099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.485106:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.485109:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.485110:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.485113:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.485113:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.485114:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.485115:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.485116:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.485117:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.485118:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.485121:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.485122:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.485125:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.485126:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.485126:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.485128:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.485128:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.485129:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.485129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.485134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.485141:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.485144:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.485145:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.485146:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.485147:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.485150:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.485154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.485160:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.485162:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.485164:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.485165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.485167:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.485168:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.485170:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499703] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.485171:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.485172:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.485173:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.485175:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.485177:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.485180:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.485180:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.485181:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.485183:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.485183:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.485184:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.485185:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.485189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.485197:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.485199:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.485200:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.485201:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:5.2:1619677475.485202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.0:1619677475.485203:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.485205:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677475.485208:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.485211:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.485212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.485214:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.485215:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.485216:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.485218:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.485219:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.485220:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.485221:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.485224:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.485226:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.485227:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.485228:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.485229:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.485230:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.485231:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.485232:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.485233:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.485234:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.485236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.485239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.485245:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.485246:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.485246:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.485246:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.485246:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.485247:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.485248:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.485248:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.485249:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677475.485250:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.485255:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.485262:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.485265:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.485265:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.485266:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.485266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.485267:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.485828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.485833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.485836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.485836:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677475.485838:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.485840:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.485841:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.485842:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.485843:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.485844:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.485844:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.485847:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.485848:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.485851:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.485851:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.485852:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.485853:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.485854:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.485854:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.485855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.485860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.485869:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.485871:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.485872:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.485873:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.485875:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.485876:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.485949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.485955:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.485958:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.485959:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.485960:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.485963:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.485964:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.485965:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.485966:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.485967:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.485968:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.485971:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677475.485973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:6.0:1619677475.485973:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.485975:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.485976:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.485977:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.485978:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677475.485979:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:6.0:1619677475.485979:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.485980:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.485980:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:18.0:1619677475.485982:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.485983:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:25.2:1619677475.485985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677475.485985:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.485987:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.485988:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677475.485989:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.485991:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.485992:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.485993:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:18.0:1619677475.485993:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:20.0:1619677475.485995:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:18.0:1619677475.485995:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677475.485996:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.485997:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677475.485998:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.485998:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.486000:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677475.486001:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.486002:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677475.486003:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677475.486005:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677475.486005:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.486006:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677475.486007:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677475.486008:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677475.486011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.2:1619677475.486013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:26.0:1619677475.486018:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:2.0:1619677475.486019:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:26.0:1619677475.486021:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:2.0:1619677475.486021:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677475.486022:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:10.0:1619677475.486022:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:2.0:1619677475.486022:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:26.0:1619677475.486023:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.486025:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.486028:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677475.486039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677475.486045:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.486048:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.486049:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.486049:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.486050:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.486051:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677475.486221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677475.486230:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.486232:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677475.486235:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.486237:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677475.486237:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.486239:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677475.486240:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.486241:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.486242:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.486243:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.486246:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.486248:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.486251:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.486252:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677475.486253:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.486254:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677475.486255:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.486256:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:30.0:1619677475.486257:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:30.0:1619677475.486258:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:9.2:1619677475.486260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.486264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:2.0:1619677475.486268:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.486271:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:31.0:1619677475.486272:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:18.0:1619677475.486272:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.486272:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.486274:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000400:00000200:2.0:1619677475.486274:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.486275:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677475.486276:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:2.0:1619677475.486278:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.486285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.486292:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.486297:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.486298:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.486299:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.486299:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.486300:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.486307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.486315:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.486318:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.486321:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.486323:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.486323:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.486325:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.486326:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.486328:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.486329:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.486330:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.486332:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.486334:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.486336:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.486337:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.486338:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.486339:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.486340:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.486341:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.486343:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.486343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.486345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.486349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.486353:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.486355:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.486356:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.486356:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.486357:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.486358:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.486359:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.486360:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.486361:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677475.486361:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.486863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.486870:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.486872:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.486874:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.486877:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.486877:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.486879:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.486880:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.486881:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.486882:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.486883:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.486886:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.486887:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.486890:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.486891:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.486892:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.486893:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.486894:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.486895:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.486895:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.486899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.486908:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.486910:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.486911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.486912:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.486914:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.486917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.486919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.486926:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.486928:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.486929:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.486929:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.486930:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.486931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.487053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.487058:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.487060:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.487061:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.487063:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.487065:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.487067:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.487068:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.487069:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.487070:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.487071:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.487074:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.487076:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.487079:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.487080:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.487080:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.487082:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.487082:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.487083:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.487084:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.487085:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.487088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.487089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.487095:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.487096:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.487097:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.487097:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.487098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.487099:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.487100:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.487102:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.487103:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.487104:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.487115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.487122:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.487124:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.487125:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.487126:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.487126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.487128:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.487510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.487518:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.487521:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.487523:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.487526:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.487526:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.487528:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.487529:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.487531:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.487532:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.487533:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.487536:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.487538:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.487540:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.487541:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.487541:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.487543:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.487544:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.487545:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.487545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.487549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.487556:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.487559:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.487560:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.487561:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.487563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.487564:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.487658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.487665:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.487668:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.487669:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.487670:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.487673:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.487675:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.487676:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.487677:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.487678:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.487679:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.487682:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.487684:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.487686:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.487687:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.487688:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.487689:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.487690:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.487690:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.487691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.487695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.487703:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.487706:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.487707:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.487707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.487710:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.487712:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.487957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.487965:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.487968:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.487970:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.487971:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.487972:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.487973:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.487974:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.487976:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.487977:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.487978:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.487981:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.487982:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.487985:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.487986:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.487987:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.487988:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.487989:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.487991:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.487992:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.487993:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.487994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.487998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.488001:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.488004:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.488005:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.488005:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.488006:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:6.0:1619677475.488007:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.488008:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.488009:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:6.0:1619677475.488010:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.488011:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.488044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.488049:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.488051:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.488052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.488054:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.488056:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.488058:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.488059:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.488060:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.488061:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.488062:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.488066:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.488068:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.488070:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.488071:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.488071:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.488073:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.488074:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.488074:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.488075:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.488079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.488087:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.488090:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.488090:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.488092:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.488093:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.488094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.488568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.488576:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.488579:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.488579:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.488581:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.488583:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.488585:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.488586:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.488587:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.488588:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.488589:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.488593:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.488594:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.488597:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.488598:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.488599:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.488600:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.488601:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.488602:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.488602:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.488607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.488615:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.488618:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.488619:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.488619:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.488622:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.488625:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.488819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.488827:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.488830:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.488832:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.488835:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.488835:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.488836:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.488838:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.488839:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.488840:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.488841:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.488844:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.488845:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.488848:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.488849:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.488850:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.488852:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.488852:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.488853:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.488854:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.488859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.488867:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.488870:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.488871:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.488873:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.488874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.488876:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.489186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.489194:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.489197:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.489199:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.489200:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.489202:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.489204:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677475.489205:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.489206:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.489207:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.489208:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677475.489211:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.489213:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.489216:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.489217:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677475.489217:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677475.489219:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677475.489220:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.489220:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.489221:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.489225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.489233:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.489236:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.489237:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.489238:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.489240:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.489243:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.489352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.489358:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.489361:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.489361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.489363:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.489366:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.489367:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.489368:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.489370:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.489371:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.489372:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.489375:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.489377:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.489379:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.489380:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.489381:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.489382:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.489383:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.489383:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.489384:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.489389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.489397:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.489399:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.489400:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.489402:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.489403:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.489404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.489422:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.489426:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.489429:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.489431:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.489431:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.489432:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.489434:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.489435:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.489437:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.489438:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.489439:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.489442:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.489444:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.489446:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.489447:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.489447:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.489449:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.489450:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.489451:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.489452:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.489453:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.489455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.489456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677475.489460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:6.0:1619677475.489463:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.489464:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:6.0:1619677475.489465:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677475.489466:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:6.0:1619677475.489466:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.489467:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.489467:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.489468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:21.0:1619677475.489468:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.489469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.489469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677475.489469:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000400:00000200:10.0:1619677475.489471:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677475.489472:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677475.489474:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.489476:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.489477:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.489478:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.489479:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.489480:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.489484:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.489485:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.489488:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.489489:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.489490:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.489491:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.489492:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.489493:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.489494:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.489495:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677475.489498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.489500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677475.489506:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677475.489507:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:21.0:1619677475.489508:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.489509:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.489510:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.489510:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677475.489510:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:10.0:1619677475.489511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.0:1619677475.489512:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.489513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.489589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.489596:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.489599:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.489600:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.489601:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.489604:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.489606:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.489607:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.489608:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.489609:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.489610:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.489613:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.489614:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.489617:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677475.489617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:26.0:1619677475.489618:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.489618:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.489620:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.489620:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677475.489620:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:26.0:1619677475.489621:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.489622:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:20.0:1619677475.489622:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.489624:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.489625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:25.2:1619677475.489626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677475.489626:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.489628:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.489629:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.489631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677475.489631:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.489632:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.489633:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677475.489633:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:22.0:1619677475.489636:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:20.0:1619677475.489636:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.489637:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.489637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.489638:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:30.0:1619677475.489639:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677475.489639:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.489640:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.489641:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677475.489641:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677475.489642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.489642:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.489643:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:30.0:1619677475.489644:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.489644:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:25.2:1619677475.489645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.489645:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.489645:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677475.489646:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.489646:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.489648:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.489649:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677475.489650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:30.0:1619677475.489651:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.489651:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.489652:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.489653:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.489655:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.489655:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.489656:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.489657:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.489657:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.489658:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:6.0:1619677475.489658:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.489659:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677475.489660:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.489661:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.489661:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677475.489662:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.489662:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.489663:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677475.489663:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.489664:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.489665:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.489666:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677475.489666:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677475.489667:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.489670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.489677:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.489679:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.489680:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.489680:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.489683:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.489685:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.489877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.489884:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.489886:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.489887:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.489889:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.489892:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.489894:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677475.489895:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.489897:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.489898:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.489899:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677475.489902:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.489904:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677475.489907:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.489908:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677475.489909:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677475.489911:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:18.0:1619677475.489911:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.489912:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.489913:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677475.489916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.489923:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.489925:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.489926:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.489927:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.489929:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.489932:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.489936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.489942:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.489945:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.489945:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.489946:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.489947:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.489947:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.490035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677475.490043:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.490045:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677475.490048:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.490050:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.490050:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677475.490051:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677475.490052:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.490054:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.490055:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.490056:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677475.490059:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.490060:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.490063:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.490064:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677475.490065:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677475.490066:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677475.490067:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.490069:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:31.0:1619677475.490070:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677475.490071:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:9.2:1619677475.490073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.490076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:2.0:1619677475.490080:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.490082:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:30.0:1619677475.490084:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000800:00000200:18.0:1619677475.490084:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.490084:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.490086:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000400:00000200:2.0:1619677475.490086:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.490087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:30.0:1619677475.490088:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:2.0:1619677475.490089:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.490270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.490277:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.490280:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.490282:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.490284:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.490285:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.490286:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.490288:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.490289:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.490290:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.490291:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.490300:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.490302:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.490303:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.490304:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.490304:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.490306:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.490306:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.490307:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.490309:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.490310:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.490312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.490316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.490321:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.490322:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.490323:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.490324:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677475.490325:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.490325:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.490325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.490327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.490328:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.490328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.490506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.490512:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.490515:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.490516:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.490518:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.490520:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.490521:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.490523:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.490524:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.490525:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.490526:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.490529:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.490530:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.490533:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.490534:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.490534:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.490536:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.490536:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.490537:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.490538:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.490542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.490550:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.490552:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.490553:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.490555:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.490556:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.490557:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.490732:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.490739:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.490742:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.490743:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.490744:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.490747:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.490749:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.490750:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.490751:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.490752:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.490753:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.490757:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.490759:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.490762:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.490762:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.490763:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.490765:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.490765:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.490766:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.490767:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.490768:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.490771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.490773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.490779:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677475.490781:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.490782:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.490783:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.490783:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.490784:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677475.490784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677475.490784:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.490786:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.490789:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.490793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.490801:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.490803:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:20.2:1619677475.490804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:17.0:1619677475.490804:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.490804:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.490805:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.490806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:18.0:1619677475.490811:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.490813:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.490814:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.490816:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.490819:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.490820:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.490822:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.490823:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.490824:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.490825:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.490828:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.490830:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.490833:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.490834:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.490835:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.490837:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.490837:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.490838:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.490839:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.490843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.490850:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.490853:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.490854:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.490856:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.490858:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.490858:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.490864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677475.490871:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.490874:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.490875:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.490875:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.490876:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.490877:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677475.490932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.490934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.490937:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.490939:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.490940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.490942:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.490943:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.490945:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.490946:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.490947:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.490948:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.490951:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.490953:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.490955:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.490956:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.490956:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.490958:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.490959:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.490960:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.490960:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.490964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.490972:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.2:1619677475.490975:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677475.490977:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.490979:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677475.490980:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.490981:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.490982:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677475.490983:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677475.490985:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.490987:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.490989:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.490989:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.490991:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.490992:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.490994:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.490995:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.490996:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.490999:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.491001:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.491004:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.491005:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.491006:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.491007:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.491008:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.491009:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.491010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.491014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.491021:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.491024:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.491025:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.491026:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.491028:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.491031:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.491429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.491435:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.491438:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.491439:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.491441:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.491443:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.491445:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.491446:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.491447:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.491448:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.491449:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.491453:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.491455:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.491456:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.491457:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.491458:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.491459:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.491460:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.491460:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.491461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.491466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.491473:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.491475:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.491476:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.491477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.491478:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.491479:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.491548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.491555:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.491557:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.491558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.491560:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.491562:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.491563:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.491565:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.491566:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.491567:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.491568:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.491571:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.491572:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.491575:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.491575:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.491576:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.491578:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.491578:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.491579:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.491580:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.491580:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.491585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.491585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.491590:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.491592:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.491592:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.491593:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.491594:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.491594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.491595:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.491597:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.491598:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.491599:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.491613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.491620:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.491623:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.491624:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.491624:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.491625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.491626:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.491976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.491984:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.491987:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.491990:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.491992:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.491993:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.491994:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.491996:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.491997:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.491998:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.491999:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.492002:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.492004:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.492007:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.492008:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.492008:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.492010:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.492011:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.492012:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.492012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.492016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.492022:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.492025:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.492026:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.492026:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.492028:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.492030:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.492382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.492387:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.492389:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.492392:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.492392:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.492394:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.492396:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677475.492397:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.492399:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.492400:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.492401:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.492404:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.492406:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.492409:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.492409:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.492410:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.492412:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.492413:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.492414:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.492415:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.492415:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677475.492418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.492421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.492424:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:31.0:1619677475.492426:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677475.492426:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677475.492427:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:31.0:1619677475.492428:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677475.492428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677475.492429:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.492431:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.492433:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.492434:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.492440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.492446:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.492449:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.492450:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.492450:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.492451:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.492452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.492530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.492533:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.492536:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.492538:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.492539:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.492541:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.492543:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.492544:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.492545:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.492546:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.492547:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.492550:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.492552:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.492555:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.492556:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.492556:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.492558:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.492558:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.492559:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.492560:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.492564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.492571:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.492574:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.492574:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.492575:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.492577:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.492580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.492580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677475.492586:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.492589:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677475.492590:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677475.492592:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.492594:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.492596:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677475.492597:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.492598:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.492599:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.492600:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677475.492604:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.492606:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677475.492608:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.492609:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677475.492609:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677475.492611:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677475.492612:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.492613:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677475.492614:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677475.492614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:9.2:1619677475.492617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.2:1619677475.492619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.492624:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:2.0:1619677475.492625:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:6.0:1619677475.492626:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677475.492626:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677475.492627:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:2.0:1619677475.492627:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677475.492628:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.492629:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.492630:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.492633:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.492820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.492825:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.492828:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.492829:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.492830:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.492833:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.492834:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.492835:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.492836:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.492837:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.492838:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.492841:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.492842:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.492844:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.492845:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.492845:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.492847:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.492847:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.492848:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.492848:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.492853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.492860:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.492863:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.492864:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.492865:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.492866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.492868:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.492989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.492997:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.492999:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.493002:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.493004:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.493004:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.493006:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.493008:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.493009:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.493010:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.493012:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.493016:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.493018:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.493020:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.493021:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.493022:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.493023:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.493024:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.493025:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.493026:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.493029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.493036:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.493038:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.493039:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.493041:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.493042:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.493043:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.493157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:4.2:1619677475.493158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677475.493164:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.493166:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677475.493167:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.493169:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.493169:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.493170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677475.493171:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:8.0:1619677475.493171:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.493172:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.493173:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.493174:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.493174:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.493176:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.493176:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.493178:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.493178:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.493179:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.493179:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.493181:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.493182:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.493183:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.493183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:30.0:1619677475.493185:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.493187:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000400:00000200:8.0:1619677475.493187:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.493188:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.493189:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.493189:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.493189:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:8.0:1619677475.493189:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.493191:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.493192:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.493192:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677475.493192:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.493192:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677475.493193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.0:1619677475.493193:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.493193:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677475.493194:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.493194:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677475.493196:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677475.493196:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677475.493196:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677475.493197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677475.493197:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96499663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677475.493197:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.493197:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:20.0:1619677475.493199:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.493199:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.493201:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.493204:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677475.493205:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:20.0:1619677475.493206:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.493208:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000800:00000200:20.0:1619677475.493208:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.493208:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677475.493209:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:20.0:1619677475.493209:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (24)++
00000400:00000200:6.0:1619677475.493209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.493210:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.493210:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (25)++
00000400:00000200:22.0:1619677475.493211:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.493211:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (25)--
00000400:00000200:6.0:1619677475.493211:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.493212:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.493212:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.493213:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677475.493215:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (24)--
00000800:00000200:22.0:1619677475.493216:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.493216:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.493218:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.493219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.493219:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.493220:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.493223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.493232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.493239:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.493241:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.493242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.493243:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.493243:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.493244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.493258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.493264:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.493267:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.493267:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.493270:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.493272:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.493274:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.493275:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.493277:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.493278:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.493279:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.493282:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.493284:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.493285:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.493286:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.493287:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.493289:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.493289:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.493290:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.493291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.493307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.493314:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.493316:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.493317:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.493318:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.493319:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.493320:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.493447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.493456:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.493459:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.493459:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677475.493462:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.493464:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.493466:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.493467:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.493468:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.493469:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.493470:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.493473:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.493475:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.493478:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.493479:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.493479:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.493481:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.493481:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.493482:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.493483:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.493488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.493496:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.493499:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.493499:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.493500:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.493502:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.493505:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.493507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.493514:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.493517:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.493517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.493518:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.493519:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.493519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.493580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.493587:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.493590:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.493590:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.493592:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.493595:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.493596:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.493597:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.493599:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.493600:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.493601:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.493604:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.493606:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.493608:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.493609:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.493609:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.493611:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.493612:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.493613:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.493613:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677475.493618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.493627:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.493629:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.493630:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.493630:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.493632:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.493634:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.493964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.493969:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.493972:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.493973:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.493974:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.493977:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.493978:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.493979:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.493980:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.493981:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.493982:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.493985:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.493987:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.493989:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.493990:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.493991:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.493993:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.493993:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.493994:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.493994:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.493999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.494006:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.494008:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.494009:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.494011:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.494012:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.494013:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.494135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.494143:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.494146:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.494148:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.494151:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.494151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.494153:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.494154:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.494155:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.494156:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.494157:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.494161:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.494163:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.2:1619677475.494163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.494165:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.494166:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.494166:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.494168:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.494168:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.494169:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.494170:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.494171:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.494173:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:25.2:1619677475.494173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677475.494175:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.494176:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.494177:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.494178:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.494178:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.494179:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.494179:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677475.494180:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.494181:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.494182:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.494182:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.494183:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:31.0:1619677475.494185:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.494186:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:30.0:1619677475.494187:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.494188:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677475.494191:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677475.494191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.494192:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677475.494192:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677475.494194:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677475.494195:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.494196:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.494197:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.494198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.494199:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:9.2:1619677475.494200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677475.494201:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.494202:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.494203:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.494203:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.2:1619677475.494204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.494206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.494208:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:2.0:1619677475.494208:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:31.0:1619677475.494211:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:2.0:1619677475.494211:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:31.0:1619677475.494212:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:18.0:1619677475.494212:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.494212:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.494213:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:2.0:1619677475.494214:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.494218:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677475.494222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677475.494229:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.494231:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677475.494232:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.494232:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.494234:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.494234:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.494406:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.494415:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.494418:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.494418:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.494420:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.494423:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.494425:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.494426:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.494427:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.494428:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.494429:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.494432:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.494434:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.494437:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.494437:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.494438:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.494439:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.494440:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.494441:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.494441:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.494446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.494454:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.494457:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.494458:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.494458:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.494460:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.494463:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.494687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.494695:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.494698:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.494700:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.494703:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.494703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.494704:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.494705:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.494706:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.494707:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.494708:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.494711:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.494713:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.494715:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.494715:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.494716:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.494717:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.494717:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.494719:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.494720:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.494721:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.494722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.494726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.494731:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.494732:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.494733:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.494735:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.494735:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677475.494735:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.494736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.494736:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.494737:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.494738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.495029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.495036:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.495039:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.2:1619677475.495040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:31.0:1619677475.495041:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.495043:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.495043:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.495044:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677475.495045:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.495045:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677475.495047:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.495048:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.495048:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.495048:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:31.0:1619677475.495049:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:6.0:1619677475.495051:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.495052:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.495053:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.495054:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677475.495055:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:6.0:1619677475.495056:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.495057:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.495058:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.495059:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.495059:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.495060:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677475.495060:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677475.495062:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677475.495062:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:6.0:1619677475.495062:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.495063:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.495064:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.495064:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.495066:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.495067:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:9.2:1619677475.495068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:6.0:1619677475.495068:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.495069:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.495070:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.495070:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.495071:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.495076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677475.495077:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.495080:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.495081:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.495081:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.495083:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:2.0:1619677475.495083:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.495086:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:2.0:1619677475.495086:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:20.0:1619677475.495087:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.495088:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.495090:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.495090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:9.2:1619677475.495107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.495114:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.495117:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.495118:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.495118:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.495119:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.495119:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.495367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.495372:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.495374:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.495376:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.495377:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.495379:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.495380:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.495382:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.495383:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.495384:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.495385:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.495388:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.495390:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.495392:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.495393:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.495394:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.495395:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.495396:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.495397:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.495398:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.495399:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.495402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.495404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.495407:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.495409:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.495410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.495410:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.495411:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.495414:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.495415:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.495415:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.495417:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.495419:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.495507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.495515:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.495518:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.495519:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.495521:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.495524:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.495525:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.495526:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.495527:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.495528:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.495529:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.495532:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.495534:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.495537:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.495538:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.495538:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.495540:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.495541:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.495542:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.495543:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.495544:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677475.495546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.495551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677475.495556:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677475.495558:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:20.0:1619677475.495558:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.495559:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.495560:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.495561:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:20.0:1619677475.495561:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.495563:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.0:1619677475.495563:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.495564:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.495893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677475.495893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.495900:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677475.495901:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677475.495903:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:30.0:1619677475.495904:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677475.495904:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.495907:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677475.495908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.495909:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.495910:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.495911:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677475.495912:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.495913:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:18.0:1619677475.495913:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.495915:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.495916:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.495917:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.495918:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.495918:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677475.495919:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.495920:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:18.0:1619677475.495922:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677475.495924:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.495925:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677475.495928:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:18.0:1619677475.495928:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.495929:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.495929:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677475.495931:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.495931:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.495931:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677475.495932:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:18.0:1619677475.495932:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677475.495933:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:18.0:1619677475.495933:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:30.0:1619677475.495934:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.495935:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.495936:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.495937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.495937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:9.2:1619677475.495941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677475.495945:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677475.495947:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677475.495947:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.495948:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.495948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.495950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.495950:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:1.0:1619677475.495950:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:8.0:1619677475.495951:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.495952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:8.0:1619677475.495953:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.495953:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677475.495955:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:12.0:1619677475.495961:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.495965:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677475.495969:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.111@o2ib7 : GET - for me
00000800:00000200:13.0:1619677475.495971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677475.495972:0:19943:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.111@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:12.0:1619677475.495975:0:19943:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.111@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:12.0:1619677475.495977:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.495979:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677475.495981:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677475.495987:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : REPLY try# 0
00000800:00000200:12.0:1619677475.495990:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.495992:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.495993:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.495994:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.495997:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.495998:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:25.2:1619677475.496002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.496011:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.496014:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.496015:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.496017:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: REPLY: OK
00000800:00000200:13.0:1619677475.496020:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677475.496020:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.2:1619677475.496242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.496248:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.496251:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.496251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.496254:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.496256:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.496257:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.496259:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.496260:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.496261:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.496262:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.496266:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.496268:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.496270:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.496271:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.496272:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.496273:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.496274:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.496275:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.496276:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.496276:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.496279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.496281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.496287:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.496288:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.496288:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.496289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.496290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.496291:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.496292:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.496294:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677475.496299:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.496300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.496302:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.496304:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.496306:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.496307:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.496307:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.496308:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.496308:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:4.2:1619677475.496512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.496516:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.496518:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.496521:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.496521:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.496523:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.496524:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.496526:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.496527:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.496528:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.496529:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.496532:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.496534:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.496536:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.496537:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.496538:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.496539:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.496540:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.496541:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.496541:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.496557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.496564:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.496566:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.496567:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.496568:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.496570:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.496572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.496711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.496718:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.496721:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.496721:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.496723:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.496726:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.496728:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.496729:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.496730:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.496731:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.496732:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.496735:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.496737:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.496740:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.496741:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.496741:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.496743:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.496744:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.496744:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.496745:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.496749:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.496758:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.496761:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.496762:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.496762:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.496764:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.496767:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.496931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.496937:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677475.496941:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677475.496942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.496944:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.496946:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.496949:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:21.0:1619677475.496949:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.496951:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.496952:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.496953:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:21.0:1619677475.496954:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.496956:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.496957:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.496957:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.496957:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677475.496960:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.496961:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:21.0:1619677475.496961:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.496962:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677475.496963:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.496964:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.496965:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.496966:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.496966:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.496966:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.496967:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.496968:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.496969:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.496969:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677475.496970:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.496970:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677475.496972:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.496974:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677475.496974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:31.0:1619677475.496975:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.496976:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.496977:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.496978:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.496979:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.496979:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.496982:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:25.2:1619677475.496983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677475.496984:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.496985:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.496987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.496987:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.496989:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677475.496990:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.496992:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.496993:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.496995:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.496996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.496997:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:8.2:1619677475.496997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.497004:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.497007:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.497008:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.497009:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.497009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.497010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.497037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.497045:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.497048:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.497050:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.497051:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.497052:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.497054:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.497055:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.497057:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.497058:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.497059:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.497062:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.497064:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.497067:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.497068:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.497068:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.497070:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.497071:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.497071:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.497072:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.497076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.497084:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.497087:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.497088:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.497088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.497090:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.497094:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.497101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.497107:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.497109:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.497111:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.497112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.497113:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.497115:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.497116:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.497117:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.497118:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.497119:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.497122:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.497124:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.497127:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.497127:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.497128:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.497130:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.497130:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.497132:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.497133:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.497133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677475.497139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.497142:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.497145:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.497145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.497146:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.497146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.497153:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.497156:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.497157:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.497157:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.497159:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.497162:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.497256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.497261:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.497264:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.497264:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.497266:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.497269:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.497270:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.497271:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.497272:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.497273:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.497274:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.497277:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.497279:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.497282:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.497282:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.497283:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.497285:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.497285:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.497286:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.497286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.497291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.497299:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.497302:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.497302:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.497304:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.497305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.497307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.497881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677475.497887:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.497890:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677475.497891:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677475.497892:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.497895:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.497897:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677475.497898:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.497899:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.497900:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.497901:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677475.497905:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.497907:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677475.497909:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.497910:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677475.497911:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.497913:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677475.497913:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.497914:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677475.497914:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:25.2:1619677475.497920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.497928:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.497930:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.497931:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.497931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.497933:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.497936:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.498038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.498047:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.498050:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.498050:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.498052:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.498055:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.498057:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.498058:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.498059:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.498060:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.498061:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.498065:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.498067:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.498069:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.498070:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.498071:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.498073:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.498073:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.498074:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.498074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.498078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.498086:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.498088:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.498089:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.498089:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.498091:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.498094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.498097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.498103:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.498106:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.498107:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.498107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.498108:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.498109:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.498109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.498115:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.498118:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.498119:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.498120:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.498122:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.498123:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.498124:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.498126:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.498127:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.498128:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.498130:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.498132:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.498134:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.498135:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.498135:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.498137:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.498137:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.498138:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.498138:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.498143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.498151:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.498153:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.498153:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.498154:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.498155:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.498156:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.498162:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.498170:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.498172:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.498173:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.498173:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.498174:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677475.498174:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:4.2:1619677475.498343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677475.498348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:10.0:1619677475.498350:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.498353:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.498354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677475.498354:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.498356:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.498357:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677475.498357:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.498359:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.498360:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.498361:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:6.0:1619677475.498361:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.498363:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.498363:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.498364:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.498365:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.498366:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.498367:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.498367:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.498369:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:6.0:1619677475.498369:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.498374:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.498375:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.498377:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.498378:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.498380:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.498380:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:10.0:1619677475.498381:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.498381:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677475.498382:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.498383:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.498383:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.498384:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:10.0:1619677475.498385:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.498385:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.498385:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.498385:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:10.0:1619677475.498386:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.498387:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.498390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677475.498394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677475.498398:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.498400:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.498401:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:21.0:1619677475.498401:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.498402:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.498404:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:20.0:1619677475.498404:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:22.0:1619677475.498405:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.498405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:6.0:1619677475.498405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.498407:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.498410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.498419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.498427:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.498430:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.498432:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.498435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.498435:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.498437:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.498438:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.498439:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.498440:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.498442:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.498445:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.498447:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.498449:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.498449:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.498450:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.498451:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.498451:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.498452:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.498453:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.498457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.498463:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.498466:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.498466:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.498468:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.498470:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.498470:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.498571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.498578:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.498581:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.498582:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.498584:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.498587:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.498588:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:26.0:1619677475.498589:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.498591:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.498592:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.498593:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:26.0:1619677475.498596:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.498598:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:26.0:1619677475.498600:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.498601:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:26.0:1619677475.498602:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677475.498603:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677475.498604:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.498605:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.498605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.498609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.498617:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.498619:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677475.498620:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.498622:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.498624:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677475.498625:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677475.498825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:4.2:1619677475.498825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.498832:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:1.0:1619677475.498832:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:26.0:1619677475.498835:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:17.0:1619677475.498836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:10.0:1619677475.498836:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:1.0:1619677475.498836:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:10.0:1619677475.498840:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.498840:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.498842:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.498843:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.498843:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.498845:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:1.0:1619677475.498845:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.498847:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.498847:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.498848:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.498849:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.498849:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.498851:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.498853:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:1.0:1619677475.498853:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.498857:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.498857:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.498859:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.498860:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:1.0:1619677475.498860:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.498861:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.498862:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.498862:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.498863:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:1.0:1619677475.498863:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.498863:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:10.0:1619677475.498864:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677475.498864:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:10.0:1619677475.498865:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.498866:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.498866:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.498867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.498868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677475.498872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:6.0:1619677475.498876:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.498878:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.498879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:12.0:1619677475.498879:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:6.0:1619677475.498879:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.498881:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.498882:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.498883:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.498884:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.498884:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:12.0:1619677475.498887:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.498888:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.498910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.498918:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.498921:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.498923:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.498925:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.498925:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.498926:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.498928:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.498929:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.498930:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.498931:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.498935:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.498936:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.498939:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.498940:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.498941:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.498942:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.498943:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.498944:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.498944:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.498948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.498955:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.498958:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.498959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.498960:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.498962:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.498964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.499322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.499328:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.499331:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.499332:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.499334:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.499337:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.499338:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.499340:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499630] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.499341:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.499342:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.499343:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.499347:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.499349:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.499351:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.499352:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.499353:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.499354:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.499355:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.499357:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.499358:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.499358:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.499361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.499364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677475.499369:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.499369:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.499371:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:20.0:1619677475.499372:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.499373:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.499373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:1.0:1619677475.499373:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:20.0:1619677475.499375:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.499376:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.499377:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.499600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.499606:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.499609:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.499610:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.499612:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.499614:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.499615:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.499616:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.499617:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.499618:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.499619:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.499622:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.499624:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.499626:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.499627:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.499628:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.499629:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.499630:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.499630:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.499631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.499636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.499644:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.499646:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.499646:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.499648:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.499649:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.499650:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.499701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.499709:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.499711:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.499712:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.499714:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.499717:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.499718:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.499719:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.499721:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.499722:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.499723:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.499726:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.499728:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.499731:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.499731:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.499732:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.499734:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.499734:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.499735:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.499737:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.499737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.499740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.499743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.499747:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.499750:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677475.499750:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677475.499751:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677475.499751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:6.0:1619677475.499751:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.499752:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.499753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.499753:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.499753:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677475.499755:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677475.499758:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.499761:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.499762:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.499764:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.499766:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.499767:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.499768:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.499770:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.499771:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.499772:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.499775:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.499777:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.499779:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.499779:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.499780:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.499781:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.499782:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.499783:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.499783:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.499788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.499795:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.499797:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.499798:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.499798:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.499800:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.499801:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.499824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.499831:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.499834:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.499834:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.499835:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.499835:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.499836:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.500288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:21.2:1619677475.500291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677475.500299:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677475.500301:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.500301:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.500302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:30.0:1619677475.500304:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677475.500305:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.500306:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.500308:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.500309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.500309:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.500309:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.500310:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.500310:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.500312:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.500312:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.500314:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.500314:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.500315:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.500315:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.500316:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.500319:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.500320:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.500322:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.500322:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.500325:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.500325:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.500326:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.500326:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.500326:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.500327:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.500328:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.500328:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.500329:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.500329:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.500330:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:30.0:1619677475.500331:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677475.500331:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:30.0:1619677475.500333:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.500333:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.500335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:25.2:1619677475.500336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.500339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.500343:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677475.500344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.500345:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.500346:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.500347:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677475.500347:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677475.500347:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.500348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.500348:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.500349:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:24.0:1619677475.500349:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.500350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.500351:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677475.500351:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.500352:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:25.2:1619677475.500353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.500356:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.500358:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.500360:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.500361:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.500361:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.500362:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.500362:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.500690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.500695:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.500697:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.500698:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.500700:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.500702:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.500704:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.500705:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499624] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.500706:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.500707:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.500708:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.500712:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.500714:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.500716:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.500717:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.500717:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.500719:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.500720:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.500721:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.500722:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.500722:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.500725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.500727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.500732:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.500733:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.500734:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.500734:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.500735:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.500735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.500736:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.500738:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.500739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.500740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.500745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.500752:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.500755:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.500756:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.500756:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.500757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.500758:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.500943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.500949:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.500952:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.500953:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.500955:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.500957:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.500958:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.500959:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.500960:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.500962:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.500963:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.500966:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.500968:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.500971:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.500972:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.500973:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.500974:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.500975:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.500976:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.500976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.500980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.500987:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.500990:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.500991:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.500993:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.500994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.500996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.501267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.501274:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.501277:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.501279:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.501281:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.501281:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.501283:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.501284:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.501286:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.501287:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.501287:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.501290:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.501292:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.501301:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.501302:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.501302:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.501304:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.501305:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.501307:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.501308:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.501309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.501311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.501320:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.501323:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.501323:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.501324:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.501326:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.501329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.501371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.501380:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.501383:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.501383:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.501384:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677475.501492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.501497:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.501500:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.501502:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.501502:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.501505:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.501507:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.501508:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.501509:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.501510:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.501511:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.501515:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.501517:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.501519:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.501520:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.501521:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.501523:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.501523:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.501524:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.501524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.501529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.501538:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.501540:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.501541:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.501542:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.501543:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.501546:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.501602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.501610:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.501613:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.501615:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.501617:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.501618:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.501619:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.501621:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.501622:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.501623:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.501624:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.501626:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.501628:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.501630:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.501631:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.501631:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.501633:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.501633:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.501634:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.501635:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.501638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.501645:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.501648:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.501648:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.501650:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.501652:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.501653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.501656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.501663:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.501665:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:21.2:1619677475.501665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.501666:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.501666:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.501667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.501668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677475.501670:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.501673:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.501673:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.501675:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.501678:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.501679:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.501680:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.501682:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.501683:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.501684:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.501687:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.501689:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.501691:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.501692:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.501693:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.501694:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.501695:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.501695:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.501696:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.501700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.501707:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.501710:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.501710:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.501712:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.501713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.501714:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.501968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.501976:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.501979:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.501980:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.501981:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.501984:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.501986:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.501987:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.501989:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.501990:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.501991:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.501994:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.501995:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.501998:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.501999:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.502000:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.502001:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.502002:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.502003:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.502004:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.502004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.502007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.502010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.502015:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.502017:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.502017:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.502018:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.502019:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677475.502019:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.502020:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.502021:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.502021:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.502024:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.502329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.502337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.502340:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.502342:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.502344:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.502344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.502346:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.502347:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.502348:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.502349:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.502350:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.502353:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.502354:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.502356:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.502357:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.502357:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.502359:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.502360:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.502360:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.502361:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.502366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.502372:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.502375:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.502375:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.502377:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.502379:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.502380:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.502693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.502699:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.502702:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.502704:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.2:1619677475.502705:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:21.0:1619677475.502706:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.502706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.502707:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:21.0:1619677475.502708:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:20.0:1619677475.502709:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.502710:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.502711:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.502712:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.502713:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677475.502717:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.502718:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677475.502719:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.502720:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.2:1619677475.502721:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:20.0:1619677475.502721:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.502722:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.502723:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.502724:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.502725:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677475.502727:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:20.0:1619677475.502729:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.502730:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:6.0:1619677475.502730:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.502731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.502732:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.502733:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677475.502733:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.502733:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.502734:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677475.502734:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.502734:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677475.502735:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.502736:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677475.502736:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.502737:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.502737:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)++
00000400:00000200:6.0:1619677475.502737:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:21.0:1619677475.502738:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.502738:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000400:00000200:6.0:1619677475.502738:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96499614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.502739:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:6.0:1619677475.502739:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.502740:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:6.0:1619677475.502740:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.502741:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:6.0:1619677475.502741:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:25.2:1619677475.502742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.2:1619677475.502744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677475.502744:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.502746:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.502748:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.502749:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:6.0:1619677475.502749:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.502750:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.502751:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:6.0:1619677475.502751:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.502752:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:6.0:1619677475.502752:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677475.502753:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.502753:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:6.0:1619677475.502753:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.502753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.502754:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.502755:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.502756:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:20.0:1619677475.502756:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.502757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:25.2:1619677475.502758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677475.502760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.0:1619677475.502761:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.502762:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.502763:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.502764:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.502765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.502766:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.502768:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.502947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.502954:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.502957:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.502958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.502960:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.502963:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.502964:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.502965:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.502967:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.502968:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.502968:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.502971:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.502973:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.502976:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.502977:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.502977:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.502979:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.502979:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.502980:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.502980:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.502985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:5.2:1619677475.502986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:22.0:1619677475.502992:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677475.502992:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677475.502995:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677475.502995:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677475.502996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.502996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:22.0:1619677475.502997:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.502998:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.502999:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.503000:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.503001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:17.0:1619677475.503001:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.503003:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.503004:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.503005:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.503006:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.503009:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.503011:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.503013:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.503013:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.503014:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.503015:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.503016:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.503017:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.503017:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.503022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.503023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:1.0:1619677475.503030:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:31.0:1619677475.503031:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:1.0:1619677475.503032:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.503033:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.503034:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677475.503035:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:1.0:1619677475.503035:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.503037:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.503037:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677475.503038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.503039:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.503041:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.503042:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.503044:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.503045:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.503046:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.503049:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.503051:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.503054:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.503055:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.503056:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.503057:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.503058:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.503059:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.503060:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.503063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.503071:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.503074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.503074:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.503075:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.503077:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.503080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.503084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.503090:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.503093:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.503094:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.503094:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.503095:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.503096:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.503216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.503224:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.503227:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.503228:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.503230:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.503232:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.503234:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.503235:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.503236:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.503237:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.503238:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.503241:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.503242:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.503245:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.503246:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.503246:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.503248:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.503249:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.503250:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.503251:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.503251:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677475.503257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.2:1619677475.503258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677475.503264:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:30.0:1619677475.503265:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677475.503265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.503266:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.503266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.503267:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677475.503268:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.503270:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.503272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677475.503272:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:30.0:1619677475.503273:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.503274:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.503275:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.503275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.503275:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.503276:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.503277:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.503278:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.503278:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.503279:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:6.0:1619677475.503281:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:30.0:1619677475.503283:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.503285:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.503287:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.503288:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.503288:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.503290:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.503290:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.503291:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.503292:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.503297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.503304:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.503307:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.503307:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.503309:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.503311:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.503312:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.503344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677475.503351:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.503354:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677475.503356:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.503358:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.503358:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677475.503359:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677475.503361:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.503362:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.503363:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.503363:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677475.503366:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.503368:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677475.503371:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.503372:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677475.503372:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677475.503374:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677475.503374:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.503375:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677475.503376:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677475.503380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.503389:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.503392:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.503392:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.503393:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.503395:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.503398:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.503644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.503650:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.503653:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.503654:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.503656:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.503658:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.503659:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.503661:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.503662:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.503663:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.503664:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.503667:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.503669:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.503672:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.503673:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.503673:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.503675:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.503676:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.503677:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.503677:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.503681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.503689:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.503691:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.503692:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.503694:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.503695:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.503696:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.504502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.504508:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.504511:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.504512:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.504513:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.504515:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.504517:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.504518:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.504519:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.504520:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.504521:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.504524:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.504525:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.504527:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.504527:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.504528:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.504530:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.504530:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.504531:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.504532:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.504532:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.504536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.504537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.504542:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.504544:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.504544:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.504545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.504546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.504547:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.504547:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.504549:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.504550:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.504551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.504559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.504566:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.504569:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.504570:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.504570:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.504571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.504572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.504659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.504667:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.504670:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.504671:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.504673:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.504676:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.504677:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.504678:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.504679:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.504680:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.504681:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.504685:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.504687:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.504690:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.504690:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.504691:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.504693:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.504693:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.504694:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.504694:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.504699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.504706:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.504709:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.504710:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.504710:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.504712:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.504715:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.505118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.505126:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.505129:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.505131:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.505134:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.505134:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.505136:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.505137:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.505138:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.505139:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.505140:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.505143:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.505145:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.505148:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.505148:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.505149:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.505151:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.505151:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.505152:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.505153:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.505157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.505164:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.505167:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.505168:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.505168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.505170:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.505172:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.505520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.505525:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.505528:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.505529:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.505531:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.505532:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.505534:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.505535:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.505536:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.505537:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.505538:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.505541:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.505543:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.505545:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.505546:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.505547:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.505548:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.505549:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.505549:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.505550:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.505555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.505563:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.505566:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.505566:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.505568:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.505569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.505570:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.506246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:21.2:1619677475.506250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677475.506253:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.506256:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677475.506257:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:31.0:1619677475.506258:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677475.506258:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677475.506261:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:1.0:1619677475.506261:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.506262:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.506264:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.506264:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.506265:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.506266:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677475.506266:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.506267:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.506268:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.506270:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.506272:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.506272:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677475.506273:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.506275:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.506275:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.506276:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677475.506277:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.506278:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.506279:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677475.506280:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.506281:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.506281:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:31.0:1619677475.506282:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:1.0:1619677475.506282:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677475.506282:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:31.0:1619677475.506286:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.506286:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.506287:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.506289:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.506289:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.506291:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.506293:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.2:1619677475.506296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:25.2:1619677475.506298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.506299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (23)++
00000800:00000200:31.0:1619677475.506302:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.506303:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.506305:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.506305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677475.506305:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.506305:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677475.506306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.506307:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:22.0:1619677475.506308:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:20.0:1619677475.506308:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.506309:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.506309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.506310:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.506310:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.506311:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.506313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677475.506314:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677475.506316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.506323:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.506326:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.506327:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.506327:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.506328:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.506329:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.506433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.506438:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.506441:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.506442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.506444:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.506446:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.506447:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.506448:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.506450:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.506451:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.506452:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.506455:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.506457:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.506460:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.506461:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.506462:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.506463:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.506463:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.506464:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.506464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.506469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.506477:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.506479:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.506480:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.506482:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.506482:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.506484:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.506613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677475.506620:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677475.506622:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677475.506623:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677475.506625:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677475.506628:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.506629:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:17.0:1619677475.506630:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.506632:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.506633:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.506634:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:17.0:1619677475.506638:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.506639:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:17.0:1619677475.506642:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.506643:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:17.0:1619677475.506643:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:17.0:1619677475.506646:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:17.0:1619677475.506646:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677475.506647:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677475.506647:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:25.2:1619677475.506651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.506658:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.506661:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.506662:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.506664:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.506665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.506667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.506926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677475.506932:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.506935:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677475.506936:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677475.506937:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677475.506940:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.506941:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677475.506943:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.506944:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.506945:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.506946:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677475.506949:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677475.506950:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677475.506953:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.506953:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677475.506954:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.506955:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.506956:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.506957:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677475.506958:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677475.506959:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:9.2:1619677475.506962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.506969:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.506971:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.506973:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.506973:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.506975:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.506978:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.507015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.507020:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:17.0:1619677475.507022:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:17.0:1619677475.507023:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677475.507024:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:4.2:1619677475.507177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.507185:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.507188:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.507189:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.507191:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.507194:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.507195:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.507196:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.507197:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.507198:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677475.507199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:10.0:1619677475.507199:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.507203:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.507204:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:30.0:1619677475.507206:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:10.0:1619677475.507207:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.507208:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.507208:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:30.0:1619677475.507209:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:10.0:1619677475.507210:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.507210:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:30.0:1619677475.507211:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.507211:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.507211:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677475.507213:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.507213:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.507215:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:25.2:1619677475.507215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:30.0:1619677475.507216:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.507217:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.507218:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.507219:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.507222:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.507223:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:30.0:1619677475.507224:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:12.0:1619677475.507225:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.507226:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.507227:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.507228:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.507228:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:12.0:1619677475.507228:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.507230:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:12.0:1619677475.507230:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:30.0:1619677475.507231:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.507231:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:13.0:1619677475.507231:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677475.507232:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.507238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.507246:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.507249:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.507250:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.507250:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.507252:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.507255:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.507256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.507263:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.507266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.507266:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.507267:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.507268:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.507268:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.507345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.507350:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.507353:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.507354:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.507355:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.507358:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.507359:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.507360:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.507362:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.507363:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.507364:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.507367:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.507369:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.507371:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.507372:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.507372:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.507374:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.507375:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.507375:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.507375:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.507381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.507388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677475.507389:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.507391:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.507392:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.507394:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.507395:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.507396:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.507396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.507399:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.507401:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.507404:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.507404:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.507406:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.507407:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.507408:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.507409:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.507410:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.507414:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.507415:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.507418:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.507419:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.507419:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.507421:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.507421:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.507422:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.507423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.507427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.507434:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.507436:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.507437:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.507437:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.507439:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.507442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.507449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.507455:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.507457:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.507458:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.507458:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.507459:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.507460:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.507582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.507587:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.507590:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.507592:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.507592:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.507595:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.507596:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.507597:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.507598:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.507600:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.507600:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.507604:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.507604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.0:1619677475.507605:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.507608:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.507609:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.507609:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.507611:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.507611:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.507612:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.0:1619677475.507612:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677475.507614:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.507614:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677475.507614:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.507615:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.507617:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.507620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:8.0:1619677475.507622:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.507625:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.507626:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:21.0:1619677475.507627:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:8.0:1619677475.507627:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.507628:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.507628:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:8.0:1619677475.507628:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.507629:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.507629:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:8.0:1619677475.507629:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.507630:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.507631:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.507631:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:20.0:1619677475.507632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.507632:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677475.507633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:8.0:1619677475.507635:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.507637:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.507640:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.507641:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.507641:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.507643:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.507644:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.507644:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.507645:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.507649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.507658:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.507661:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.507662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.507662:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.507664:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.507667:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.507669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.507677:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.507679:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.507680:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.507680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.507681:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.507682:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.508019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.508026:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.508029:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.508031:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.508033:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.508034:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.508036:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.508037:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.508038:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.508039:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.508040:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.508043:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.508045:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.508048:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.508049:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.508049:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.508051:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.508052:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.508053:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.508053:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.508057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.508065:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.508068:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.508069:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.508070:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.508071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.508073:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.508231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.508239:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.508242:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.508243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.508245:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.508248:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.508249:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.508250:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.508251:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.508252:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.508253:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.508256:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.508258:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.508261:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.508262:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.508262:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.508264:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.508264:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.508265:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.508265:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.508281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.508289:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.508291:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.508292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.508293:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.508302:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.508305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.508415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.2:1619677475.508420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:22.0:1619677475.508421:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.508423:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.508424:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.508426:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.508427:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:22.0:1619677475.508428:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.508429:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:18.0:1619677475.508430:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:22.0:1619677475.508431:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677475.508431:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:22.0:1619677475.508432:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.508434:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.508434:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.508435:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.508437:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.508439:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.508439:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.508440:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.508441:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.508442:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.508443:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.508444:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677475.508444:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677475.508445:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.508445:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.508447:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.508447:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.508448:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:18.0:1619677475.508448:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.508449:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.508450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677475.508450:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.508451:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.508452:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.508453:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:25.2:1619677475.508454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.508454:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.2:1619677475.508455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:18.0:1619677475.508455:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.508456:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.508457:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.508460:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677475.508461:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.508462:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.508462:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.508463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.508464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.508465:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.508465:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.508467:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.508468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677475.508468:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677475.508469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.508470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677475.508470:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.508471:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.508472:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.508473:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.508475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677475.508477:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.508480:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.508481:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.508481:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.508483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.508483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677475.508658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.508666:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.508669:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:20.2:1619677475.508670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.0:1619677475.508670:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.508672:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.508676:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.508677:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:18.0:1619677475.508677:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:24.0:1619677475.508678:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.508680:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677475.508680:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.508680:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:24.0:1619677475.508681:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.508682:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.508683:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.508685:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.508686:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.508687:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:24.0:1619677475.508688:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.508689:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.508690:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.508691:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.508692:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677475.508692:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:24.0:1619677475.508693:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.508693:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.508695:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.508696:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.508696:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677475.508696:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677475.508697:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677475.508698:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.508701:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677475.508701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.508702:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.508702:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.508704:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.508705:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.508706:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.508707:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.508708:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677475.508709:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:25.2:1619677475.508710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677475.508712:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:20.2:1619677475.508713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677475.508713:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.508713:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.508715:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.508717:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677475.508718:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:1.0:1619677475.508718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:12.0:1619677475.508720:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.508721:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.508721:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.508721:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.508722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:12.0:1619677475.508723:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.508724:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.508725:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.508728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677475.508735:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.508737:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.508738:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.508739:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.508740:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.508741:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.508812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.508820:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.508823:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.508826:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.508828:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.508828:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.508830:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.508831:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.508832:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.508833:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.508835:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.508838:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.508840:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.508842:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.508843:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.508844:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.508846:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.508846:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.508847:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.508848:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.508851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.508858:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.508860:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.508861:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.508862:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.508864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.508865:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.509237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.509244:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.509247:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.509249:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.509251:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.509251:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.509253:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.509254:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.509255:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.509256:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.509257:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.509260:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.509262:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.509265:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.509266:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.509266:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.509268:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.509269:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.509270:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.509271:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.509272:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.509275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.509277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.509282:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.509285:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.509285:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.509286:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.509286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.509288:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.509288:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677475.509288:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.509289:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.509292:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.509304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.509309:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.509312:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.509313:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.509315:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.509316:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.509318:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.509319:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.509320:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.509321:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.509322:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.509326:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.509328:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.509330:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.509331:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.509332:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.509334:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.509334:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.509335:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.509335:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.509340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.509349:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.509351:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.509352:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.509353:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.509354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.509355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.509805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.509813:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.509816:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.509816:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677475.509818:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.509821:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.509823:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.509824:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.509825:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.509826:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.509827:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.509831:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.509833:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.509836:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.509837:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.509837:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.509839:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.509839:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.509840:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.509841:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.509845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.509853:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.509855:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.509856:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.509857:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.509859:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.509862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.509953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.509959:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.509962:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.509963:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.509964:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.509967:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.509968:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.509970:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.509971:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.509972:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.509973:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.509976:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.509978:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.509980:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.509981:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.509981:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.509983:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.509983:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.509985:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.509985:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.509986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.509989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.509992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.509998:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.509999:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.509999:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.510000:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677475.510000:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.510001:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.510002:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.510002:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.510003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677475.510003:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.510220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.510226:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.510228:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.510229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.510231:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.510233:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.510235:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.510236:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.510237:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.510238:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.510239:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.510242:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.510244:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.510247:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.510247:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.510248:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.510250:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.510250:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.510251:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.510251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.510256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.510264:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.510266:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.510267:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.510268:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.510269:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.510270:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677475.510547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.510555:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.510558:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.510559:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.510559:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.510560:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.510560:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677475.510827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.510830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.510833:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.510835:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.510836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.510837:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.510838:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.510840:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.510841:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.510842:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.510843:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.510846:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.510848:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.510850:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.510851:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.510852:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.510854:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.510854:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.510855:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.510856:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.510860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.510867:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.510870:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.510871:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.510871:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.510873:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.510875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.511069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.511080:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.511084:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.511088:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.511089:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.511092:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.511095:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.511097:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.511100:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.511102:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.511103:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.511109:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.511112:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.511117:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.511118:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.511119:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.511121:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.511122:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.511124:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.511125:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.511128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.511136:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.511139:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.511140:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.511141:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.511143:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.511146:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.511329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.511335:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.511337:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.511338:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.511340:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.511342:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.511343:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.511344:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.511346:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.511347:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.511348:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.511351:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.511353:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.511355:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.511356:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.511356:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.511358:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.511358:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.511359:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.511359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.511364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.511373:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.511375:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.511376:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.511377:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.511378:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.511380:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.511565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.511568:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.511571:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.511573:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.511574:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.511576:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.511577:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.511579:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.511580:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.511581:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.511582:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.511585:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.511587:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.511589:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.511590:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.511590:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.511592:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.511592:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.511593:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.511594:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.511598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.511606:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.511608:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.511609:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.511610:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.511611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.511613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.511618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.511626:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.511628:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.511629:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.511630:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.511630:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.511631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.511695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.511703:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.511706:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.511708:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.511711:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.511712:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.511713:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.511714:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.511715:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.511716:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.511717:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.511721:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.511722:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.511724:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.511725:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.511726:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.511727:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.511728:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.511730:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.511731:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.511732:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.511733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.511737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.511742:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.511744:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.511745:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.511746:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.511746:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.511747:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.511748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677475.511748:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.511749:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.511750:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.511751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.511758:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.511761:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.511762:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.511762:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.511763:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.511764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.512212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.512217:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.512220:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.512221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.512223:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.512226:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.512227:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.512228:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.512230:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.512231:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.512232:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.512235:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.512237:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.512239:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.512240:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.512241:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.512242:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.512243:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.512244:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.512245:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.512245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.512248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.512250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.512255:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.512257:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.512257:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.512258:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.512259:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.512259:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.512260:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.512262:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.512262:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.512264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.512265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.512272:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.512275:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.512276:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.512276:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.512277:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.512278:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.512536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.512542:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.512545:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.512545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.512548:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.512549:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.512551:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.512552:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.512553:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.512554:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.512555:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.512559:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.512560:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.512563:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.512564:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.512564:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.512566:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.512566:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.512567:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.512567:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.512572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.512581:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.512583:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.512584:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.512585:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.512587:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.512588:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.512682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.512690:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.512693:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.512695:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.512697:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.512697:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.512699:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.512700:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.512701:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.512702:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.512704:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.512707:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.512709:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.512711:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.512712:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.512712:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.512714:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.512714:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.512715:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.512716:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.512720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.512727:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.512729:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.512730:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.512732:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.512733:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.512734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.512771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.512779:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.512782:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.512783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.512785:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.512788:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.512790:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.512791:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.512792:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.512793:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.512794:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.512798:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.512799:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.512802:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.512803:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.512803:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.512805:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.512805:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.512807:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.512808:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.512808:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.512811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.512814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.512819:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.512821:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.512822:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677475.512823:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677475.512823:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.512823:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.512824:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.512825:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.512825:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.512828:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.512832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.512839:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.512841:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.512842:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.512843:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.512844:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.512844:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.513061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.513066:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.513069:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.513070:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.513071:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.513073:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.513075:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.513076:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.513077:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.513078:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.513079:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.513083:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.513084:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.513087:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.513088:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.513088:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.513090:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.513090:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.513091:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.513091:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.513096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.513103:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.513105:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.513106:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.513108:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.513109:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.513111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.513186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.513193:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.513196:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.513199:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.513200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.513201:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.513202:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.513204:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.513205:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.513206:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.513207:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.513210:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.513212:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.513215:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.513216:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.513216:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.513218:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.513219:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.513220:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.513220:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.513224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.513231:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.513234:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.513235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:20.2:1619677475.513235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:6.0:1619677475.513235:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.513237:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.513238:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677475.513240:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.513241:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.513243:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.513244:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.513245:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.513246:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.513248:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.513249:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.513250:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.513251:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:21.2:1619677475.513254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:20.0:1619677475.513255:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.513256:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.513258:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.513259:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.513260:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.513261:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:31.0:1619677475.513262:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.513262:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.513263:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.513263:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677475.513265:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.513267:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.513268:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.513268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:31.0:1619677475.513270:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.513271:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.513273:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.513274:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.513275:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.513275:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:31.0:1619677475.513276:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.513277:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.513278:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.513280:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.513281:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.513281:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:31.0:1619677475.513282:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.513283:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677475.513285:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.513285:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.513286:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.513287:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.513288:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.513289:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.513290:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.513293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.513300:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.513303:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.513303:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.513305:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.513307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.513308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.513429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.513437:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.513439:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.513440:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.513442:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.513445:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.513447:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.513448:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.513449:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.513451:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.513451:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.513455:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.513457:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.513459:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.513459:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.513460:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.513462:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.513463:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.513464:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.513464:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.513469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.513477:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.513479:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.513480:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.513480:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.513482:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.513485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.514111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.514117:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.514120:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.514120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.514122:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.514125:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.514126:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.514127:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.514128:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.514129:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.514130:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.514133:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.514135:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.514137:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.514138:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.514138:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.514140:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.514140:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.514141:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.514141:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.514146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.514154:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.514156:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.514158:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.514159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.0:1619677475.514160:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.514160:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.514163:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677475.514167:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.514169:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.514170:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.514172:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.514175:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.514176:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.514178:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499564] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.514179:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.514180:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.514181:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.514184:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.514186:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.514189:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.514190:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.514190:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.514192:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.514193:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.514193:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.514194:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.514198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.514207:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.514209:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.514210:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.514210:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.514213:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.514215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677475.514259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.514264:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.514267:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.514268:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.514270:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.514273:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.514274:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.514276:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.514277:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.514278:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.514279:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.514282:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.514284:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.514286:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.514286:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.514287:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.514289:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.514290:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.514290:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.514291:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.514296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.514304:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.514306:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.514306:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.514307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.514308:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.514309:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.514449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.514457:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.514460:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.514462:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.514464:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.514464:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.514465:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.514467:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.514468:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.514469:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.514470:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.514473:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.514474:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.514477:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.514478:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.514478:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.514480:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.514481:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.514482:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.514482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.514486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.514493:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.514496:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.514497:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.514497:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.514499:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.514501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.514872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.514879:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.514882:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.514884:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.514886:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.514886:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.514888:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.514889:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.514890:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.514891:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.514892:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.514895:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.514897:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.514899:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.514900:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.514901:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.514902:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.514903:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.514904:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.514904:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.514909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.514917:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.514920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.514920:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.514921:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.514923:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.514926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.515004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.515009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.515012:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.515012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.515014:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.515016:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.515018:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.515019:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.515020:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.515021:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.515022:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.515025:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.515027:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.515029:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.515030:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.515031:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.515032:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.515033:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.515033:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.515034:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.515039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.515046:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.515048:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.515049:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.515051:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.515052:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.515053:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.515096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.515104:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.515107:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.515108:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.515110:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.515113:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.515114:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.515115:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.515117:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.515118:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.515119:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.515123:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.515125:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.515128:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.515128:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.515129:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.515131:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.515131:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.515132:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.515132:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.515136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.515144:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.515147:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.515147:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.515148:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.515150:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.515153:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.515835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.515841:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.515843:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.515845:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.515846:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.515849:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.515850:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.515852:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.515853:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.515854:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.515855:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.515858:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.515860:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.515862:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.515863:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.515863:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.515865:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.515866:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.515866:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:4.2:1619677475.515867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:2.0:1619677475.515867:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.515868:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.515872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.515875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:10.0:1619677475.515875:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.515877:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.515878:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.515880:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.515881:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.515882:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.515882:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.515882:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:10.0:1619677475.515883:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.515883:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.515884:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.515885:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.515885:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:2.0:1619677475.515885:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.515886:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677475.515886:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:10.0:1619677475.515887:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.515888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:10.0:1619677475.515888:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.515889:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.515890:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.515892:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.515894:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:10.0:1619677475.515894:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.515897:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:10.0:1619677475.515897:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.515898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677475.515899:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.515899:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677475.515900:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.515901:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.515902:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.515902:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677475.515903:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.515903:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677475.515904:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.515905:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.515906:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.515907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677475.515907:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.515908:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.2:1619677475.515908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:22.0:1619677475.515913:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.515915:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.515916:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677475.515916:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.515917:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.515918:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.515918:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677475.515919:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677475.515919:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:6.0:1619677475.515919:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.515921:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:6.0:1619677475.515921:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.515922:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677475.515923:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.515924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677475.515924:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.515924:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.515925:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:30.0:1619677475.515926:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.515926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.515926:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:30.0:1619677475.515927:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:25.2:1619677475.515927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.515928:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.515930:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677475.515930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677475.515931:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.515932:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:25.2:1619677475.515932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000400:00000200:30.0:1619677475.515934:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.515934:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677475.515936:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:20.0:1619677475.515937:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677475.515938:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.515938:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:20.0:1619677475.515938:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.515939:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677475.515939:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:30.0:1619677475.515940:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677475.515940:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.515940:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.515941:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.515941:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677475.515942:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.515942:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.515942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.515942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677475.515943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.515943:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677475.515944:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677475.515944:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:30.0:1619677475.515945:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:6.0:1619677475.515945:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:30.0:1619677475.515946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677475.515946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.515947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:9.2:1619677475.515947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.515953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.515955:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.515958:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.515959:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:21.0:1619677475.515959:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677475.515959:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.515961:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.515961:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.515962:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.515962:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677475.515963:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677475.515963:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677475.515963:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.515964:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.515964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.515965:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.515966:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677475.516593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.516598:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.516600:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.516602:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.516603:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.516604:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.516605:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.516607:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.516608:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.516609:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.516610:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.516613:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.516615:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.516617:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.516618:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.516619:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.516620:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.516621:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.516621:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.516622:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.516627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.516636:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.516639:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.516640:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.516640:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.516642:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.516644:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.516864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.516872:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.516875:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.516876:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.516878:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.516881:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.516883:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:26.0:1619677475.516884:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.516885:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.516886:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.516887:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:26.0:1619677475.516890:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.516892:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:26.0:1619677475.516894:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.516895:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:26.0:1619677475.516895:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:26.0:1619677475.516897:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:26.0:1619677475.516897:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.516898:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.516899:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.516905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677475.516910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.516915:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.516916:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.516917:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.516918:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.516919:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.516920:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677475.516921:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.516922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:30.0:1619677475.516922:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.516923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.516924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000400:00000200:30.0:1619677475.516925:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.516926:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.516927:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.516929:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.516930:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:30.0:1619677475.516930:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.516930:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.516931:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.516932:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.516933:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.516933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.516934:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:30.0:1619677475.516934:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.516936:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.516938:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.516939:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.516940:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.516941:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.516942:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.516944:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.516945:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.516946:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.516948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.516951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.516956:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.2:1619677475.516957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.516959:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.516959:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.516960:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.516961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.516962:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.516962:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677475.516962:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.516962:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.516963:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.516964:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.516965:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677475.516966:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.516966:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.516967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000400:00000200:6.0:1619677475.516968:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.516971:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.516972:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.516974:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.516975:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.516976:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677475.516976:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.516977:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.516978:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.516979:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.516980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.516981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:6.0:1619677475.516981:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.516983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677475.516983:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.516986:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.516986:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.516987:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.516989:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.516989:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.516990:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.516990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677475.516991:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.516993:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677475.516995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.516996:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.516997:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.516999:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.517000:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.517002:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.517002:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:31.0:1619677475.517003:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.517004:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.517005:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677475.517005:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.517005:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.517007:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.517009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677475.517010:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.517010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.517012:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.517014:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.517015:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.517016:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.517017:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.517018:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.517018:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.517019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.517022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.517029:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.517032:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.517032:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.517034:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.517035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.517036:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.517848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.517853:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.517856:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.517857:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.517859:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.517862:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.517863:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.517864:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.517866:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.517867:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.517868:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.517871:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.517873:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.517875:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.517876:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.517877:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.517878:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.517878:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.517879:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.517880:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.517885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.517894:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.517896:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.517896:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.517898:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.517899:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.517901:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.517958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.517967:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.517969:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.517972:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.517974:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.517974:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.517975:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.517977:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.517978:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.517979:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.517980:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.517983:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.517984:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.517987:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.517987:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.517988:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.517989:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.517990:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.517991:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.517991:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.517995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.518002:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.518004:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.518005:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.518006:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.518008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.518009:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.518012:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.518019:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.518021:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.518022:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.518023:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.518024:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.518024:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.518283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.518291:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.518295:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.518299:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677475.518302:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.518304:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.518306:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.518307:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.518309:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.518310:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.518311:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.518315:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.518316:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.518319:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.518320:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.518321:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.518322:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.518323:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.518323:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.518324:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.518328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.518337:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.518340:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.518341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.518342:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.518344:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.518346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.518347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.518355:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.518358:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.518359:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.518359:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.518359:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:4.2:1619677475.518360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:1.0:1619677475.518360:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677475.518367:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.518370:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.518372:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.518375:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.518375:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.518376:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677475.518377:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.518379:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.518380:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.518380:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.518384:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.518385:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.518388:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.518389:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677475.518390:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.518391:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677475.518392:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.518393:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:30.0:1619677475.518395:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677475.518396:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677475.518398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.518401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.518409:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:2.0:1619677475.518409:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:31.0:1619677475.518412:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677475.518412:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677475.518413:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:2.0:1619677475.518414:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.518416:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.518417:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.518419:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.518424:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.518559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.518566:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.518571:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.518572:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.518575:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:21.2:1619677475.518578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:18.0:1619677475.518578:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.518580:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.518582:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.518584:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.518585:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.518586:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:18.0:1619677475.518587:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.518589:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677475.518592:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:18.0:1619677475.518593:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.518596:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:31.0:1619677475.518598:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.518601:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.518601:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.518602:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677475.518602:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.518603:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:31.0:1619677475.518604:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.518605:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.518606:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677475.518606:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:31.0:1619677475.518607:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:18.0:1619677475.518607:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.518608:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.518609:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677475.518610:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.518612:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:25.2:1619677475.518612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.518615:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.518616:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.518617:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.518618:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.518618:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.518619:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.518620:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.518620:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.518621:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:30.0:1619677475.518621:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.518622:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:30.0:1619677475.518622:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.518625:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.518625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.518633:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.518636:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.518637:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.518638:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.518640:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.518643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.518647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.518653:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.518656:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.518656:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.518658:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.518660:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.518661:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.518663:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.518664:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.518665:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.518666:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.518668:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.518670:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.518672:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.518673:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.518673:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.518675:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.518675:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.518676:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.518677:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.518681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.518688:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.518690:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.518690:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.518691:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.518692:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.518693:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.518699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.518707:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.518709:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.518710:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.518710:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.518710:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.518711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.518716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.518722:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.518724:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.518725:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.518727:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.518729:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.518731:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.518732:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.518733:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.518734:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.518735:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.518738:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.518740:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.518742:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.518743:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.518744:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.518745:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.518746:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.518746:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.518747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.518752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.518759:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.518762:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.518762:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.518764:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.518766:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.518767:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.518817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.518825:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.518827:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.518830:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.518832:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.518833:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.518834:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.518836:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.518837:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.518838:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.518839:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.518843:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.518845:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.518847:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.518847:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.518848:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.518850:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.518850:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.518851:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.518852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.518856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.518862:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.518864:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.518865:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.518867:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.518868:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.518870:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.518985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.518990:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.518993:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.518995:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.518995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.518997:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.518999:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.519000:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.519001:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.519002:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.519003:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.519007:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.519009:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.519011:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.519012:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.519012:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.519014:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.519015:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.519016:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.519017:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.519018:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.519020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.519023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.519030:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.519031:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.519031:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.519031:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.519032:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677475.519033:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.519033:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.519034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.519035:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677475.519036:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677475.519042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.519046:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.519048:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.519050:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.519051:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.519052:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.519054:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.519055:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.519056:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.519057:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.519058:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.519061:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.519063:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.519065:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.519066:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.519066:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.519068:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.519069:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.519069:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.519070:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.519074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.519082:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.519084:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.519084:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.519086:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.519087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.519088:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.519093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.519101:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.519104:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.519105:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.519105:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.519106:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.519107:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.519126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.519134:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.519137:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.519138:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.519141:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.519143:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.519145:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677475.519146:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.519147:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.519148:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.519149:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677475.519153:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.519155:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.519158:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.519159:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677475.519160:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677475.519161:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677475.519162:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.519163:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.519164:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.519164:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677475.519167:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677475.519170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:1.0:1619677475.519174:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677475.519177:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:1.0:1619677475.519177:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.519178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.519179:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:1.0:1619677475.519179:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.519180:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.519181:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:1.0:1619677475.519181:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.519184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.519497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.519505:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.519508:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.519508:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.519510:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.519512:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.519514:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.519515:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.519516:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.519517:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.519518:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.519521:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.519522:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.519524:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.519525:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.519525:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.519527:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.519527:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.519528:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.519528:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.519533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.519541:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.519544:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.519545:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.519546:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.519549:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677475.519549:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:4.2:1619677475.519557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.519565:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.519568:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.519569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.519571:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.519574:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.519575:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.519576:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.519577:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.519578:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.519579:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.519582:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.519584:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.519586:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.519587:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.519588:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.519589:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.519590:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.519590:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.519591:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.519595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.519604:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.519606:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.519607:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.519607:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.519609:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.519612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.519775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.519781:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.519784:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.519785:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.519787:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.519790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.519791:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.519792:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.519793:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.519795:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.519795:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.519799:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.519800:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.519803:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.519803:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.519804:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.519806:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.519806:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.519807:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.519808:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.519809:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.519812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.519814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.519819:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.519821:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.519821:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.519821:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.519822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.519823:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.519824:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.519826:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.519826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.519827:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.519830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.519837:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.519840:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.519841:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.519842:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.519842:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.519843:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.519935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.519943:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.519945:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.519946:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.519948:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.519950:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.519952:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.519953:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.519955:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.519955:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.519957:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.519960:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.519962:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.519965:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.519966:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.519966:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.519968:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.519968:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.519969:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.519970:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.519971:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.519974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.519976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.519981:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.519983:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.519984:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.519985:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.519985:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677475.519985:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.519986:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.519987:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.519987:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.519990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.520015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677475.520018:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.520021:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677475.520023:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.520024:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677475.520025:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.520027:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.520028:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.520029:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.520030:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.520031:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.520035:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.520036:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.520039:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.520040:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.520040:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.520042:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.520043:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.520043:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677475.520044:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:25.2:1619677475.520048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.520055:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.520058:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.520059:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.520059:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.520061:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.520063:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.520528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.520537:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.520539:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.520541:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.520544:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.520544:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.520545:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.520546:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.520548:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.520549:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.520550:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.520553:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.520554:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.520557:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.520558:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.520558:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.520560:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.520560:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.520562:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.520563:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.520564:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.520566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.520569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.520574:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.520576:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:30.0:1619677475.520577:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.520577:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.520577:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.520579:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.520580:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.520580:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.520581:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677475.520582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.520653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.520661:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.520663:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.520664:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.520666:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.520669:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.520670:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.520671:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.520673:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.520673:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.520674:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.520677:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.520679:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.520682:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.520683:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.520683:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.520685:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.520685:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.520686:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.520686:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.520691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.520699:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.520701:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.520702:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.520702:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.520704:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.520707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.520708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.520713:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.520715:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.520716:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.520718:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.520720:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.520721:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.520722:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.520724:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.520725:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.520726:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.520729:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.520731:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.520733:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.520734:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.520735:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.520737:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.520737:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.520738:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.520738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.520743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.520750:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.520752:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.520753:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.520755:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.520756:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.520757:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.520906:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.520913:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.520916:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.520918:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.520920:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.520921:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.520923:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.520924:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.520925:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.520926:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.520927:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.520931:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.520933:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.520936:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.520937:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.520937:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.520939:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.520940:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.520941:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.520941:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.520945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.520952:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.520955:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.520956:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.520956:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.520958:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.520961:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.520966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.520973:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.520976:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.520977:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.520977:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.520978:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.520979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.521577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.521585:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.521588:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.521590:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.521592:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.521593:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.521594:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.521595:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.521597:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.521598:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.521598:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.521601:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.521603:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.521605:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.521606:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.521606:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.521608:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.521609:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.521610:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.521610:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.521615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.521621:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.2:1619677475.521624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.0:1619677475.521624:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.521625:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.521625:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.521627:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.521629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677475.521630:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.521632:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.521633:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:9.2:1619677475.521634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:6.0:1619677475.521635:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.521637:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.521638:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:24.0:1619677475.521640:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:6.0:1619677475.521640:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.521641:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.521642:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.521643:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677475.521643:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:24.0:1619677475.521644:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.521644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.521645:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.521645:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:6.0:1619677475.521646:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.521648:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.521650:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.521651:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.521652:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.521653:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.521654:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.521655:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.521655:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.521659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.521667:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.521669:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.521670:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.521672:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.521673:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.521674:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.522179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.522186:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.522189:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.522191:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.522193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.522194:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.522195:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.522197:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.522198:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.522199:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.522200:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.522204:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.522205:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.522208:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.522209:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.522210:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.522211:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.522212:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.522213:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.522213:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.522217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.522223:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.522226:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.522226:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.522227:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.522228:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.522230:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.522479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.522486:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.522489:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.522489:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677475.522492:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.522494:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.522495:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.522496:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.522498:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.522499:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.522500:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.522503:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.522505:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.522507:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.522508:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.522509:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.522511:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.522511:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.522512:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.522512:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.522518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.522526:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.522528:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.522529:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.522530:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.522532:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.522533:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.522592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677475.522599:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.522602:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677475.522603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677475.522605:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677475.522607:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.522609:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.522610:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.522612:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.522613:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.522614:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.522617:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677475.522619:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.522622:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.522623:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.522623:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.522625:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.522625:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.522626:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677475.522627:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677475.522632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.522640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.522643:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.522643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.522644:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.522646:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.522649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.522730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.522735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.522738:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.522739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.522741:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.522742:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.522744:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.522745:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.522746:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.522747:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.522748:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.522751:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.522753:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.522755:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.522756:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.522757:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.522759:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.522759:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.522760:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.522760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.522765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.522772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.522774:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.522775:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677475.522776:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:21.0:1619677475.522777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.522780:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.522782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677475.522782:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.522785:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.522786:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.522789:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.522791:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.522793:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.522794:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.522795:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.522796:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.522797:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.522800:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.522802:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.522804:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.522805:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.522805:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.522807:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.522808:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.522808:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.522809:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.522813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.522820:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.522822:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.522823:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.522823:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.522825:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.522825:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.522840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.2:1619677475.522841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.522846:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.522848:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677475.522848:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000400:00000200:21.0:1619677475.522851:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.522851:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:2.0:1619677475.522851:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.522852:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.522852:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.522852:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000400:00000200:21.0:1619677475.522853:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.522853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:21.0:1619677475.522855:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.522856:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.522857:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.522859:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.522860:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.522863:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.522864:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.522867:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.522867:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.522868:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.522870:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.522871:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.522871:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.522872:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.522876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.522877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677475.522884:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:31.0:1619677475.522886:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677475.522887:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:31.0:1619677475.522888:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677475.522888:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.522888:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.522890:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.522892:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.522893:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677475.522893:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:31.0:1619677475.522894:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.522895:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.522897:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.522898:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.522899:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.522900:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.522903:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.522905:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.522907:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.522908:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.522908:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.522910:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.522910:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.522911:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.522912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.522915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.522922:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.522925:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.522925:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.522927:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.522929:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.522930:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677475.522970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.522976:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.522979:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.522980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.522982:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.522984:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.522985:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.522986:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.522987:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.522988:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.522989:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.522992:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.522994:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.522995:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.522996:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.522997:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.522998:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.522999:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.522999:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.523000:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.523004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.523011:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.523013:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.523014:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.523015:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.523016:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.523017:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.523076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.523084:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.523087:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.523089:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.523091:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.523092:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.523093:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.523095:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.523096:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.523097:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.523098:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.523101:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.523103:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.523107:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.523107:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.523108:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.523110:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.523110:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.523111:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.523112:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.523116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.523123:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.523126:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.523127:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.523127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.523130:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.523132:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.523414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.523422:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.523425:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.523426:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.523428:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.523431:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.523432:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.523433:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.523434:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.523436:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.523436:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.523440:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.523441:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.523444:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.523445:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.523445:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.523447:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.523448:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.523449:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.523450:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.523450:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.523454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.523456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.523462:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.523462:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.523464:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.523465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677475.523465:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.523466:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.523466:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.523467:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677475.523469:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.523471:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.523622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.523627:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.523630:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.523630:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.523632:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.523635:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.523636:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.523637:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.523638:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.523639:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.523640:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.523643:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.523645:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.523647:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.523648:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.523649:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.523650:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.523651:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.523652:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.523653:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.523653:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.523657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.523658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.523663:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.523664:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.523665:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.523666:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.523666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.523667:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.523667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.523669:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.523670:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.523671:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.523676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.523684:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.523686:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.523687:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.523688:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.523689:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.523689:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.524503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.524509:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.524512:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.524513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.524514:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.524517:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.524518:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.524519:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499515] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.524521:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.524522:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.524523:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.524526:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.524527:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.524529:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.524530:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.524530:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.524532:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.524532:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.524533:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.524533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.524538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.524546:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.524548:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.524549:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.524551:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.524552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.524553:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.524579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.524583:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.524585:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677475.524588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.524588:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.524590:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.524592:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677475.524593:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.524594:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.524595:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.524597:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.524600:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.524602:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.524604:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.524605:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.524606:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.524608:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677475.524608:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.524609:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.524611:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677475.524611:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677475.524614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.524616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.524622:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:18.0:1619677475.524622:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677475.524624:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.524625:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:18.0:1619677475.524625:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:21.0:1619677475.524626:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:18.0:1619677475.524626:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.524626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.524628:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.524631:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.524754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.524760:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.524762:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.524764:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.524765:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.524767:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.524769:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.524770:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.524771:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.524772:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.524773:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.524776:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.524778:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.524780:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.524780:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.524781:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.524782:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.524783:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.524784:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.524784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.524789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.524796:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.524799:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.524799:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.524801:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.524802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.524803:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.525024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.525032:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.525034:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.525037:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.525037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:30.0:1619677475.525040:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.525041:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.525042:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677475.525042:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.525043:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.525044:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.525044:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.525045:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.525046:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.525049:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.525051:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.525052:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.525053:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:30.0:1619677475.525053:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.525055:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.525056:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.525057:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.525057:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.525061:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.525061:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.525062:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.525062:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.525063:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.525064:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.525065:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.525065:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.525066:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.525066:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.525067:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.525068:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.525068:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.525069:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.525069:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.525070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.2:1619677475.525070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.525071:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (23)++
00000800:00000200:31.0:1619677475.525072:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.525073:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.525075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.525076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677475.525079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.525082:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.525084:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:6.0:1619677475.525085:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.525086:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:22.0:1619677475.525086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.525086:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:6.0:1619677475.525086:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.525087:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.525088:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:21.0:1619677475.525088:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:6.0:1619677475.525088:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.525089:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.525089:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:21.0:1619677475.525089:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.525090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.525091:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:21.0:1619677475.525091:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.525091:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.525092:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:25.2:1619677475.525094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.525094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677475.525101:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.525104:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.525105:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.525105:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.525106:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.525107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.525458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.525466:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.525469:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.525470:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.525472:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.525475:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.525476:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.525477:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.525478:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.525480:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.525480:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.525484:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.525485:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.525488:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.525489:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.525490:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.525491:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.525492:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.525493:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.525494:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.525495:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.525498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.525500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.525506:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.525507:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.525508:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677475.525509:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677475.525510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.525510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677475.525510:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.525511:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.525512:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.525515:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.525777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.525784:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.525787:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.525787:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.525789:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.525792:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.525794:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.525795:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.525796:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.525797:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.525798:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.525801:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.525802:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.525805:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.525806:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.525806:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.525808:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.525808:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.525809:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.525809:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.525814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.525821:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.525824:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.525825:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.525825:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.525827:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.525830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.526047:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.526053:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.526055:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.526056:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.526057:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.526059:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.526061:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.526062:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.526063:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.526064:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.526065:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.526068:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.526069:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.526071:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.526072:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.526073:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.526074:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.526075:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.526076:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.526076:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.526080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.526088:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.526090:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.526091:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.526093:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.526094:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.526095:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.526218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.526225:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.526227:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.526229:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.526231:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.526234:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.526235:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.526237:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.526239:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.526240:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.526241:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.526244:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.526246:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.526248:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.526249:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.526249:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.526251:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.526252:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.526252:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.526253:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.526258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.526266:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.526269:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.526270:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.526270:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.526272:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.526274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.526373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677475.526380:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.526383:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677475.526384:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677475.526386:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677475.526389:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.526390:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.526392:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.526393:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.526394:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.526395:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.526399:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677475.526401:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.526403:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.526404:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.526405:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.526407:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.526407:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.526408:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:8.0:1619677475.526410:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677475.526410:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:25.2:1619677475.526413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.526416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:22.0:1619677475.526421:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.526424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:22.0:1619677475.526424:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.526425:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.526425:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.526426:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677475.526426:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677475.526427:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:22.0:1619677475.526427:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:21.2:1619677475.526429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.526430:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.526437:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.526440:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.526442:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.526445:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.526445:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.526446:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.526448:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.526449:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.526450:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.526451:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.526454:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.526455:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.526457:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.526458:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.526458:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.526460:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.526460:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.526461:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.526462:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.526465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.526472:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.526474:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.526475:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.526477:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.526478:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.526480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.526797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.526805:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.526807:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.526809:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.526811:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.526812:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.526813:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.526814:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.526815:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.526816:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.526817:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.526820:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.526822:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.526825:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.526825:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.526826:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.526827:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.526828:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.526829:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.526829:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.526833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.526841:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.526843:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.526844:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.526845:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.526847:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.526849:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.526905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.526913:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.526916:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.526918:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.526920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.526921:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.526922:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.526924:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.526925:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.526926:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.526927:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.526930:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.526931:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.526935:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.526935:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.526936:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.526938:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.526938:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.526939:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.526940:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.526944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.526953:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.526955:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.526956:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.526958:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.526958:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.526961:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677475.527227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.527233:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.527236:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.527237:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.527239:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.527240:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.527241:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.527243:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.527244:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.527245:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.527246:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.527249:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.527251:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.527252:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.527253:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.527254:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.527255:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.527256:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.527257:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.527258:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.527258:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.527261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.527264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.527270:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.527271:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.527271:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.527271:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.527272:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.527272:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.527273:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.527273:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.527274:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677475.527275:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.527281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.527288:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.527290:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.527291:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.527292:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.527292:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.527292:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.527365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.527374:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.527376:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.527378:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.527381:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.527382:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.527383:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.527384:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.527385:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.527386:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.527387:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.527390:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.527392:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.527394:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.527395:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.527396:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.527397:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.527398:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.527399:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.527399:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.527403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.527410:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.527413:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.527413:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.527415:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.527417:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.527417:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.527419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.527427:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.527429:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.527430:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.527431:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.527431:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.527432:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.527768:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.527775:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.527778:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.527779:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.527781:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.527784:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.527785:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.527786:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.527787:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.527788:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.527789:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.527792:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.527793:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.527796:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.527796:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.527797:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.527798:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.527799:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.527800:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.527801:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.527802:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.527804:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.2:1619677475.527806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:4.2:1619677475.527808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.2:1619677475.527810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.527813:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:12.0:1619677475.527813:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677475.527815:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:22.0:1619677475.527815:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.527816:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677475.527817:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677475.527817:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:12.0:1619677475.527817:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:26.0:1619677475.527818:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:22.0:1619677475.527818:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677475.527818:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.527819:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677475.527820:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:22.0:1619677475.527821:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.527821:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:31.0:1619677475.527823:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.527823:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:13.0:1619677475.527823:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:30.0:1619677475.527824:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:12.0:1619677475.527824:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:31.0:1619677475.527825:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.527825:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.527826:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.527827:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.527828:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.527828:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.527829:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.527831:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.527833:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.527833:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.527834:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.527836:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.527838:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.527839:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.527839:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:31.0:1619677475.527840:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.527840:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.527842:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.527842:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.527842:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.527843:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677475.527843:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.527843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677475.527844:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.527845:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.527846:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.527847:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:25.2:1619677475.527848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.527849:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.527850:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.527852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.527855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.527856:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.527858:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.527859:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.527859:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:20.0:1619677475.527860:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.527862:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677475.527862:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:30.0:1619677475.527863:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677475.527863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677475.527863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677475.527863:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.527865:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.527866:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.527867:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.527868:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677475.527868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.527894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.527901:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.527903:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.527905:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.527906:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.527908:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.527910:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.527911:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.527912:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.527913:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.527914:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.527917:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.527919:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.527922:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.527923:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.527923:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.527925:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.527926:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.527926:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.527926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.527931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.527939:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.527942:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.527943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.527943:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.527946:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.527948:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.527950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.527957:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.527960:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.527960:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.527961:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.527962:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.527963:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.528309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677475.528316:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.528319:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[2] from 10.0.10.111@o2ib7
00000400:00000200:13.0:1619677475.528322:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.5.6@o2ib3(10.0.10.237@o2ib7) <- 10.0.10.111@o2ib7 : PUT - routed
00000800:00000200:12.0:1619677475.528323:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.528324:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.528825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.528833:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.528836:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.528838:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.528840:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.528840:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.528842:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.528843:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.528844:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.528845:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.528846:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.528849:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.528850:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.528853:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.528854:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.528855:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.528856:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.528857:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.528858:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.528859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.528863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.528872:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.528874:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.528875:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.528875:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.528878:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.528880:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.529160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677475.529165:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.529168:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677475.529169:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677475.529171:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.529172:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.529174:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:21.2:1619677475.529174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:22.0:1619677475.529175:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.529177:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.529178:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.529179:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.529180:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.529181:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677475.529183:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.529184:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.529185:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:6.0:1619677475.529186:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.529187:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677475.529188:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.529188:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.529189:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677475.529190:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:6.0:1619677475.529190:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.529191:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.529192:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:6.0:1619677475.529192:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.529193:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.529194:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677475.529194:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:22.0:1619677475.529195:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.529195:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.529197:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.2:1619677475.529199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677475.529199:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.529200:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.529200:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.529202:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.529202:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.529203:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.529204:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.529205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:17.0:1619677475.529207:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:25.2:1619677475.529208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.529210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677475.529210:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.529211:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.529211:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.529213:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.529215:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677475.529216:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.529217:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.529218:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.529219:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.529219:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.529220:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.529220:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.529222:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.529223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.529224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.529226:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.529233:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.529235:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.529236:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.529237:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.529238:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.529239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.529505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.529507:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.529509:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.529512:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.529513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.529513:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.529515:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:20.0:1619677475.529516:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.529518:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.529519:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.529520:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:20.0:1619677475.529523:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.529524:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677475.529527:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.529528:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:20.0:1619677475.529529:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677475.529531:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:20.0:1619677475.529531:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.529532:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.529532:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:25.2:1619677475.529537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.529545:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.529548:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.529549:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.529551:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.529553:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.529554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.530609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.530617:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.530619:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.530622:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.530624:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.530625:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.530626:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.530628:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.530629:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.530630:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.530631:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.530635:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.530637:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.530640:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.530641:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.530641:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.530643:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.530644:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.530644:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.530645:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.530649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.530658:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.530661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.530661:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.530662:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.530664:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.530667:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.530675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.530680:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.530682:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.530683:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.530685:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.530687:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.530688:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.530690:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.530691:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.530692:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.530693:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.530696:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.530698:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.530700:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.530701:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.530702:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.530704:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.530704:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.530705:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.530705:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.530709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.530717:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.530719:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.530720:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.530722:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.530723:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.530724:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.530760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.530768:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.530770:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.530771:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.530773:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.530776:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.530777:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.530779:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.530780:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.530781:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.530782:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.530785:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.530787:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.530790:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.530791:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.530791:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.530793:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.530793:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.530794:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.530795:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677475.530797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:25.2:1619677475.530799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.530804:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.530806:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677475.530807:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.530809:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.530809:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.530810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.530812:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.530812:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.530814:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677475.530815:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.530816:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.530818:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.530819:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:22.0:1619677475.530822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.530823:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677475.530823:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.530824:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677475.530826:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677475.530829:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.530829:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677475.530830:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677475.530832:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677475.530832:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.530833:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.530833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677475.530838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.530845:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.530847:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.530848:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.530850:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.530851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.530854:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.531150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.531157:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.531159:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.531160:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.531162:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.531165:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.531166:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.531167:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.531169:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.531170:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.531171:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.531174:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.531175:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.531178:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.531179:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.531179:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.531181:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.531182:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.531182:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.531183:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.531187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.531194:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.2:1619677475.531194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.531197:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.531198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.531199:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.531201:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677475.531201:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.531203:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.531204:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.531206:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.531208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.531209:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.531211:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.531212:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.531213:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.531214:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.531215:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.531219:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.531220:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.531222:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.531223:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.531223:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.531225:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.531226:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.531227:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.531227:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.531231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.531238:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.531240:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.531241:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.531242:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.531244:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.531245:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.531370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.531377:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.531380:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.531381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.531383:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.531385:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.531387:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.531388:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.531389:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.531391:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.531391:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.531395:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.531397:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.531400:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.531401:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.531402:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.531404:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.531404:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.531405:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.531406:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.531407:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.531410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677475.531413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.531418:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:12.0:1619677475.531420:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:2.0:1619677475.531420:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.531420:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.531421:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:12.0:1619677475.531423:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.531424:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.531427:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.531428:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.531430:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677475.531612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.531618:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.531621:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.531622:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.531624:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.531626:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.531628:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.531629:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.531630:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.531632:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.531632:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.531635:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.531637:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.531638:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.531639:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.531640:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.531642:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.531642:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.531643:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.531643:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.531648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.531656:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.531658:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.531658:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.531660:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.531660:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.531661:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.531959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.531967:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.531970:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.531972:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.531975:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.531975:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.531976:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.531977:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.531979:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.531980:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.531981:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.531984:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.531986:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.531989:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.531989:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.531990:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.531991:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.531992:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.531993:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.531994:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.531997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.532004:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.532006:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.532007:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.532007:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.532009:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.532011:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.532308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677475.532316:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.532318:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677475.532319:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677475.532321:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677475.532324:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.532325:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677475.532326:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.532327:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.532328:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.532329:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677475.532332:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677475.532334:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.532336:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.532336:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.532337:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.532338:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.532339:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.532339:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677475.532340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677475.532344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.532351:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.532353:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.532353:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.532354:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.532355:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.532356:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.532363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.532368:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.532371:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.532372:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.532372:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.532372:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.532373:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.532397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:21.2:1619677475.532403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:10.0:1619677475.532405:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.532408:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.532409:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:6.0:1619677475.532409:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677475.532410:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.532411:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.532412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677475.532413:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.532414:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.532415:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677475.532416:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.532417:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.532417:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.532418:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.532418:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:10.0:1619677475.532419:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677475.532420:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.532422:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.532423:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.532425:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.532425:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.532428:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:6.0:1619677475.532429:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.532430:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.532431:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677475.532431:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677475.532431:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.532433:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.532433:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.532434:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:6.0:1619677475.532434:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.532435:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677475.532435:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.532435:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.532437:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.532438:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.532438:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:9.2:1619677475.532439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677475.532439:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.532443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677475.532445:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.532447:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.532447:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.532448:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.532449:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.532451:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:26.0:1619677475.532452:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.0:1619677475.532453:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.532454:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.532456:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.532457:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.532458:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.532502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.532511:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.532514:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.532516:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.532518:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.532518:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.532520:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.532521:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.532523:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.532524:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.532525:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.532528:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.532530:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.532532:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.532532:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.532533:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.532534:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.532535:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.532536:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.532536:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.532540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.532547:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.532549:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.532550:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.532551:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.532553:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.532554:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.532631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.532639:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.532641:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.532642:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.532644:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.532647:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.532649:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.532650:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.532651:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.532652:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.532653:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.532656:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.532658:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.532660:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.532661:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.532662:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.532663:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.532664:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.532665:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.532665:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.532669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.532676:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.532678:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.532679:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.532680:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.532681:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.532682:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.532763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.532771:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.532774:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.532774:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.532776:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.532778:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.532780:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.532781:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.532782:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.532783:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.532784:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.532787:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.532789:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.532792:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.532793:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.532793:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.532795:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.532796:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.532797:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.532797:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.532801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.532809:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.532812:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.532812:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.532813:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.532815:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.532817:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.532821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.532828:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.532830:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.532831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.532831:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.532833:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.532833:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.532873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.532880:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.532882:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.532885:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.532886:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.532887:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.532888:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.532889:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.532891:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.532892:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.532893:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.532896:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.532898:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.532900:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.532900:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.532901:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.532902:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.532903:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.532904:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.532906:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.532906:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.532908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.532912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.532916:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.532918:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.532918:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.532920:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.532920:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.532921:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.532922:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.532923:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.532924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.532925:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.532927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.532934:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.532936:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.532937:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.532938:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.532938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.532939:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.533088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.533096:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.533099:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.533101:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.533103:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.533103:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.533104:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.533105:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.533107:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.533107:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.533109:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.533112:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.533114:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.533117:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.533117:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.533118:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.533119:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.533120:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.533122:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.533123:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.533123:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.533125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.533129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.533133:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.533136:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.533136:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.533137:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.533137:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.533139:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000400:00000200:22.0:1619677475.533139:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.533140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.533141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.533142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.533782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.533787:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.533790:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.533791:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.533793:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.533794:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.533796:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.533797:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.533798:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.533799:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.533800:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.533803:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.533805:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.533807:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.533808:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.533809:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.533810:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.533811:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.533811:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.533812:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.533817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.533825:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.533828:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.533828:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.533830:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.533831:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.533832:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.534781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.534784:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.534786:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.534788:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.534789:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.534791:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.534792:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.534794:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.534796:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.534797:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.534798:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.534801:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.534803:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.534805:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.534805:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.534806:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.534807:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.534808:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.534809:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.534810:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.534811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.534814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.534817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.534821:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.534821:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.534822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:20.0:1619677475.534822:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.534823:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677475.534823:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.534824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.534825:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.534826:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.534827:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.2:1619677475.534886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.534894:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.534897:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.534899:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.534901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.534902:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.534903:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.534905:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.534906:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.534907:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.534908:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.534912:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.534913:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.534916:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.534917:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.534918:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.534920:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.534920:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.534921:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.534922:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.534926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.534934:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.534937:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.534938:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.534938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.534940:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.534943:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.535015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.535021:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.535024:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.535025:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.535027:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.535028:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.535030:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.535031:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.535033:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.535034:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.535035:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.535038:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.535039:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.535042:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.535042:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.535043:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.535045:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.535045:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.535046:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.535046:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.535051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.535058:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.535060:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.535061:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.535063:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.535064:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.535066:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.535073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.535081:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.535083:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.535084:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.535085:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.535086:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.535087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.535093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.535101:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.535104:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.535105:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.535107:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.535109:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.535110:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.535111:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.535112:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.535113:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.535114:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.535118:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.535119:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.535122:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.535123:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.535124:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.535125:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.535126:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.535126:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.535127:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.535131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.535139:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.535141:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.535142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.535143:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.535145:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.535148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.535151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.535157:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.535160:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.535161:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.535161:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.535162:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.535163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.535256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.535262:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.535265:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.535266:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.535268:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.535270:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.535272:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.535273:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.535275:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.535276:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.535277:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.535281:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.535283:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.535285:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.535286:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.535287:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.535289:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.535289:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.535291:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.535292:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.535293:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.535297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.535298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.535305:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.535307:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.535307:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.535308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.535309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.535309:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.535310:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.535312:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.535313:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.535314:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.535323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.535330:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.535333:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.535334:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.535335:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.535336:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.535337:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.535942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.535950:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.535954:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.535956:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.535959:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.535962:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.535964:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.535966:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677475.535966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:17.0:1619677475.535969:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.535970:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.535972:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:26.0:1619677475.535974:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.535977:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:17.0:1619677475.535977:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.535977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.535980:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:17.0:1619677475.535981:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:26.0:1619677475.535982:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.535983:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677475.535984:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:17.0:1619677475.535985:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:26.0:1619677475.535986:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499468] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.535986:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:26.0:1619677475.535987:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.535988:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677475.535988:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:26.0:1619677475.535989:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677475.535989:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.535990:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.535991:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:26.0:1619677475.535992:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.535994:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.2:1619677475.535995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.535997:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.535998:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.535998:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.536000:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.536001:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.536001:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.536002:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:1.0:1619677475.536004:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:25.2:1619677475.536006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.536006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677475.536007:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.536009:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.536010:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.536012:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.536013:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:1.0:1619677475.536014:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:13.0:1619677475.536016:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677475.536016:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.536017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.536018:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.536019:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.536020:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.536021:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.536022:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:13.0:1619677475.536023:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:10.0:1619677475.536024:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:12.0:1619677475.536025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:10.0:1619677475.536025:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.536026:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.536027:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:13.0:1619677475.536029:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:10.0:1619677475.536031:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.536032:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.536035:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.536036:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.536036:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.536038:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.536038:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.536039:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.536040:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.536044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.536052:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.536055:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.536055:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.536056:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.536058:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.536061:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.536229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.536239:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.536243:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.536245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.536248:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.536251:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.536253:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.536255:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.536258:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.536260:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.536261:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.536267:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.536270:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.536274:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.536275:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.536276:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.536278:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.536279:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.536281:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.536282:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.536284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.536292:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.536300:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.536302:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:31.0:1619677475.536303:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.536305:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.536309:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.536447:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.536454:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.536457:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.536459:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.536461:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.536462:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.536463:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.536464:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.536465:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.536466:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.536467:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.536470:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.536472:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.536474:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.536475:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.536475:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.536477:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.536477:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.536478:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.536479:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.536483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.536490:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.536492:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.536493:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.536494:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.536496:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.536497:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.536621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.536629:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.536632:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.536632:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.536634:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.536637:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.536638:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.536639:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.536641:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.536641:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.536642:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:4.2:1619677475.536644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000400:00000200:26.0:1619677475.536645:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.536647:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.536648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:26.0:1619677475.536649:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.536650:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.536650:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:26.0:1619677475.536651:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.536652:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.536652:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.536653:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.0:1619677475.536653:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:26.0:1619677475.536654:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.536654:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:20.0:1619677475.536654:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.536656:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.536657:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.536658:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677475.536659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:20.0:1619677475.536660:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.536661:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677475.536662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:20.0:1619677475.536665:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.536667:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.536667:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.536669:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.536669:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.536670:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.536670:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.536670:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:21.0:1619677475.536671:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.536672:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.536673:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.536674:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.536674:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.536676:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.536676:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.0:1619677475.536677:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:25.2:1619677475.536678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.536678:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677475.536679:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.536680:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000400:00000200:21.0:1619677475.536681:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.536681:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.536681:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.536682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.536683:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.536683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:21.0:1619677475.536684:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:22.0:1619677475.536686:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:21.0:1619677475.536686:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.536687:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.536688:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:21.0:1619677475.536688:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.536689:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.536689:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677475.536689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.536691:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.536692:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.536693:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.0:1619677475.536695:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.536697:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.536698:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.536698:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:25.2:1619677475.536700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.536700:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.536701:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.536702:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.536702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.536707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677475.536708:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.536711:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.536711:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.536712:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.536713:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.536713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677475.536714:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.536716:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.536717:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.536718:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.536720:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.536722:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.536822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.536830:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.536832:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.536833:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.536835:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.536838:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.536840:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677475.536841:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.536842:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.536844:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.536844:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677475.536848:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.536850:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677475.536853:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.536854:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677475.536854:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677475.536856:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677475.536856:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.536857:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:4.2:1619677475.536858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.536861:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.536862:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:9.2:1619677475.536862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677475.536864:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.536866:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.536866:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.536868:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.536870:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:10.0:1619677475.536870:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.536871:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.536872:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:10.0:1619677475.536872:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.536873:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677475.536873:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.536874:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.536874:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677475.536877:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.536879:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.536881:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677475.536881:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.536884:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.536884:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.536885:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.536886:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:9.2:1619677475.536886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677475.536887:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.536888:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.536889:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.536890:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.536892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.536894:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.536895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:18.0:1619677475.536897:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.536897:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.536898:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.536899:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.536899:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:22.0:1619677475.536900:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.536903:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:22.0:1619677475.536903:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.536904:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.536904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.536905:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.536906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:22.0:1619677475.536906:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.536906:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677475.536909:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.536918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.536926:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.536929:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.536929:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.536930:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:26.0:1619677475.536931:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.536932:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.536934:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.536934:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.536935:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677475.536937:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.536938:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.536939:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.536940:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677475.536942:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.536944:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:10.0:1619677475.536944:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.536945:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.536946:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.536947:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.536948:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.536949:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.536950:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.536951:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.536952:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.536952:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.536953:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (23)++
00000800:00000200:26.0:1619677475.536954:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (24)--
00000800:00000200:10.0:1619677475.536954:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.536955:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.536956:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.536956:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.536957:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:9.2:1619677475.536957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.536958:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.536960:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.536961:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.536961:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.536962:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677475.536964:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.536964:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.536965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.536965:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.536966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677475.536966:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.536968:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:6.0:1619677475.536973:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.536975:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.536976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.536977:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.536979:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.536981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.537005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.537012:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.537015:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.537017:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.537019:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.537019:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.537020:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.537022:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.537023:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.537024:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.537025:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.537027:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.537029:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.537030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.537032:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.537033:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.537033:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.537035:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.537036:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.537037:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.537037:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677475.537038:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.537041:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677475.537041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.537042:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.537044:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.537045:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.537046:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.537047:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.537048:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.537048:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677475.537049:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.537050:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.537051:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.537052:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.537052:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677475.537054:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.537055:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.537056:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.537058:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.537059:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.537060:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.537060:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.537062:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.537063:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.537063:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.537064:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.537068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.537074:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.2:1619677475.537075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.537077:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.537078:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.537079:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.537080:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677475.537081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.537082:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:6.0:1619677475.537083:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.537084:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.537085:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.537088:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.537089:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.537090:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.537092:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.537093:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.537094:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.537097:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.537099:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.537101:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.537102:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.537103:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.537105:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.537105:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.537106:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.537106:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:4.2:1619677475.537108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:25.2:1619677475.537111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677475.537115:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.537118:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:20.0:1619677475.537118:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677475.537118:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.537120:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.537120:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.537121:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.537123:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.537123:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.537124:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677475.537125:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677475.537125:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:26.0:1619677475.537126:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.537127:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.537128:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.537130:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.537133:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.537135:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.537137:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.537138:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.537139:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.537140:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.537141:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.537142:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.537142:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.537146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.537154:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.537156:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.537157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.537158:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.537160:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.537162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.537177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.537184:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.537187:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.537187:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.537188:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.537189:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.537189:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.537648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.537656:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.537659:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.537660:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.537662:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.537665:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.537666:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.537668:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.537669:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.537670:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.537671:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.537674:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.537675:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.537678:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.537679:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.537679:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.537681:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.537681:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.537682:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.537682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.537688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.537696:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.537698:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.537699:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.537699:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.537702:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.537704:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.537716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677475.537717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.537724:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:30.0:1619677475.537725:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677475.537727:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:30.0:1619677475.537728:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:2.0:1619677475.537728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:30.0:1619677475.537731:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.537731:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.537732:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:18.0:1619677475.537733:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.537734:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.537736:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:18.0:1619677475.537736:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:18.0:1619677475.537737:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.537738:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.537739:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.537740:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.537741:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.537741:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.537742:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.537743:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677475.537747:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.537749:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.537750:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.537751:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677475.537753:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.537754:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.537754:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.537755:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:18.0:1619677475.537755:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.537756:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677475.537756:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.537757:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:18.0:1619677475.537757:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:30.0:1619677475.537758:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677475.537758:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:30.0:1619677475.537759:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.537759:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677475.537759:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677475.537763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.2:1619677475.537763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.537770:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.0:1619677475.537771:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677475.537772:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.537773:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.537773:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.537774:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.537775:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.537776:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677475.537777:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.537777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.537778:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.0:1619677475.537780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.538394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.538401:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.538404:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.538407:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.538409:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.538409:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.538410:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.538412:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.538413:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.538414:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.538415:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:21.2:1619677475.538416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:30.0:1619677475.538418:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.538419:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.538422:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.538422:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677475.538423:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.538424:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.538425:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:22.0:1619677475.538425:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.538425:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:30.0:1619677475.538426:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.538427:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:6.0:1619677475.538427:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.538428:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.538429:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:6.0:1619677475.538429:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.538431:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.2:1619677475.538432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:6.0:1619677475.538432:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.538433:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.2:1619677475.538434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:6.0:1619677475.538434:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.538435:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.538438:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.538440:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:6.0:1619677475.538440:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.538442:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677475.538442:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.538443:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.538443:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.538444:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.538444:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.538445:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:20.0:1619677475.538445:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.538446:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.538446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:6.0:1619677475.538446:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.538447:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677475.538447:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.538447:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.538448:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677475.538448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.538453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.538460:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.538462:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.538463:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.538465:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.538466:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.538467:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.538853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.538861:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.538863:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.538866:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.538867:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.538869:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.538870:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.538872:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.538873:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.538874:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.538875:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.538878:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.538880:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.538883:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.538884:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.538885:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.538886:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.538887:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.538888:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.538889:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.538902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.538910:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.538912:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.538913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.538914:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.538916:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.538918:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.539355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.539363:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.539366:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.539367:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.539368:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.539372:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.539373:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.539374:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.539375:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.539376:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.539377:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.539380:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.539381:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.539384:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.539384:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.539385:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.539386:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.539387:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.539388:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.539389:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.539390:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677475.539392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.539396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:8.0:1619677475.539399:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.539401:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.539402:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.539402:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.539403:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:8.0:1619677475.539404:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.539405:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.539405:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.539406:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.0:1619677475.539407:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.539413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.539419:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.539422:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.539422:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.539423:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.539423:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.539424:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.539433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.539439:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.539442:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.539443:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.539445:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.539448:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.539449:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.539450:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.539452:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.539453:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.539454:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.539458:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.539459:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.539461:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.539462:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.539463:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.539464:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.539465:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.539466:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.539466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.539471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.539477:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.539480:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.539480:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.539481:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.539482:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.539483:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.539655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.539660:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.539663:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.539664:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.539665:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.539667:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.539668:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:2.0:1619677475.539669:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.539670:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.539671:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.539672:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:2.0:1619677475.539674:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.539676:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:2.0:1619677475.539678:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.539679:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:2.0:1619677475.539679:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:2.0:1619677475.539681:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:2.0:1619677475.539681:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.539683:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677475.539683:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677475.539684:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.2:1619677475.539687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677475.539689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.539695:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:20.0:1619677475.539697:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:18.0:1619677475.539697:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677475.539698:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.539699:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:20.0:1619677475.539700:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.539701:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.539702:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.539704:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.539705:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.539908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.539913:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.539916:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.539917:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.539919:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.539921:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.539922:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.539923:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.539925:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.539926:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.539927:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.539930:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.539932:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.539934:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.539935:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.539936:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.539937:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.539938:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.539938:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.539938:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.539943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.539951:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.539953:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.539954:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.539955:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.539956:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.539957:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.540166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.540172:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.540175:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.540175:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.540177:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.540179:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.540180:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.540181:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.540182:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.540183:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.540184:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.540187:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.540188:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.540190:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.540191:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.540191:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.540193:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.540193:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.540194:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.540194:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.540199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.540207:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.540209:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.540210:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.540211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.540211:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.540212:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.540218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.540225:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.540227:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.540228:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.540228:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.540229:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.540230:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.540355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.540363:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.540366:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.540367:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.540369:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.540371:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.540372:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.540374:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.540375:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.540376:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.540377:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.540380:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.540382:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.540385:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.540386:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.540387:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.540388:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.540389:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.540389:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.540390:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.540395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.540403:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.540406:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.540407:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.540407:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.540409:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.540412:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.540439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.540446:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.540449:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.540450:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.540452:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.540455:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.540456:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.540457:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.540458:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.540460:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.540460:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.540463:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.540465:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.540467:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.540468:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.540468:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.540470:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.540470:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.540471:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.540472:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.540487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.540494:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.540497:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.540498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.540498:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.540500:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.540503:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.540637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.540639:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.540642:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.540644:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.540645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.540647:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.540648:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.540650:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.540651:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.540652:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.540653:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.540656:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.540658:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.540660:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.540661:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.540662:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.540664:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.540664:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.540665:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.540665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.540670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.540678:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.540680:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.540681:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.540683:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.540684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.540685:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.540844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.540851:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.540854:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.540855:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.540857:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.540860:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.540861:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.540862:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.540863:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.540865:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.540866:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.540869:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.540870:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.540873:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.540874:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.540874:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.540876:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.540876:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.540878:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.540879:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.540879:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.540882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.540885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.540890:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.540892:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.540893:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.540893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.540894:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:22.0:1619677475.540894:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.540895:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.540896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.540896:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.540898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.540901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.540908:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.540911:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.540911:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.540912:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.540913:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.540913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.541060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.541067:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.541070:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.541071:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.541073:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.541075:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.541077:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.541078:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.541080:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.541081:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.541082:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.541085:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.541086:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.541089:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.541090:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.541090:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.541092:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.541092:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.541094:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.541095:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.541095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.541098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.541101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.541106:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.541108:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.541109:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.541110:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677475.541110:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.541110:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.541111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.541112:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677475.541112:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.541115:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.541509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.541517:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.541520:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.541522:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.541524:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.541525:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.541526:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.541527:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.541528:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.541529:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.541530:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.541533:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.541535:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.541537:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.541537:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.541538:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.541539:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.541540:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.541541:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.541541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.541546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.541553:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.541555:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.541556:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.541557:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.541559:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.541560:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.541563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.541569:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.541572:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.541573:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.541573:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.541574:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.541575:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.541634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.541640:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.541642:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.541643:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.541645:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.541647:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.541649:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.541650:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.541652:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.541653:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.541654:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.541656:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.541658:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.541660:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.541661:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.541662:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.541664:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.541664:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.541665:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.541666:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.541667:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.541670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.541672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.541677:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.541677:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.541678:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.541679:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.541679:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.2:1619677475.541680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:20.0:1619677475.541680:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.541680:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.541682:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.541684:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.541685:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.541687:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.541690:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.541692:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.541694:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.541694:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.541695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000400:00000200:31.0:1619677475.541696:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.541697:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.541698:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.541699:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.541700:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.541702:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.541703:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.541705:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.541706:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.541706:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.541706:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.541707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.541708:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.541709:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.541709:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.541710:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.541712:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.541712:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.541714:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.541715:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.541716:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.541718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.541722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.541725:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.541728:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.541729:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.541729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.541730:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:22.0:1619677475.541731:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.541732:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.541733:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677475.541734:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.541735:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.542230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.542236:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.542238:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.542240:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.542241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.542242:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.542244:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.542245:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.542247:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.542248:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.542249:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677475.542252:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.542253:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.542256:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.542256:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.2:1619677475.542259:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.0:1619677475.542262:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.542264:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.542265:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.542265:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:30.0:1619677475.542266:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.542266:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:30.0:1619677475.542269:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:9.2:1619677475.542270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:30.0:1619677475.542271:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.542273:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.542274:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.542276:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.542277:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677475.542277:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:30.0:1619677475.542279:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.542279:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677475.542280:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:30.0:1619677475.542281:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677475.542281:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.542281:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.542283:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000400:00000200:30.0:1619677475.542286:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.542287:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:30.0:1619677475.542288:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.542290:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.542291:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.542291:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.542298:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:25.2:1619677475.542299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.542300:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.542301:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.542302:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.542303:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.542304:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.542305:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.542307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677475.542307:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.542308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.542784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.542787:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.542789:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.542791:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.542792:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.542793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.542795:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.542796:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.542798:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.542799:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.542800:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.542803:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.542805:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.542807:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.542807:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.542808:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.542809:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.542810:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.542811:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.542811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.542816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.542818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.542826:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.542826:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:31.0:1619677475.542829:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.542829:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.542829:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.542831:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.542832:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.542832:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:30.0:1619677475.542833:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.542834:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:31.0:1619677475.542835:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.542837:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.542838:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.542840:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.542841:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.542842:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.542845:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.542847:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.542849:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.542850:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.542851:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.542852:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.542853:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.542854:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.542854:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.542858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.542865:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.542867:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.542868:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.542870:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.542871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.542872:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.542941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.542947:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.542949:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.542950:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.542952:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.542954:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.542956:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.542957:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.542958:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.542959:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.542960:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.542963:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.542965:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.542967:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.542968:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.542969:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.542970:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.542971:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.542971:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.542972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.542976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.542983:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.542985:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.542986:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.542988:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.542989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.542990:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.543346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.543354:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.543357:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.543359:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.543361:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.543362:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.543362:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.543364:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.543365:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.543366:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.543367:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.543371:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.543372:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.543375:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.543376:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.543376:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.543378:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.543379:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:4.2:1619677475.543379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677475.543380:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.543380:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.543385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.543389:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.543392:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.543392:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.543394:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.543394:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.543395:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.543396:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.543398:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.543399:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.543400:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.543402:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.543403:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.543404:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.543405:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.543406:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.543407:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:8.0:1619677475.543407:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.543409:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.543412:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.543413:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.543413:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.543415:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.543416:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.543416:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.543417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.543421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.543429:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.543432:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.543433:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.543433:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.543435:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.543438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.543442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.543448:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.543450:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.543452:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.543453:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.543454:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.543455:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.543457:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.543458:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.543459:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.543460:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.543462:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.543464:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.543467:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.543468:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.543468:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.543470:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.543471:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.543472:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.543472:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.543476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.543482:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.543485:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.543486:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.543487:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.543489:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.543492:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.543797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.543804:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.543807:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.543808:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.543809:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.543812:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.543813:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.543815:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.543816:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.543817:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.543818:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.543820:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.543822:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.543824:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.543825:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.543826:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.543827:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.543828:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.543829:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.543829:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.543834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.543842:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.543845:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.543846:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.543846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.543847:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.543849:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.543852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.543859:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.543862:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.543862:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.543862:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.543863:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.543864:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.544139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.2:1619677475.544142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:10.0:1619677475.544147:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677475.544150:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677475.544150:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.544150:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:31.0:1619677475.544152:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:10.0:1619677475.544153:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:31.0:1619677475.544155:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.544156:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:10.0:1619677475.544156:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.544157:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.544157:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.544158:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.544159:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.544160:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.544160:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.544161:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.544161:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.544162:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.544162:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677475.544163:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.544166:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:31.0:1619677475.544168:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.544168:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677475.544170:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.544170:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.544171:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.544172:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:31.0:1619677475.544173:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.544173:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677475.544174:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.544174:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:31.0:1619677475.544175:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.544175:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.544175:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:31.0:1619677475.544176:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.544177:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.544178:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:25.2:1619677475.544179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:31.0:1619677475.544180:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.544181:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.544183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.544186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:12.0:1619677475.544187:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.544189:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:20.0:1619677475.544190:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:12.0:1619677475.544190:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.544191:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.544193:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677475.544194:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677475.544194:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.544194:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.544195:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677475.544196:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:20.0:1619677475.544196:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.544197:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.544197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.544198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.544199:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.544212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.544219:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.544222:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.544223:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.544224:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.544224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.544225:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.544459:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.544465:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.544467:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.544468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.544470:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.544472:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.544473:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.544474:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.544476:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.544477:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.544478:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.544481:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.544483:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.544486:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.544486:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.544487:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.544489:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.544489:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.544490:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.544490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:5.2:1619677475.544495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:25.2:1619677475.544496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677475.544502:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.0:1619677475.544504:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:17.0:1619677475.544505:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.0:1619677475.544506:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:1.0:1619677475.544506:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:21.0:1619677475.544507:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.544507:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.544509:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.544509:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.544510:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:17.0:1619677475.544510:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:21.0:1619677475.544511:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:17.0:1619677475.544512:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.544513:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.544514:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.544515:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.544518:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.544520:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.544522:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.544523:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.544523:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.544525:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.544525:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.544527:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.544528:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.544528:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.544532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.544535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.544540:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.544541:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.544541:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.544542:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:1.0:1619677475.544542:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.544543:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.544544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.544544:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.544545:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677475.544545:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.545403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.545411:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.545413:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.545414:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.545416:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.545418:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.545420:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.545421:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.545422:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.545423:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.545424:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.545427:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.545429:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.545432:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.545433:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.545434:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.545435:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.545436:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.545436:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.545438:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.545438:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.545442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.545444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.545450:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.545452:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677475.545452:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677475.545453:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.545453:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677475.545453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.545454:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677475.545455:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.545455:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677475.545458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.545658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.545666:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.545669:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.545672:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.545674:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.545675:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.545676:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.545678:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.545679:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.545680:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.545681:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.545684:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.545686:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.545688:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.545689:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.545690:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.545691:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.545692:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.545693:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.545693:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.545697:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.545705:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.545708:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.545709:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.545709:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.545711:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.545713:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.545726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677475.545734:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.545737:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677475.545739:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.545740:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677475.545741:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.545743:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:31.0:1619677475.545744:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.545745:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.545746:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.545747:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:31.0:1619677475.545750:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.545752:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677475.545754:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.545755:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:31.0:1619677475.545755:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:31.0:1619677475.545757:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677475.545758:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.545759:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677475.545759:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677475.545763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.545769:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.545771:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.545772:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.545772:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.545774:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.545777:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:5.2:1619677475.545909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:4.2:1619677475.545914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677475.545915:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677475.545918:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677475.545919:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:24.0:1619677475.545920:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677475.545921:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.545922:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.545923:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:17.0:1619677475.545924:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000800:00000200:8.0:1619677475.545924:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677475.545925:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.545926:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.545927:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.545928:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.545929:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.545929:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:24.0:1619677475.545931:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.545933:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.545933:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677475.545934:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.545935:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.545936:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.545936:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677475.545939:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.545940:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:24.0:1619677475.545941:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.545941:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677475.545943:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677475.545943:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:24.0:1619677475.545944:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.545944:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677475.545945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:24.0:1619677475.545946:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.545947:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.545948:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:9.2:1619677475.545948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:24.0:1619677475.545950:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.545950:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.545951:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.545952:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:10.0:1619677475.545955:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:8.2:1619677475.545956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677475.545957:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.545958:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.545958:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.545960:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.545963:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:1.0:1619677475.545964:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.545966:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.545967:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:9.2:1619677475.545967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000400:00000200:1.0:1619677475.545968:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.545969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:1.0:1619677475.545970:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.545973:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:26.0:1619677475.545975:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.545977:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:10.0:1619677475.545977:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.545978:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.545979:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677475.545979:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.545980:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.545981:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:10.0:1619677475.545981:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.545982:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.545984:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.545986:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:10.0:1619677475.545987:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.545988:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.545990:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.545990:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:10.0:1619677475.545994:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.545995:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:10.0:1619677475.545997:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.545998:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:10.0:1619677475.545999:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677475.546000:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677475.546000:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.546001:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.546002:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.546006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.546012:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.546015:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.546015:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.546017:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.546019:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.546020:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.546216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.546223:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.546225:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.546226:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.546228:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.546230:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.546231:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.546233:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.546234:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.546235:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.546235:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.546238:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.546240:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.546242:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.546243:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.546243:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.546245:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.546245:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.546246:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.546246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.546251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.546258:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.546261:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.546262:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.546262:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.546265:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.546267:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.546313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.546317:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.546319:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.546322:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.546322:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.546324:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.546326:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.546327:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.546328:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.546329:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.546330:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.546334:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.546335:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 504 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.546338:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.546339:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.546339:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.546341:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.546342:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.546342:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.546343:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.546347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.546355:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.546358:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.546359:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.546359:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:22.0:1619677475.546361:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.546361:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.546362:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (23)++
00000800:00000200:6.0:1619677475.546363:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677475.546363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.546364:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.546365:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.546451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.546459:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.546462:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.546464:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.546466:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.546466:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.546468:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.546469:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.546470:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.546471:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.546472:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.546476:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.546478:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.546480:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.546481:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.546482:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.546483:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.546484:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.546485:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.546485:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.546489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.546496:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.546498:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.546499:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.546500:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.546502:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.546503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.546507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.546514:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.546516:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.546517:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.546518:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.546518:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.546519:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.547153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.547160:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.547162:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.547164:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.547166:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.547167:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.547167:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.547168:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.547169:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.547170:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.547171:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.547173:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.547175:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.547176:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.547177:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.547178:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.547179:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.547179:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.547180:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.547181:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.547184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.547191:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.547194:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.547194:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.547196:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.547198:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.547199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.547271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.547278:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.547281:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.547283:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.547284:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.547285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.547286:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.547287:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.547288:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.547289:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.547289:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.547293:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.547299:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.547302:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.547302:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.547303:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.547304:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.547305:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.547306:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.547306:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.547311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.547318:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.547320:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.547321:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.547322:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.547323:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.547326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.547333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.547338:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.547341:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.547341:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.547342:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.547342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.547343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.547435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.547443:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.547445:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.547446:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.547448:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.547450:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.547452:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.547453:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.547455:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.547455:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.547456:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.547459:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.547461:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.547464:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.547465:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.547465:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.547467:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.547467:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.547468:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.547468:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.547473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.547481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.547484:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.547484:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.547485:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.547487:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.547490:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.547602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.547608:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.547610:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.547611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.547613:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.547615:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.547616:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.547617:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.547618:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.547619:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.547620:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.547623:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.547625:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.547627:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.547628:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.547629:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.547630:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.547631:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.547631:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.547632:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.547637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.547644:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.547646:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.547647:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.547648:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.547650:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.547651:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.548125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.548128:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.548130:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.548133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.548133:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.548135:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.548137:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.548138:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.548139:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.548140:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.548141:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.548145:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.548147:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.548149:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.548149:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.548150:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.548152:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.548152:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.548153:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.548153:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.548158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.548166:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.548168:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.548169:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.548171:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.548172:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.548173:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.548224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.548227:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.548230:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.548232:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.548233:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.548234:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.548236:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.548237:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.548238:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.548239:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.548240:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.548244:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.548245:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.548248:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.548248:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.548249:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.548251:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.548251:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.548252:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.548254:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.548254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.548256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.548259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677475.548263:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677475.548265:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677475.548265:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.548266:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.548267:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:8.0:1619677475.548267:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.548268:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.548269:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677475.548269:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.548271:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.548300:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.548308:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.548311:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.548312:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.548313:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.548316:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.548317:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.548318:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.548319:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.548320:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.548321:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.548324:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.548326:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.548329:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.548329:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.548330:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.548332:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.548332:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.548333:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.548333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.548338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.548346:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.548349:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.548350:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.548350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.548352:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.548355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.548380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.548388:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.548391:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.548393:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677475.548395:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.548397:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.548397:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677475.548398:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.548400:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.548400:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.548401:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677475.548405:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.548406:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677475.548409:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.548409:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677475.548410:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677475.548412:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677475.548413:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.548413:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.548414:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.548418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.548425:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.548428:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.548429:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.548429:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.548430:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.548433:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.548736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.548745:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.548747:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.548750:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.548752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.548752:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.548753:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.548754:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.548756:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.548756:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.548757:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.548761:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677475.548761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677475.548762:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.548765:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.548766:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.548766:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.548767:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.548768:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:17.0:1619677475.548768:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:30.0:1619677475.548769:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.548769:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677475.548771:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.548771:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:9.2:1619677475.548773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677475.548774:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.548776:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.548777:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.548779:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677475.548779:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677475.548780:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.548781:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.548782:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677475.548782:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.548783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.548783:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.548785:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.548786:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.548788:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677475.548788:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677475.548790:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.548791:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.548792:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.548793:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.548793:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.548794:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.548795:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.548799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.548807:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.548808:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.548809:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.548810:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.548811:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.548812:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.548986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.548992:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.548995:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.548995:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.548997:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.548999:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.549001:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.549002:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.549003:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.549004:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.549006:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.549009:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.549011:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.549013:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.549014:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.549014:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.549016:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.549017:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.549018:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.549019:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.549019:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.549023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.549024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.549029:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.549031:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.549032:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.549032:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.549033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.549034:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.549034:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.549036:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.549037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.549038:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.549048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.549055:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.549058:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.549059:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.549060:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.549060:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.549061:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.549302:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.549310:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.549313:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.549316:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.549317:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.549318:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.549319:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.549321:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.549322:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.549323:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677475.549323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:31.0:1619677475.549324:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.549327:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677475.549328:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.549329:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:31.0:1619677475.549330:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.549332:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.549332:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:31.0:1619677475.549333:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.549334:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.549334:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677475.549334:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.549336:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677475.549336:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:31.0:1619677475.549337:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:20.0:1619677475.549337:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.549338:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.549339:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000400:00000200:20.0:1619677475.549339:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.549340:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:20.0:1619677475.549340:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.549341:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:9.2:1619677475.549342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677475.549344:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.2:1619677475.549346:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677475.549346:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677475.549349:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.549350:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.549350:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677475.549350:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:24.0:1619677475.549352:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677475.549352:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:24.0:1619677475.549353:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.549353:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:8.0:1619677475.549353:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.549354:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:20.0:1619677475.549354:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.549354:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:24.0:1619677475.549355:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.549356:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.549357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677475.549357:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.549358:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.549358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.549363:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.549365:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.549365:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.549366:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:31.0:1619677475.549367:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.549370:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.549377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.549384:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.549386:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.549387:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.549388:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.549389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.549390:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677475.549482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.549489:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.549492:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.549493:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.549495:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.549497:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.549498:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.549500:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.549501:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.549502:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.549503:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.549506:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.549508:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.549510:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.549511:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.549512:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.549514:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.549514:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.549515:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.549515:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.2:1619677475.549520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.549529:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.549531:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.549532:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.549532:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.549535:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.549537:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.549558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.549564:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.549567:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.549568:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.549569:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.549572:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.549573:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.549574:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.549575:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.549576:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.549577:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.549580:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.549582:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.549585:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.549585:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.549586:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.549587:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.549588:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.549589:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.549589:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.549593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677475.549600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.549603:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.549605:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.549606:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.549607:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.549608:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.549609:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677475.549610:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.549611:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000400:00000200:30.0:1619677475.549611:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.549613:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.549613:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.549615:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.549616:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.549617:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.549618:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.549619:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.549622:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.549623:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.549626:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.549627:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.549627:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.549629:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.549629:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.549631:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.549632:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.549633:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.549634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.549638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.549641:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.549643:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.549644:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.549644:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.549646:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.549646:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.549648:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.549649:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.549649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.549650:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.549836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.549842:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.549845:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.549846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.549847:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.549849:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.549851:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.549852:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.549854:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.549855:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.549856:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.549859:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.549861:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.549864:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.549864:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.549865:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.549867:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.549867:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.549868:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.549870:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.549870:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.549873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.549875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.549881:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677475.549883:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677475.549883:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.549883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.549884:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677475.549885:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.549886:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.549888:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.549888:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.549891:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.549907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.549915:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.549918:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.549918:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.549919:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.549920:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.549921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.550245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.550253:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.550255:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.550256:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.550258:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.550261:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.550262:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.550264:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.550265:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.550266:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.550267:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.550270:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.550272:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.550275:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.550275:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.550276:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.550278:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.550278:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.550279:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.550279:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.550284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.550292:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.550299:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.550300:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.550301:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.550303:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.550306:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.550411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.550420:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.550423:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.550425:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.550428:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.550428:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.550429:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.550430:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.550432:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.550433:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.550434:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.550437:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.550438:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.550441:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.550441:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.550442:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.550444:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.550444:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.550445:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.550445:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.550450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.550457:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.550460:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.550461:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.550461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.550462:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.550464:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.550475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.550481:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.550484:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.550485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.550487:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.550489:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.550490:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.550492:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.550493:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.550495:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.550496:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.550499:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.550500:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.550503:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.550504:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.550504:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.550506:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.550507:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.550507:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.550508:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.550512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.550520:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.550522:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.550523:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.550525:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.550526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.550527:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.550998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.551006:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.551009:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.551011:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.551012:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.551013:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.551015:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.551016:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.551017:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.551018:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.551019:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.551023:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.551024:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.551027:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.551027:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.551028:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.551030:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.551030:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.551031:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.551032:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.551035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.551042:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.551044:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.551045:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.551045:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.551047:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.551049:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.551073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.551081:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.551084:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.551085:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.551087:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.551089:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.551091:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.551092:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.551093:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.551094:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.551095:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.551098:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.551100:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.551102:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.551103:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.551104:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.551105:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.551106:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.551106:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.551107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.551111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.551119:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.551121:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.551122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.551123:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.551125:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.551127:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.551562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.551568:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.551571:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.551572:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.551574:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.551576:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.551577:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677475.551579:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.551580:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.551581:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.551582:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677475.551585:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.551586:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677475.551589:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.551590:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677475.551591:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677475.551593:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677475.551593:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.551594:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:1.0:1619677475.551595:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677475.551596:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677475.551600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.2:1619677475.551604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:10.0:1619677475.551607:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:17.0:1619677475.551609:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:26.0:1619677475.551610:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.551610:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:17.0:1619677475.551611:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:10.0:1619677475.551611:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.551612:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.551613:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:10.0:1619677475.551613:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.551617:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677475.551761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.551769:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.551772:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.551773:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.551775:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.551777:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.551779:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.551780:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.551781:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.551782:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.551783:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.551786:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.551788:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.551791:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.551791:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.551792:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.551794:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.551794:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.551795:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.551797:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.551797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.551800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.551803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.551808:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:8.0:1619677475.551810:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.551811:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.551812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.551812:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677475.551812:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.551813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.551814:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677475.551814:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.551818:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.551981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.551985:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.551987:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.551990:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.551991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.551992:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.551993:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.551995:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.551996:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.551997:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.551998:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.552001:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.552003:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.552005:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.552006:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.552007:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.552008:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.552009:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.552010:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.552010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.552014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.552023:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.552025:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.552026:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.552026:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.552029:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.552031:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.552032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.552039:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.552042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.552042:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.552042:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.552043:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.552044:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.552045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.552050:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.552052:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.552054:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.552055:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.552057:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.552059:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.552060:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.552061:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.552062:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.552063:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.552066:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.552068:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.552070:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.552070:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.552071:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.552073:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.552074:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.552074:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.552076:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.552076:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.552079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.552085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.552090:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.552092:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.552092:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.552093:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.552093:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.552093:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.552094:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.552095:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.552096:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677475.552096:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.552215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.552220:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.552223:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.552224:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.552225:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.552228:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.552229:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.552230:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.552232:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.552232:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.552234:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.552237:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.552238:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.552241:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.552241:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.552242:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.552244:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.552244:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.552245:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.552245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.552250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.552258:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.552260:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.552261:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.552262:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.552263:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.552264:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.552318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.552325:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.552328:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.552329:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.552331:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.552334:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.552335:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.552336:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.552338:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.552339:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.552340:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.552343:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.552345:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.552347:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.552348:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.552349:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.552350:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.552351:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.552352:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.552352:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.552357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.552364:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.552367:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.552367:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.552369:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.552371:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.552372:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.552379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677475.552385:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.552388:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.552389:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.552390:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.552390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.552392:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677475.552398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677475.552407:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.552410:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677475.552412:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677475.552412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000400:00000200:31.0:1619677475.552414:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.552415:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677475.552416:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677475.552417:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.552418:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.552419:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677475.552419:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000400:00000200:31.0:1619677475.552420:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:26.0:1619677475.552422:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677475.552422:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:31.0:1619677475.552424:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.552425:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.552426:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.552427:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.552429:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677475.552429:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677475.552430:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000400:00000200:10.0:1619677475.552430:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.552431:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:10.0:1619677475.552432:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.552432:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.552433:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677475.552433:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000400:00000200:10.0:1619677475.552433:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677475.552434:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677475.552435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000400:00000200:10.0:1619677475.552436:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.552438:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:9.2:1619677475.552439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677475.552441:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.552442:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677475.552442:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677475.552444:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677475.552444:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677475.552445:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677475.552446:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:2.0:1619677475.552447:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.552450:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.552451:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:8.2:1619677475.552451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:2.0:1619677475.552451:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.552453:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.552457:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677475.552459:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.552462:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.552463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.552464:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.552466:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.552468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677475.552922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.552928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:21.2:1619677475.552930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:18.0:1619677475.552931:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.552932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.552934:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.552936:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.552937:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677475.552938:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:18.0:1619677475.552939:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.552940:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.552941:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:18.0:1619677475.552941:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.552943:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.552944:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.552945:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.552947:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.552949:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.552949:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.552950:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:18.0:1619677475.552951:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.552952:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.552952:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:30.0:1619677475.552953:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677475.552953:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:30.0:1619677475.552954:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677475.552954:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:30.0:1619677475.552955:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677475.552955:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.552956:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.552957:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.552958:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:30.0:1619677475.552959:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677475.552960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.552961:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.552963:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677475.552963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677475.552964:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.552965:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.552966:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.552967:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.552968:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.552968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:2.0:1619677475.552969:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.552970:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.552970:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.552971:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.552972:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.552972:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.552972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:2.0:1619677475.552973:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.552974:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.552975:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677475.552975:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:8.0:1619677475.552978:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.552981:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.552982:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.552982:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.552984:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.552987:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.552990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.552996:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.552999:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.553000:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.553000:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.553001:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.553001:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677475.553081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.553087:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.553090:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.553091:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.553092:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.553095:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.2:1619677475.553096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:1.0:1619677475.553096:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.553097:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.553099:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.553100:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.553101:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:18.0:1619677475.553102:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:1.0:1619677475.553104:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.553105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677475.553105:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:1.0:1619677475.553105:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:2.0:1619677475.553106:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:1.0:1619677475.553107:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677475.553108:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.553108:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.553109:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:18.0:1619677475.553110:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.553110:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.553111:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.553111:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:18.0:1619677475.553112:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:1.0:1619677475.553112:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677475.553113:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677475.553113:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.553114:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.553115:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.553116:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677475.553116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:18.0:1619677475.553117:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677475.553119:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.553119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.553121:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.553121:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.553123:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677475.553123:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:31.0:1619677475.553124:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96499378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.553124:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:31.0:1619677475.553125:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.553126:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677475.553126:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:31.0:1619677475.553127:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677475.553127:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677475.553127:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.553128:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.553128:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677475.553128:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.553129:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.553130:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:31.0:1619677475.553131:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.553131:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:17.0:1619677475.553131:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677475.553132:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.553132:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:31.0:1619677475.553134:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.553136:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.553136:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:25.2:1619677475.553136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:31.0:1619677475.553137:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.2:1619677475.553137:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677475.553139:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.553139:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.553140:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.553141:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:12.0:1619677475.553143:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:25.2:1619677475.553144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:1.0:1619677475.553145:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:12.0:1619677475.553146:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.553147:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.553148:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.553149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:12.0:1619677475.553149:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.553149:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:13.0:1619677475.553150:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:1.0:1619677475.553150:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:30.0:1619677475.553151:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:1.0:1619677475.553151:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:12.0:1619677475.553152:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:30.0:1619677475.553154:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.553154:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.553156:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.553158:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.553158:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.553322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.553330:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.553332:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.553333:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.553335:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.553338:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.553339:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.553340:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.553341:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.553342:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.553343:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.553347:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.553348:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.553351:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.553352:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.553353:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.553355:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.553355:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.553356:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.553356:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.553361:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.553369:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.553372:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.553373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.553373:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.553376:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.553379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.553539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.553544:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.553547:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.553547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.553549:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.553552:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.553553:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.553554:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.553556:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.553557:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.553558:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.553562:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.553563:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.553566:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.553567:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.553567:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.553569:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.553569:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.553570:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.553570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.553575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.553582:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.553585:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.553586:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.553587:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.553588:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.553590:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.553814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.553821:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.553824:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.553825:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.553827:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.553830:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.553831:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.553833:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.553834:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.553835:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.553836:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.553839:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.553841:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.553844:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.553845:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.553846:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.553847:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.553848:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.553849:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.553850:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.553850:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.553853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.553856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:21.2:1619677475.553858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677475.553860:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.553863:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:6.0:1619677475.553863:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.553864:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.553864:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.553866:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.553866:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:6.0:1619677475.553866:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.553867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.553868:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677475.553869:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677475.553869:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677475.553871:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.553873:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.553874:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677475.553874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.553876:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.553878:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.553879:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.553879:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.553882:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.553884:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.553886:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.553886:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.553887:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.553888:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.553889:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.553890:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.553890:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.553894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.553901:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.553903:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.553904:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.553905:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.553907:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.553908:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.554093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.554100:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.554103:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.554104:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.554106:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.554109:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.554110:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.554111:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.554113:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.554114:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.554115:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.554118:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.554120:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.554123:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.554124:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.554125:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.554127:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.554127:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.554128:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.554129:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.554130:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.554133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677475.554135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677475.554140:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677475.554141:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.554143:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:13.0:1619677475.554143:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:18.0:1619677475.554144:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:13.0:1619677475.554144:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.554144:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677475.554146:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.554147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.554149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.554348:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.554356:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.554358:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.554360:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.554362:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.554363:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.554363:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.554365:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.554366:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.554367:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.554368:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.554372:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.554373:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.554376:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.554376:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.554377:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.554379:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.554380:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.554381:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.554381:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.554386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.554393:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.554396:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.554397:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.554397:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.554398:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.554401:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.554787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.554795:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.554798:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.554800:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.554802:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.554802:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.554804:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.554805:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.554806:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.554807:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.554808:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.554811:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.554812:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.554815:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.554816:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.554817:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.554819:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.554819:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.554820:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.554822:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.554823:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.554825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.554829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.554837:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.554838:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677475.554839:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.554840:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.554840:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677475.554841:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677475.554841:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.554843:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.554844:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.554847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.554997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.555000:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.555003:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.555005:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.555006:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.555007:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.555009:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.555010:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.555011:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.555012:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.555013:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.555017:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.555018:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.555021:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.555021:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.555022:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.555024:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.555025:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.555025:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.555026:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.555030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.555039:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.555041:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.555042:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.555042:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.555044:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.555046:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.555065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.555071:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.555074:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.555074:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.555076:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677475.555077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677475.555079:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.555080:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.555081:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.555082:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.555083:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.555084:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.555084:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.555088:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677475.555088:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.555088:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677475.555090:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:10.0:1619677475.555091:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.555093:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.555093:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.555094:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:10.0:1619677475.555094:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.555095:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:22.0:1619677475.555096:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.555096:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.555097:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:10.0:1619677475.555097:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.555098:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:10.0:1619677475.555098:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.555099:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:10.0:1619677475.555099:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.555100:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677475.555102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.555103:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677475.555104:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.555105:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.555108:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.555109:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.555109:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.555110:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677475.555110:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.555111:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.555111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:10.0:1619677475.555112:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.555112:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.555112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.555113:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.555113:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.555113:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.555113:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:21.0:1619677475.555115:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.555116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:25.2:1619677475.555118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677475.555118:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.555124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.555125:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.555127:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.555128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.555129:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.555130:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:22.0:1619677475.555131:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.555133:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677475.555133:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.555134:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.555135:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.555135:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.555137:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.555138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.555145:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.555148:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.555148:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.555149:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.555150:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.555150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.555202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.555207:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.555209:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.555212:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.555212:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.555214:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.555215:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.555217:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.555218:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.555219:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.555220:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.555224:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.555225:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.555228:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.555229:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.555230:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.555232:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.555233:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.555233:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.555234:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.555238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.555244:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.555247:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.555248:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.555248:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.555251:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.555253:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.555652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.555658:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.555661:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.555662:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.555664:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.555665:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.555667:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.555668:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.555669:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.555670:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.555671:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.555674:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.555676:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.555677:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.555678:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.555679:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.555680:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.555680:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.555681:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.555682:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677475.555687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.555695:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.555697:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.555697:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.555698:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.555699:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.555700:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.555830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.555839:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.555842:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.555844:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.555846:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.555847:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.555848:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.555850:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.555851:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.555852:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.555853:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.555856:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.555858:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.555861:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.555862:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.555862:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.555864:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.555864:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.555866:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.555867:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.555868:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.555870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.555873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.555876:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.555879:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.555880:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.555880:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.555881:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.555882:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.555884:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677475.555884:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.555885:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.555885:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.556108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.556114:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.556117:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.556118:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.556120:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.556122:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.556124:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677475.556126:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.556127:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.556128:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.556129:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677475.556132:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.556134:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.556136:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.556137:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677475.556138:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677475.556140:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677475.556140:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.556141:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.556142:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677475.556146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.556153:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.556156:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.556157:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.556158:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.556160:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.556161:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.556168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.556175:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.556178:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.556179:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.556179:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.556180:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.556181:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.556209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.556213:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.556216:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.556218:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.556219:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.556221:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.556223:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677475.556224:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.556225:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.556226:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.556227:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.556230:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.556232:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.556234:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.556235:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.556236:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.556237:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677475.556238:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.556239:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.556240:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677475.556240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:9.2:1619677475.556243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.556246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.556251:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:18.0:1619677475.556252:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.0:1619677475.556254:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.556255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.556255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:18.0:1619677475.556255:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.556256:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.556256:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.556258:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.556261:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.556265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.556272:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.556275:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.556276:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.556276:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.556277:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.556277:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.556413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.556421:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.556424:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.556425:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.556427:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.556430:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.556431:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.556432:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.556433:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.556434:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.556435:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.556438:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.556440:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.556443:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.556443:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.556444:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.556446:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.556446:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.556447:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.556447:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.556452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.556460:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.556463:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.556464:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.556464:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.556467:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.556469:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.556824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.556829:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.556831:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.556832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.556834:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.556836:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.556838:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.556839:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.556840:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.556841:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.556842:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.556845:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.556847:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.556849:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.556850:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.556850:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.556852:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.556852:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.556853:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.556853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.556858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.556866:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.556868:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.556869:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.556871:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.556871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.556873:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.556955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.556961:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.556964:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.556965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.556966:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.556968:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.556969:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.556971:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.556972:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.556973:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.556974:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.556977:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.556979:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.556981:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.556982:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.556982:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.556984:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.556984:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.556985:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.556986:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.556991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.556999:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.557000:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.557001:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.557002:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.557003:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.557004:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.557198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.557203:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.557206:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.557207:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.557208:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.557209:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.557211:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677475.557212:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.557213:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.557213:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.557214:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677475.557217:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.557218:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.557219:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.557220:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.557220:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.557222:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.557222:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.557223:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.557223:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677475.557228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.557235:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.557237:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.557237:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.557239:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.557239:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.557240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.557247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.557255:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.557257:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.557258:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.557259:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.557259:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.557259:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.557325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.557328:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.557330:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.557332:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.557334:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.557335:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.557336:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.557337:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.557339:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.557339:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.557341:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.557344:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.557346:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.557348:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.557349:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.557349:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.557351:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.557351:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.557352:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.557353:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.557357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.557365:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.557367:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.557368:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.557370:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.557370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.557372:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.557416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.557424:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.557427:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.557429:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.557431:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.557431:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.557433:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.557434:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.557435:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.557436:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.557438:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.557440:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.557442:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.557445:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.557445:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.557446:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.557448:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.557448:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.557449:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.557450:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.557454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.557461:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.557464:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.557465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.557465:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.557467:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.557469:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.557474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.557480:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.557482:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.557483:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.557483:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.557484:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.557484:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.557989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.557997:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.558000:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.558002:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.558004:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.558004:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.558005:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.558007:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.558008:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.558009:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.558010:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.558012:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.558014:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.558016:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.558017:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.558017:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.558019:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.558019:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.558020:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.558021:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.558025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.558031:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.558033:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.558034:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.558034:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.558036:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.558038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.558046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.558052:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.558054:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.558055:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.558055:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.558056:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.558056:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.558088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.558094:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.558097:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.558098:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.558100:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.558101:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.558103:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.558104:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.558105:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.558106:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.558108:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.558110:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.558112:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.558115:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.558115:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.558116:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.558118:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.558118:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.558119:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.558119:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.558124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.558131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:20.0:1619677475.558135:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.558137:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:20.0:1619677475.558137:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.558138:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.558139:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:20.0:1619677475.558140:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.558141:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.558143:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677475.558144:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.558144:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:21.0:1619677475.558145:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:20.0:1619677475.558145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677475.558146:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.558148:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.558149:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.558150:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.558153:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.558155:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.558157:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.558158:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.558159:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.558160:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.558161:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.558162:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677475.558162:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677475.558166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.558173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.0:1619677475.558174:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.558176:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.558177:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.558178:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.558178:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.558179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.558181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677475.558181:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.558182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.2:1619677475.558183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:6.0:1619677475.558184:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.558186:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.558187:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.558188:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.558190:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.558190:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677475.558190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.0:1619677475.558191:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677475.558191:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.558194:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.558195:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.558195:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000400:00000200:6.0:1619677475.558195:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.558196:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.558197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677475.558197:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.558197:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.558199:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.558200:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677475.558201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.558201:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677475.558201:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677475.558203:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.558203:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.558203:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.558204:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.558205:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:8.0:1619677475.558206:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.558207:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:25.2:1619677475.558209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677475.558209:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.558210:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.558211:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.558212:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.558215:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.558216:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.0:1619677475.558217:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.558218:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.558219:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.558219:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.558220:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:20.0:1619677475.558221:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.558221:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.558222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.558223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677475.558223:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.558223:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.558224:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.558225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.558229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.558236:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.558239:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.558240:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.558240:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.558242:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.558245:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.558250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.558257:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.558259:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.558260:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.558261:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.558262:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.558262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.558635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.558643:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.558646:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.558648:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.558651:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.558651:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.558652:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.558654:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.558655:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.558656:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.558657:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.558661:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.558662:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.558664:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.558665:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.558665:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.558667:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.558667:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.558668:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.558669:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.558673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.558680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.558683:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.558683:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.558685:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.558686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.558688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.559116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.559124:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.559127:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.559129:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.559130:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.559131:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.559132:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.559133:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.559135:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.559136:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.559137:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.559140:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.559142:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.559144:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.559144:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.559145:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.559146:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.559147:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.559148:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.559148:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.559152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.559159:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.559161:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.559162:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.559163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.559164:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.2:1619677475.559164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677475.559167:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.559171:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.559174:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.559176:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.559178:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.559178:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.559179:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.559180:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.559181:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.559182:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.559183:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.559186:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.559187:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.559189:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.559189:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.559190:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.559191:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.559192:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.559193:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.559193:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.559197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.559204:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.559206:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.559206:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.559208:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.559210:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.559210:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.559640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.559645:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.559648:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.559649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.559650:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.559653:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.559655:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.559656:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.559657:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.559658:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.559659:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.559662:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.559664:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.559666:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.559667:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.559668:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.559669:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.559670:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.559670:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.559671:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.559676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.559684:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.559686:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.559687:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.559689:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.559690:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.559691:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.559784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.559792:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.559796:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.559796:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677475.559798:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.559801:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.559803:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.559804:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.559805:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.559806:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.559807:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.559811:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.559813:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.559815:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.559816:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.559817:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.559818:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.559819:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.559819:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.559820:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.559824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.559832:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.559834:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.559835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.559836:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.559838:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.559841:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.559989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.559997:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.560000:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.560001:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.560002:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.560005:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.560006:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.560008:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499346] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.560009:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.560010:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.560011:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.560013:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.560015:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.560017:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.560018:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.560019:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.560020:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.560021:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.560021:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.560022:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.560026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.560033:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.560036:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.560037:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.560037:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.560039:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.560042:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.560721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677475.560726:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.560729:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677475.560730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677475.560732:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.560734:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.560735:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:6.0:1619677475.560736:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.560738:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.560739:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.560740:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:6.0:1619677475.560743:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.560745:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:6.0:1619677475.560748:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.560749:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:6.0:1619677475.560750:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677475.560751:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677475.560752:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.560753:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.560753:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677475.560758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.560765:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.560768:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.560769:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.560769:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.560771:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.560774:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.560809:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.560812:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677475.560813:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677475.560814:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.560816:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.560818:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.560820:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.560820:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.560821:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.560822:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677475.560822:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.560823:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.560825:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677475.560825:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677475.560829:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.560829:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.560830:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.560830:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.560832:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677475.560833:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.560834:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.560835:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.560836:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.560836:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.560836:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:31.0:1619677475.560837:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.560838:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:20.0:1619677475.560838:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.560839:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.560840:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.560840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677475.560841:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.560843:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.560845:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677475.560845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677475.560846:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.560847:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.560848:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.560849:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.560850:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.560852:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.560852:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.560853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677475.560853:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.560856:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.560857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.560857:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.560858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677475.560859:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.560860:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677475.560861:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677475.560862:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.560863:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.560864:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:24.0:1619677475.560865:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.560866:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.560868:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677475.560868:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.560869:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677475.560869:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.560870:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677475.560872:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.560875:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.560875:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.560876:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.560877:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.560877:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.560951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:4.2:1619677475.560955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.560960:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.560962:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.560963:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.560964:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677475.560966:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.560967:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.560969:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.560970:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.560971:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.560972:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.560973:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.560975:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.560976:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.560977:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.560978:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.560979:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677475.560980:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.560981:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677475.560981:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.560982:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.560984:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000400:00000200:26.0:1619677475.560986:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.560986:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.560986:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.560987:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.560988:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.560989:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677475.560990:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.560990:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (23)++
00000800:00000200:26.0:1619677475.560991:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.560992:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.560992:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (24)--
00000800:00000200:10.0:1619677475.560992:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.560994:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.560994:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (22)++
00000800:00000200:25.2:1619677475.560994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.560994:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.560995:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.560995:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.560996:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:4.2:1619677475.560998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:25.2:1619677475.561000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:12.0:1619677475.561003:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677475.561005:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:12.0:1619677475.561006:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:22.0:1619677475.561007:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.561007:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.561007:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.561008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:12.0:1619677475.561008:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.561010:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:12.0:1619677475.561010:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:22.0:1619677475.561011:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.561011:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:13.0:1619677475.561012:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:22.0:1619677475.561013:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.561014:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677475.561017:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.561121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.561129:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.561132:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.561134:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.561136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.561136:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.561138:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.561139:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.561140:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677475.561140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000400:00000200:30.0:1619677475.561141:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.561142:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.561145:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000400:00000200:30.0:1619677475.561146:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.561147:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.561148:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:30.0:1619677475.561150:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.561150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:30.0:1619677475.561151:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:21.0:1619677475.561151:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.561152:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.561153:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.561154:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:21.0:1619677475.561154:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.561155:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.561155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:21.0:1619677475.561155:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677475.561156:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.561157:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.561158:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.561159:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:9.2:1619677475.561159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:21.0:1619677475.561162:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.561164:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:8.0:1619677475.561165:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677475.561166:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.561167:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:24.0:1619677475.561168:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677475.561168:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677475.561168:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677475.561169:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:8.0:1619677475.561169:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.561170:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.561170:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677475.561171:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000400:00000200:8.0:1619677475.561171:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.561173:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.2:1619677475.561175:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.561183:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.561186:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.561187:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.561188:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.561189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.561190:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.561263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.561270:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.561273:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.561274:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.561276:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.561279:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.561280:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.561282:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.561283:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.561284:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.561285:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.561288:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.561289:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.561292:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.561293:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.561299:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.561300:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.561301:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.561302:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.561303:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.561304:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.561307:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.561310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.561314:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.561317:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.561317:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.561318:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.561318:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.561319:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.561320:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.561321:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.561321:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:5.2:1619677475.561323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:6.0:1619677475.561324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677475.561329:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.561332:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.561333:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.561335:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.561337:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.561338:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.561339:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.561340:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.561342:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.561343:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.561346:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.561347:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.561349:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.561350:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.561351:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.561352:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.561352:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.561354:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.561355:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.561355:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.561359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.561362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:5.2:1619677475.561362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:22.0:1619677475.561368:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:17.0:1619677475.561368:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.561369:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.561369:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:22.0:1619677475.561370:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:20.2:1619677475.561370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:17.0:1619677475.561370:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.561370:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:6.0:1619677475.561371:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:1.0:1619677475.561371:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.561372:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.561372:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:22.0:1619677475.561374:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.561374:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677475.561374:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:22.0:1619677475.561377:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.561377:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:22.0:1619677475.561378:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.561379:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.561380:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677475.561381:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677475.561381:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677475.561382:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.561383:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.561384:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:22.0:1619677475.561386:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.561386:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.561388:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.561388:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.561389:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.561390:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.561391:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677475.561391:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.561392:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:18.0:1619677475.561392:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:22.0:1619677475.561393:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.561394:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.561395:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.561396:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:18.0:1619677475.561396:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677475.561397:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.561398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677475.561398:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.561400:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677475.561401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.561401:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.561401:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.2:1619677475.561403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:18.0:1619677475.561403:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.561404:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.561405:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.561405:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.561409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.561409:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.561410:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.561411:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.561411:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.561412:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.561412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.561413:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.561415:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.561415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.561417:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677475.561417:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:25.2:1619677475.561419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:1.0:1619677475.561420:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.561421:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.561421:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.561423:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.561425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.0:1619677475.561426:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.561429:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.561430:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.561430:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.561431:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.561432:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.561984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.561990:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.561993:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.561993:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.561995:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.561998:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.562000:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677475.562001:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.562002:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.562003:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.562004:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677475.562007:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.562009:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.562012:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.562013:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677475.562013:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677475.562015:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677475.562016:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.562017:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677475.562018:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677475.562018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677475.562021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.562023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:24.0:1619677475.562027:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677475.562028:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:24.0:1619677475.562030:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.562031:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677475.562031:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000400:00000200:24.0:1619677475.562032:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.562032:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.562033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:24.0:1619677475.562034:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.562036:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.562172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.562180:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.562183:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.562185:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.562187:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.562188:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.562189:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.562190:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.562192:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.562193:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.562194:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.562197:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.562199:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.562202:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.562203:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.562204:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.562206:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.562206:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.562207:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.562208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.562212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.562221:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.562223:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.562224:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.562226:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.562226:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.562229:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.562613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.562619:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.562622:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.562623:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.562625:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.562628:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.562629:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.562630:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.562632:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.562632:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.562633:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.562636:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.562638:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.562639:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.562640:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.562641:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.562642:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.562643:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.562644:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.562644:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.562649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677475.562653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:18.0:1619677475.562657:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.562659:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.562659:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.562659:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677475.562660:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.562661:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.562662:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677475.562662:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.562663:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.562665:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.562668:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.562669:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.562671:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.562672:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.562673:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.562674:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.562677:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.562678:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.562681:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.562682:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.562682:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.562684:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.562684:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.562685:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.562686:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.562690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.562698:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.562700:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.562701:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.562702:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.562704:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.562704:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.562745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198623bcc00] (20)++
00000800:00000200:22.0:1619677475.562750:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)++
00000800:00000200:22.0:1619677475.562753:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.10@o2ib3
00000800:00000200:6.0:1619677475.562754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (22)--
00000400:00000200:22.0:1619677475.562755:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.10@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.562758:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.562760:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:22.0:1619677475.562761:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.562762:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.562763:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.562764:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677475.562767:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.562769:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 2520 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.562771:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.562772:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677475.562773:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677475.562775:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677475.562775:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198623bcc00] (21)++
00000800:00000200:22.0:1619677475.562776:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198623bcc00] (22)--
00000800:00000200:22.0:1619677475.562776:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198623bcc00] (21)--
00000800:00000200:25.2:1619677475.562781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.562789:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.562792:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.562793:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.562794:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.562796:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.562798:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.562951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.562959:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.562961:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.562962:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.562964:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.562966:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.562968:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.562969:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.562970:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.562971:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.562972:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.562976:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.562978:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.562980:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.562981:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.562982:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.562983:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.562984:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.562985:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.562985:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677475.562989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677475.562994:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.562997:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.562997:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:25.2:1619677475.563000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677475.563000:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.563002:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.563005:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677475.563006:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.563008:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:6.0:1619677475.563009:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.563010:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.563010:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.563011:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.563011:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677475.563013:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.563015:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.563016:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.563017:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.563018:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:6.0:1619677475.563021:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.563022:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677475.563023:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677475.563024:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:25.2:1619677475.563025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.563025:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.563025:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.563029:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677475.563030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677475.563030:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.563032:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.563032:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.563033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.563033:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.563034:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:10.0:1619677475.563038:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.563041:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.563042:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.563043:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.563045:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.563048:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.563082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.563090:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.563093:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.563095:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.563096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.563097:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.563099:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.563100:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.563101:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.563103:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.563103:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.563107:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.563108:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.563111:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.563111:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.563112:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.563114:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.563114:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.563116:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.563117:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.563118:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.563119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.563123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.563126:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.563128:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.563129:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.563130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.563131:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:20.2:1619677475.563131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:24.0:1619677475.563132:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.563133:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.563134:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677475.563134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.563135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677475.563136:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.563139:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.563141:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.563142:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.2:1619677475.563144:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:21.0:1619677475.563148:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.563149:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.563151:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.563152:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:21.0:1619677475.563152:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.563153:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.563154:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677475.563155:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677475.563159:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.563159:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.563160:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.563161:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677475.563163:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.563164:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.563165:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:21.0:1619677475.563165:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:30.0:1619677475.563166:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.563166:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:30.0:1619677475.563167:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.563167:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:30.0:1619677475.563168:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.563168:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:30.0:1619677475.563169:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.563169:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.563170:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.563171:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677475.563173:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677475.563174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677475.563175:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.563177:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.563177:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:20.2:1619677475.563177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677475.563178:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.563179:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.563180:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.563181:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.563182:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677475.563184:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.563184:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:25.2:1619677475.563185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677475.563185:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.563185:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.563186:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.563186:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.563187:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.563188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.563189:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.563190:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677475.563192:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.563195:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.563196:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.563198:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.563199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.563200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.563203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.563210:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.563212:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.563213:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.563213:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.563214:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.563215:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.563317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.563322:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.563325:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.563327:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.563327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.563329:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.563330:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.563331:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.563333:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.563334:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.563335:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.563338:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.563339:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.563342:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.563342:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.563343:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.563345:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.563346:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.563347:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.563347:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.563352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.563359:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.563362:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.563363:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.563363:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.563365:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.563367:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.563476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.563480:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.563483:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.563485:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.563486:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.563487:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.563489:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.563490:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.563491:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.563492:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.563493:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.563497:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.563498:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.563501:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.563501:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.563502:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.563503:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.563504:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.563505:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.563505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.563510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.563518:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.563520:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.563520:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.563522:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.563523:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.563524:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.563728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677475.563733:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.563736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677475.563736:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677475.563739:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.563741:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.563743:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677475.563744:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.563745:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.563746:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.563747:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677475.563750:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.563751:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677475.563754:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.563755:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677475.563755:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677475.563757:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677475.563757:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.563758:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677475.563758:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677475.563763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.563770:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.563773:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.563774:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.563775:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.563776:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.563779:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.563786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.563794:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.563797:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.563799:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.563801:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.563801:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.563802:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.563803:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.563805:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.563806:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.563807:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.563810:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.563812:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.563814:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.563815:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.563815:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.563817:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.563818:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.563819:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.563819:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.563823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.563829:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.563832:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.563832:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.563834:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.563835:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.563836:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.564110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.564116:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.564119:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.564119:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.564121:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.564124:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.564125:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.564126:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.564128:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.564129:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.564130:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.564132:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.564134:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.564136:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.564137:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.564138:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.564139:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.564140:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.564141:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.564141:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.564146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.564153:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.564155:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.564156:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.564158:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.564159:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.564160:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.564321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.564329:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.564331:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.564332:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.564334:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.564337:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.564338:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677475.564339:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.564341:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.564342:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.564343:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677475.564346:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.564347:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.564349:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.564350:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.564350:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.564351:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.564352:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.564353:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.564353:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.564358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.564364:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.564365:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.564366:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.564367:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.564368:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.564368:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.564712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.564720:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.564723:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.564725:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.564727:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.564728:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677475.564728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.564729:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.564730:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.564731:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.564732:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.564735:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.564737:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.564739:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.564740:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.564741:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.564742:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.564743:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.564743:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.564744:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.564748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.564755:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.564758:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.564759:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.564759:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.564761:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.564762:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.564766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.564772:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.564774:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.564775:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.564775:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.564776:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.564777:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.565458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.565466:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.565468:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.565469:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.565470:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.565473:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.565475:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.565476:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.565477:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.565478:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.565479:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.565482:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.565483:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.565486:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.565487:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.565487:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.565489:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.565490:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.565491:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.565492:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.565492:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.565495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.565499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677475.565503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677475.565504:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677475.565507:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677475.565507:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.565508:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.565508:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677475.565509:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:24.0:1619677475.565510:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677475.565510:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.565510:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.565512:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.565512:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.565513:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677475.565513:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:26.0:1619677475.565514:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.565515:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677475.565515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677475.565517:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.565518:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.565519:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.565520:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.565522:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.565524:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.565527:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.565528:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.565528:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.565530:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.565530:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.565532:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.565533:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.565533:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.565537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.565539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.565545:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.565546:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.565547:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.565548:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677475.565548:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.565549:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.565549:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.565550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.565552:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.565554:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.565587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.565595:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.565598:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.565600:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.565603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.565603:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.565604:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.565605:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.565607:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.565608:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.565609:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.565612:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.565614:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.565616:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.565617:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.565617:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.565619:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.565619:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.565621:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.565622:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.565623:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.565624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.565628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.565631:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.565633:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.565634:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.565634:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.565636:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.565636:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.565638:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.565639:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.565640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.565640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.565648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.565654:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.565657:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.565658:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.565658:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.565658:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.565659:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.565875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.565880:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.565883:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.565883:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.565885:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.565887:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.565889:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.565890:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.565891:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.565892:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.565893:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.565896:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.565897:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.565900:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.565901:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.565901:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.565903:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.565903:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.565904:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.565904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.565909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.565917:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.565919:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.565920:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.565922:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.565923:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.565924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.566030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677475.566038:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.566041:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677475.566043:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.566045:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.566045:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677475.566047:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.566048:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.566049:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.566050:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.566051:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.566054:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.566056:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.566058:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.566059:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.566060:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.566061:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.566062:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.566063:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677475.566063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.566067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.566073:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.566075:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.566076:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.566076:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.566078:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.566080:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.566122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.566129:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.566132:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.566133:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.566135:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.566138:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.566139:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.566140:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.566141:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.566142:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.566143:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.566146:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.566148:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:5.2:1619677475.566149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:26.0:1619677475.566150:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.566151:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.566152:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.566153:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.566154:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.566155:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.566155:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677475.566155:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.566158:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.566159:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:25.2:1619677475.566160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:1.0:1619677475.566160:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.566162:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.566163:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.566164:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.566166:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.566167:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.566167:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.566168:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.566170:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:1.0:1619677475.566170:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.566171:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.566171:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.566172:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:6.0:1619677475.566174:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.566174:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.566175:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.566176:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677475.566177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:1.0:1619677475.566177:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.566178:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.566178:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.566179:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.566183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.566191:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.566193:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.566194:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.566194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.566196:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.566197:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.566272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.566279:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.566281:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.566282:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.566284:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.566287:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.566289:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677475.566290:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.566291:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.566292:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.566296:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677475.566300:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.566301:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677475.566303:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.566304:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677475.566304:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.566306:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.566306:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.566307:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.566309:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.566309:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677475.566312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677475.566315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.566321:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.566322:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.566322:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.566324:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.566324:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.566325:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.566325:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.566326:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.566327:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677475.566327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.2:1619677475.566604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.566607:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.566609:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.566611:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.566613:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.566613:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.566615:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.566616:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.566617:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.566618:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.566619:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.566622:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.566624:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.566626:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.566627:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.566628:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.566629:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.566630:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.566631:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.566631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.566636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.566644:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.566647:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.566648:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.566648:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.566650:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.566652:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.566927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.566935:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.566938:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.566940:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.566942:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.566943:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.566944:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.566945:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.566947:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.566948:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.566949:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.566952:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.566954:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.566957:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.566957:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.566958:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.566960:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.566960:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.566961:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.566962:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.566965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.566972:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.566974:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.566975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.566975:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.566977:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.566979:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.567044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.567050:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.567052:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.567054:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.567055:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.567056:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.567058:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.567059:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.567060:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.567061:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.567062:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.567065:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.567067:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.567069:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.567070:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.567070:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.567072:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.567072:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.567073:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.567074:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.567075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.567078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.567080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.567084:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:22.0:1619677475.567086:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.567086:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.567087:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:22.0:1619677475.567088:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677475.567088:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:22.0:1619677475.567089:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.567090:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.567091:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.567094:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.567115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.567122:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.567125:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.567127:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.567129:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.567130:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.567131:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.567132:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.567133:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.567134:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.567135:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.567138:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.567140:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.567142:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.567143:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.567144:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.567145:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.567146:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.567147:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.567148:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.567151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.567154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:24.0:1619677475.567157:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.567160:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677475.567160:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677475.567161:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.567161:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677475.567162:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.567163:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677475.567164:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.567166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:6.0:1619677475.567166:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.567168:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.567169:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.567171:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.567172:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.567173:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.567174:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.567177:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.567179:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.567182:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.567182:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.567183:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.567185:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.567185:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.567186:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.567187:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.567188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.567191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.567193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.2:1619677475.567197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677475.567198:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.567200:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.567200:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.567201:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.567202:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.567204:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:18.0:1619677475.567204:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677475.567205:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.567207:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.567207:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.567207:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677475.567208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.567210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:18.0:1619677475.567210:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:25.2:1619677475.567211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000400:00000200:18.0:1619677475.567212:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.567213:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.567214:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.567216:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.567217:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.567218:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677475.567218:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.567221:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.567222:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.567222:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000400:00000200:18.0:1619677475.567222:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.567223:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.567224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677475.567224:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.567227:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.567228:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.567229:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.567231:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.567232:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.567233:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.567233:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.567237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.567244:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.567247:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.567248:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.567250:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:30.0:1619677475.567250:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.567253:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.567347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.567355:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.567358:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.567359:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.567361:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.567363:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.567364:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677475.567365:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.567366:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.567367:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.567368:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677475.567371:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.567373:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.567374:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.567375:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677475.567376:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.567377:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.567377:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.567378:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.567378:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.567383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.567390:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.567392:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.567392:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.567393:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677475.567394:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.567396:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677475.567787:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.567795:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.567798:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.567799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.567800:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.567803:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.567805:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.567806:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.567807:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.567808:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.567809:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.567812:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.567814:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.567817:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.567817:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.567818:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.567820:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.567820:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.567821:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.567821:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.567826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.567834:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.567837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.567837:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.567838:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.567840:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.567843:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.567844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.567851:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.567853:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.567854:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.567855:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.567856:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.567856:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.567888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.567896:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.567899:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.567901:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.567903:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.567904:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.567905:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.567906:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.567907:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.567908:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.567909:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.567912:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.567914:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.567916:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.567917:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.567917:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.567919:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.567919:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.567920:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.567921:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.567924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.567931:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.567933:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.567934:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.567934:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.567936:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.567938:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.568357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.568365:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.568367:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.568369:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.568371:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.568372:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.568373:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.568374:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.568375:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.568376:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.568377:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.568380:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.568382:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.568383:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.568384:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.568384:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.568386:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.568386:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.568387:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.568388:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.568392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.568400:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.568402:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.568403:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.568404:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.568406:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.568407:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.568633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.568638:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.568641:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.568642:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.568644:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.568646:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.568648:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.568649:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.568650:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.568651:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.568652:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.568656:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.568657:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.568660:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.568660:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.568661:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.568663:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.568663:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.568664:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.568664:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.568669:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.568676:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.568678:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.568679:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.568681:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.568682:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.568683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.568760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.568768:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.568771:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.568773:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.568774:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.568775:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.568776:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.568777:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.568779:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.568780:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.568781:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.568784:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.568786:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.568788:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.568789:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.568790:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.568791:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.568792:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.568793:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.568793:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.568797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.568804:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.568807:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.568808:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.568808:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.568810:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.568812:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.568881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677475.568882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.568885:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.568887:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677475.568888:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:20.0:1619677475.568889:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.568890:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677475.568891:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:20.0:1619677475.568892:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.568893:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:31.0:1619677475.568895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677475.568895:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.568896:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.568897:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.568898:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.568898:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.568899:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677475.568900:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.568901:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.568903:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.568903:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.568904:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.568905:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:20.0:1619677475.568906:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.568908:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.568909:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.568909:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.568910:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.568911:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:30.0:1619677475.568912:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.0:1619677475.568912:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.568913:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677475.568914:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.568914:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:20.0:1619677475.568914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677475.568915:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.568916:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.568917:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677475.568917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677475.568918:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.568920:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.568921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.568922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.568927:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.568930:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:31.0:1619677475.568931:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.0:1619677475.568932:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:31.0:1619677475.568933:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:21.0:1619677475.568933:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.568934:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.568935:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677475.568935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:30.0:1619677475.568937:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.568937:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.568937:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.568938:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.568939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.568939:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677475.568940:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677475.568941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.569366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.569370:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.569373:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.569375:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.569376:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.569377:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.569379:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677475.569380:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.569381:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.569382:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.569383:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677475.569387:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.569389:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677475.569392:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.569392:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677475.569393:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677475.569395:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677475.569395:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.569396:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.569397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677475.569401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.569409:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.569412:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.569413:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.569414:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.569416:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.569417:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.569560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.569568:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.569571:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.569573:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.569575:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.569575:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.569576:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:31.0:1619677475.569578:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.569579:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.569580:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.569581:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:31.0:1619677475.569584:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.569586:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:31.0:1619677475.569589:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.569590:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:31.0:1619677475.569590:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:31.0:1619677475.569592:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677475.569592:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.569593:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.569594:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.569598:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.569607:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.569610:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.569611:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.569613:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.569616:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.569618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.569621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677475.569629:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.569633:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.569635:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.569636:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.569637:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.569638:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:5.2:1619677475.569893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.569900:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.569903:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.569903:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.569906:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.569907:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.569908:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.569910:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.569911:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.569912:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.569913:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.569916:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.569918:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.569920:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.569921:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.569921:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.569923:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.569923:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.569924:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.569925:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.569929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.569937:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.569939:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.569940:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.569941:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.569942:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.569943:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.569949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.569957:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.569959:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.569960:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.569960:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.569961:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.569961:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.570153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.570159:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.570161:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.570164:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.570164:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.570166:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.570168:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.570169:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.570171:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.570172:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.570173:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.570176:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.570178:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.570181:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.570182:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.570182:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.570184:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.570185:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.570186:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.570186:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.570190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.570196:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.570199:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.570200:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.570200:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.570202:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.570205:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.570208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.570214:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.570217:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.570218:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.570218:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.570219:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.570220:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.570314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.570322:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.570325:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.570325:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.570328:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.570330:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.570331:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.570332:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.570333:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.570335:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.570336:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.570339:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.570341:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.570344:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677475.570344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.570345:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.570345:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.570348:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.570348:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.570349:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.570350:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:26.0:1619677475.570351:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.570352:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.570352:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.570354:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677475.570354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:26.0:1619677475.570356:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.570357:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.570358:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.570359:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.570360:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.570361:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:20.0:1619677475.570362:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:26.0:1619677475.570363:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.570364:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677475.570365:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:20.0:1619677475.570365:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.570367:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:26.0:1619677475.570368:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.570368:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.570369:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.570369:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.570371:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677475.570371:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:26.0:1619677475.570372:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.570372:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.570373:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677475.570374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:25.2:1619677475.570378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.2:1619677475.570384:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:21.0:1619677475.570387:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677475.570388:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:21.0:1619677475.570389:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.570390:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:21.0:1619677475.570390:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.570390:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:6.0:1619677475.570390:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.570391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:22.0:1619677475.570392:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.570392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677475.570392:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:22.0:1619677475.570394:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.570395:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.570397:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.570397:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.570398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:22.0:1619677475.570399:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.570399:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.570400:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.570401:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.570402:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.570404:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.570407:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.570409:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.570411:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.570412:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.570413:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.570414:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.570415:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.570415:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.570416:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.570420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.570427:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.570430:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.570430:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.570432:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.570433:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.570434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.570444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.570450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.570453:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.570454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.570456:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.570459:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.570460:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.570462:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.570463:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.570464:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.570465:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.570468:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.570469:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.570471:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.570472:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.570472:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.570474:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.570474:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.570475:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.570475:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.570480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.570487:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.570489:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.570490:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.570491:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.570492:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.570493:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.570796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.570804:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.570807:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.570809:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.570811:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.570812:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.570813:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.570814:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.570816:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.570817:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.570818:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.570821:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.570823:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.570826:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.570826:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.570827:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.570829:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.570829:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.570831:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.570832:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.570833:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.570834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.570838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.570841:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.570843:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.570844:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.570844:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.570846:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.570846:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.570848:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.570849:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.570849:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.570850:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.570873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.570879:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.570882:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.570883:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.570885:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.570886:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.570888:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677475.570889:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.570891:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.570891:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.570893:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677475.570896:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.570897:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.570900:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.570901:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.570901:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.570903:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.570904:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.570904:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.570905:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677475.570909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.570917:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.570919:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.570920:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.570920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.570922:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.570925:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.571010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.571017:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.571020:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.571020:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677475.571022:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.571025:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.571027:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.571028:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.571029:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.571030:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.571031:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.571035:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.571037:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.571040:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.571041:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.571041:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.571043:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.571043:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.571044:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.571045:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.571049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.571056:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.571059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.571059:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.571060:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.571062:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.571065:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.571090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91946139b200] (20)++
00000800:00000200:21.0:1619677475.571096:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946139b200] (21)++
00000800:00000200:4.2:1619677475.571097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.0:1619677475.571099:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.2.24@o2ib3
00000400:00000200:21.0:1619677475.571101:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.24@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.571102:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946139b200] (22)--
00000400:00000200:21.0:1619677475.571104:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.571104:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677475.571106:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:21.0:1619677475.571107:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677475.571107:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.571108:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:21.0:1619677475.571109:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.571110:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.571111:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.571112:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:8.0:1619677475.571114:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.571117:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.571118:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000400:00000200:8.0:1619677475.571119:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.571121:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.571122:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677475.571122:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.571123:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:21.0:1619677475.571123:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000400:00000200:8.0:1619677475.571123:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.571124:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:21.0:1619677475.571125:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:21.0:1619677475.571126:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946139b200] (21)++
00000800:00000200:21.0:1619677475.571127:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946139b200] (22)--
00000800:00000200:21.0:1619677475.571127:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946139b200] (21)--
00000400:00000200:8.0:1619677475.571128:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.571130:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.2:1619677475.571132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:8.0:1619677475.571133:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.571134:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.571135:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677475.571135:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.571137:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:8.0:1619677475.571137:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:24.0:1619677475.571138:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.571138:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.571139:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.571140:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.571140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677475.571141:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677475.571143:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:8.2:1619677475.571143:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.571151:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.571154:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.571155:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.571155:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.571157:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.571160:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.571351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677475.571359:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.571362:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677475.571364:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.571367:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.571367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677475.571368:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:31.0:1619677475.571369:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.571371:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.571372:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.571373:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:31.0:1619677475.571377:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.571378:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:31.0:1619677475.571381:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.571382:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:31.0:1619677475.571382:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:31.0:1619677475.571384:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677475.571385:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.571385:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677475.571386:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.571391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.571399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.571402:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.571403:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.571403:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.571405:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.571409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.571506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.571514:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.571517:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.571517:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.571520:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.571522:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.571524:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.571525:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.571526:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.571527:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.571528:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.571531:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.571533:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.571535:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.571536:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.571537:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.571538:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.571539:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.571540:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.571540:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.571545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.571553:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.571555:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.571556:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.571557:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.571559:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.571561:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.571673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.571678:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.571680:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.571681:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.571683:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.571685:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.571686:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.571687:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.571689:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.571690:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.571691:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.571695:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.571697:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.571699:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.571700:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.571700:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.571702:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.571702:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.571703:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.571704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.571708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.571715:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.571717:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.571718:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.571720:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.571721:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.571722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.571761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.571769:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.571772:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.571774:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.571776:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.571776:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.571778:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.571779:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.571781:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.571782:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.571783:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.571786:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.571788:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.571791:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.571792:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.571793:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.571795:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.571795:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.571796:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.571797:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.571801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.571809:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.571812:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.571813:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.571814:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.571815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.571818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.572075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677475.572082:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677475.572085:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677475.572086:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677475.572088:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.572090:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.572092:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677475.572093:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.572094:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.572095:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.572096:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677475.572099:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.572101:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677475.572103:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.572104:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677475.572104:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677475.572106:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677475.572106:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677475.572107:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677475.572107:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:20.2:1619677475.572110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.2:1619677475.572111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:20.0:1619677475.572113:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.2:1619677475.572114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:8.0:1619677475.572115:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:20.0:1619677475.572116:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:8.0:1619677475.572117:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:24.0:1619677475.572118:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:20.0:1619677475.572118:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.572118:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.572120:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.572120:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.572120:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.572121:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000400:00000200:20.0:1619677475.572122:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.572123:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677475.572123:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:1.0:1619677475.572123:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.572124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:20.0:1619677475.572125:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.572126:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.572127:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.572128:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.572129:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.572132:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.2:1619677475.572133:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:20.0:1619677475.572134:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.572134:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677475.572135:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.572136:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:20.0:1619677475.572136:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.572137:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:1.0:1619677475.572137:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.572138:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:1.0:1619677475.572138:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.572139:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.572139:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.572140:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:20.0:1619677475.572140:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.0:1619677475.572140:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677475.572141:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (21)++
00000800:00000200:20.0:1619677475.572141:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.572141:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:24.0:1619677475.572142:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677475.572142:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000400:00000200:1.0:1619677475.572143:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.572145:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.2:1619677475.572146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.572147:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.572148:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.572149:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.572151:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.572152:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.572153:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.572154:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:17.0:1619677475.572155:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.572156:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:25.2:1619677475.572157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:17.0:1619677475.572157:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.572157:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.572159:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.572162:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677475.572165:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.572167:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.572168:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.572168:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.572170:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.572173:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.572179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.572186:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.572188:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.572189:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.572190:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.572191:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.572191:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.572433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.572439:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.572442:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:21.2:1619677475.572443:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:17.0:1619677475.572443:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.572445:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.572447:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.572448:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.572450:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.572451:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:1.0:1619677475.572451:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.572452:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.572453:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677475.572454:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.572457:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.572458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:1.0:1619677475.572458:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.572460:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677475.572461:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.572462:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.572463:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:1.0:1619677475.572463:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.572463:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:31.0:1619677475.572464:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.572465:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.572465:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:31.0:1619677475.572466:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.572466:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000400:00000200:31.0:1619677475.572467:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677475.572467:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.572467:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:31.0:1619677475.572471:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677475.572472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:31.0:1619677475.572473:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.572475:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.572476:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.572477:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.572478:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.572479:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.572480:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.572480:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677475.572480:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.572483:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.572484:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.572484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:21.0:1619677475.572485:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.572486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.572488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.572490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:8.0:1619677475.572495:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.572498:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677475.572498:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.572499:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.572499:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.572501:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.572502:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000400:00000200:8.0:1619677475.572502:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.572503:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.572503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677475.572504:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677475.572505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.572778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.572786:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.572788:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.572790:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.572792:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.572793:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.572794:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.572795:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.572797:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.572798:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.572799:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.572802:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.572803:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.572806:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.572807:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.572808:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.572809:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.572810:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.572811:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.572811:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.572815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.572823:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.572825:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.572826:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.572828:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.572830:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.572831:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.572967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.572974:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.572977:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.572978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.572979:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.572982:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.572983:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.572985:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.572986:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.572987:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.572988:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.572992:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.572994:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.572997:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.572998:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.572998:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.573000:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.573001:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.573002:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.573002:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.573006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.573014:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.573017:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.573018:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.573020:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.573021:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.573022:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.573106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.573112:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.573115:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.573116:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.573118:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.573120:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.573121:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.573122:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.573124:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.573125:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.573125:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:4.2:1619677475.573126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677475.573128:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.573130:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.573132:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.573133:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677475.573134:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.573134:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.573135:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677475.573136:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:22.0:1619677475.573136:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.573137:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:8.0:1619677475.573137:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677475.573138:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:24.0:1619677475.573139:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.573139:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.573141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:24.0:1619677475.573142:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.573143:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.2:1619677475.573144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677475.573145:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.573146:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.573147:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.573148:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.573149:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:24.0:1619677475.573151:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.573151:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.573151:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.573151:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.573152:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677475.573153:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.573153:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.573153:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.573155:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.573156:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.573156:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677475.573157:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.573158:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.573158:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.573160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:24.0:1619677475.573160:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.573160:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.573161:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.573162:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.573166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.573167:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.573170:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.573171:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.573171:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.573172:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.573173:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.573173:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:22.0:1619677475.573176:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.573177:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.573177:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.573179:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.573182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.573280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.573288:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.573291:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677475.573295:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.573297:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.573300:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.573301:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.573303:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.573304:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.573305:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.573306:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.573309:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.573311:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.573313:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.573314:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.573314:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.573316:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.573316:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.573318:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.573319:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.573320:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.573322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.573326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.573331:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.573332:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.573333:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.573335:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.573335:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.573336:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.573336:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.573337:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.573338:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.573338:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.573340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.573347:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.573350:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.573351:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.573351:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.573352:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.573353:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.573656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e6e00] (20)++
00000800:00000200:30.0:1619677475.573664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e6e00] (21)++
00000800:00000200:30.0:1619677475.573667:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.11@o2ib3
00000400:00000200:30.0:1619677475.573669:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.11@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.573671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e6e00] (22)--
00000400:00000200:30.0:1619677475.573671:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.573673:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677475.573674:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.573675:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.573676:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.573677:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:30.0:1619677475.573680:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.573682:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.573685:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.573686:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677475.573687:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677475.573689:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.573689:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e6e00] (21)++
00000800:00000200:30.0:1619677475.573690:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e6e00] (22)--
00000800:00000200:30.0:1619677475.573690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e6e00] (21)--
00000800:00000200:25.2:1619677475.573695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.573702:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.573705:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.573706:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.573707:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.573708:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.573710:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.573740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.573748:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.573751:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.573751:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.573753:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.573756:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.573756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677475.573757:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.573759:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.573760:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.573761:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.573762:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.573764:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677475.573765:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.573766:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677475.573767:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.573768:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.573770:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677475.573770:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.573770:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:31.0:1619677475.573771:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.573771:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677475.573772:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.573773:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677475.573773:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.573773:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.573774:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.573774:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:31.0:1619677475.573775:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.573775:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.573776:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.573779:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.573779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.573781:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.573783:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.573783:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.573784:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.573785:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.573786:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.573787:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.573787:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677475.573787:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.573789:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.573790:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.573790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.573791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:6.0:1619677475.573792:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.573795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.573797:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.573800:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.573801:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.573803:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.573804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.573804:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.574171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.574174:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.574176:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.574178:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.574179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.574181:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.574182:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.574183:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.574185:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.574186:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.574187:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.574190:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.574191:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.574194:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.574195:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.574195:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.574197:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.574198:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.574199:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:5.2:1619677475.574199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677475.574200:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.574201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.574204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.574205:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:20.2:1619677475.574207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677475.574208:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.574208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:20.0:1619677475.574210:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.574211:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.574211:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:17.0:1619677475.574211:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.574212:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.574213:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.574213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677475.574213:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.574214:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.574214:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:21.0:1619677475.574215:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.574216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:17.0:1619677475.574216:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.574217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677475.574217:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.574218:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.574219:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.574222:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.574224:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.574226:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.574226:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.574227:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.574228:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.574229:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.574229:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.574230:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.574234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.574241:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.574243:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.574244:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.574245:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.574246:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.574247:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.574292:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677475.574301:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.574303:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677475.574304:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677475.574306:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.574309:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.574311:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677475.574312:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.574313:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.574315:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.574315:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677475.574319:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.574321:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677475.574324:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.574325:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677475.574326:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677475.574327:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677475.574328:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.574329:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677475.574329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677475.574334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.574342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.574345:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.574346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.574347:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.574349:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.574353:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.574356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.574363:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.574366:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.574367:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.574367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.574368:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.574369:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.574382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.574390:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.574393:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.574395:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.574397:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.574398:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.574399:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.574400:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.574402:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.574403:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.574404:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.574407:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.574409:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.574411:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.574412:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.574412:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.574414:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.574415:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.574416:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.574416:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.574420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.574427:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.574429:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.574430:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.574430:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.574432:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.574435:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.574827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.574834:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.574837:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.574837:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.574839:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.574841:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.574843:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.574844:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.574845:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.574846:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.574847:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.574851:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.574853:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.574855:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.574856:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.574857:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.574858:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.574859:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.574859:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.574860:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.574864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.574872:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.574874:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.574875:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.574876:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.574878:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.574878:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.575143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.575150:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.575153:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.575154:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.575155:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.575158:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.575160:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.575161:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.575163:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.575164:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.575165:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.575168:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.575170:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.575173:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.575173:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.575174:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.575176:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.575176:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.575177:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.575178:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.575182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.575190:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.575192:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.575193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.575194:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.575196:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.575199:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.575202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.575209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.575212:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.575213:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.575213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.575214:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.575215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.575708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677475.575714:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.575717:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677475.575718:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677475.575719:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.575722:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.575724:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.575725:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.575726:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.575727:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.575728:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.575732:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.575733:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.575736:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.575737:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.575738:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.575740:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.575740:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.575741:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677475.575742:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:25.2:1619677475.575746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.575754:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.575757:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.575758:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.575760:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.575761:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.575763:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.575765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.575772:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.575774:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.575775:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.575776:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.575777:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.575778:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.575964:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.575972:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.575975:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.575977:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.575979:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.575979:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.575980:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.575981:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.575983:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.575984:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.575985:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.575988:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.575989:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.575992:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.575992:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.575993:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.575995:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.575995:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.575996:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.575996:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.576000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.576008:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.576010:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.576011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.576011:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.576013:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.576016:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.576020:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.576026:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.576028:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.576029:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.576029:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.576030:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.576031:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.576102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.576108:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.576111:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.576112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.576113:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.576116:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.576117:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.576118:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.576120:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.576121:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.576122:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.576125:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.576127:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.576129:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.576130:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.576131:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.576132:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.576133:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.576134:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.576134:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.576139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.576146:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.576148:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.576149:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.576151:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.576152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.576153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.576155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.576165:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.576168:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.576172:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.576173:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.576176:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.576178:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:12.0:1619677475.576180:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.576183:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.576184:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.576186:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:12.0:1619677475.576192:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.576195:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:12.0:1619677475.576199:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.576200:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677475.576201:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:12.0:1619677475.576204:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677475.576205:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.576206:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.576207:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.576209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.576216:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.576219:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.576220:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.576220:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.576222:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.576224:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677475.576231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677475.576237:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.576240:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.576241:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.576241:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.576242:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.576242:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677475.576350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.576357:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.576360:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.576361:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.576362:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.576365:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.576366:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.576367:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.576369:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.576370:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.576371:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.576373:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.576375:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.576378:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.576379:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.576379:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.576381:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.576381:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.576382:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.576382:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.576387:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.576395:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.576398:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.576399:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.576399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.576401:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.576404:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.576454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919461bd0c00] (20)++
00000800:00000200:20.0:1619677475.576457:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919461bd0c00] (21)++
00000800:00000200:20.0:1619677475.576460:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.3@o2ib3
00000400:00000200:20.0:1619677475.576462:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.2.3@o2ib3 : GET - for me
00000800:00000200:21.0:1619677475.576463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919461bd0c00] (22)--
00000400:00000200:20.0:1619677475.576464:0:19934:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.2.3@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:20.0:1619677475.576467:0:19934:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.2.3@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:20.0:1619677475.576468:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919461bd0c00] (21)++
00000800:00000200:20.0:1619677475.576469:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919461bd0c00] (22)--
00000400:00000200:20.0:1619677475.576471:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.2.3@o2ib3 local destination
00000400:00000200:20.0:1619677475.576475:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.2.3@o2ib3(10.51.2.3@o2ib3:10.51.2.3@o2ib3) : REPLY try# 0
00000800:00000200:20.0:1619677475.576477:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.2.3@o2ib3
00000800:00000200:20.0:1619677475.576479:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9197c81a8a00] -> 10.51.2.3@o2ib3 (2) version: 12
00000800:00000200:20.0:1619677475.576480:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919461bd0c00] (21)++
00000800:00000200:20.0:1619677475.576481:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919461bd0c00] (22)++
00000800:00000200:20.0:1619677475.576482:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919461bd0c00] (23)--
00000800:00000200:20.0:1619677475.576483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919461bd0c00] (22)--
00000800:00000200:20.2:1619677475.576488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919461bd0c00] (21)++
00000800:00000200:21.0:1619677475.576493:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919461bd0c00] (22)++
00000800:00000200:21.0:1619677475.576495:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919461bd0c00] (23)--
00000400:00000200:21.0:1619677475.576496:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.576498:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.2.3@o2ib3: REPLY: OK
00000800:00000200:20.0:1619677475.576499:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919461bd0c00] (22)--
00000800:00000200:21.0:1619677475.576500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919461bd0c00] (21)--
00000800:00000200:4.2:1619677475.576783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.576791:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.576794:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.576794:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.576796:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.576799:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.576800:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.576802:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.576804:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.576805:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.576805:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.576809:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.576810:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.576813:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.576814:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.576815:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.576816:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.576817:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.576817:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.576819:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.576819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.576822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.576825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677475.576831:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.576832:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.576834:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677475.576834:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.576835:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677475.576835:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.576836:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677475.576836:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.576838:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.576841:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.576842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.576850:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.576852:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.576853:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.576853:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.576854:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.576855:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.577349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.577356:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.577359:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.577360:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.577362:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.577365:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.577367:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677475.577368:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.577369:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.577370:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.577371:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.577374:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.577376:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677475.577379:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.577379:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677475.577380:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677475.577382:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677475.577382:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.577383:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.577383:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677475.577388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.577397:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.577400:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.577401:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.577401:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.577404:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677475.577407:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677475.577412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677475.577419:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.577422:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677475.577423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.577423:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.577424:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.577425:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.577573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.577579:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.577582:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.577583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.577584:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.577586:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.577588:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.577589:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.577590:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.577591:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.577592:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.577596:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.577597:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.577600:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.577601:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.577602:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.577603:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.577604:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.577604:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.577605:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.577609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.577617:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.577620:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.577621:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.577622:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.577623:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.577625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.577627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.577634:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.577637:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.577638:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.577638:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.577639:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.577640:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.577682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.577691:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.577694:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.577696:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.577698:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.577698:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.577700:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.577701:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.577702:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.577703:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.577704:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.577707:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.577709:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.577711:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.577712:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.577713:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.577714:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.577715:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.577716:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.577716:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.577720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.577726:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.577729:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.577730:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.577731:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.577733:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.577733:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.577880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.577888:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.577891:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.577892:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.577894:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.577895:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677475.577895:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.577897:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.577898:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.577899:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.577899:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.577902:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.577904:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.577906:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.577907:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.577907:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.577909:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.577909:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.577911:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.577912:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.577913:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.577914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.577918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.577922:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.577924:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.577925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.577925:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.577926:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.577927:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.577928:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.577929:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.577930:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.0:1619677475.577930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.577965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.577972:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.577974:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.577976:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.577977:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.577980:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.577981:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.577982:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.577983:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.577984:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.577985:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.577988:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.577989:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.577992:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.577993:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.577993:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.577995:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.577996:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.577996:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.577997:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.578001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.578009:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.578011:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.578012:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.578013:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.578015:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.578018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.578021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.578028:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.578030:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.578031:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.578032:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.578033:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.578033:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.578293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.578302:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.578305:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.578308:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.578310:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.578310:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.578311:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.578312:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.578314:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.578315:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.578316:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.578319:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.578320:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.578322:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.578323:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.578323:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.578325:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.578325:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.578326:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.578327:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.578331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.578338:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.578341:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.578341:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.578343:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.578345:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.578346:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.578475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.578481:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.578484:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.578486:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.578486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.578489:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.578490:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.578492:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.578493:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.578494:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.578495:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.578499:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.578500:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.578503:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.578504:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.578504:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.578506:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.578507:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.578507:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.578508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677475.578513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677475.578516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.578523:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.578526:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.578526:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.578527:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.578528:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.578529:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.578530:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.578530:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.578533:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.578534:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.578535:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.578536:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.578537:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.578540:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.578541:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.578542:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.578542:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.578543:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677475.578544:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.578545:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.578545:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.578547:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.578548:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.578549:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.578551:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.578551:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.578553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.578556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.578562:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.578563:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.578563:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.578563:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.578563:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.578564:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.578565:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.578565:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.578566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677475.578566:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.578573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.578580:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.578582:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.578583:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.578584:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.578584:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.578584:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.578806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.578814:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.578817:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.578819:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.578821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.578821:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.578823:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.578824:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.578825:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.578826:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.578827:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.578831:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.578833:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.578835:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.578836:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.578836:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.578838:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.578839:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.578840:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.578842:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.578842:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.578844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.578848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.578851:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.578853:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.578854:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.578854:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.578856:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:8.0:1619677475.578856:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.578858:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:8.0:1619677475.578858:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:31.0:1619677475.578859:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.578860:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.578929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.578937:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.578940:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.578942:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.578944:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.578945:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.578947:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677475.578948:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.578949:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.578950:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.578951:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677475.578954:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.578956:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677475.578959:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.578959:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677475.578960:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677475.578961:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677475.578962:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.578962:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:13.0:1619677475.578963:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.578967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.578974:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.578976:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677475.578977:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.578978:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.578979:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.578982:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677475.579178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.579185:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.579188:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.579189:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.579191:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.579194:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.579195:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.579196:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.579198:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.579199:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.579200:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.579203:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.579205:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.579207:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.579208:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.579208:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.579210:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.579210:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.579211:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.579212:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.579221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.579228:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.579230:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.579231:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.579231:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.579233:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.579234:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.579277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.579283:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.579285:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.579286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.579288:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.579291:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.579292:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.579299:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.579301:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.579302:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.579303:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.579306:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.579308:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.579310:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.579311:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.579312:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.579314:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.579314:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.579316:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.579317:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.579317:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.579321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.579323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.579328:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.579328:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.579330:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.579330:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.579331:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.579331:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.579332:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.579333:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.579335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.579335:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.579431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.579436:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.579439:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.579441:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.579441:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.579443:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.579445:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677475.579446:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.579448:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.579449:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.579450:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.579453:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.579455:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.579458:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.579459:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.579460:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.579461:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.579462:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.579463:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.579464:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677475.579468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.579475:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.579478:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.579479:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.579480:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.579482:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.579483:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677475.579484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.2:1619677475.579491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:18.0:1619677475.579491:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.579494:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.579495:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.579497:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:30.0:1619677475.579499:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:18.0:1619677475.579499:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.579501:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677475.579502:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:18.0:1619677475.579502:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.579504:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.579505:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.579506:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.579506:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.579506:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677475.579508:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.579510:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:18.0:1619677475.579510:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000400:00000200:30.0:1619677475.579511:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.579513:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677475.579513:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:30.0:1619677475.579515:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677475.579515:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.579516:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:18.0:1619677475.579516:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.579517:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.579518:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:30.0:1619677475.579519:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.579519:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677475.579521:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:18.0:1619677475.579521:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.579522:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:30.0:1619677475.579523:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.579523:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:30.0:1619677475.579524:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.579525:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677475.579525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677475.579526:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.579527:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.579528:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:20.2:1619677475.579528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677475.579529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.579532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:2.0:1619677475.579533:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677475.579534:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677475.579535:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.579536:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:17.0:1619677475.579537:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677475.579537:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677475.579538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:17.0:1619677475.579538:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.579538:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.579539:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.579541:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.579542:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.579542:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.0:1619677475.579542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.579544:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.579546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.579942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.579950:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.579953:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.579955:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.579957:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.579957:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.579958:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.579959:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.579961:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.579962:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.579963:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.579966:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.579968:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.579970:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.579971:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.579972:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.579973:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.579974:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.579975:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.579975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.579979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.579985:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.579987:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.579988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.579988:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.579990:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.579992:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.579996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.580002:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.580004:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.580005:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.580005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.580006:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.580007:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.580032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.580039:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.580042:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.580043:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.580045:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.580048:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.580049:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.580050:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.580051:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.580052:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.580053:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.580057:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.580058:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.580061:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.580062:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.580062:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.580064:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.580064:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.580065:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.580066:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.580070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.580077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:22.0:1619677475.580078:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.580081:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.580082:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.580082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.580084:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:22.0:1619677475.580084:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.580086:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.580087:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:22.0:1619677475.580088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:26.0:1619677475.580089:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.580092:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.580094:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677475.580095:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.580096:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.580097:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.580098:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677475.580101:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.580102:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.580104:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.580104:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677475.580105:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.580106:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.580106:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.580108:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.580109:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.580109:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677475.580111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677475.580115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.580120:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.580121:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.580122:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.580124:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:10.0:1619677475.580124:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.580125:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.580125:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.580126:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:10.0:1619677475.580126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.580127:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.2:1619677475.580326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.580329:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.580332:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.580335:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.580335:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.580337:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.580339:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.580340:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.580341:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.580342:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.580343:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.580346:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.580348:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.580350:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.580350:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.580351:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.580353:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.580353:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.580354:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.580355:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.580359:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.580367:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.580369:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.580370:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.580372:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.580373:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.580374:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.580571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.580577:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.580580:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.580581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.580583:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.580585:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.580586:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.580587:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.580589:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.580590:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.580591:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.580593:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.580595:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.580597:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.580598:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.580599:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.580600:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.580601:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.580601:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.580602:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.580607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.580614:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.580617:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.580618:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.580619:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.580620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.580621:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.581009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.581017:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.581020:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.581023:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.581025:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.581025:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.581027:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.581028:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.581029:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.581030:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.581031:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.581034:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.581036:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.581040:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.581040:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.581041:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.581043:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.581043:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.581044:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.581045:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.581049:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.581056:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.581059:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.581060:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.581061:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.581061:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.581065:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.581098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.581106:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.581109:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.581110:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.581112:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.581114:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.581115:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.581117:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.581118:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.581119:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.581120:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.581123:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.581124:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.581126:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.581127:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.581127:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.581129:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.581129:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.581130:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.581130:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:4.2:1619677475.581132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:25.2:1619677475.581134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.581139:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.581141:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.581142:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:12.0:1619677475.581142:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:10.0:1619677475.581143:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.581144:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.581145:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.581145:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.581146:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:10.0:1619677475.581147:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677475.581148:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677475.581149:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:10.0:1619677475.581149:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.581150:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.581151:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677475.581153:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.581155:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:10.0:1619677475.581155:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677475.581156:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.581157:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677475.581157:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.581158:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.581159:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.581160:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.581160:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.581164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.581170:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.581172:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.581173:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.581173:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.581175:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.581176:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.581495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.581503:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.581505:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.581507:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.581510:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.581510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.581511:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.581512:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.581513:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.581514:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.581515:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.581518:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.581519:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.581522:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.581522:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.581523:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.581524:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.581525:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.581526:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.581526:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.581531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.581538:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.581541:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.581542:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.581542:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.581543:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.581546:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.581703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:4.2:1619677475.581707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.581711:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:10.0:1619677475.581711:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677475.581714:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:10.0:1619677475.581714:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.581715:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:21.0:1619677475.581717:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.581717:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.581717:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.581720:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.581720:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.581721:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.581722:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.581723:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.581724:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.581725:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.581726:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.581727:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.581728:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.581729:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.581729:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.581733:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.581734:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.581738:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.581739:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.581739:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.581740:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.581740:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.581742:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.581742:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677475.581743:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.581743:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:20.0:1619677475.581744:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.581744:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:20.0:1619677475.581745:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.581745:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677475.581746:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677475.581747:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:25.2:1619677475.581748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.581748:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.581748:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:4.2:1619677475.581751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.581755:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (23)++
00000800:00000200:26.0:1619677475.581758:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.581758:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:6.0:1619677475.581758:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (24)++
00000400:00000200:22.0:1619677475.581759:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.581759:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (24)--
00000800:00000200:26.0:1619677475.581760:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677475.581760:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.581761:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677475.581761:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.581762:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.581762:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.581764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.581765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.581766:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.582010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.582015:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.582017:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.582018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.582020:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.582022:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.582023:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.582025:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.582026:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.582027:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.582028:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.582032:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.582033:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.582036:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.582037:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.582037:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.582039:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.582039:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.582040:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.582040:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.582045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.582052:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.582054:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.582055:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.582057:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.582057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.582059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.582232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.582239:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.582242:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.582243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.582245:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.582248:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.582249:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.582251:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.582252:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.582253:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.582254:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.582257:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.582259:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.582262:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.582263:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.582263:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.582265:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.582265:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.582266:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.582267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.582271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.582279:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.582282:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.582283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.582283:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.582285:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.582288:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.582531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.582539:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.582541:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.582544:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.582546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.582546:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.582548:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.582549:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.582550:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.582551:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.582552:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.582555:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.582557:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.582559:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.582560:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.582560:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.582562:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.582562:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.582563:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.582563:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.582568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.582575:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.582577:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.582578:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.582579:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.582581:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.582582:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.582589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.582595:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.582598:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.582599:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.582599:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.582600:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.582601:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.582767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.582774:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.582777:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.582779:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.582781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.582781:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.582782:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.582783:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.582785:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.582785:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.582786:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.582789:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.582790:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.582792:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:5.2:1619677475.582792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:30.0:1619677475.582793:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.582793:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.582795:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.582796:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.582796:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.582797:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:17.0:1619677475.582798:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.582801:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:9.2:1619677475.582801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.582801:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.582803:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.582805:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.582806:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.582807:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.582808:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.582808:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:17.0:1619677475.582809:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.582810:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.0:1619677475.582810:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.582811:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.582811:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.582814:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.582814:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.582816:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677475.582816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677475.582818:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.582818:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.582819:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.582820:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:9.2:1619677475.582820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677475.582821:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.582822:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.582823:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677475.582826:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677475.582827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677475.582829:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.582830:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.582831:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.582831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.582832:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677475.582835:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.582837:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.582837:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.582838:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.582840:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.582841:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.583256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.583264:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.583266:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.583267:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.583269:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.583271:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.583273:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.583274:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.583275:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.583277:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.583278:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.583281:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.583283:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.583286:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.583287:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.583288:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.583290:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.583290:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677475.583291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.583291:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.583291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.583297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.583300:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.583302:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.583304:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.583306:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677475.583307:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.583307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.583308:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:22.0:1619677475.583308:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:31.0:1619677475.583309:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.583309:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.583309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677475.583310:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.583311:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.583312:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.583313:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.583315:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677475.583316:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.583318:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.583320:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.583321:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.583321:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.583323:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.583323:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.583325:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.583326:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.583327:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.583328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.583333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.583338:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.583339:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.583340:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.583342:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.583342:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.583343:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.583343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.583344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.583345:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.583346:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.583730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.583735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.583738:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.583738:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.583740:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.583742:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.583743:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.583745:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.583746:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.583747:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.583748:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.583751:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.583753:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.583755:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.583756:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.583757:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.583759:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.583759:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.583760:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.583760:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.583765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.583772:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.583774:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.583775:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.583777:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.583778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.583779:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.583783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.583790:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.583792:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.583793:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.583794:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.583795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.583795:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.583871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91946139b200] (20)++
00000800:00000200:21.0:1619677475.583876:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946139b200] (21)++
00000800:00000200:21.0:1619677475.583878:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.24@o2ib3
00000400:00000200:21.0:1619677475.583880:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.24@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.583881:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946139b200] (22)--
00000400:00000200:21.0:1619677475.583883:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.583884:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:21.0:1619677475.583885:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.583887:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.583888:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.583889:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:21.0:1619677475.583892:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.583894:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:21.0:1619677475.583897:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.583898:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:21.0:1619677475.583899:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:21.0:1619677475.583901:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:21.0:1619677475.583901:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946139b200] (21)++
00000800:00000200:21.0:1619677475.583902:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946139b200] (22)--
00000800:00000200:21.0:1619677475.583903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946139b200] (21)--
00000800:00000200:24.2:1619677475.583907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.583911:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677475.583914:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677475.583915:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.583915:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677475.583917:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.583920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677475.584698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.584702:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.584704:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.584707:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.584707:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.584709:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.584711:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.584712:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.584713:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.584714:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.584715:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.584719:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.584721:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.584723:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.584724:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.584725:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.584726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.0:1619677475.584727:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.584727:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.584728:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.584729:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.584733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.584734:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.584737:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.584739:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.584740:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:31.0:1619677475.584741:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.584742:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.584743:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:8.0:1619677475.584743:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:31.0:1619677475.584744:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.584744:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.584744:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.584746:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.584747:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.584747:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:31.0:1619677475.584749:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.584752:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.584754:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.584754:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.584757:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.584757:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.584758:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.584759:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.584760:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.584761:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.584763:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.584763:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.584765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.584769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.584771:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.584774:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.584774:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.584775:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.584776:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.584776:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.584778:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.584779:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.584779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.584780:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677475.584783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.584788:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.584791:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.584793:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.584793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.584795:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.584797:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.584798:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.584800:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.584801:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.584802:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.584805:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.584807:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.584809:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.584810:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.584811:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.584812:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.584813:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.584813:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.584814:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.584818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.584820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.584827:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677475.584827:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:18.0:1619677475.584830:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:17.0:1619677475.584830:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:4.2:1619677475.584830:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:17.0:1619677475.584831:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.584831:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:1.0:1619677475.584831:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:18.0:1619677475.584834:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677475.584834:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:18.0:1619677475.584836:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.584836:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:18.0:1619677475.584837:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.584838:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.584840:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.584840:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:18.0:1619677475.584841:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.584842:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:10.0:1619677475.584845:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.584846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:18.0:1619677475.584846:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.584848:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677475.584850:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.584852:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.584852:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.584853:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:10.0:1619677475.584854:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.584855:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.584856:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.584857:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.584857:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:10.0:1619677475.584857:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.584859:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.584861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:10.0:1619677475.584861:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.584862:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.584864:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:30.0:1619677475.584868:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:10.0:1619677475.584869:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.584870:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.584871:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.584872:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:30.0:1619677475.584873:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.584874:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.584876:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:10.0:1619677475.584876:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.584877:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.584878:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.584881:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.584881:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.584883:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.584885:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.584886:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.584887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.584891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.584895:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.584898:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.584898:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.584899:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.584900:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677475.584901:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.584903:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:22.0:1619677475.584903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:26.0:1619677475.584905:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.584906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.2:1619677475.585027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.585033:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.585035:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.585036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.585038:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.585040:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.585041:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.585042:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.585043:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.585044:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.585045:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.585048:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.585050:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.585052:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.585053:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.585054:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.585056:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.585056:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.585057:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.585058:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.585058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.585062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.585063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.585068:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.585069:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.585070:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.585070:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.585071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.585071:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.585072:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.585073:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.585074:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.585076:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.585854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.585862:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.585865:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.585866:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.585868:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.585870:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.585872:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.585873:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.585874:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.585875:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.585876:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.585879:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.585881:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.585883:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.585884:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.585885:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.585886:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.585887:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.585888:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.585888:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.585893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.585901:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.585903:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.585904:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.585905:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:4.2:1619677475.585907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:13.0:1619677475.585908:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.585909:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677475.585914:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.585917:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.585918:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.585919:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.585922:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.585923:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677475.585924:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.585925:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.585926:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.585927:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677475.585930:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.585931:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677475.585934:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677475.585934:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:26.0:1619677475.585935:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677475.585935:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677475.585937:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677475.585937:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.585938:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677475.585940:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.585940:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:1.0:1619677475.585940:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.585943:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.585944:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:4.2:1619677475.585946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:1.0:1619677475.585946:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.585948:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.585949:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.585951:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.585952:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.585953:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:1.0:1619677475.585953:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677475.585954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:1.0:1619677475.585954:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:10.0:1619677475.585955:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.585955:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.585956:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:1.0:1619677475.585957:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.585959:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:2.0:1619677475.585961:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677475.585961:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.585962:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.585963:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677475.585964:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:1.0:1619677475.585964:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:18.0:1619677475.585965:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.585965:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.585965:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.585966:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.585966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:2.0:1619677475.585967:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.585970:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:8.2:1619677475.585971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.585980:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.585982:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.585983:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.585985:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.585985:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.585987:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.586345:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.586348:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.586350:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.586352:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.586353:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.586355:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.586356:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.586357:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.586359:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.586360:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.586361:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.586364:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.586366:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.586368:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.586369:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.586370:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.586371:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.586372:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.586373:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.586375:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.586375:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.586378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.586381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.586384:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.586385:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.586385:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.586386:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.586387:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:20.0:1619677475.586387:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.586388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.586389:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.586389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.586390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.586394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.586402:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.586404:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.586405:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.2:1619677475.586407:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677475.586407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.586411:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.586411:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:31.0:1619677475.586415:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.586418:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677475.586418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:31.0:1619677475.586421:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.586423:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.586423:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.586424:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677475.586425:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:31.0:1619677475.586426:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.586426:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.586427:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.586428:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.586429:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.586432:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.586434:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.586434:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.586436:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.586436:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.586437:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.586438:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.586439:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.586440:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.586443:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.586443:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.586444:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.586445:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.586445:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.586447:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.586447:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.586447:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.586448:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.586448:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.586449:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.586449:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.586450:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.586450:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.586451:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.586452:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.586452:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677475.586453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:9.2:1619677475.586457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.586461:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.586463:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.586464:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.586464:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677475.586466:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.586466:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.586466:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.586467:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.586467:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.586469:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:8.0:1619677475.586470:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.2:1619677475.586472:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:8.0:1619677475.586477:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677475.586480:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.586482:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.586483:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.586484:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.586485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.586486:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.586524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.586530:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.586532:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.586533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.2:1619677475.586535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:6.0:1619677475.586535:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.586537:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.586539:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.586540:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.586541:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:6.0:1619677475.586542:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.586543:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677475.586544:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:6.0:1619677475.586544:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:2.0:1619677475.586544:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.586548:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.586549:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.586551:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.586551:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.586552:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.586554:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.586554:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.586555:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:18.0:1619677475.586556:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.586556:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.586556:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.586557:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:18.0:1619677475.586558:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677475.586558:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.586558:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.586559:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:18.0:1619677475.586561:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.586563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.586563:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.586565:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.586566:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.586566:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.586568:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.586568:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.586570:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:21.0:1619677475.586571:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677475.586571:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.586571:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.0:1619677475.586573:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.586573:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.586574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:21.0:1619677475.586575:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.586577:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.2:1619677475.586577:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:21.0:1619677475.586578:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.586578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677475.586583:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.586584:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.586585:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677475.586585:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.586586:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:26.0:1619677475.586587:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677475.586587:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.586587:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.586588:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.586589:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677475.586589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:26.0:1619677475.586590:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.586590:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.586593:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.586596:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.586597:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.586598:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.586600:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.586601:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.586602:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.586605:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.586607:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.586610:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.586611:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.586611:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.586613:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.586613:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.586614:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.586614:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.586619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.586627:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.586630:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.586631:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.586631:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.586633:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.586636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.586637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.586644:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.586647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.586647:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.586648:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.586649:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.586649:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.586745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.586750:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.586753:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.586755:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.586755:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.586758:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.586759:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.586761:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.586762:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.586763:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.586764:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.586768:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.586769:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.586772:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.586773:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.586774:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.586776:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.586776:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.586777:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.586779:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.586779:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.586782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.586784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677475.586788:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.586788:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.586790:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:24.0:1619677475.586791:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677475.586791:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:24.0:1619677475.586792:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.586792:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677475.586792:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.586794:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.586797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.586800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.586806:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.586808:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.586809:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.586809:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.586810:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.586811:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.586951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.586958:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.586961:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.586962:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.586964:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.586966:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.586967:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677475.586969:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.586970:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.586971:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.586972:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677475.586974:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.586976:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677475.586977:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.586978:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677475.586979:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.586980:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.586980:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.586981:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.586981:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.586985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.586992:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.586993:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.586994:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.586995:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.586996:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.586997:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677475.587008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677475.587014:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.587016:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.587017:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.587017:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.587018:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677475.587018:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677475.587128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.587138:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.587142:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.587145:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.587146:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.587148:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.587150:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:13.0:1619677475.587152:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.587154:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.587156:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.587157:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:13.0:1619677475.587162:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.587165:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:13.0:1619677475.587169:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.587170:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:13.0:1619677475.587171:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:13.0:1619677475.587173:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:13.0:1619677475.587174:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.587176:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677475.587178:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677475.587179:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:8.2:1619677475.587179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677475.587185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:17.0:1619677475.587188:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.587191:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.587192:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.587192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:12.0:1619677475.587193:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:17.0:1619677475.587195:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.587195:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677475.587196:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.587197:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:17.0:1619677475.587198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.587442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.587450:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.587453:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.587455:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.587457:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.587458:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.587459:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.587460:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.587461:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.587462:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.587463:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.587466:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.587468:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.587470:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.587470:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.587471:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.587472:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.587473:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.587474:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.587474:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.587479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.587486:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.587488:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.587489:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.587490:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.587492:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.587493:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677475.587551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.587557:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.587560:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.587561:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.587563:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.587564:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.587566:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.587567:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.587568:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.587569:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.587571:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.587574:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.587575:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.587577:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.587578:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.587579:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.587580:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.587581:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.587582:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.587582:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.587587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.587594:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.587596:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.587596:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.587597:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.587598:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.587599:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.587607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.587614:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.587616:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.587617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.587617:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.587617:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.587618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.587821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.587829:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.587832:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.587832:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.587834:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.587836:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.587837:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677475.587838:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.587840:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.587841:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.587842:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677475.587845:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.587847:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.587848:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.587849:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677475.587849:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.587851:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.587851:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.587853:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.587854:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.587854:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:9.2:1619677475.587856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:4.2:1619677475.587860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.587865:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.587866:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.587867:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.587868:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000400:00000200:10.0:1619677475.587868:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.587869:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.587870:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.587870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.587871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:10.0:1619677475.587871:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677475.587876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.587883:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.587886:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.587887:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.587888:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.587891:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.587892:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.587893:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.587895:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.587896:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.587896:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.587900:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.587901:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.587904:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.587905:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.587905:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.587907:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.587907:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.587908:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.587910:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.587910:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.587913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.587916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.587921:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.587923:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.587924:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.587925:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.587925:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:6.0:1619677475.587925:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.587926:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.587927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.587927:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.587930:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.588027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.588035:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.588038:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.588040:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.588041:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.588042:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.588043:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.588044:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.588046:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.588047:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.588048:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.588051:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.588052:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.588054:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.588055:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.588055:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.588057:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.588057:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.588058:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.588059:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.588062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.588069:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.588071:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.588072:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.588074:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.588075:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.588076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.588265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.588270:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.588273:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.588273:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.588275:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.588277:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.588279:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.588280:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.588282:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.588282:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.588284:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.588287:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.588289:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.588291:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.588292:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.588293:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.588298:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.588298:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.588299:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.588299:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.588304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.588312:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.588314:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.588315:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.588317:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.588318:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.588319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.588454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.588459:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.588461:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.588463:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.588464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.588466:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.588468:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.588469:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.588470:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.588471:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.588472:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.588476:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.588478:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.588480:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.588481:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.588482:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.588484:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.588484:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.588485:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.588485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.588500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.588507:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.588510:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.588510:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.588511:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.588513:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.588515:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.588645:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677475.588651:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.588654:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677475.588654:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677475.588656:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.588658:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.588659:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677475.588661:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.588662:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.588663:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.588664:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677475.588666:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.588668:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.588671:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.588671:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677475.588672:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.588674:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.588674:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.588675:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677475.588676:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:25.2:1619677475.588680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.588688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.588690:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.588691:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.588692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.588693:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.588696:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.588763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.588766:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.588769:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.588771:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.588772:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.588774:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.588776:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677475.588777:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.588778:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.588779:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.588780:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677475.588784:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.588786:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677475.588789:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.588790:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677475.588790:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677475.588793:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677475.588793:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.588794:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.588796:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677475.588796:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677475.588799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.588801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677475.588805:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:21.0:1619677475.588807:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677475.588808:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.588809:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.588809:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.588809:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:30.0:1619677475.588811:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.588811:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:31.0:1619677475.588812:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.588814:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.588866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.588873:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.588876:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.588879:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.588880:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.588881:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.588882:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.588884:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.588885:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.588886:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.588887:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.588890:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.588892:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.588895:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.588896:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.588896:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.588898:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.588898:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.588900:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.588901:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.588901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.588903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.588907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.588909:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.588912:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.588913:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.588913:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.588915:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:24.0:1619677475.588915:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.588917:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:24.0:1619677475.588917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.588918:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.588919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.589046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.589052:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.589054:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.589056:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.589057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.589059:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.589061:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.589062:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.589064:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.589065:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.589066:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.589068:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.589070:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.589073:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.589074:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.589074:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.589076:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.589077:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.589078:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.589078:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.589082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.589088:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.589091:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.589092:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.589092:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.589094:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.589096:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.589278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.589285:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.589288:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.589290:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.589292:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.589293:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.589301:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.589302:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.589303:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.589304:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.589305:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.589308:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.589310:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.589313:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.589313:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.589314:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.589315:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.589316:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.589317:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.589317:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.589322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.589329:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.589331:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.589332:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.589332:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.589333:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.589336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.589638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.589644:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.589647:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.589647:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.589649:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.589652:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.589653:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.589654:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.589655:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.589656:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.589658:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.589661:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.589663:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.589665:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.589666:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.589666:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.589668:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.589668:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.589669:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.589669:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.589674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.589681:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.589683:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.589684:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.589686:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.589687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.589688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.589693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.589700:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.589702:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.589703:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.589704:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.589704:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.589705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.589705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.589713:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.589716:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.589717:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.589719:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.589720:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.589722:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.589723:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.589724:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.589725:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.589726:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.589730:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.589731:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.589734:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.589735:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.589736:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.589737:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.589738:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.589739:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.589739:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.589744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.589751:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.589754:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.589754:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.589755:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.589757:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.589760:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.590165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.590172:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.590175:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.590176:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.590178:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.590180:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.590181:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677475.590183:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.590184:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.590185:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.590186:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677475.590188:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.590190:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.590191:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.590191:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.590192:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.590193:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.590193:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.590194:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.590194:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.590198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.590205:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.590206:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.590207:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.590208:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.590209:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.590210:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677475.590218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.590224:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.590226:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.590227:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.590227:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.590228:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.590228:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:20.2:1619677475.590613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.590619:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.590621:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.590623:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.590624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.590626:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.590627:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.590628:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.590630:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.590631:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.590632:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.590634:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.590636:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.590638:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.590639:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.590640:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.590641:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.590642:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.590642:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.590643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.590648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.590656:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.590658:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.590659:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.590661:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.590662:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.590663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.591078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.591084:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.591087:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.591088:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.591089:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.591091:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.591093:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.591094:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.591095:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.591096:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.591097:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.591100:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.591101:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.591104:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.591104:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.591105:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.591107:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.591107:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.591108:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.591109:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.591110:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.591113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.591115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.591120:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.2:1619677475.591121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:22.0:1619677475.591122:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.591122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.591123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.591128:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677475.591129:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677475.591130:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677475.591131:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:21.0:1619677475.591131:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.591133:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.591133:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677475.591135:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.591135:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.591137:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.591137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.591138:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.591140:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.591141:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.591142:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.591143:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.591146:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.591148:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.591151:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.591152:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.591152:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.591154:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.591155:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.591156:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.591156:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.591160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.591167:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.591170:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.591171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.591171:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.591173:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.591175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.591181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.591187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.591190:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.591191:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.591191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.591192:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.591192:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.591467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.591475:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.591480:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.591482:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.591485:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.591489:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.591491:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.591493:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.591495:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.591497:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.591499:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.591504:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.591506:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.591511:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.591512:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.591513:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.591516:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.591517:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.591518:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.591519:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.591533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.2:1619677475.591533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677475.591540:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:12.0:1619677475.591543:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:1.0:1619677475.591543:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677475.591544:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:12.0:1619677475.591546:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:1.0:1619677475.591546:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:12.0:1619677475.591547:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.591548:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.591550:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:13.0:1619677475.591551:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:12.0:1619677475.591551:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:1.0:1619677475.591551:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499181] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.591553:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.591554:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677475.591555:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:1.0:1619677475.591555:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:25.2:1619677475.591556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000400:00000200:1.0:1619677475.591558:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677475.591560:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:13.0:1619677475.591563:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677475.591563:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.591563:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.591564:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:13.0:1619677475.591566:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677475.591566:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.591566:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:13.0:1619677475.591567:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:1.0:1619677475.591567:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:13.0:1619677475.591568:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:1.0:1619677475.591568:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:13.0:1619677475.591569:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.591571:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.591572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.591580:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.591583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.591583:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.591584:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.591586:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.591588:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.591827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.591830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.591833:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.591835:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.591836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.591836:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.591838:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.591839:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.591840:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.591841:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.591842:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.591845:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.591847:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.591849:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.591850:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.591850:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.591852:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.591853:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.591854:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.591855:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677475.591856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.2:1619677475.591859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.591862:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.591865:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.591866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:21.0:1619677475.591867:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:17.0:1619677475.591867:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.591870:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.591870:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.591870:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.591871:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.591873:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.591873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:17.0:1619677475.591873:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.591874:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.591875:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:17.0:1619677475.591875:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.591877:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.591880:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.591882:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.591883:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.591884:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.591884:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.591886:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.591887:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.591887:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.591888:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.591892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.591900:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.591902:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.591903:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.591904:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.591905:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.591906:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.592033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.592041:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.592045:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.592046:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.592047:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.592049:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.592051:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.592052:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.592054:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.592054:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.592056:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.592059:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.592061:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.592064:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.592065:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.592066:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.592067:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.592068:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.592069:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.592069:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.592073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.592081:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.592084:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.592084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.592085:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.592087:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.592090:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.592095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:21.2:1619677475.592100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677475.592102:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.592104:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.592105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.592105:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.592106:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.592107:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.592108:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.592110:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.592112:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.592115:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.592115:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.592116:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.592117:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.592120:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.592121:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.592121:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.592124:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.592126:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.592129:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.592130:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.592131:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.592132:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.592133:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.592134:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.592135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.592139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.592147:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.592149:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.592150:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.592152:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.592153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.592155:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.592751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.592758:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.592761:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.592762:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.592764:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.592767:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.592768:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.2:1619677475.592769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:26.0:1619677475.592770:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.592771:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.592772:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.592773:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.592775:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:26.0:1619677475.592776:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.592777:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.592778:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.592779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:26.0:1619677475.592780:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.592781:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:22.0:1619677475.592781:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.592782:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677475.592783:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.592784:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.592784:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.592785:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.592785:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:22.0:1619677475.592785:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.592786:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.592787:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.592788:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.592789:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.2:1619677475.592790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677475.592793:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.592794:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.592798:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.592798:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.592799:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.592799:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:22.0:1619677475.592801:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.592801:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.592801:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:22.0:1619677475.592802:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677475.592802:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.592803:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.592803:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.592805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.592806:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:25.2:1619677475.592807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.592815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.592817:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.592818:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.592819:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.592820:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.592822:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.592898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.592906:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.592909:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.592912:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.592914:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.592914:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.592916:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.592917:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.592918:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.592919:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.592920:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.592924:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.592925:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.592928:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.592928:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.592929:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.592931:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.592931:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.592932:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.592933:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.592936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.592943:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.592945:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.592946:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.592948:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.592949:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.592950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.592956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.592963:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.592966:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.592967:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.592967:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.592968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.592969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.593203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.593211:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.593213:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.593215:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.593216:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.593217:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.593218:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.593219:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.593220:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.593221:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.593222:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.593224:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.593226:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.593228:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.593229:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.593229:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.593231:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.593231:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.593233:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.593234:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.593235:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.593237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.593241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.593243:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.593246:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.593247:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.593247:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.593249:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.593249:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.593251:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677475.593251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.593252:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.593253:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.593303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.593311:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.593313:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.593315:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.593316:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.593319:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.593320:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.593321:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.593323:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.593324:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.593325:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.593328:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.593330:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.593333:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.593333:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.593334:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.593336:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.593336:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.593337:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.593337:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.593342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.593350:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.593353:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.593354:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.593354:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.593356:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.593359:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.593469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.593478:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.593481:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.593482:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.593485:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.593485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.593486:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.593487:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.593489:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.593490:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.593491:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.593494:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.593496:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.593498:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.593498:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.593499:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.593500:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.593501:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.593502:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.593502:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.593506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.593513:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.593515:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.593516:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.593518:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.593519:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.593520:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.594002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.594009:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.594012:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.594013:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.594016:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.594018:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.594020:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.594021:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.594022:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.594023:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.594024:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.594027:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.594029:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.594030:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.594031:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.594032:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.594033:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.594034:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.594035:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.594035:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.594039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.594046:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.594048:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.594049:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.594050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.594050:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.594051:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.594073:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.594078:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.594081:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.594082:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.594084:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.594086:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.594087:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.594089:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.594090:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.594091:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.594092:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.594096:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.594097:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.594100:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.594101:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.594101:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.594103:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.594103:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.594104:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.594104:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.594109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.594117:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.594119:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.594120:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.594122:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.594122:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.594124:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.594520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.594527:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.594530:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.594531:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.594533:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.594536:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.594538:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.594539:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.594540:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.594541:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.594542:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.594546:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.594548:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.594550:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.594551:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.594552:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.594553:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.594554:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.594555:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.594555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.594560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.594568:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.594571:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.594572:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.594572:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.594574:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.594577:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.594580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.594587:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.594590:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.594591:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.594591:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.594592:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.594593:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.594927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.594935:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.594938:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.594938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.594940:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.594942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:26.0:1619677475.594943:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.594944:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.594946:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.594947:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.594948:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.594949:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.594951:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:26.0:1619677475.594952:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.594953:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:26.0:1619677475.594953:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.594955:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.594956:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.594957:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.594957:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.594957:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.594958:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.594958:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.594959:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677475.594959:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677475.594960:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677475.594960:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.594960:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000400:00000200:30.0:1619677475.594961:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677475.594961:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:30.0:1619677475.594962:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.594965:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.594965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.594967:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.594969:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.594969:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.594970:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.594972:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.594972:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.594972:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.594973:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.594973:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.594975:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.594976:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.594976:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.594977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:6.0:1619677475.594978:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.594981:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.0:1619677475.594985:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.594987:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.594988:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.594988:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.594990:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.594993:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.595551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.595557:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.595560:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.595560:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.595562:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.595565:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.595566:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.595567:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.595569:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.595570:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.595571:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.595574:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.595576:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.595578:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.595579:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.595580:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.595581:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.595582:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.595582:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.595583:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.595588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.595597:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.595599:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.595600:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.595602:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.595603:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.595604:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.595606:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.595613:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.595616:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.595616:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.595617:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.595618:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.595619:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.595836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.595842:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.595845:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.595846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.595848:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.595849:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.595851:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.595852:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.595854:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.595855:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.595856:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.595859:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.595861:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.595863:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.595864:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.595865:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.595866:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.595867:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.595868:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.595869:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.595870:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.595873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.595875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.595880:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.595882:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677475.595883:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.595883:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.595884:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677475.595885:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.595886:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.595887:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.595889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.595890:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677475.596102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.596108:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.596110:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.596111:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.596113:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.596115:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.596116:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.596117:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.596119:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.596119:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.596120:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.596123:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.596124:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.596125:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.596126:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.596126:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.596128:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.596128:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.596129:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.596130:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.596130:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.596134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.596136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.596142:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.596143:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.596144:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.596144:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.596144:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.596144:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.596145:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.596146:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677475.596146:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.596147:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.596166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.596173:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.596176:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.596177:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.596178:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.596181:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.596183:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.596184:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.596186:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.596186:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.596188:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.596191:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.596193:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.596196:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.596196:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.596197:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.596199:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.596199:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.596200:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.596200:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.596204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.596213:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.596216:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.596217:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.596217:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.596219:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.596222:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.596285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91946f783000] (20)++
00000800:00000200:20.0:1619677475.596289:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f783000] (21)++
00000800:00000200:20.0:1619677475.596291:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.4@o2ib3
00000800:00000200:21.0:1619677475.596294:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000400:00000200:20.0:1619677475.596298:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.13.4@o2ib3 : GET - for me
00000400:00000200:20.0:1619677475.596301:0:19934:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.13.4@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:20.0:1619677475.596303:0:19934:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.13.4@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:20.0:1619677475.596304:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91946f783000] (21)++
00000800:00000200:20.0:1619677475.596305:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91946f783000] (22)--
00000400:00000200:20.0:1619677475.596307:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.13.4@o2ib3 local destination
00000400:00000200:20.0:1619677475.596311:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.13.4@o2ib3(10.51.13.4@o2ib3:10.51.13.4@o2ib3) : REPLY try# 0
00000800:00000200:20.0:1619677475.596313:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.13.4@o2ib3
00000800:00000200:20.0:1619677475.596315:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9197eee63e00] -> 10.51.13.4@o2ib3 (2) version: 12
00000800:00000200:20.0:1619677475.596316:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91946f783000] (21)++
00000800:00000200:20.0:1619677475.596317:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91946f783000] (22)++
00000800:00000200:20.0:1619677475.596318:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91946f783000] (23)--
00000800:00000200:20.0:1619677475.596319:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000800:00000200:4.2:1619677475.596324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91946f783000] (21)++
00000800:00000200:21.0:1619677475.596329:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91946f783000] (22)++
00000800:00000200:21.0:1619677475.596331:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91946f783000] (23)--
00000400:00000200:21.0:1619677475.596332:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.596334:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.13.4@o2ib3: REPLY: OK
00000800:00000200:20.0:1619677475.596334:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (22)--
00000800:00000200:21.0:1619677475.596336:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91946f783000] (21)--
00000800:00000200:20.2:1619677475.596633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.596635:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.596638:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.596640:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.596641:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.596642:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.596643:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.596645:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.596646:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.596647:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.596648:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.596651:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.596652:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.596654:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.596655:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.596656:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.596657:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.596658:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.596658:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.596659:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.596664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.596672:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.596675:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.596676:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.596676:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.596678:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.596680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.596879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.596887:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.596890:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.596891:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.596893:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.596896:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.596897:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.596898:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.596899:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.596900:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.596901:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.596905:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.596907:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.596910:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.596910:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.596911:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.596913:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.596913:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.596913:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.596914:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.596919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.596927:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.596930:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.596931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.596931:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.596933:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.596936:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.596990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.596998:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.597002:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.597004:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.597006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.597007:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.597008:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.597009:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.597010:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.597012:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.597012:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.597016:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.597017:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.597020:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.597020:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.597021:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.597022:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.597023:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.597024:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.597024:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.597029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.597035:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.597038:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.597038:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.597039:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.597041:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.597043:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.597271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.597276:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.597279:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.597280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.597282:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.597284:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.597285:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.597287:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.597288:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.597289:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.597290:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.597293:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.597299:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.597301:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.597302:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.597303:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.597304:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.597305:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.597306:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.597307:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.597308:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.597311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.597314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.597319:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.597321:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.597321:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.597321:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.597322:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.597323:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.597324:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.597326:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.597326:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.597328:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.597363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.597371:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.597373:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.597374:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.597376:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.597379:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.597380:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.597382:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.597383:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.597384:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.597385:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:21.2:1619677475.597385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677475.597388:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.597390:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.597392:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.597393:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677475.597393:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.597394:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:30.0:1619677475.597395:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677475.597395:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.597396:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.597397:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.597397:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:30.0:1619677475.597398:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.597400:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.597400:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.597401:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:25.2:1619677475.597401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:30.0:1619677475.597402:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.597403:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.597405:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.597405:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.597409:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.597409:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:30.0:1619677475.597410:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677475.597411:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.597411:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.597412:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.597413:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.597413:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:12.0:1619677475.597413:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.597415:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.597415:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677475.597416:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677475.597416:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:30.0:1619677475.597417:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.597418:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.597419:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.597420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.597424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.597429:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.597430:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.597431:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.597432:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.597433:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.597433:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.597434:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:4.2:1619677475.597434:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677475.597435:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.597436:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.597437:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.597438:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.597440:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677475.597443:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.597443:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.597445:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.597447:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.597448:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.597449:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.597451:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.597451:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.597455:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.597456:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.597458:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.597459:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.597460:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.597461:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.597462:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.597463:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.597463:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.597468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.597476:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.597478:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.597479:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.597481:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.597481:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.597483:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.597487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.597494:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.597497:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.597498:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.597498:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.597499:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.597500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.597546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.597554:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.597557:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.597557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.597559:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.597561:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.597563:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.597564:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.597565:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.597566:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.597567:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.597570:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.597572:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.597574:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.597575:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.597576:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.597577:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.597578:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.597579:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.597579:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.597583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.597591:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.597594:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.597595:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.597595:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.597597:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.597600:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.597603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.597610:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.597613:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.597614:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.597614:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.597615:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.597615:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.597659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677475.597664:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.597667:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677475.597668:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677475.597670:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.597672:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.597674:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:22.0:1619677475.597675:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.597676:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.597678:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.597679:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:22.0:1619677475.597682:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.597684:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:22.0:1619677475.597686:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.597687:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677475.597688:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:22.0:1619677475.597689:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677475.597690:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.597690:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677475.597691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677475.597695:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.597703:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.597705:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677475.597706:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.597707:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.597708:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.597711:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677475.597895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.597900:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.597903:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.597905:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.597906:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.597908:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.597909:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.597910:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.597912:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.597913:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.597914:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.597917:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.597919:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.597920:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.597921:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.597922:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.597923:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.597924:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.597925:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.597926:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.597926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.597929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.597930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677475.597933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677475.597937:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.597939:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:30.0:1619677475.597940:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.597940:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.597941:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.597941:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.597942:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.597942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677475.597943:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.597943:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.597944:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677475.597944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.597945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:30.0:1619677475.597946:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.597947:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:20.0:1619677475.597947:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:30.0:1619677475.597948:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.597950:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.597951:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.597952:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.597955:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.597957:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.597959:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.597959:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.597960:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.597961:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.597962:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.597963:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.597964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.597967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.597974:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.597976:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.597977:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.597979:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.597980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.597981:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.598180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.598187:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.598190:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.598191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.598192:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.598195:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.598197:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.598198:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.598199:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.598200:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.598201:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.598204:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.598206:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.598208:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.598209:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.598210:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.598211:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.598212:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.598212:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.598213:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.598217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.598225:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.598228:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.598228:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.598229:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.598231:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.598234:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.598235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.598242:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.598245:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.598246:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.598246:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.598247:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.598247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.598414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.598422:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.598425:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.598427:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.598428:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.598429:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.598430:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.598431:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.598433:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.598434:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.598435:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.598438:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.598439:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.598442:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.598443:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.598444:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.598446:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.598446:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.598447:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.598448:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.598452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.598461:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.598463:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.598464:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.598466:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.598467:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.598469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.598592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.598598:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.598601:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.598602:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.598603:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.598605:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.598607:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.598608:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.598609:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.598610:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.598611:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.598615:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.598616:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.598619:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.598619:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.598620:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.598622:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.598622:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.598623:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.598624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.598628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.598636:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.598638:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.598639:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.598640:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.598642:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.598642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.598657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.598664:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.598667:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.598670:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.598671:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.598673:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.598674:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.598676:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.598677:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.598678:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.598679:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.598683:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.598684:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.598687:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.598688:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.598688:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.598690:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.598691:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.598692:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.598692:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.598696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.598702:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.598705:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.598706:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.598706:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.598708:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.598710:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.598716:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.598722:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.598724:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.598725:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.598725:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.598726:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.598726:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.599010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677475.599015:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.599017:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677475.599020:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.599020:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677475.599021:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.599023:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:21.0:1619677475.599024:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.599026:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.599027:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.599028:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677475.599031:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.599033:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.0:1619677475.599036:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.599037:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677475.599038:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677475.599040:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677475.599041:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.599042:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677475.599043:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677475.599044:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:25.2:1619677475.599046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.2:1619677475.599048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.599051:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677475.599053:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:12.0:1619677475.599053:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:20.0:1619677475.599054:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677475.599055:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:12.0:1619677475.599056:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677475.599057:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.599059:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.599061:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677475.599062:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:4.2:1619677475.599227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.599235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.599238:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.599238:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677475.599241:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.599243:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.599244:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.599246:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.599247:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.599248:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.599249:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.599252:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.599254:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.599257:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.599258:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.599259:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.599261:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.599261:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.599262:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.599262:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.599267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.599275:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.599277:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.599278:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.599280:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.599281:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.599283:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.599287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.599295:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.599301:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.599302:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.599302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.599303:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.599304:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.599342:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.599350:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.599352:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.599355:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.599357:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.599357:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.599358:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.599360:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.599361:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.599362:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.599363:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.599366:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.599368:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.599371:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.599371:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.599372:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.599374:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.599374:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.599375:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.599376:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.599380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.599387:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.599389:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.599390:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.599390:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.599392:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.599394:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.599418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677475.599425:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677475.599427:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677475.599429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677475.599430:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.599432:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.599434:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:24.0:1619677475.599435:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.599436:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.599437:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.599438:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:24.0:1619677475.599440:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.599442:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:24.0:1619677475.599444:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.599445:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:24.0:1619677475.599445:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:24.0:1619677475.599447:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:24.0:1619677475.599447:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677475.599448:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677475.599448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:9.2:1619677475.599453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.599459:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.599461:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.599462:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.599462:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.599464:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.599466:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677475.599945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.599951:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.599954:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.599955:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.599956:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.599958:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.599960:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.599961:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.599962:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.599963:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.599964:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.599967:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.599968:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.599971:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.599972:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.599972:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.599974:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.599974:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.599975:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.599975:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.599990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.599999:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.600001:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.600002:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.600003:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.600005:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.600008:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.600046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:20.0:1619677475.600048:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.600051:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:20.0:1619677475.600053:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.600054:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677475.600056:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.600057:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:20.0:1619677475.600059:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.600060:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.600061:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.600062:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000800:00000200:21.2:1619677475.600065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:20.0:1619677475.600066:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.600067:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:6.0:1619677475.600070:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.600071:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.600071:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:20.0:1619677475.600072:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:6.0:1619677475.600073:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.600074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.600074:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:20.0:1619677475.600075:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.600076:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677475.600076:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:6.0:1619677475.600076:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.600078:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.600079:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:25.2:1619677475.600080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:6.0:1619677475.600080:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.600082:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.600082:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.600084:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:13.0:1619677475.600087:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:6.0:1619677475.600087:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.600089:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:13.0:1619677475.600090:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677475.600091:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.600091:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.600091:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:13.0:1619677475.600092:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.600092:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:12.0:1619677475.600094:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:6.0:1619677475.600094:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.600095:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.600095:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:13.0:1619677475.600096:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:6.0:1619677475.600096:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.600100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.600108:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.600110:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.600111:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.600112:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.600113:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.600115:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.600452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.600458:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.600461:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.600462:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.600464:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.600466:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.600467:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.600468:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.600470:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.600470:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.600472:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.600474:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.600476:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.600478:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.600479:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.600479:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.600481:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.600481:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.600482:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.600483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.600488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.600496:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.600497:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.600498:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.600499:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.600500:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.600501:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.600505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.600512:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.600514:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.600515:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.600515:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.600516:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.600516:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.600771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.600780:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.600782:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.600785:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.600787:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.600787:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.600788:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.600790:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.600791:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.600792:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.600793:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.600797:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.600798:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.600801:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.600802:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.600803:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.600804:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.600805:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.600806:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.600808:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.600808:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.600811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.600814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.600818:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.600820:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.600821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.600821:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.600822:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.600823:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.600824:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.600825:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677475.600825:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.600826:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.601372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.601378:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.601381:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.601382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.601384:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.601386:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.601387:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.601388:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.601389:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.601391:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.601391:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.601395:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.601397:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.601400:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.601400:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.601401:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.601402:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.601403:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.2:1619677475.601403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:24.0:1619677475.601404:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.601405:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.601405:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.601408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677475.601409:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:4.2:1619677475.601411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.601412:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.601412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.601414:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.601416:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:22.0:1619677475.601417:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.601418:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.601419:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677475.601419:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.601419:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:22.0:1619677475.601420:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.601420:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:22.0:1619677475.601421:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.601421:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.601421:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677475.601421:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.601422:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.601423:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:1.0:1619677475.601423:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.601426:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.601426:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:22.0:1619677475.601427:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.601430:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.601431:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.601431:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.601433:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.601433:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.601434:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.601435:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.601440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.601448:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.601450:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.601451:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.601452:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.601454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.601455:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.601465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.601473:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.601475:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.601476:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.601476:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.601477:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.601478:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.601545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.601553:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.601556:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.601557:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.601559:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.601561:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.601563:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.601564:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.601565:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.601566:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.601567:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.601571:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.601572:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.601575:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.601576:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.601577:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.601578:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.601579:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.601580:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.601581:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.601582:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.601584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.601587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677475.601592:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.601594:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:6.0:1619677475.601594:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.601595:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.601596:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:6.0:1619677475.601596:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.601597:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.601598:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:6.0:1619677475.601598:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.601601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.601718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.601725:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.601728:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.601729:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.601730:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.601734:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.601735:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.601736:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.601737:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.601738:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.601739:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.601742:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.601743:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.601745:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.601746:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.601747:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.601748:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.601749:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.601750:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.601751:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.601751:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.601754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.601757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.601761:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.601764:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677475.601764:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677475.601765:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.601765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.601766:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.601767:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.601767:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.601767:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677475.601770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.601993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.601998:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.602001:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.602003:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.602004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.602006:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.602007:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.602009:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.602010:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.602011:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.602012:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.602015:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.602016:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.602018:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.602019:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.602020:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.602021:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.602022:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.602022:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.602023:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.602027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.602035:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.602037:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.602038:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.602039:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.602041:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.602041:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.602326:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.602332:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.602335:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.602336:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.602337:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.602339:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.602341:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.602342:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.602343:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.602344:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.602345:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.602348:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.602350:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.602352:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.602353:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.602354:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.602355:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.602356:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.602357:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.602358:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.602358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.602362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.602364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.602370:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.602371:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.602371:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.602372:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.602373:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.602373:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.602374:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.602376:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.602377:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.602378:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.602405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677475.602411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.602415:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.602417:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.602419:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.602419:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.602420:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.602422:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.602423:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.602423:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.602424:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.602426:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.602427:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.602428:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.602430:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.602431:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.602432:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.602433:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.602434:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.602434:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.602435:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.602436:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.602439:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.602441:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.602442:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.602443:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.602443:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.602444:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.602444:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.602444:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.602446:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.602446:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.602447:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.602447:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.602448:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.602449:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.602449:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (23)++
00000800:00000200:31.0:1619677475.602450:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677475.602450:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (24)--
00000800:00000200:30.0:1619677475.602451:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.602452:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.602452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:25.2:1619677475.602453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.602456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.602460:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:30.0:1619677475.602461:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.602462:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:24.0:1619677475.602462:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:30.0:1619677475.602463:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.602463:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.602464:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.602464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.602465:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.602466:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.602466:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:24.0:1619677475.602466:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.602467:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.602468:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:24.0:1619677475.602468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.602469:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.602472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.602479:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.602481:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.602482:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.602483:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.602483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.602484:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.602727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.602735:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.602738:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.602738:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.602741:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.602743:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.602745:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677475.602746:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.602747:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.602748:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.602749:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.602752:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.602753:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677475.602756:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.602757:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677475.602757:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677475.602759:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677475.602760:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.602760:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.602761:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677475.602765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.602773:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.602776:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677475.602777:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.602778:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.602780:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677475.602783:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677475.602942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.602948:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.602951:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.602952:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.602953:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.602955:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.602956:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.602957:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.602959:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.602960:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.602961:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.602964:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.602966:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.602968:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.602969:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.602970:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.602972:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.602972:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.602973:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.602974:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.602975:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.602978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677475.602980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677475.602985:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677475.602986:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677475.602987:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.602988:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.602988:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.602989:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.602989:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677475.602990:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.602992:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.602993:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.602998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.603005:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.603005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677475.603008:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.603009:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.603009:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.603010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.603011:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:8.0:1619677475.603012:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.603015:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.603016:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.603017:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.603020:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.603021:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.603022:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.603024:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.603025:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.603026:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.603029:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.603031:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.603033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.603034:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.603035:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.603036:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.603037:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.603038:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.603039:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.603039:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677475.603041:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.603044:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677475.603044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.603046:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.603047:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.603048:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.603049:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.603050:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.603051:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.603051:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677475.603052:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.603053:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.603054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.603054:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.603055:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.603056:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.603058:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:6.0:1619677475.603058:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.603060:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.603060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.603061:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.603061:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.603063:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.603063:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.603064:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.603065:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.603068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.603075:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.603077:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.603078:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.603080:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.603081:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.603082:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.603129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.603135:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.603137:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.603138:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.603140:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.603143:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.603144:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.603145:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.603147:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.603148:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.603149:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.603152:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.603154:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.603156:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.603157:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.603157:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.603158:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.603159:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.603160:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.603160:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.603164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.603171:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.603173:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.603174:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.603175:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.603176:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.603177:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.603190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.603197:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.603200:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.603200:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.603200:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.603201:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.603201:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.603262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.603269:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.603272:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.603272:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.603275:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.603277:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.603279:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677475.603280:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.603281:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.603283:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.603284:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677475.603287:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.603289:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677475.603292:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.603293:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677475.603300:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677475.603302:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677475.603303:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.603304:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.603304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.603309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.603318:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.603321:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.603322:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.603322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.603324:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.603327:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.603396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.603401:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.603404:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.603405:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.603407:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.603409:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.603410:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.603412:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.603413:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.603414:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.603415:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.603418:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.603419:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.603421:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.603422:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.603423:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.603424:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.603425:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.603426:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.603426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.603431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.603439:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.603442:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.603442:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.603444:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.603445:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.603447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.603878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677475.603879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.603885:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677475.603887:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.603887:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677475.603890:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:21.0:1619677475.603890:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.603890:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677475.603893:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.603893:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.603894:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.603894:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677475.603895:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.603896:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.603897:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.603898:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.603898:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.603899:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.603900:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.603900:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677475.603901:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.603902:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677475.603904:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.603906:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:30.0:1619677475.603909:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.603911:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.603912:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.603913:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.603913:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677475.603915:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.603915:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.603915:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:30.0:1619677475.603916:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.603916:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.603916:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.603917:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:30.0:1619677475.603918:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.603918:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.603919:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.603920:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.603921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.2:1619677475.603924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.603929:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.603931:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.603932:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:20.0:1619677475.603932:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.603933:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.603933:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.603934:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.603936:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.603936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.603937:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.603938:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.603939:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.604203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.604211:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.604214:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.604214:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000400:00000200:10.0:1619677475.604217:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.604219:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.604221:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677475.604222:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.604223:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.604224:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.604225:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677475.604229:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.604230:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677475.604233:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.604234:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677475.604235:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677475.604236:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677475.604237:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.604237:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.604238:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.604242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.604250:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.604253:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.604254:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.604254:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.604256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:1.0:1619677475.604257:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.604259:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677475.604262:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.604264:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.604265:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.604267:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.604269:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.604271:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.604272:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.604273:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.604274:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.604275:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.604278:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.604280:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.604282:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.604283:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.604283:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.604285:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.604285:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.604286:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.604286:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.604291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.604300:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.604303:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.604303:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.604305:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.604306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.604307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.604374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.604382:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.604385:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.604388:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.604389:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.604390:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.604392:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.604393:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.604395:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.604396:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.604397:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.604400:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.604402:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.604404:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.604405:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.604406:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.604407:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.604408:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.604409:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.604409:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.604414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.604421:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.604424:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.604425:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.604425:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.604427:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.604429:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.604754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677475.604760:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.604763:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677475.604763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677475.604765:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.604768:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.604770:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677475.604771:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.604772:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.604773:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.604774:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:4.2:1619677475.604777:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:22.0:1619677475.604778:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.604780:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677475.604782:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.604783:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677475.604783:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.604785:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.604785:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677475.604785:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.604787:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:22.0:1619677475.604787:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:26.0:1619677475.604788:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:22.0:1619677475.604788:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:10.0:1619677475.604788:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.604791:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677475.604791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:26.0:1619677475.604793:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.604793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000400:00000200:26.0:1619677475.604795:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.604796:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.604798:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.604798:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000400:00000200:26.0:1619677475.604799:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677475.604799:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:26.0:1619677475.604800:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.604800:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:17.0:1619677475.604801:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:6.0:1619677475.604801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677475.604802:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:26.0:1619677475.604803:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677475.604803:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.604803:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.604805:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.604806:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:17.0:1619677475.604808:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:26.0:1619677475.604809:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.604810:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.604811:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.604812:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.604813:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.604814:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.604815:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.604816:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.604819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.604821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.604826:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.604828:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.604829:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.604830:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.604830:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677475.604830:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.604831:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.604832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677475.604832:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.604835:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.605136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.605141:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.605143:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.605144:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.605146:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.605148:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.605149:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.605150:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499115] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.605152:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.605153:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.605154:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.605157:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.605159:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.605161:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.605162:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.605163:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.605164:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.605164:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.605165:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.605165:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.605170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.605178:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.605180:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.605181:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.605183:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.605183:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.605185:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.605189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.605196:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.605198:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.605200:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.605200:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.605201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.605202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.606022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.606030:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.606033:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.606035:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.606037:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.606037:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.606039:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.606040:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.606041:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.606042:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.606043:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.606047:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.606049:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.606051:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.606052:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.606053:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.606054:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.606055:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.606056:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.606057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.606061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.606068:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.606071:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.606071:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.606072:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.606073:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.606075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.606085:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.606091:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.606094:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.606095:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.606095:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.606096:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.606096:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.606152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.606158:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.606161:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.606162:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.606164:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.606166:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.606167:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.606169:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.606170:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.606171:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.606172:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.606175:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.606176:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.606179:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.606179:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.606180:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.606182:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.606182:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.606183:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.606184:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.606185:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.606188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.606190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.606194:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.606196:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.606196:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.606197:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.606198:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.606198:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.606199:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.606201:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.606202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.606203:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.606995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677475.606999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:8.0:1619677475.607005:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.607005:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.607009:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.607009:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:8.0:1619677475.607010:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.607011:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677475.607013:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.607015:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.607016:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.607016:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:21.2:1619677475.607017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:1.0:1619677475.607018:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.607019:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.607020:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.607020:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.607021:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:22.0:1619677475.607023:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677475.607023:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:22.0:1619677475.607026:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:8.0:1619677475.607026:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.607026:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:1.0:1619677475.607026:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.607028:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:22.0:1619677475.607029:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.607029:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.607031:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.607031:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.607032:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.607032:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.607032:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677475.607032:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:22.0:1619677475.607033:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.607034:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.607034:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:22.0:1619677475.607036:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.607037:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.607038:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677475.607040:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677475.607041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:22.0:1619677475.607043:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.607043:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:8.0:1619677475.607044:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.607044:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677475.607045:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:22.0:1619677475.607046:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677475.607046:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:22.0:1619677475.607048:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.607049:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.0:1619677475.607049:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.607049:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:22.0:1619677475.607050:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.607050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:8.0:1619677475.607050:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.607051:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677475.607052:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.607052:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:4.2:1619677475.607052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:2.0:1619677475.607052:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:22.0:1619677475.607053:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677475.607053:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:8.0:1619677475.607053:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:2.0:1619677475.607053:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.607054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677475.607054:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.607055:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:2.0:1619677475.607055:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.607056:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:17.0:1619677475.607057:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:25.2:1619677475.607058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677475.607058:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:17.0:1619677475.607059:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:9.2:1619677475.607059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:8.2:1619677475.607059:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677475.607060:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677475.607062:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:10.0:1619677475.607062:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.607063:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677475.607063:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677475.607064:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:10.0:1619677475.607065:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.607066:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:21.0:1619677475.607067:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:17.0:1619677475.607067:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.607068:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.607068:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.607068:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.607069:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:21.0:1619677475.607070:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:18.0:1619677475.607070:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.607070:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:21.0:1619677475.607071:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.607071:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.607071:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:18.0:1619677475.607072:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.607072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:21.0:1619677475.607073:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.607073:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677475.607073:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96499109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.607075:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:10.0:1619677475.607075:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.607076:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677475.607076:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.607077:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.607080:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677475.607080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677475.607082:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.607085:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.607086:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.607086:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.607088:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.607089:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.607089:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:1.0:1619677475.607089:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677475.607090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:1.0:1619677475.607092:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.607093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.607093:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:25.2:1619677475.607094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.607094:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.607095:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677475.607102:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.607105:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.607105:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.607106:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.607108:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.607111:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.607116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.607123:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.607126:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.607126:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.607127:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.607128:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.607128:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.607217:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.607227:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.607231:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.607234:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.607237:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.607238:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.607240:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.607242:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.607244:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.607246:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.607247:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.607253:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.607255:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.607258:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.607259:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.607260:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.607262:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.607263:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.607265:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.607266:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.607268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.607275:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.607277:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.607278:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.607280:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.607282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.607283:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.607391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.607399:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.607401:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.607403:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.607404:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.607405:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:31.0:1619677475.607406:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.607406:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.607408:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.607408:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.607409:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.607412:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.607414:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.607416:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.607417:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.607417:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.607419:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.607419:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.607421:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.607422:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.607423:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.607425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.607428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.607432:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.607434:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.607435:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.607436:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.0:1619677475.607436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.607437:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.607439:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677475.607439:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.607440:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.607440:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.607492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.607498:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.607501:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.607502:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.607504:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.607505:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.607507:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.607508:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.607510:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.607510:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.607512:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.607514:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.607516:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.607518:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.607519:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.607520:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.607522:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.607522:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.607523:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.607523:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.607528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.607536:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.607538:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.607539:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.607540:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.607542:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.607543:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.607548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.607556:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.607559:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.607560:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.607560:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.607561:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.607562:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.607821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.607826:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.607828:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.607830:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.607831:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.607833:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.607834:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.607835:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.607837:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.607838:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.607839:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.607842:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.607844:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.607846:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.607847:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.607848:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.607849:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.607850:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.607850:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.607851:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.607855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.607863:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.607865:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.607866:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.607867:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.607869:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:5.2:1619677475.607870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:20.0:1619677475.607871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677475.607876:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.607879:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.607880:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.607881:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.607883:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.607884:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.607886:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.607887:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.607888:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.607889:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.607892:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.607893:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.607895:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.607896:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.607897:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.607898:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.607899:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.607899:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.607900:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677475.607901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.2:1619677475.607904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677475.607906:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.607909:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.607910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677475.607910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:17.0:1619677475.607912:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:6.0:1619677475.607912:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.607914:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:6.0:1619677475.607914:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.607915:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.607915:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:1.0:1619677475.607916:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677475.607917:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677475.607917:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.607918:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.607919:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:6.0:1619677475.607919:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677475.607920:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.607920:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:6.0:1619677475.607920:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.607921:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677475.607924:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.607925:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.607927:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.607928:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:6.0:1619677475.607928:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677475.607929:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.607930:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677475.607931:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.607931:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:8.0:1619677475.607932:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.607932:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.607933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:8.0:1619677475.607933:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677475.607934:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.607934:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.607935:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.607935:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:8.0:1619677475.607936:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.607938:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:31.0:1619677475.607940:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677475.607940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677475.607941:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.607942:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.607942:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677475.607943:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677475.607944:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.607944:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677475.607945:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.607945:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.607946:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677475.607947:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.607947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.607948:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:31.0:1619677475.607949:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.607950:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96499101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.607950:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.2:1619677475.607950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:31.0:1619677475.607951:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.607951:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.607953:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.607953:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.607953:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677475.607954:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.607955:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677475.607957:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:31.0:1619677475.607958:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.607959:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:1.0:1619677475.607960:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677475.607961:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.607962:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.607962:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:17.0:1619677475.607962:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:31.0:1619677475.607963:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:1.0:1619677475.607963:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.607964:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.607965:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.607966:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677475.607966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677475.607968:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.607968:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.607970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.607974:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.607978:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.607980:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.607981:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.607982:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.607983:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677475.607983:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.607984:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.607985:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.607986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.607987:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.608091:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.608098:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.608100:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.608101:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.608103:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.608106:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.608107:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.608108:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.608110:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.608110:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.608111:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.608114:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.608116:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.608119:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.608119:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.608120:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.608122:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.608122:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.608123:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.608123:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.608128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.608135:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.608138:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.608139:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.608139:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.608141:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.608145:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.608896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.608905:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.608908:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.608910:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.608912:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.608913:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:30.0:1619677475.608913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.608915:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.608916:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.608917:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.608918:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.608922:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.608923:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.608925:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.608926:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.608927:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.608928:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.608929:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.608930:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.608930:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.608935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.608942:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:21.2:1619677475.608943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:24.0:1619677475.608944:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.608945:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.608945:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.608947:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.608948:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677475.608949:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677475.608951:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.608952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.608954:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.608956:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.608957:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.608958:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.608959:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.608960:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.608961:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.608964:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.608966:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.608968:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.608969:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.608969:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.608971:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.608971:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.608972:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.608972:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.608977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.608984:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.608987:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.608987:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.608989:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.608991:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.608992:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.608998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.609005:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.609007:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.609008:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.609009:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.609009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.609010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.609367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.609372:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.609374:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.609376:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.609377:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.609378:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.609380:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.609381:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.609383:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.609383:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.609385:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.609388:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.609389:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.609391:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.609392:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.609393:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.609394:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.609395:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.609396:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.609396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.609401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.609408:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.609410:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.609411:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.609413:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.609414:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.609415:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.609663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.609669:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.609672:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.609673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.609675:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.609677:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.609679:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:4.2:1619677475.609680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:2.0:1619677475.609680:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.609682:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.609683:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.609684:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.609686:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.609688:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:2.0:1619677475.609688:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.609690:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.609691:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:2.0:1619677475.609691:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.609692:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.609693:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:10.0:1619677475.609694:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677475.609695:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:10.0:1619677475.609696:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.609696:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.609697:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.609697:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:10.0:1619677475.609698:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.609699:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.609700:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.609701:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.609702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:10.0:1619677475.609702:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.609706:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.609708:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.609710:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.609711:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677475.609711:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.609712:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:12.0:1619677475.609714:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677475.609714:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.609714:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.609715:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:12.0:1619677475.609716:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.609716:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:12.0:1619677475.609718:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.609719:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:25.2:1619677475.609720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:12.0:1619677475.609722:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:22.0:1619677475.609727:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.609730:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.609731:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.609731:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.609734:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.609736:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.609746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.609754:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.609757:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.609759:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.609761:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.609762:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.609763:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.609764:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.609766:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.609767:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.609768:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.609771:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.609773:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.609775:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.609776:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.609776:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.609778:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.609778:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.609779:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.609780:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.609784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.609790:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.609793:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.609793:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.609794:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.609796:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.609797:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.609822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.609828:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.609830:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.609831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.609833:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.609835:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.609836:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.609838:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.609839:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.609840:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.609841:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.609843:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.609845:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.609848:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.609848:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.609849:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.609851:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.609851:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.609852:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.609853:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.609853:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.609857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.609858:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.609863:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.609865:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.609866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.609866:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.609867:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.609868:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.609869:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.609871:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.609871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.609873:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.609917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.609925:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.609928:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.609928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.609931:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.609933:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.609934:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.609936:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.609937:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.609938:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.609939:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.609942:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.609944:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.609946:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.609947:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.609948:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.609950:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.609950:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.2:1619677475.609950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:26.0:1619677475.609951:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677475.609952:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.609953:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.609955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.609957:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:4.2:1619677475.609958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:18.0:1619677475.609959:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.609960:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.609962:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.609962:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.609964:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:18.0:1619677475.609965:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.609965:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.609965:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:18.0:1619677475.609966:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.609968:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.609968:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:10.0:1619677475.609968:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.609968:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:18.0:1619677475.609969:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.609970:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:12.0:1619677475.609970:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.609970:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:18.0:1619677475.609971:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:13.0:1619677475.609971:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:18.0:1619677475.609975:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.609977:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.609978:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.609979:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.609980:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.609981:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.609982:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.609983:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.609983:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:21.2:1619677475.609986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:9.2:1619677475.609987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:17.0:1619677475.609993:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:2.0:1619677475.609994:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:17.0:1619677475.609996:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:2.0:1619677475.609996:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:1.0:1619677475.609996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:2.0:1619677475.609997:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.609998:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:17.0:1619677475.609999:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.610000:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.610001:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.610002:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:17.0:1619677475.610003:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677475.610004:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.610005:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.610006:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.610007:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677475.610011:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.610012:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677475.610015:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.610015:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677475.610016:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677475.610018:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677475.610018:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.610019:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677475.610020:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677475.610024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.610031:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.610034:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.610035:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.610035:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.610037:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.610039:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.610063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.610070:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.610073:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.610075:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.610077:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.610077:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.610079:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.610080:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.610082:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.610083:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.610084:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.610087:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.610089:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.610092:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.610093:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.610094:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.610096:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.610096:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.610097:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.610098:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.610102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.610110:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.610112:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.610113:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.610115:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.610116:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.610118:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.610508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.610514:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.610516:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.610518:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.610519:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.610520:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.610522:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.610523:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.610525:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.610526:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.610526:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.610530:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.610531:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.610534:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.610535:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.610535:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.610537:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.610538:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.610539:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.610540:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.610541:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.610544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.610546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.610548:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.610550:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.610551:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.610552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677475.610552:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.610555:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.610556:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.610557:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.610558:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.610560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.610677:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.610683:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.610686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.610686:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.610688:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.610690:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.610692:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.610693:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.610694:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.610695:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.610696:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.610699:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.610700:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.610702:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.610703:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.610704:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.610705:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.610706:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.610707:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.610707:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.610712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.610720:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.610722:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.610723:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.610724:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.610726:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.610727:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.610980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677475.610987:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677475.610990:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677475.610991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677475.610993:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.610995:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.610997:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677475.610998:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.610999:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.611000:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.611002:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677475.611004:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.611006:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677475.611009:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.611009:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677475.611010:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677475.611012:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677475.611012:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677475.611013:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677475.611013:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677475.611017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.611021:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677475.611023:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677475.611024:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.611025:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677475.611026:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.611028:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677475.611574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.611582:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.611585:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.611587:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.611590:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.611590:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.611591:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.611592:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.611593:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.611594:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.611595:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.611598:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.611599:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.611602:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.611603:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.611603:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.611605:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.611605:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.611606:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.611607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.611611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.611619:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.611621:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.611622:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.611622:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.611624:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.611626:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.611629:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.611635:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.611637:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.611638:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.611638:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.611639:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.611639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.611655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677475.611663:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.611666:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677475.611668:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.611670:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.611670:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677475.611671:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677475.611673:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.611674:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.611675:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.611676:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677475.611679:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.611680:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677475.611683:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.611683:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:31.0:1619677475.611684:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677475.611686:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:31.0:1619677475.611686:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.611687:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677475.611688:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:25.2:1619677475.611691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.611699:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.611702:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677475.611702:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.611704:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.611706:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677475.611707:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:25.2:1619677475.611709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:12.0:1619677475.611716:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.611718:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:12.0:1619677475.611719:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.611719:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677475.611720:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677475.611721:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677475.611729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677475.611734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.611737:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.611739:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677475.611739:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677475.611741:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.611742:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:31.0:1619677475.611743:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.611743:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.611743:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:30.0:1619677475.611745:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677475.611745:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.611746:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.611748:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.611748:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.611748:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.611749:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677475.611750:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677475.611751:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.611753:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.611754:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.611755:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:30.0:1619677475.611756:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.611759:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677475.611759:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.611762:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.611762:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:22.0:1619677475.611762:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.611763:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.611764:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677475.611764:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.611765:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.611765:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677475.611766:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677475.611766:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677475.611767:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.611768:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.611768:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.611769:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.611770:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:9.2:1619677475.611770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:25.2:1619677475.611774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677475.611776:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.611779:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.611780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.611780:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.611782:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677475.611782:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.611784:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677475.611784:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:21.0:1619677475.611785:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.611787:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.611788:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.611789:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.611930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.611938:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.611940:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.611941:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.611943:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.611946:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.611947:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.611948:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.611950:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.611950:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.611952:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.611955:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.611957:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.611960:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.611960:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.611961:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.611963:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.611963:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.611964:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.611964:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.611969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.611976:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.611979:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.611980:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.611980:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.611982:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.611986:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.612216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.612223:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.612226:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.612227:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.612229:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.612231:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.612232:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.612233:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.612235:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.612236:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.612237:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.612240:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.612242:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.612244:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.612244:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.612245:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.612247:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.612247:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.612248:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.612248:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.612253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.612261:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.612262:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.612263:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.612264:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.612265:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.612266:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.612271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.612279:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.612281:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.612282:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.612282:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.612283:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.612283:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.612324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.612332:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.612335:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.612337:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.612340:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.612340:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.612341:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.612342:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.612343:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.612345:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.612345:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.612349:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.612351:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.612353:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.612354:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.612354:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.612356:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.612356:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.612358:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.612359:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.612360:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.612362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677475.612364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677475.612366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.612370:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.612372:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.612372:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.612372:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677475.612374:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.612374:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.612375:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.612375:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677475.612375:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.612375:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677475.612376:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.612377:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:24.0:1619677475.612377:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.612378:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.612380:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000400:00000200:24.0:1619677475.612380:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.612382:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.612383:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.612384:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.612385:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.612386:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:24.0:1619677475.612386:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.612389:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:24.0:1619677475.612389:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.612390:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:24.0:1619677475.612390:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.612391:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.612391:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.612392:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:24.0:1619677475.612393:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.612394:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.612395:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.612396:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.612397:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.612398:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.612399:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.612400:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.612404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.612405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.612411:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.612412:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.612414:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.612415:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.612415:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.612415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:6.0:1619677475.612415:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.612417:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.612417:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.612421:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.612609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.612614:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.612617:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.612618:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.612620:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.612621:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.612623:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.612624:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.612625:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.612626:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.612627:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.612630:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.612632:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.612634:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.612635:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.612636:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.612637:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.612638:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.612638:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.612639:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.612643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.612651:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.612653:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.612654:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.612656:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.612657:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.612658:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.612664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.612671:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.612674:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.612675:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.612675:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.612676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.612677:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.612688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.612694:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.612697:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.612698:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.612700:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.612702:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.612703:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677475.612705:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.612706:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.612707:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.612708:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677475.612712:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.612713:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677475.612717:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.612718:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677475.612718:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.612720:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.612720:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.612721:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.612722:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677475.612726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.612733:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.612735:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.612736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.612737:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.612739:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.612742:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.612745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.612751:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.612753:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.612754:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.612754:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.612755:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.612756:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.612946:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.612953:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.612956:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.612958:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.612960:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.612960:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.612962:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.612963:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.612964:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.612965:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.612966:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.612969:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.612971:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.612973:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.612974:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.612975:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.612976:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.612977:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.612978:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.612978:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.612982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.612989:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.612991:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.612992:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.612993:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.612995:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.612996:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.613083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.613089:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.613091:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.613093:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.613094:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.613096:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.613098:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.613099:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499074] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.613100:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.613101:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.613102:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.613106:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.613108:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.613110:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.613111:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.613112:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.613113:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.613114:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.613115:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.613115:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.613120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.613127:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.613129:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.613130:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.613132:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.613133:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.613134:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.613201:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677475.613208:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.613211:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677475.613212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677475.613214:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677475.613216:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.613218:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.613219:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.613220:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.613221:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.613222:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.613227:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677475.613229:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.613232:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.613233:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.613233:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.613235:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.613236:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.613236:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677475.613237:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677475.613241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.613249:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.613251:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.613252:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.613253:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.613255:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.613257:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.613259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.613266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.613269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.613269:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.613270:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.613271:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.613271:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.613285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.613293:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.613299:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.613301:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.613303:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.613306:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.613307:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.613308:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.613309:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.613310:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.613311:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.613314:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.613316:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.613319:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.613319:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.613320:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.613321:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.613322:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.613323:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.613323:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.613327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.613336:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.613339:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.613340:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.613340:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.613342:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.613345:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.613478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.613484:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.613487:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.613487:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.613489:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.613492:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.613493:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.613494:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.613495:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.613496:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.613497:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.613500:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.613502:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.613504:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.613505:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.613506:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.613507:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.613508:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.613509:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.613510:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.613510:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.613514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.613515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.613520:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.613522:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.613522:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.613522:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.613523:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.613524:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.613525:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.613527:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.613528:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.613529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.613972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.613980:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.613983:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.613985:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.613987:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.613988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.613989:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.613990:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.613991:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.613992:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.613993:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.613997:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.613999:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.614001:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.614002:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.614003:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.614004:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.614005:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.614006:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.614007:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.614010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.614017:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.614020:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.614021:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.614021:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.614023:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.614025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.614441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.614448:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.614450:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.614451:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.614453:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.614456:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.614457:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.614458:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.614459:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.614460:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.614461:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.614464:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.614466:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.614468:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.614469:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.614470:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.614471:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.614472:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.614472:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.614473:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.614478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.614487:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.614490:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.614490:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.614491:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.614493:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:21.2:1619677475.614494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.614497:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.614499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.614501:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.614502:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.614503:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.614504:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000400:00000200:22.0:1619677475.614505:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.614505:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.614506:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.614506:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.614507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.614508:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.614508:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:22.0:1619677475.614509:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.614511:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.614512:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.614513:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.614514:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.614517:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.614519:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.614521:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.614522:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.614522:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.614524:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.614524:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.614525:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.614525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.614530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.614537:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.614540:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.614541:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.614542:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.614543:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.614545:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.614877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.614883:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.614886:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.614887:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.614888:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.614891:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.614893:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677475.614894:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.614895:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.614896:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.614897:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:2.0:1619677475.614900:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.614902:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:2.0:1619677475.614905:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.614906:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677475.614907:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677475.614908:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:2.0:1619677475.614909:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.614910:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.614910:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:9.2:1619677475.614914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.614921:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.614923:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.614924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.614925:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.614927:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.614930:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677475.615102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.615108:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.615111:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.615112:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.615114:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.615116:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.615117:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.615118:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.615120:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.615121:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.615122:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.615125:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.615127:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.615130:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.615131:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.615131:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.615133:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.615133:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.615134:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.615134:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:20.2:1619677475.615140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:8.2:1619677475.615140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:18.0:1619677475.615146:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.0:1619677475.615148:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:18.0:1619677475.615149:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.615150:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.0:1619677475.615151:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.615152:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.615152:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677475.615153:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.615154:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:18.0:1619677475.615154:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.615156:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:18.0:1619677475.615156:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.615157:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.615158:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.615159:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.615161:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.615164:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.615166:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.615168:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.615169:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.615169:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.615171:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.615171:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.615172:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.615173:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.615177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.615186:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.615188:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.615189:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.615189:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.615191:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.615194:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.615265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.615272:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.615275:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.615276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.615278:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.615281:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.615282:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.615283:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.615285:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.615286:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.615287:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.615290:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.615292:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.615299:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.615300:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.615301:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.615302:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.615303:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.615304:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.615305:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.615306:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.615309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.615312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:6.0:1619677475.615317:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.615320:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:6.0:1619677475.615320:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.615321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.615321:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.615322:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.615323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:6.0:1619677475.615323:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.615324:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:6.0:1619677475.615326:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.615356:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.615361:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.615364:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.615364:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.615366:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.615368:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.615370:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.615371:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.615372:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.615373:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.615374:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.615378:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.615379:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.615382:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.615383:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.615383:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.615385:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.615385:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.615386:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.615386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.615391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.615398:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.615400:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.615401:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.615403:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.615404:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.615405:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.615573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.615581:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.615584:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.615587:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.615589:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.615589:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.615591:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.615592:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.615593:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.615594:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.615595:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.615599:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.615601:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.615603:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.615604:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.615605:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.615606:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.615607:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.615608:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.615610:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.615610:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.615612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.615616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.615619:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.615622:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.615623:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.615623:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.615624:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.615625:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.615626:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.615627:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677475.615627:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.615628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.615698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.615705:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.615708:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.615710:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.615712:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.615712:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.615714:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.615715:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.615717:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.615718:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.615719:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.615722:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.615724:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.615727:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.615728:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.615728:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.615730:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.615731:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.615732:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.615732:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.615736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.615745:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.615747:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.615748:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.615750:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.615750:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.615753:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.615755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.615763:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.615765:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.615766:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.615767:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.615768:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.615769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.616134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.616141:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.616143:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.616144:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.616146:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.616148:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.616150:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677475.616151:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.616153:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.616154:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.616155:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677475.616158:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.616160:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.616162:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.616163:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677475.616164:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.616165:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.616166:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.616167:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677475.616168:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677475.616169:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:25.2:1619677475.616171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.616174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:6.0:1619677475.616179:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677475.616180:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:6.0:1619677475.616181:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:1.0:1619677475.616182:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:22.0:1619677475.616183:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.616183:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.616183:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.616184:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:6.0:1619677475.616185:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.616188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.616199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.616204:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.616207:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.616208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.616210:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.616212:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.616213:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.616214:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.616215:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.616216:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.616217:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.616221:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.616223:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.616225:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.616226:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.616226:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.616228:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.616228:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.616229:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.616229:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.616234:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.616241:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.616243:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.616244:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.2:1619677475.616246:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.616247:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.616248:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:20.0:1619677475.616248:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.616249:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:20.0:1619677475.616250:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:21.0:1619677475.616251:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.616252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.616252:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.616254:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.616255:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.616257:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.616258:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.616259:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.616259:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677475.616260:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.616261:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:21.0:1619677475.616263:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.616263:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.616263:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.616264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.616264:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677475.616266:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.616268:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.616269:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.616269:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.616271:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.616272:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.616273:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.616273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.616278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.616286:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.616288:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.616289:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.616289:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.616291:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.616301:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:5.2:1619677475.616539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.616545:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.616548:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.616549:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.616550:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.616552:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.616554:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.616555:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.616556:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.616557:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.616558:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.616561:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.616563:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.616565:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.616566:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.616566:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.616567:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.616568:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.616569:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.616570:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.616570:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.616574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.616577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.616582:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.616583:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.616583:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677475.616584:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.616584:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.616585:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.616586:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.616586:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.616587:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677475.616587:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677475.616740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677475.616748:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.616751:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677475.616752:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677475.616753:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677475.616756:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.616758:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.616759:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.616761:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.616761:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.616763:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.616766:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677475.616768:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.616771:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.616772:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.616772:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.616774:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.616774:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.616775:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677475.616776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677475.616781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.616789:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.616792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.616792:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.616793:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.616795:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.616798:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.616851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.616859:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.616862:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.616865:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.616867:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.616868:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.616869:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.616870:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.616872:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.616873:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.616874:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.616877:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.616879:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.616881:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.616882:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.616882:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.616884:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.616884:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.616885:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.616885:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.616889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.616896:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.616898:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.616899:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.616901:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.616902:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.616903:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.617242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.617248:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.617251:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.617251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.617253:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.617255:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.617257:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.617258:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.617259:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.617260:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.617261:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.617265:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.617266:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.617269:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.617270:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.617270:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.617272:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.617272:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.617274:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.617275:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.617275:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677475.617277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677475.617278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.617280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.617285:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677475.617285:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.617285:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:10.0:1619677475.617287:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:6.0:1619677475.617287:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:26.0:1619677475.617288:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:20.0:1619677475.617288:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.617288:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.617289:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.617290:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.617290:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.617291:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677475.617291:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.617298:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677475.617298:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.617300:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.617301:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.617303:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.617303:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.617305:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.617308:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.617310:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.617313:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.617314:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.617314:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.617316:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.617316:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.617317:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.617318:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677475.617319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:25.2:1619677475.617323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.617327:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677475.617329:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677475.617330:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677475.617331:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.617331:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:26.0:1619677475.617334:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.617334:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:26.0:1619677475.617335:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.617335:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.617335:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:26.0:1619677475.617337:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.617338:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:26.0:1619677475.617339:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.617340:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.617340:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:25.2:1619677475.617341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677475.617343:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.617344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.617345:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.617345:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.617348:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.617348:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.617349:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.617349:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.617349:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.617350:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.617350:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.617350:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677475.617351:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677475.617352:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677475.617353:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677475.617354:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:25.2:1619677475.617357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.617365:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.617367:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.617368:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.617370:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.617370:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.617372:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.617431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.617439:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.617442:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.617444:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.617446:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.617446:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.617447:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.617449:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.617450:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.617451:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.617452:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.617455:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.617457:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.617459:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.617460:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.617460:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.617462:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.617462:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.617464:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.617465:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.617466:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.617467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.617471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.617476:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.617477:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.617478:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.617479:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.617480:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.617481:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.617481:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.617482:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.617483:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.617484:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.617500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.617508:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.617511:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.617512:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.617514:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.617516:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.617518:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.617519:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.617520:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.617521:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.617522:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.617525:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.617526:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.617529:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.617530:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.617530:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.617532:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.617532:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.617533:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.617534:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.617538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.617546:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.617549:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.617550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.617550:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.617552:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.617555:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.617556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.617564:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.617566:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.617567:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.617567:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.617568:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.617569:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.617746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.617753:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.617756:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.617758:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.617759:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.617760:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.617761:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.617762:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.617763:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.617764:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.617765:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.617768:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.617769:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.617772:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.617773:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.617773:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.617775:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.617775:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.617777:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.617777:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.617791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.617797:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.617800:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.617801:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.617801:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.617802:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.617805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.617863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.617870:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.617873:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.617875:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.617877:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.617877:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.617878:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.617879:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.617881:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.617881:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.617882:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.617886:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.617887:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.617889:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.617889:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.617890:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.617891:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.617892:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.617893:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.617893:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.617897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.617904:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.617906:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.617907:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.617908:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.617910:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.617911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.617913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.617920:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.617923:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.617924:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.617924:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.617925:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.617925:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.618115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.618121:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.618123:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.618124:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.618126:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.618128:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.618129:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.618131:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.618132:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.618133:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.618134:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.618137:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.618138:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.618141:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.618142:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.618142:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.618144:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.618145:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.618145:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.618146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.618150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.618157:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.618160:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.618160:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.618162:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.618163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.618165:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.618251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.618256:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.618260:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.618260:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677475.618262:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.618264:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.618265:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.618267:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.618268:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.618269:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.618270:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.618273:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.618275:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.618278:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.618278:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.618279:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.618281:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.618281:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.618282:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.618282:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.618287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.618295:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.618300:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.618301:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.618302:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:21.0:1619677475.618303:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.618305:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.618309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:4.2:1619677475.618315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:20.0:1619677475.618318:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.618321:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.618322:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.618323:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:10.0:1619677475.618323:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.0:1619677475.618324:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.618325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677475.618326:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.618327:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.618329:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.618331:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.618333:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.618334:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.618336:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.618337:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.618338:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.618341:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.618343:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.618345:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.618346:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.618347:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.618349:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.618349:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.618350:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.618350:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.618355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.618364:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.618368:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.618369:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.618371:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.618374:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.618375:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.618380:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.618381:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.618382:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.618383:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.618385:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.618766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.618773:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.618775:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.618776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.618778:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.618781:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.618782:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.618783:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96499044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.618784:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.618785:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.618786:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.618789:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.618791:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.618794:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.618795:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.618795:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.618797:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.618797:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.618798:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.618798:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677475.618798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:8.2:1619677475.618803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:12.0:1619677475.618807:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:20.2:1619677475.618811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:12.0:1619677475.618811:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:17.0:1619677475.618812:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:20.0:1619677475.618813:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677475.618814:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.618815:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.618815:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.618815:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:20.0:1619677475.618816:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:13.0:1619677475.618816:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:17.0:1619677475.618818:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:12.0:1619677475.618818:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.618819:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.618819:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677475.618820:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:20.0:1619677475.618822:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.618822:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:20.0:1619677475.618823:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:12.0:1619677475.618823:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.618825:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.618826:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.618827:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.618829:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.618829:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.618830:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:12.0:1619677475.618831:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:20.0:1619677475.618835:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.618837:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:12.0:1619677475.618838:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.618840:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.618841:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.618842:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:12.0:1619677475.618842:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:20.0:1619677475.618843:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.618844:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.618845:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.618845:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:12.0:1619677475.618845:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.618847:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:12.0:1619677475.618848:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.2:1619677475.618849:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:12.0:1619677475.618850:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:12.0:1619677475.618851:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.618852:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.618853:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.618855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.0:1619677475.618858:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.618860:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.618861:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.618862:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:21.0:1619677475.618863:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.618863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:26.0:1619677475.618864:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:21.0:1619677475.618865:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:10.0:1619677475.618865:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.618866:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.618868:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.618871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677475.618873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677475.618880:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.618882:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.618883:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.618883:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.618884:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.618884:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.618970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.618978:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.618981:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.618983:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.618983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:30.0:1619677475.618985:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.618986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.618986:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.618987:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.618989:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.618989:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677475.618990:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.618991:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.618992:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.618993:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:30.0:1619677475.618996:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.618996:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.618998:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.618999:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:22.0:1619677475.619000:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:30.0:1619677475.619002:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.619002:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.619003:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.619003:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677475.619003:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.619004:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.619005:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.619005:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677475.619005:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.619006:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.619007:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677475.619009:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677475.619010:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677475.619011:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.619013:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.619014:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.619015:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.619016:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677475.619017:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677475.619017:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.619017:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.619018:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677475.619019:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.619020:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.619021:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:25.2:1619677475.619022:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:24.0:1619677475.619022:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.619025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.0:1619677475.619030:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.619032:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.619033:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.619035:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.619036:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.619037:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.619043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.619051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.619053:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.619054:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.619055:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.619055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.619056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.619115:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.619121:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.619124:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.619125:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.619127:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.619130:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.619131:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.619132:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.619133:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.619135:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.619136:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.619139:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.619141:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.619143:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.619144:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.619144:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.619145:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.619146:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.619147:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.619148:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.619149:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.619151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.619154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.619160:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.619161:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.619162:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.619162:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.619162:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.619162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.619163:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.619164:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.619164:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677475.619166:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.619231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.619238:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.619240:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.619241:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.619243:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.619246:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.619247:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.619248:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.619250:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.619251:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.619252:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.619254:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.619256:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.619259:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.619260:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.619260:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.619262:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.619262:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.619263:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.619264:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.619268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.619276:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.619278:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.619279:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.619280:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.619282:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.619285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.619475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.619483:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.619486:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.619488:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.619491:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.619491:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.619492:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.619493:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.619494:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.619496:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.619496:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.619499:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.619501:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.619503:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.619504:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.619504:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.619506:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.619506:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.619507:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.619508:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.619512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.619520:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.619522:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.619523:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.619523:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.619525:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.619527:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.619532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.619538:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.619541:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.619542:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.619542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.619543:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.619543:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.619916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677475.619923:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.619926:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.619928:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.619930:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.619930:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.619932:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677475.619933:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.619934:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.619935:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.619936:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677475.619939:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.619941:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.619942:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.619943:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.619943:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.619945:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.619946:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.619946:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677475.619947:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:25.2:1619677475.619951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.619958:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.619960:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.619961:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.619962:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.619964:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.619965:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.620005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.620010:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.620013:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.620013:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.620015:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.620018:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.620019:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.620021:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.620022:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.620023:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.620024:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.620027:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.620028:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.620031:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.620031:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.620032:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.620034:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.620034:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.620035:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.620035:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.620040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.620048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.620050:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.620051:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.620053:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.620053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.620055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.620211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.620219:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.620222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.620222:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.620225:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.620227:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.620229:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677475.620230:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.620231:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.620232:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.620234:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.620237:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.620239:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677475.620242:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.620242:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677475.620243:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677475.620245:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677475.620245:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.620246:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.620246:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677475.620251:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.620260:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.620262:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.620264:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.620264:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.620266:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.620269:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677475.620611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.620617:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.620620:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.620621:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.620623:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.620624:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.620626:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.620627:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.620628:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.620629:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.620631:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.620634:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.620636:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.620639:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.620640:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.620640:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.620642:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.620643:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.620644:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.620644:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.620649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.620657:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.620659:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.620660:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.620662:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.620664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.620665:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:5.2:1619677475.620797:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.620803:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.620806:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.620806:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677475.620808:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.620811:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.620812:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.620813:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.620815:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.620816:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.620817:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.620820:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.620822:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.620823:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.620824:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.620825:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.620826:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.620827:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.620828:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.620828:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.620833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.620841:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.620843:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.620843:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.620844:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.620845:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.620847:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.620886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.620892:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.620895:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.620895:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.620897:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.620899:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.620901:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.620902:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.620903:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.620904:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.620905:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.620908:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.620910:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.620913:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.620913:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.620914:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.620916:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.620916:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.620917:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.620918:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.620919:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.620922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.620924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.620929:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.620929:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.620931:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.620931:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.620931:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.620932:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.620932:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.620934:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.620935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.620936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.621532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677475.621535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.621540:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.621542:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677475.621544:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677475.621544:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.621545:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.621547:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.621548:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.621548:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.621549:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.621551:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.621552:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.621553:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.621555:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.621556:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.621557:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.621558:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.621559:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.621559:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.621560:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.621561:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.621564:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.621565:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.621567:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.621567:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.621568:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.621568:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.621569:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.621569:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.621570:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.621571:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.621571:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.621571:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.621573:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.621573:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.621573:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.621574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.621575:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677475.621576:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.621578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:9.2:1619677475.621578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.621585:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:8.0:1619677475.621586:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:31.0:1619677475.621587:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.621587:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.621588:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:31.0:1619677475.621589:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.621589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.621589:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.621591:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.621591:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:8.0:1619677475.621591:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.621593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.621721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.621727:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.621730:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.621730:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.621732:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.621734:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.621735:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.621737:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.621738:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.621739:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.621740:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.621743:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.621745:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.621747:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.621748:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.621748:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.621750:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.621751:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.621752:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.621752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.621756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.621764:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.621767:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.621768:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.621769:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.621770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.621771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.622141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.622149:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.622152:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.622154:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.622156:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.622156:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.622158:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.622159:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.622160:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.622161:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.622162:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.622165:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.622167:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.622169:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.622169:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.622170:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.622172:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.622172:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.622173:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.622175:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.622175:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.622177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.622181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.622186:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.622187:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.622188:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.622189:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.622190:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.622190:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.622191:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.622192:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.622193:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.622194:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.622522:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.622532:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.622536:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.622537:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.622540:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.622545:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.622547:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.622549:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.622551:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.622552:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.622554:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.622559:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.622562:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.622566:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.622567:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.622568:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.622570:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.622571:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.622572:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.622573:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.622577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.622580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:22.0:1619677475.622585:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:31.0:1619677475.622588:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:22.0:1619677475.622588:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.622589:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.622589:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.622591:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:22.0:1619677475.622592:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.622594:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.622594:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677475.622596:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.622596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677475.622597:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.622599:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.622600:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.622601:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.622602:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.622605:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.622606:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.622609:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.622610:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.622610:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.622612:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.622612:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.622614:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677475.622615:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677475.622616:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:9.2:1619677475.622617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.622622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:24.0:1619677475.622625:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.622627:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.622628:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.622629:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:8.0:1619677475.622629:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.622630:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.622631:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:30.0:1619677475.622632:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677475.622633:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:24.0:1619677475.622633:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.622668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.622676:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.622678:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.622681:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.622683:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.622683:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.622684:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.622685:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.622687:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.622688:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.622689:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.622692:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.622693:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.622696:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.622696:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.622697:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.622698:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.622699:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.622700:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.622700:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.622704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.622710:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.622713:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.622714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.622714:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.622715:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.622717:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.622725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.622728:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.622731:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.622733:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.622734:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.622735:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.622737:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.622738:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.622740:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.622741:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.622742:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.622745:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.622747:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.622749:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.622750:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.622751:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.622752:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.622753:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.622754:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.622755:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.622756:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.622759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.622761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.622767:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:1.0:1619677475.622767:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.2:1619677475.622770:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:1.0:1619677475.622770:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.622771:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.622771:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.622772:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.622773:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:1.0:1619677475.622773:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.622774:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:1.0:1619677475.622775:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677475.622776:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.2:1619677475.622779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677475.622779:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.622780:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.622781:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.622783:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.622785:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.622786:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.622787:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:6.0:1619677475.622787:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.622788:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677475.622790:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677475.622790:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677475.622790:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.622791:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.622792:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.622793:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:6.0:1619677475.622793:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.622794:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.622797:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.622797:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.622798:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.622800:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.622800:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.622801:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.622802:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.622806:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.622814:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.622816:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.622817:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.622819:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.622820:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.622821:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.622824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.622832:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.622834:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.622835:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.622836:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.622837:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.622838:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.623122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.623129:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.623133:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.623135:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.623136:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.623137:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.623139:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.623140:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.623141:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.623142:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.623144:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.623147:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.623149:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.623151:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.623152:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.623153:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.623154:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.623155:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.623157:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.623158:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.623159:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.623160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.623164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.623167:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.623169:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.623170:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.623170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.623172:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.623172:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.623174:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.623175:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.623175:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.623176:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.623349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.623357:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.623359:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.623362:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.623364:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.623364:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.623365:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.623367:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.623368:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.623369:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.623370:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.623373:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.623375:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.623376:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.623377:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.623377:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.623379:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.623379:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.623380:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.623380:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.623385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.623392:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.623394:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.623395:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.623396:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.623398:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.623399:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.623625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.623630:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.623633:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.623634:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.623635:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677475.623635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:22.0:1619677475.623638:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.623639:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.623640:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.623642:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.623642:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677475.623643:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.623644:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677475.623645:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.623646:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677475.623648:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.623649:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.623650:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.623652:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:8.0:1619677475.623652:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.623653:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.623654:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677475.623654:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.623655:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.623656:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.623656:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.623656:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:8.0:1619677475.623656:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.623657:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:8.0:1619677475.623657:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.623658:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.623661:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.623662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677475.623663:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.623666:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.623667:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.623667:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.623669:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.0:1619677475.623669:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.623670:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.623671:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:21.0:1619677475.623672:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.623672:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.623672:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.623673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:21.0:1619677475.623674:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.623675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.623675:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.623676:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.623678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.623683:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.623685:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.623685:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.623686:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.623686:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.623687:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.623688:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677475.623688:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.623689:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677475.623691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.623821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.623828:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.623831:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.623832:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.623834:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.623836:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.623838:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:10.0:1619677475.623839:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.623841:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.623842:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.623843:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:10.0:1619677475.623846:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.623848:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677475.623851:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.623851:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:10.0:1619677475.623852:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:10.0:1619677475.623854:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:10.0:1619677475.623854:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.623855:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.623855:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.623860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.623868:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.623871:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.623872:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.623872:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.623874:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.623878:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.624004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.624012:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.624014:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.624015:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.624017:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.624020:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.624021:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.624023:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.624024:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.624025:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.624026:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.624029:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.624030:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.624032:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.624033:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.624034:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.624035:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.624036:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.624037:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.624038:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.624039:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.624041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.624045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.624049:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.624052:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677475.624052:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:22.0:1619677475.624053:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.624053:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.624054:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.624055:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.624055:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.624055:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677475.624057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.624470:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.624477:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.624480:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.624481:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.624482:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.624486:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.624487:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.624488:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.624489:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.624490:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.624491:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.624494:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.624495:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.2:1619677475.624497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:10.0:1619677475.624498:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.624499:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.624499:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.624501:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.624501:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.624502:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.624502:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:10.0:1619677475.624504:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.624504:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677475.624505:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.624506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.624507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677475.624508:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.624510:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.624510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000400:00000200:6.0:1619677475.624512:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.624513:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96499014] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.624515:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.624516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:6.0:1619677475.624516:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677475.624517:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.624517:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.624518:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.624518:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:26.0:1619677475.624519:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.624520:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677475.624520:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.624521:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.624521:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.624523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.624523:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.624525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:6.0:1619677475.624526:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.624527:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.624528:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.624529:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.624530:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.624531:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.624532:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.624533:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.624536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.624537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.624542:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.624543:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.624544:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.624545:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.624545:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.624546:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.624546:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.624548:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.624549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.624550:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.624611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:4.2:1619677475.624614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.0:1619677475.624617:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.624620:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.624622:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.624622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677475.624622:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677475.624624:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.624625:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.624626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:21.0:1619677475.624626:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.624627:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499013] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.624628:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.624629:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.624630:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.624631:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:8.0:1619677475.624631:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.624633:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.624635:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.624635:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96499012] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.624636:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.624638:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:8.0:1619677475.624638:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.624639:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.624640:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.624641:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.624642:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.624643:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:8.0:1619677475.624643:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.624644:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.624645:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.0:1619677475.624645:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.624646:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.0:1619677475.624648:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.624648:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.2:1619677475.624651:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677475.624654:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.624656:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.624657:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.624657:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677475.624658:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677475.624658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:20.0:1619677475.624660:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.624661:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.624662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677475.624663:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.624664:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.624665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:6.0:1619677475.624669:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.624672:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.624672:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.624673:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.624675:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.624678:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.624683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.624690:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.624692:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.624693:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.624694:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.624695:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.624695:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.625149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.625156:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.625158:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.625159:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.625161:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.625163:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.625164:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.625165:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499011] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.625166:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.625168:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.625169:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.625171:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.625173:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677475.625174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:1.0:1619677475.625174:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.625175:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.625176:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.625178:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.625178:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.625179:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.625179:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:31.0:1619677475.625182:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.2:1619677475.625184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677475.625185:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.625187:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.625189:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.625190:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677475.625190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:17.0:1619677475.625191:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:31.0:1619677475.625192:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96499010] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.625193:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.625193:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:31.0:1619677475.625194:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.625194:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.625195:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677475.625195:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.625197:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.625198:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:31.0:1619677475.625199:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.625201:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.625204:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.625205:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.625205:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.625207:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.625208:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.625208:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.625209:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.625213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.625219:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.625222:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.625223:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.625223:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.625225:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.625227:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.625231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.625238:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.625240:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.625241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.625241:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.625242:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.625242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.625516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.625522:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.625525:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.625525:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.625527:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.625529:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.625531:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.625532:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499009] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.625533:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.625534:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.625536:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.625539:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.625540:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.625543:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.625543:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.625544:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.625546:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.625546:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.625547:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.625547:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.625552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.625561:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.625563:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.625564:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.625566:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.625567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.625568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.625968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.625975:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.625977:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.625978:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.625980:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.625983:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.625984:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.625985:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499008] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.625986:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.625987:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.625988:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.625992:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.625994:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.625995:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.625996:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.625997:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.625999:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.625999:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.626000:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.626001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.626005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.626012:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.626014:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.626014:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.626015:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.626016:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.626018:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.626037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.626044:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.626046:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.626047:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.626047:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.626048:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.626048:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.626176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.626184:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.626187:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.626190:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.626192:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.626192:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.626193:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.626195:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499007] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.626196:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.626197:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.626198:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.626201:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.626203:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.626206:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.626206:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.626207:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.626209:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.626209:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.626210:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.626211:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.626214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.626222:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.626224:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.626225:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.626225:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.626227:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.626229:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.626248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.626256:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.626259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.626259:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.626262:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.626264:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.626265:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.626266:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499006] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.626268:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.626269:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.626269:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.626272:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.626274:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.626276:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.626277:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.626277:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.626279:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.626279:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.626280:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.626280:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.626285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.626301:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.626305:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.626306:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.626308:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.626312:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.626313:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.626418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.626425:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.626427:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.626428:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.626430:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.626433:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.626434:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.626435:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499005] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.626436:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.626437:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.626438:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.626441:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.626443:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.626445:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.626446:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.626447:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.626448:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.626449:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.626450:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.626450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.626455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.626463:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.626465:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.626466:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.626468:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.626469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.626470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.626475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.626482:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.626484:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.626485:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.626486:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.626487:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.626488:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.626788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.626796:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.626799:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.626801:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.626803:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.626803:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.626805:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.626806:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499004] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.626807:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.626808:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.626809:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.626813:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.626814:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.626816:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.626817:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.626818:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.626819:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.626820:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.626821:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.626821:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.626825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.626831:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.626834:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.626835:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.626836:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.626838:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.626839:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.626845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.626852:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.626855:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.626856:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.626856:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.626857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.626858:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.627306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.627312:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.627315:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.627316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.627318:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.627320:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.627321:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.627323:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499003] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.627324:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.627325:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.627326:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.627329:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.627331:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.627333:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.627334:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.627334:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.627336:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.627336:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.627337:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.627337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.627343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.627351:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.627353:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.627354:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.627356:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.627357:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.627358:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.627385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.627394:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.627396:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.627399:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.627401:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.627401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.627402:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.627404:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499002] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.627405:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.627406:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.627407:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.627411:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.627412:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.627416:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.627417:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.627417:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.627419:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.627420:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.627421:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.627423:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.627423:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.627426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.627429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.627434:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677475.627437:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.0:1619677475.627437:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.627438:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.627439:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:20.0:1619677475.627439:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.627440:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.627440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.627441:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.627443:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.627886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.627894:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.627897:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.627899:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.627901:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.627902:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.627903:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.627905:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96499001] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.627906:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.627907:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.627907:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.627910:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.627912:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.627914:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.627915:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.627915:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.627917:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.627917:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.627918:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.627919:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.627922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.627929:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.627931:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.627932:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.627933:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.627934:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.627936:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.627978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.627984:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.627987:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.627988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.627990:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.627992:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.627993:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.627995:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96499000] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.627996:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.627997:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.627998:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.628001:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.628003:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.628005:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.628006:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.628007:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.628008:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:1.0:1619677475.628008:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.628009:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.628009:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.628010:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:2.0:1619677475.628013:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677475.628014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677475.628016:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.628017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.628019:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.628021:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.628022:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:2.0:1619677475.628022:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.628024:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498999] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.628025:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.628025:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.628025:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.628026:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.628027:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:21.0:1619677475.628028:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.628028:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.628031:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:2.0:1619677475.628031:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.628033:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.628036:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.628037:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.628038:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.628039:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.628040:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.628041:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.628042:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.628043:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.628045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677475.628048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:13.0:1619677475.628053:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:18.0:1619677475.628054:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.628056:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:13.0:1619677475.628056:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.628056:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.628057:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.628058:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:13.0:1619677475.628058:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.628060:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.628061:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.628109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.628117:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.628120:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.628122:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.628124:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.628124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.628126:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.628127:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498998] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.628128:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.628130:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.628130:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.628134:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.628136:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.628138:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.628139:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.628140:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.628141:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.628142:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.628143:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.628143:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.628147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.628153:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.628155:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.628156:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.628157:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.628158:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.628160:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.628312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.628318:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.628320:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.628321:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.628323:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.628325:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.628327:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.628328:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498997] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.628329:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.628330:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.628331:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.628334:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.628335:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.628338:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.628339:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.628339:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.628341:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.628341:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.628342:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.628343:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.628344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.628347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.628349:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.628354:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.628355:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.628356:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.628357:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.628357:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.628358:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.628358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.628360:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.628361:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.628362:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.628455:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.628464:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.628466:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.628468:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.628471:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.628471:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.628472:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.628473:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498996] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.628475:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.628476:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.628476:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.628480:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.628481:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.628484:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.628485:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.628485:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.628487:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.628487:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.628488:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.628489:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.628492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.628499:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.628502:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.628503:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.628503:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.628505:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.628507:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.628512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.628518:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.628520:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.628521:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.628521:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.628522:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.628522:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.628529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.628534:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.628537:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.628539:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.628539:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.628541:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.628543:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.628544:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498995] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.628546:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.628547:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.628548:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.628551:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.628552:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.628554:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.628555:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.628556:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.628558:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.628558:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.628559:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.628560:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.628564:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.628572:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.628575:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.628576:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.628576:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.628579:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.628581:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.628703:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.628710:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.628713:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.628714:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.628716:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.628718:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.628720:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.628721:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498994] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.628722:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.628723:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.628724:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.628727:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.628729:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.628732:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.628732:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.628733:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.628735:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.628735:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.628736:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.628736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.628741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.628748:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.628751:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.628752:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.628752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.628754:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.628757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.628889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.628897:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.628900:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.628901:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.628903:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.628906:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.628907:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.628908:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498993] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.628909:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.628910:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.628911:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.628915:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.628916:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.628919:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.628920:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.628921:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.628922:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.628923:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.628923:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.628924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.628928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.628935:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.628938:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.628939:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.628939:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.628942:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.628944:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.628968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:10.0:1619677475.628975:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677475.628977:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677475.628978:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:10.0:1619677475.628980:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.628983:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.628984:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.628986:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498992] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.628987:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.628988:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.628988:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.628992:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.628993:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.628996:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.628997:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.628997:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.628999:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.629000:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677475.629001:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:10.0:1619677475.629002:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (23)--
00000800:00000200:10.0:1619677475.629002:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:8.2:1619677475.629006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.629009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:20.0:1619677475.629015:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677475.629016:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)++
00000800:00000200:20.0:1619677475.629017:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677475.629018:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75ca00] (23)--
00000400:00000200:20.0:1619677475.629018:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.629019:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:20.0:1619677475.629019:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.629020:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:21.0:1619677475.629020:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.629022:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.629028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.629035:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.629038:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.629039:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.629040:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.629043:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.629044:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.629046:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498991] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.629047:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.629048:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.629048:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.629051:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.629053:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.629055:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.629056:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.629056:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.629058:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.629058:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.629060:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.629061:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.629061:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.629064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.629067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:22.0:1619677475.629072:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.629074:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:22.0:1619677475.629074:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.629075:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.629075:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.629076:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.629077:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:22.0:1619677475.629077:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.629078:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677475.629080:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.629179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.629184:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.629187:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.629188:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.629190:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.629192:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.629193:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.629194:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498990] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.629195:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.629196:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.629198:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.629201:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.629202:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.629205:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.629205:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.629206:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.629208:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.629208:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.629209:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.629209:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.629214:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.629221:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.629223:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.629224:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.629226:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.629227:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.629228:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.629291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.629300:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.629302:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.629303:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.629305:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.629308:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.629309:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.629311:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498989] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.629312:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.629313:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.629314:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.629317:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.629319:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.629322:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.629323:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.629323:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.629325:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.629326:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.629326:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.629327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.629332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.629340:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.629343:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.629344:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.629344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.629346:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.629348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.629391:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:24.0:1619677475.629398:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.629401:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:8.0:1619677475.629402:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:24.0:1619677475.629404:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.629407:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.629409:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:24.0:1619677475.629410:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498988] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.629411:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.629412:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.629413:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:24.0:1619677475.629416:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.629418:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:24.0:1619677475.629420:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.629421:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677475.629422:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677475.629423:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:24.0:1619677475.629424:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.629425:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fee39d800] (22)++
00000800:00000200:24.0:1619677475.629426:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (23)--
00000800:00000200:24.0:1619677475.629427:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.2:1619677475.629430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:4.2:1619677475.629432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (21)++
00000800:00000200:5.2:1619677475.629436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:8.0:1619677475.629439:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (22)++
00000800:00000200:8.0:1619677475.629441:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fee39d800] (23)--
00000800:00000200:1.0:1619677475.629441:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:8.0:1619677475.629442:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677475.629443:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:1.0:1619677475.629443:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.629444:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:1.0:1619677475.629444:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.629445:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677475.629446:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.629448:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.629449:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.629450:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.629450:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677475.629451:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.629452:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.629453:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498987] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.629455:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.629456:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.629457:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.629460:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.629462:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.629464:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.629464:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.629465:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.629466:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.629467:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.629467:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.629468:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.629472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.629480:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.629481:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.629482:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.629483:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.629483:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.629485:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.629490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.629497:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.629500:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.629500:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.629500:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.629501:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.629501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.629510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.629519:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.629522:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.629524:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.629526:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.629526:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.629528:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.629529:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498986] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.629530:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.629531:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.629532:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.629535:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.629537:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.629540:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.629540:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.629541:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.629542:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.629543:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.629544:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.629545:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.629549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.629555:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.629557:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.629558:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.629558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.629560:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.629562:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.629597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.629605:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.629608:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.629610:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.629612:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.629612:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.629613:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.629614:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498985] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.629615:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.629616:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.629617:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.629620:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.629622:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.629624:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.629625:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.629625:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.629627:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.629627:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.629629:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.629630:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.629631:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.629632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.629636:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.629639:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.629641:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.629642:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.629642:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.629644:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.629645:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.0:1619677475.629646:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.629647:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.629648:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.629649:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.629693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.629700:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.629702:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.629704:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.629705:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.629707:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.629709:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.629710:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498984] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.629711:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.629712:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.629713:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.629716:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.629717:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.629720:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.629721:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.629721:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.629723:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.629723:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.629724:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.629724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.629729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.629737:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.629740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.629740:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.629741:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.629743:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.629746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.629747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.629754:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.629756:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.629757:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.629758:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.629758:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.629759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.630032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.630037:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.630040:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.630040:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.630042:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.630044:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.630045:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.630046:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498983] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.630048:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.630049:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.630050:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.630053:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.630055:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.630058:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.630058:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.630059:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.630061:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.630061:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.630062:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.630062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.630067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.630074:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.630076:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.630077:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.630079:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.630080:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.630081:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.630087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.630094:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.630097:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.630098:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.630098:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.630099:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.630100:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.630106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.630114:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.630117:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.630118:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.630120:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.630123:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.630124:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.630126:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498982] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.630127:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.630128:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.630129:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.630132:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.630134:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.630137:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.630137:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.630138:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.630140:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.630140:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.630141:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.630141:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.630145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.630153:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.630156:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.630157:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.630157:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.630159:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.630162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.630205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.630210:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.630213:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.630215:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.630215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.630218:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.630219:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.630220:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498981] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.630222:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.630223:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.630224:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.630227:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.630228:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.630231:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.630231:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.630232:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.630233:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.630234:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.630235:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.630235:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.630239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.630247:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.630249:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.630250:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.630252:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.630253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.630254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.630262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.630270:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.630273:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.630274:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.630274:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.630275:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.630276:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.630368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.630374:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.630376:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.630378:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.630379:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.630380:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.630381:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.630382:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498980] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.630384:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.630384:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.630385:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.630388:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.630389:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.630391:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.630392:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.630392:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.630394:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.630394:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.630395:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.630396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.630400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.630409:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.630411:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.630412:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.630413:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.630414:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.630417:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.631034:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.631039:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.631042:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.631043:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.631044:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.631047:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.631048:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.631049:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498979] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.631050:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.631051:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.631052:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.631055:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.631057:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.631059:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.631060:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.631060:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.631062:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.631063:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.631063:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.631064:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.631068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.631075:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.631078:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.631078:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.631080:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.631081:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.631082:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.631557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.2:1619677475.631563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:1.0:1619677475.631563:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.631565:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.631566:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.631568:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.631570:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.631571:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:1.0:1619677475.631571:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.631572:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498978] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.631574:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:1.0:1619677475.631574:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.631575:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.631576:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677475.631578:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.631579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.631580:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.631580:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.631582:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:1.0:1619677475.631583:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677475.631584:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498977] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.631585:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.631585:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.631585:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:31.0:1619677475.631586:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.631586:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:31.0:1619677475.631587:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:1.0:1619677475.631587:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.631588:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.631589:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.631590:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.631590:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677475.631591:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.631593:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.2:1619677475.631594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677475.631595:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.631596:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.631597:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:5.2:1619677475.631597:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:31.0:1619677475.631598:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.631599:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.631600:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.631600:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:1.0:1619677475.631603:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.631604:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:9.2:1619677475.631604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.631604:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.631605:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.631605:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.631605:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.631606:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:17.0:1619677475.631607:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.631607:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.631609:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.0:1619677475.631611:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.631613:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.631614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.631615:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.631617:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.631619:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.631624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.631630:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.631632:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.631633:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.631633:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.631634:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.631635:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.631885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.631892:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.631895:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.631897:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.631899:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.631900:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.631902:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.631903:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498976] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.631904:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.631905:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.631906:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.631910:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.631911:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.631915:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.631915:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.631916:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.631918:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.631918:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.631920:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.631921:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.631922:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.631924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.631928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.631936:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.631936:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677475.631939:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677475.631939:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677475.631940:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.631940:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.631941:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677475.631942:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.631943:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.631945:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.631950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.631956:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.631959:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.631960:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.631962:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.631964:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.631965:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.631967:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498975] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.631968:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.631969:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.631970:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.631973:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.631974:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.631977:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.631978:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.631979:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.631980:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.631981:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.631982:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.631983:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.631983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.631986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.631988:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.631993:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.631994:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.631995:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.631995:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.631996:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.631997:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.631997:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.631999:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.632000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.632001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.632511:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.632520:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.632523:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.632525:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.632527:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.632528:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.632530:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.632531:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498974] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.632532:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.632533:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.632534:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.632538:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.632540:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.632542:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.632543:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.632543:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.632545:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.632546:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.632547:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.632547:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.632551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.632558:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.632561:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.632561:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.632563:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.632565:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.632566:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.632803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.632809:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.632812:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.632813:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.632815:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.632817:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.632818:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.632819:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498973] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.632821:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.632822:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.632823:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.632825:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.632827:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.632829:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.632830:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.632831:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.632833:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.632833:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.632834:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.632834:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.632839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677475.632842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:21.0:1619677475.632847:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.632849:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677475.632849:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:21.0:1619677475.632850:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.632851:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.632852:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:10.0:1619677475.632852:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.632853:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:21.0:1619677475.632854:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677475.632855:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.632858:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.632859:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.632861:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498972] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.632862:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.632863:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.632864:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.632867:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.632868:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.632871:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.632872:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.632873:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.632874:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.632875:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.632876:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.632877:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.632878:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677475.632881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.632883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677475.632890:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677475.632891:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677475.632892:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:26.0:1619677475.632893:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:20.0:1619677475.632893:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.632894:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.632895:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:20.0:1619677475.632895:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.632896:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.632898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.632900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.632908:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.632911:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.632912:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.632912:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.632913:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.632914:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.632984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.632991:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.632994:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.632996:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.632997:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.632998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.632999:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.633000:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498971] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.633001:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.633002:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.633003:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.633006:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.633008:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.633010:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.633011:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.633011:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.633013:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.633014:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.633015:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.633015:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.633019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.633025:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.633028:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.633029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.633029:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.633031:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.633033:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.633223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.633231:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.633235:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.633237:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.633240:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.633244:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.633247:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.633249:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498970] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.633251:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.633252:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.633254:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.633259:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.633262:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.633265:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.633266:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.633268:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.633270:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.633271:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.633272:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.633273:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.633276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.633285:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.633287:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.633289:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.633289:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.633291:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.633298:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.633367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.633374:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.633378:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.633380:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.633382:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.633382:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.633384:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.633385:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498969] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.633386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000400:00000200:31.0:1619677475.633387:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.633387:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.633389:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.633392:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.633392:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:31.0:1619677475.633394:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.633395:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.633396:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:31.0:1619677475.633398:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.633398:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.633399:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677475.633399:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.633401:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:22.0:1619677475.633401:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.633402:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.633403:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.633403:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677475.633403:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:22.0:1619677475.633404:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498968] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.633406:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.633407:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:5.2:1619677475.633407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000400:00000200:22.0:1619677475.633408:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:8.2:1619677475.633408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677475.633411:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.633413:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.633414:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:22.0:1619677475.633415:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.633416:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.633416:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:22.0:1619677475.633417:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677475.633417:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:1.0:1619677475.633417:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:22.0:1619677475.633418:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:22.0:1619677475.633419:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:21.0:1619677475.633419:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:17.0:1619677475.633419:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.633420:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.633420:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:21.0:1619677475.633420:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.633422:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.633422:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:17.0:1619677475.633422:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.633423:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:8.2:1619677475.633424:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.0:1619677475.633425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:17.0:1619677475.633425:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498967] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.633426:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.633428:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.633429:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677475.633434:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.633434:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.633435:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.633436:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:1.0:1619677475.633436:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.633438:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.633439:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.633440:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677475.633441:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677475.633441:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.633442:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:17.0:1619677475.633444:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677475.633445:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677475.633446:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfdf63800] (22)++
00000800:00000200:17.0:1619677475.633447:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (23)--
00000800:00000200:17.0:1619677475.633447:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000800:00000200:9.2:1619677475.633450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:5.2:1619677475.633454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (21)++
00000800:00000200:24.0:1619677475.633457:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.633459:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677475.633459:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)++
00000400:00000200:24.0:1619677475.633461:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.633461:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677475.633461:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfdf63800] (23)--
00000800:00000200:1.0:1619677475.633462:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:24.0:1619677475.633463:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.633463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:24.0:1619677475.633466:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.633821:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.633827:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.633829:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.633830:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.633832:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.633834:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.633836:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.633837:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498966] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.633838:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.633839:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.633840:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:4.2:1619677475.633842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:6.0:1619677475.633843:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.633845:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.633847:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.633848:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.0:1619677475.633849:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.633849:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.633850:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.633851:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.633851:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677475.633852:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:6.0:1619677475.633852:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:24.0:1619677475.633853:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.633854:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.633856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677475.633857:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.633859:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.633860:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498965] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.633861:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.633862:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.633863:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.633864:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.633866:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.633867:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.633867:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677475.633869:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.633869:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.633870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.633871:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677475.633872:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.633872:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.633873:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:25.2:1619677475.633875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.0:1619677475.633875:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.633876:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.633877:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.633878:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.633879:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677475.633882:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.633885:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:4.2:1619677475.633885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677475.633886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.633886:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.633886:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.633887:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.633888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677475.633893:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:22.0:1619677475.633893:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.633895:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.633896:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:22.0:1619677475.633896:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677475.633897:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.633897:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.633897:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.633899:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.633902:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.634003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d7a00] (20)++
00000800:00000200:6.0:1619677475.634009:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d7a00] (21)++
00000800:00000200:22.0:1619677475.634012:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d7a00] (22)--
00000800:00000200:6.0:1619677475.634012:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d2[1] from 10.51.2.13@o2ib3
00000400:00000200:6.0:1619677475.634015:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.13@o2ib3 : REPLY - routed
00000800:00000200:6.0:1619677475.634017:0:19937:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 137602
00000800:00000200:6.0:1619677475.634019:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91986f3d7a00] (21)++
00000800:00000200:6.0:1619677475.634021:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d7a00] (22)++
00000800:00000200:6.0:1619677475.634021:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d7a00] (23)--
00000800:00000200:6.0:1619677475.634022:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d7a00] (22)--
00000800:00000200:4.2:1619677475.634026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d7a00] (21)++
00000800:00000200:4.2:1619677475.634030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:22.0:1619677475.634031:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d7a00] (22)++
00000800:00000200:22.0:1619677475.634033:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d7a00] (23)--
00000800:00000200:22.0:1619677475.634034:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d7a00] (22)--
00000800:00000200:30.0:1619677475.634038:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.634041:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677475.634043:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.634045:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677475.634046:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.634048:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.634049:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498964] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.634050:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677475.634050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91986f3d7a00] (21)++
00000400:00000200:30.0:1619677475.634051:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.634052:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.634055:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.634055:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91986f3d7a00] (22)++
00000800:00000200:30.0:1619677475.634057:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.634057:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d5[0] from 10.51.2.13@o2ib3
00000800:00000200:6.0:1619677475.634058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d7a00] (23)--
00000800:00000200:22.0:1619677475.634059:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91986f3d7a00] (23)--
00000800:00000200:30.0:1619677475.634060:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.634060:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.634061:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.634062:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677475.634062:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:22.0:1619677475.634063:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498963] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.634064:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.634064:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000400:00000200:22.0:1619677475.634064:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.634065:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000400:00000200:22.0:1619677475.634065:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.634066:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000400:00000200:22.0:1619677475.634066:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:22.0:1619677475.634070:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : REPLY try# 0
00000800:00000200:8.2:1619677475.634070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677475.634071:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 137602 bytes in 256 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.634073:0:19936:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 256 offset 0 nob 137602
00000800:00000200:22.0:1619677475.634077:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.634078:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)++
00000800:00000200:22.0:1619677475.634078:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (23)++
00000800:00000200:21.0:1619677475.634078:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (23)++
00000800:00000200:22.0:1619677475.634080:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (25)--
00000800:00000200:22.0:1619677475.634081:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91986f3d7a00] (21)++
00000800:00000200:21.0:1619677475.634081:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (25)--
00000800:00000200:22.0:1619677475.634082:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91986f3d7a00] (22)--
00000800:00000200:22.0:1619677475.634082:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91986f3d7a00] (21)--
00000400:00000200:21.0:1619677475.634082:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.634083:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.634084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.634086:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677475.634088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.634096:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.634098:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.634099:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677475.634109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.634116:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.634119:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[2] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.634121:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:21.0:1619677475.634122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (24)--
00000800:00000200:20.0:1619677475.634126:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.634126:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:4.2:1619677475.634145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:8.2:1619677475.634147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.2:1619677475.634152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:10.0:1619677475.634153:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677475.634155:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677475.634156:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.634157:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.0:1619677475.634158:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.634159:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.634159:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:10.0:1619677475.634159:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.634161:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: REPLY: OK
00000800:00000200:20.0:1619677475.634161:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:18.0:1619677475.634162:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:10.0:1619677475.634162:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.634162:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:21.0:1619677475.634163:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:10.0:1619677475.634163:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.634164:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498962] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.634166:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.634167:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.634168:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.634169:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.634169:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677475.634170:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.634171:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498961] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.634173:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.634173:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.634174:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.634175:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:10.0:1619677475.634175:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.634178:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677475.634179:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.634179:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.634180:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:18.0:1619677475.634181:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677475.634181:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.634182:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.634182:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:18.0:1619677475.634183:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.634183:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:18.0:1619677475.634184:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.634184:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.634186:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.634186:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.634187:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.634188:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.634192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:25.2:1619677475.634198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677475.634199:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.634201:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.634202:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.634202:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.634204:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.634205:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:2.0:1619677475.634205:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:6.0:1619677475.634208:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.634209:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.634209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.634211:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.634214:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.634225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.634232:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.634235:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.634236:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.634236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.634237:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.634238:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.634242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.634249:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.634252:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.634253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.634255:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.634257:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.634259:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.634260:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498960] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.634261:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.634262:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.634263:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.634267:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.634269:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.634272:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.634272:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.634273:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.634275:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.634275:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.634276:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.634276:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.634281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.634288:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.634291:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.634291:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.634292:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.634298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:6.0:1619677475.634300:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.634304:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.634305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.634307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.634307:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.634308:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.634309:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.634309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.634656:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.634665:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.2:1619677475.634665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677475.634667:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.0:1619677475.634669:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677475.634670:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.634671:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:30.0:1619677475.634672:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.634673:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.634673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:20.0:1619677475.634673:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.634674:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.634676:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498959] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.634676:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.634677:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.634677:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677475.634678:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.634678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.634679:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498958] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.634680:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.634681:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:20.0:1619677475.634681:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.634682:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.634683:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:20.0:1619677475.634684:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677475.634686:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.634686:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.634688:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:20.0:1619677475.634689:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.634690:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.634691:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.634691:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677475.634691:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677475.634692:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.634692:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:30.0:1619677475.634693:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96498957] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.634694:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:20.0:1619677475.634694:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.634695:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.634695:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.634695:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:30.0:1619677475.634696:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.634697:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677475.634697:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.634697:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.634698:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677475.634699:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.634699:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.634699:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000400:00000200:30.0:1619677475.634700:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.634700:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:9.2:1619677475.634700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.634701:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:20.0:1619677475.634701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677475.634702:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.634704:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677475.634704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.634705:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.634706:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.2:1619677475.634706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:8.2:1619677475.634706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.634707:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.634708:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.634709:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.634710:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.634710:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.634712:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:8.0:1619677475.634712:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:22.0:1619677475.634713:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:21.0:1619677475.634714:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:6.0:1619677475.634714:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.634715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.634715:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.634715:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:8.0:1619677475.634715:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.634716:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:22.0:1619677475.634716:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.634716:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.634718:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.634718:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:8.0:1619677475.634718:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.634719:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.634719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:22.0:1619677475.634720:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:1.0:1619677475.634720:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:22.0:1619677475.634721:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96498956] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.634722:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.634722:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677475.634723:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.634724:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.634724:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:22.0:1619677475.634725:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677475.634725:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677475.634726:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.634726:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.634726:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.634730:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.634730:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.634732:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.0:1619677475.634732:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677475.634735:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.634736:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.634737:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.634738:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.634739:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.634739:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.634740:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.634744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.634752:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.634754:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.634755:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.634756:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.634757:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.634759:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.634765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.634772:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.634775:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.634776:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.634778:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.634781:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.634782:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.634784:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498955] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.634785:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.634786:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.634787:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.634791:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.634793:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.634796:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.634796:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.634797:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.634799:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.634799:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.634800:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.634800:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.634805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.634812:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.634815:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.634816:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.634816:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.634819:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.634822:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.634823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.634830:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.634832:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.634833:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.634833:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.634834:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.634834:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.635166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.635170:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.635172:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.635174:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.635175:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.635177:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.635179:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.635180:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498954] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.635181:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.635182:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.635183:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.635187:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.635189:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.635191:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.635192:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.635193:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.635195:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.635195:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.635196:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.635198:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.635198:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.635200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.635203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677475.635207:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677475.635209:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:24.0:1619677475.635210:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.635210:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677475.635211:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:8.0:1619677475.635211:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.635212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.635213:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677475.635213:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.635216:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.635571:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.635577:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.635580:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.635581:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.635583:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.635585:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.635586:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.635587:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498953] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.635589:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.635590:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.635590:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.635593:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.635595:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.635597:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.635598:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.635599:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.635600:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.635601:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.635602:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.635603:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.635603:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.635607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.635609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.635614:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.635616:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.635616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.635616:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.635617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.635619:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.635619:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.635621:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.635622:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.635623:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.635737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.635744:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.635747:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.635748:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.635750:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.635753:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.635754:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.635755:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498952] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.635757:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.635758:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.635759:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.635762:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.635764:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.635767:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.635768:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.635768:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.635770:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.635770:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.635771:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.635771:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.635786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.635793:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.635796:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.635797:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.635797:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.635799:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.635802:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.635817:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.635825:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.635828:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.635830:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.635832:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.635832:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.635834:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:12.0:1619677475.635835:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498951] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.635836:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.635837:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.635838:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:12.0:1619677475.635841:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.635842:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.635845:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.635845:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677475.635846:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.635847:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.635848:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.635849:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.635849:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677475.635853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.635859:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.635862:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.635862:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.635864:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.635865:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.635867:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:20.2:1619677475.636196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.636199:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.636201:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.636203:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.636204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.636206:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.636208:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.636209:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498950] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.636211:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.636211:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.636213:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.636216:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.636217:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.636220:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.636221:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.636221:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.636223:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.636223:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.636224:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.636225:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.636230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.636239:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.636241:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.636242:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.636243:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.636245:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:21.2:1619677475.636247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:1.0:1619677475.636247:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:31.0:1619677475.636255:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.636258:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.636261:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.636263:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.636263:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.636265:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.636266:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498949] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.636267:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.636268:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.636269:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.636272:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.636274:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.636276:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.636277:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.636278:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.636279:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.636280:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.636281:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.636283:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.636283:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.636285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.636289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.636291:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.636296:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677475.636297:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.636300:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677475.636300:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.636301:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:24.0:1619677475.636301:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.636302:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:24.0:1619677475.636302:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:9.2:1619677475.636304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.636307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.636308:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.636310:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.636311:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.636311:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.636312:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.636312:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677475.636343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.636349:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.636351:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.636352:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.636354:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.636357:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.636358:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.636359:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498948] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.636360:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.636362:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.636363:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.636365:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.636367:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.636368:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.636369:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.636370:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.636372:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.636372:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.636373:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.636373:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.636378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.636385:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.636387:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.636388:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.636389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.636389:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.636391:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.636394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.636402:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.636404:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.636404:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.636405:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.636405:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.636406:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.636576:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.636582:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.636585:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.636586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.636587:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.636590:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.636591:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.636592:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498947] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.636594:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.636594:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.636595:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.636598:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.636600:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.636602:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.636603:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.636604:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.636605:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.636606:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.636606:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.636607:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.636612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.636619:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.636621:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.636622:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.636624:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.636625:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.636626:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.636805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.636812:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.636815:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.636817:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.636819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.636819:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.636821:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.636822:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498946] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.636823:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.636824:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.636825:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.636829:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.636831:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.636832:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.636833:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.636834:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.636835:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.636836:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.636837:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.636839:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.636839:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.636841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.636845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.636849:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.2:1619677475.636850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:31.0:1619677475.636851:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.636852:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.636853:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.636853:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.636854:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.636855:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.636855:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.636856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:1.0:1619677475.636856:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:31.0:1619677475.636857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:1.0:1619677475.636858:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.636859:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.636861:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.636863:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.636864:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677475.636866:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498945] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.636867:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.636868:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.636869:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677475.636872:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.636873:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.636876:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.636877:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.636877:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.636879:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.636879:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.636880:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.636880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677475.636885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.636892:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.636895:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.636896:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.636896:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.636898:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.636900:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.637299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.637306:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.637310:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.637310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.637313:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.637315:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.637317:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.637318:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498944] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.637320:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.637321:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.637322:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.637325:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.637326:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.637329:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.637330:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.637331:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.637332:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.637333:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.637334:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.637334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677475.637339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.637348:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.637350:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.637351:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.637353:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.637354:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.637356:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.637375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.637380:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.637383:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.637385:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.637385:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.637387:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.637389:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.637390:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498943] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.637392:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.637392:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.637394:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.637397:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.637398:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.637400:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.637401:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.637402:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.637404:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.637404:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.637405:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.637405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.637409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.637416:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.637418:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.637419:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.637421:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.637422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.637423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.637437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.637443:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.637445:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.637446:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.637448:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.637451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.637452:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.637453:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498942] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.637455:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.637456:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.637457:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.637460:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.637461:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.637464:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.637465:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.637465:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.637467:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.637467:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.637468:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.637468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.637474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.637481:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.637483:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.637484:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.637486:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.637487:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.637488:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.637495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.637502:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.637505:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.637506:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.637506:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.637507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.637508:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.637600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.637608:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.637610:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.637613:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.637615:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.637615:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.637617:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.637618:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498941] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.637619:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.637620:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.637621:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677475.637623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:30.0:1619677475.637625:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.637626:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.637629:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.637630:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.637630:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.637631:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.637632:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.637633:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677475.637633:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.637634:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.637634:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:31.0:1619677475.637635:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.637636:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.637637:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.637637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:31.0:1619677475.637638:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.637639:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498940] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.637640:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.637641:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.637642:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:8.0:1619677475.637644:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:31.0:1619677475.637645:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.637647:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:8.0:1619677475.637647:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.637648:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.637648:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.637649:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.637649:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.637650:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:8.0:1619677475.637650:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.637651:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.637652:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.637652:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.637653:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.637653:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.637657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.637663:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.637666:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.637666:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.637668:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.637670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.637670:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677475.637941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.637945:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.637947:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:21.0:1619677475.637950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.637950:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.637952:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.637953:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.637955:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498939] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.637956:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.637957:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.637958:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.637962:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.637963:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.637966:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.637966:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.637967:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.637968:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.637969:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.637970:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.637971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.637975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.637982:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.637985:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.637986:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.637986:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.637988:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.637990:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.638083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.638091:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.638094:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.638095:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.638097:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.638100:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.638101:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677475.638103:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498938] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.638104:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.638105:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.638106:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677475.638109:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.638111:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677475.638113:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.638113:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677475.638114:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.638115:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.638116:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.638116:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.638117:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.638131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.638137:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.638139:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.638140:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.638141:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.638141:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.638143:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.638299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.638304:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.638307:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.638308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.638310:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.638312:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.638313:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.638315:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498937] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.638316:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.638317:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.638318:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.638321:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.638323:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.638325:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.638326:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.638326:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.638328:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.638328:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.638329:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.638329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.638335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.638343:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.638345:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.638346:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.638348:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.638349:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.638350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.638526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.638534:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.638537:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.638539:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.638541:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.638542:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.638542:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.638543:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498936] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.638545:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.638546:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.638547:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.638550:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.638552:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.638554:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.638555:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.638555:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.638557:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.638558:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.638559:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.638559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.638563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.638570:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.638572:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.638573:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.638573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.638575:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.638577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.638581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.638587:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.638589:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.638590:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.638590:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.638591:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.638592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.639221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.639228:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.639231:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.639232:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.639233:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.639235:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.639236:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.639237:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498935] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.639239:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.639240:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.639240:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.639244:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.639245:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.639248:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.639249:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.639249:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.639251:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.639252:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.639252:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.639253:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.639258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.639266:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.639268:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.639269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.639270:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.639272:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.639274:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.639275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.639282:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.639285:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.639285:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.639286:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.639287:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.639287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.639304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.639312:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.639314:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.639315:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.639317:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.639320:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.639321:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.639323:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498934] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.639324:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.639325:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.639326:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.639328:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.639330:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.639330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:24.0:1619677475.639333:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677475.639333:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:24.0:1619677475.639334:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.639334:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.639336:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.639336:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677475.639337:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.639337:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.639338:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:6.0:1619677475.639339:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.639340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677475.639340:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677475.639342:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.639343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677475.639343:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.639343:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677475.639344:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.639346:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:6.0:1619677475.639346:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.639347:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498933] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.639348:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.639349:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:6.0:1619677475.639349:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.639351:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:18.0:1619677475.639351:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498932] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.639351:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.639352:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.639352:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.639353:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.639354:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.639354:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.639355:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:22.0:1619677475.639356:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.639357:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.639360:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.639360:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.639361:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:22.0:1619677475.639362:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:18.0:1619677475.639363:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:6.0:1619677475.639363:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.639364:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.639364:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.639366:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.639366:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677475.639367:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:6.0:1619677475.639367:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:18.0:1619677475.639368:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:6.0:1619677475.639368:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:18.0:1619677475.639369:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:6.0:1619677475.639369:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:18.0:1619677475.639370:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677475.639370:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677475.639371:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.639372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.639372:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:21.2:1619677475.639375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677475.639376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677475.639379:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.639380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.639381:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.639382:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677475.639382:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.639383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.639383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677475.639384:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:20.0:1619677475.639384:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.639386:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.639387:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.639387:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.639387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.639390:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.639391:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.639393:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677475.639496:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677475.639501:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.639504:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677475.639507:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.639507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677475.639510:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.639512:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.639514:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498931] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.639515:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.639516:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.639518:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.639522:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:21.0:1619677475.639526:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.639529:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.639530:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.639532:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.639532:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.639533:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.639533:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.639535:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.639535:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.639536:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.639536:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000400:00000200:30.0:1619677475.639537:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.639537:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677475.639538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000400:00000200:30.0:1619677475.639539:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.639540:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498930] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.639541:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.639541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.639542:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.639543:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.639547:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.639548:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.639549:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.639552:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.639552:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.639553:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.639553:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677475.639553:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.639553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.639555:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.639555:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677475.639555:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.639556:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.639557:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677475.639558:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:8.2:1619677475.639561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.639569:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.639572:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.639573:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.639575:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.639576:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.639578:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.639792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:4.2:1619677475.639796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:26.0:1619677475.639800:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677475.639801:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:26.0:1619677475.639803:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:21.0:1619677475.639803:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:10.0:1619677475.639804:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:21.0:1619677475.639806:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.639806:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:26.0:1619677475.639807:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.639809:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.639810:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.639810:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:26.0:1619677475.639812:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:21.0:1619677475.639812:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.639813:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498929] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.639814:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.639816:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.639816:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.639817:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.639817:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:26.0:1619677475.639819:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:21.0:1619677475.639822:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:26.0:1619677475.639824:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.639824:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677475.639826:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.0:1619677475.639827:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.639828:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677475.639829:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.639829:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677475.639830:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.639831:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677475.639831:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.639831:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.639832:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:26.0:1619677475.639833:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.639833:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677475.639833:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:26.0:1619677475.639834:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.639835:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677475.639847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:25.2:1619677475.639850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:8.0:1619677475.639855:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.639857:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.639858:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.639858:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.639858:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:8.0:1619677475.639860:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.639861:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.639862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.639862:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.639863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:8.0:1619677475.639864:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:6.0:1619677475.639865:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.639868:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:31.0:1619677475.639870:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.639873:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.639876:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.639877:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.639879:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.639880:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.639882:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498927] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.639883:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.639884:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.639885:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.639888:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.639890:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.639893:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.639893:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.639894:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.639896:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.639896:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.639897:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.639898:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.639912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.639918:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.639921:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.639922:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.639922:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.639924:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.639926:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.640186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.640191:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.640194:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.640195:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.640197:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.640199:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.640201:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.640202:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498926] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.640203:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.640205:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.640206:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.640209:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.640211:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.640213:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.640214:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.640215:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.640217:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.640217:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.640218:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.640218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.640223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.640230:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.640232:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.640233:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.640235:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.640236:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.640237:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.640294:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.640303:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.640306:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.640307:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.640308:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.640311:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.640313:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.640314:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498925] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.640315:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.640316:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.640317:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.640321:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.640322:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.640325:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.640326:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.640326:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.640328:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.640329:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.640330:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.640330:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.640335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.640343:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.640346:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.640347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.640347:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.640350:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.640353:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.640431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.640437:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.640439:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.640442:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.640442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.640444:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.640446:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.640448:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498924] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.640449:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.640450:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.640451:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.640455:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.640457:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.640460:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.640461:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.640461:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.640463:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.640464:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.640465:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.640465:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.640471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.640481:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.640485:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.640487:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.640487:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.640489:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.640492:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.640707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.640712:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.640715:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.640716:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.640718:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.640720:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.640721:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.640722:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498923] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.640723:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.640724:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.640725:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.640728:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.640729:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.640731:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.640732:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.640732:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.640733:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.640734:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.640735:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.640735:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677475.640739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.640746:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.640748:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.640749:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.640750:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.640750:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.640751:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.641040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.641045:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.641048:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.641049:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.641051:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.641053:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.641055:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.641056:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498922] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.641057:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.641058:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.641059:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.641063:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.641064:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.641067:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.641068:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.641068:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.641070:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.641070:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.641071:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.641071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.641076:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.641082:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677475.641083:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.641085:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.641086:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.641088:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.641089:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.641090:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.641090:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.641093:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.641095:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.641097:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.641097:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.641098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000400:00000200:30.0:1619677475.641099:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.641100:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498921] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.641101:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.641102:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.641103:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.641105:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.641107:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.641108:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.641109:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.641109:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.641109:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.641110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.641111:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.641111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.641112:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.641113:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.641114:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.641115:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.641116:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.641117:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.641118:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.641120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.641124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.641128:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.641130:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.641130:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.641132:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.641132:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.641133:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.641133:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.641134:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.641135:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.641136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.641138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.641145:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.641148:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.641148:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.641149:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.641150:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.641151:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.641501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.641508:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.641511:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.641513:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677475.641515:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.641516:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.641517:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677475.641518:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498920] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.641519:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.641520:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.641521:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677475.641524:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.641526:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677475.641529:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.641529:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677475.641530:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677475.641532:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677475.641532:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.641533:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.641534:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.641539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.641547:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.641550:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.641551:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.641551:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.641553:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.641557:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.641578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.641587:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.641590:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.641592:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.641594:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.641594:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.641595:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.641596:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498919] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.641597:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.641598:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.641599:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.641603:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.641604:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.641607:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.641607:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.641608:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.641609:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.641610:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.641611:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.641611:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.641615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.641622:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.641625:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.641626:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.641626:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.641628:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.641630:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.641634:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.641640:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.641642:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.641643:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.641643:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.641644:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.641645:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677475.641907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.641913:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.641916:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.641917:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.641918:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.641920:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.641921:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.641922:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498918] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.641924:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.641925:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.641926:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.641929:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.641931:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.641932:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.641933:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.641934:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.641935:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.641935:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.641936:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.641936:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.641941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.641948:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.641950:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.641951:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.641952:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.641953:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.641954:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.642052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.642058:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.642061:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.642062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.642064:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.642066:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.642068:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.2:1619677475.642069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:6.0:1619677475.642069:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498917] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.642070:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.642071:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.642072:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.642075:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677475.642075:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.642077:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:6.0:1619677475.642077:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.642079:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.642080:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.642080:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677475.642080:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.642081:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.642082:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677475.642083:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.642083:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.642083:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677475.642084:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:6.0:1619677475.642084:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.642085:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498916] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.642087:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.642088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677475.642088:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.642089:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.642092:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.642094:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.642095:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.642096:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.642097:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.642097:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.642097:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.642098:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.642099:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:20.0:1619677475.642099:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.642100:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.642100:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.642101:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.642102:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677475.642103:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677475.642105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.642113:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.642115:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.642116:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.642117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.642118:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.642120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.642124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.642131:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.642133:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.642134:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.642134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.642135:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.642136:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.642232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.642239:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.642242:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.642243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.642245:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.642248:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.642249:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.642250:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498915] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.642252:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.642253:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.642254:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.642256:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.642258:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.642261:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.642262:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.642262:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.642264:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.642264:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.642266:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.642267:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.642267:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.642270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.642273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:22.0:1619677475.642278:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.642280:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:22.0:1619677475.642280:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.642281:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.642282:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:22.0:1619677475.642282:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.642283:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.642284:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:22.0:1619677475.642284:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.642287:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.642289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.642296:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.642299:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.642300:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.642301:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.642302:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.642302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.642339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.642345:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.642348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.642348:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000400:00000200:1.0:1619677475.642350:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.642352:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.642353:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.642355:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498914] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.642356:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.642357:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.642358:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.642361:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.642362:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.642365:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.642366:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.642366:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.642368:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.642368:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.642370:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.642371:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.642371:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.642375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.642376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.642382:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677475.642384:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677475.642384:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.642385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:21.0:1619677475.642386:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:17.0:1619677475.642386:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:21.0:1619677475.642387:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.642389:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.642390:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.642392:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.642475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.642483:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.642486:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.642488:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.642490:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.642490:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.642492:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.642493:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498913] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.642495:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.642496:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.642497:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.642500:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.642502:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.642504:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.642505:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.642506:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.642507:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.642508:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.642509:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.642510:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.642514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.642521:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.642523:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.642524:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.642526:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.642528:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.642528:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.642717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.642723:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.642726:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.642727:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.642729:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.642731:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.642733:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.642734:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498912] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.642735:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.642736:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.642737:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.642740:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.642741:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.642743:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.642744:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.642745:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.642746:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.642747:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.642748:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.642748:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.642753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.642759:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.642762:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.642762:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.642763:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.642764:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.642765:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.642921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.642927:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.642929:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.642930:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.642932:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.642934:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.642936:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.642937:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498911] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.642938:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.642939:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.642940:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.642943:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.642945:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.642948:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.642948:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.642949:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.642951:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.642951:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.642953:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.642954:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.642954:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677475.642954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:25.2:1619677475.642957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.642959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:10.0:1619677475.642961:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677475.642964:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:10.0:1619677475.642964:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:6.0:1619677475.642964:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:26.0:1619677475.642965:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:6.0:1619677475.642966:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.642967:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:10.0:1619677475.642967:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:6.0:1619677475.642967:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.642968:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.642968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.642969:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.642969:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.642971:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:10.0:1619677475.642971:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:20.0:1619677475.642972:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677475.642972:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498910] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.642974:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.642975:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.642976:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.642979:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.642981:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.642983:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.642984:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.642985:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.642986:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.642987:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.642988:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.642988:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.642992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.642999:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.643001:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.643002:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.643004:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.643006:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:12.0:1619677475.643006:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:25.2:1619677475.643014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677475.643021:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.643023:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.643024:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.643025:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.643025:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.643027:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.643130:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.643135:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.643138:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.643140:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.643140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.643143:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.643144:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.643146:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498909] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.643147:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.643148:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.643149:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.643152:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.643154:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.643156:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.643157:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.643158:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.643160:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.643160:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.643161:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.643162:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.643166:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.643173:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.643175:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.643176:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.643177:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.643179:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.643181:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.643555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.643562:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.643565:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.643567:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.643569:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.643570:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677475.643570:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.643572:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498908] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.643573:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.643574:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.643575:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.643578:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.643580:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.643583:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.643584:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.643584:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.643586:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.643587:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.643588:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.643590:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.643590:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.643593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.643596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.643600:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.643602:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.643603:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.643603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.643604:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.643605:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.643606:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.643607:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:24.0:1619677475.643607:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.643608:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.2:1619677475.643786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.643792:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.643794:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.643795:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.643797:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.643799:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.643800:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.643802:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498907] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.643803:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.643804:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.643805:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.643808:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.643810:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.643813:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.643813:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.643814:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.643816:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.643816:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.643817:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.643817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.643822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.643829:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.643832:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.643832:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.643834:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.643835:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.643836:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.644063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.644070:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.644073:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.644074:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.644076:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.644078:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.644080:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.644081:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498906] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.644082:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.644083:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.644084:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.644087:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.644089:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.644092:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.644093:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.644094:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.644095:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.644096:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.644097:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.644098:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.644098:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.644101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.644104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.644109:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.644111:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.644112:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.644112:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.644113:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.644113:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.644114:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677475.644115:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.644115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:22.0:1619677475.644118:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.644281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.644288:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.644291:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677475.644295:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.644298:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.644301:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.644302:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.644304:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498905] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.644305:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.644306:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.644307:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.644310:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.644312:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.644315:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.644316:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.644317:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.644318:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.644319:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.644320:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.644320:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.644325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.644334:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.644336:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.644337:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.644339:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.644339:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.644342:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.644344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.644351:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.644353:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.644354:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.644356:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.644356:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.644357:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.644378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.644385:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.644388:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.644389:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.644391:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.644393:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.644395:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.644396:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498904] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.644398:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.644399:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.644400:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.644403:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.644405:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.644408:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.644409:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.644409:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.644411:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.644412:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.644413:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.644414:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.644415:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.644417:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.644420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.644425:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.644427:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.644428:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.644429:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677475.644429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.644429:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.644430:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.644431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677475.644431:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.644434:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.644796:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.644802:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.644805:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.644806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.644807:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.644809:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.644811:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.644812:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498903] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.644813:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.644814:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.644815:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.644819:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.644821:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.644823:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.644824:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.644824:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.644826:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.644827:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.644827:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.644828:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.644833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.644840:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.644842:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.644843:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.644845:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.644846:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.644847:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.644854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.644861:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.644864:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.644864:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.644865:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.644866:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.644866:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.644998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.645005:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.645008:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.645009:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.645011:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.645014:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.645015:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.645016:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498902] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.645018:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.645019:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.645020:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.645023:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.645025:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.645028:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.645028:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.645029:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.645031:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.645031:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.645032:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.645033:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.645033:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.645037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.645039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.645044:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.645046:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.645047:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.645048:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.645048:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677475.645048:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.645049:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:10.0:1619677475.645049:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677475.645050:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.645053:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.645407:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.645414:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.645417:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.645418:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.645420:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.645423:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.645424:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.645425:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498901] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.645426:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.645428:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.645428:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.645431:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.645433:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.645435:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.645436:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.645437:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.645438:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.645439:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.645439:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.645440:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.645444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.645453:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.645455:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.645456:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.645457:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.645459:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.645462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.645609:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.645617:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.645620:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.645622:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.645624:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.645625:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.645627:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677475.645628:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498900] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.645629:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.645630:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.645631:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677475.645635:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.645637:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677475.645639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.645640:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.645641:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677475.645641:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677475.645643:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677475.645644:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.645645:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.645645:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.645645:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.645648:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.645649:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.2:1619677475.645649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:6.0:1619677475.645651:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.645653:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.645654:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.645656:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498899] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.645657:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.645657:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:6.0:1619677475.645658:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.645659:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677475.645659:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.645661:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.645661:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.645663:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.645664:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.645666:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:1.0:1619677475.645666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677475.645669:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.645670:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.645670:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.645672:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.645672:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.645673:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.645673:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.645679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.645686:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.645689:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.645690:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.645691:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.645693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.645694:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.645812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.645815:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.645818:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.645820:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.645821:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.645822:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.645824:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.645825:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498898] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.645827:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.645828:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.645829:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.645832:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.645834:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.645836:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.645837:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.645838:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.645840:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.645840:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.645842:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.645843:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.645844:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:4.2:1619677475.645848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.645854:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.645856:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.645856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.645856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.645857:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677475.645862:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.645865:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.645866:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.645866:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.645868:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.645871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.646111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.646119:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.646121:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.646124:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.646125:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.646127:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.646128:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.646130:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498897] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.646131:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.646132:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.646133:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.646136:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.646138:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.646141:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.646142:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.646143:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.646144:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.646145:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.646146:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.646147:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.646151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.646159:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.646161:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.646162:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.646164:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.646164:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.646167:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:5.2:1619677475.646181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.646187:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.646190:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.646191:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.646193:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.646195:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.646196:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.646198:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498896] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.646199:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.646200:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.646201:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.646204:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.646205:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.646207:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.646208:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.646209:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.646210:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.646211:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.646212:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.646212:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.646216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.646223:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.646225:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.646226:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.646227:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.646228:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.646229:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677475.646389:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.646395:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.646398:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.646399:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.646401:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.646403:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.646404:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.646406:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498895] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.646407:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.646408:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.646409:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.646412:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.646414:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.646417:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.646417:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.646418:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.646420:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.646420:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.646421:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.646422:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.646426:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.646435:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.646437:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.646438:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.646440:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.646442:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.646443:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.646507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.646513:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.646516:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.646517:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.646519:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.646521:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.646523:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.646524:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498894] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.646525:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.646526:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.646527:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.646530:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.646532:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.646534:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.646535:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.646535:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.646537:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.646538:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.646538:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.646539:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.646540:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.646543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.646545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.646550:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.646550:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.646552:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.646552:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.646553:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.646553:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.646554:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.646555:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.646556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.646558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.646673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.646678:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.646681:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.646683:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.646683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.646686:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.646687:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.646688:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498893] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.646690:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.646691:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.646692:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.646695:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.646697:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.646699:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.646700:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.646700:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.646702:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.646703:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.646704:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.646705:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.646705:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.646708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.646711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.646718:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.646720:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.646720:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.646720:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677475.646721:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.646721:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.646721:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.646722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.646724:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677475.646724:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.647079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.647087:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.647089:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.647090:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.647092:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.647095:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.647096:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677475.647098:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498892] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.647099:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.647100:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.647101:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677475.647103:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.647105:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677475.647107:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.647108:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677475.647109:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.647110:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.647110:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.647111:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.647111:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.647116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.647122:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.647124:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.647124:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.647125:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.647126:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.647128:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.647243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.647251:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.647254:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.647256:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.647258:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.647259:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.647260:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.647261:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498891] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.647262:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.647263:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.647264:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.647267:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.647268:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.647271:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.647271:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.647272:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.647274:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.647274:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.647275:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.647276:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.647280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.647286:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.647288:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.647289:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.647289:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.647291:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.647299:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.647314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.647323:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.647326:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.647328:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.647330:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.647330:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.647332:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.647333:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498890] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.647334:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.647335:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.647336:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.647339:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.647340:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.647342:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.647343:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.647343:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.647345:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.647345:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.647347:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.647348:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.647348:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.647351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.647354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.647359:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.647361:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.647362:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.647362:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.647362:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:31.0:1619677475.647363:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.647363:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.647364:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.647365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.647366:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.647497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.647503:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.647506:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.647507:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.647509:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.647511:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.647513:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.647514:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498889] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.647515:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.647516:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.647517:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.647520:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.647522:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.647524:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.647525:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.647525:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.647527:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.647527:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.647528:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.647529:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.647533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.647541:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.647543:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.647544:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.647545:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.647546:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.647547:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.647983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.647988:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.647991:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.647993:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.647994:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.647994:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.647996:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.647997:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498888] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.647998:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.647999:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.648000:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.648003:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.648005:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.648007:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.648008:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.648008:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.648010:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.648010:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.648011:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.648012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.648017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.648025:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.648028:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.648029:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.648029:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.648031:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.648033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.648331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.648336:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.648339:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.648340:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.648342:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.648344:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.648345:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.648347:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498887] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.648348:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.648349:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.648350:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.648353:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.648355:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.648357:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.648358:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.648358:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.648360:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.648360:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.648361:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.648361:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.648367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.648375:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.648377:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.648378:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.648380:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.648381:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.648382:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.648384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.648391:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.648394:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.648395:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.648395:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.648396:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.648397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.648685:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.648693:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.648696:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.648698:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.648700:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.648701:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677475.648701:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.648703:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498886] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.648704:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.648705:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.648706:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.648710:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.648711:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.648715:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.648715:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.648716:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.648718:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.648719:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.648720:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.648720:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.648725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.648733:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.648735:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.648736:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.648737:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.648739:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.648741:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.648881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.648889:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.648892:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.648894:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.648895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.648897:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.648899:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.648900:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498885] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.648901:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.648902:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.648903:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.648905:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.648907:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.648909:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.648910:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.648910:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.648912:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.648912:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.648913:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.648914:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.648918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.648924:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.648926:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.648927:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.648928:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.648929:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.648931:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.648937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.648944:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.648946:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.648947:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.648947:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.648948:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.648948:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.649178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.649183:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.649186:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.649187:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.649189:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.649191:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.649192:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.649194:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498884] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.649195:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.649196:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.649197:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.649200:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.649202:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.649204:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.649205:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.649206:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.649207:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.649208:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.649208:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.649208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.649213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.649221:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.649223:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.649224:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.649226:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.649227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.649228:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.649373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.649379:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.649381:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.649383:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.649384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.649385:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.649387:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.649388:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498883] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.649389:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.649390:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.649391:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.649394:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.649396:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.649398:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.649399:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.649400:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.649401:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.649402:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.649403:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.649404:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.649405:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677475.649408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.649411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.649416:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.649418:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.649418:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.649419:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:20.0:1619677475.649419:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.649420:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.649420:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.649421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.649422:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677475.649423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.649428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.649435:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.649438:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.649438:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.649439:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.649440:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.649441:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.649577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.649585:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.649587:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.649588:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.649590:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.649592:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.649594:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.649595:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498882] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.649596:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.649597:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.649599:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.649602:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.649603:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.649606:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.649607:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.649608:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.649609:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.649610:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.649610:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.649611:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.649616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.649624:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.649627:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.649628:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.649628:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.649630:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:4.2:1619677475.649632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677475.649633:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677475.649638:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.649641:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677475.649643:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.649643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.649647:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.649649:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.649650:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498881] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.649651:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.649652:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.649653:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.649656:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.649658:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.649660:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.649661:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.649662:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.649664:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.649664:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.649665:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.649665:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:4.2:1619677475.649666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.649674:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.649676:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.649677:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.649679:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:25.2:1619677475.649680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677475.649681:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.649683:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.649684:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498880] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.649685:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.649686:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.649686:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.649688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:26.0:1619677475.649690:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.649690:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677475.649691:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.649691:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.649692:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.649694:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.649694:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.649695:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.649696:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.649697:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.649697:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677475.649698:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.649699:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.649699:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.649714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.649722:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.649724:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.649725:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.649725:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.649727:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.649729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.649735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.649742:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.649744:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.649745:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.649745:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.649746:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.649747:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.650061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.650068:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.650071:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.650071:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:4.2:1619677475.650072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919063fd6000] (20)++
00000400:00000200:10.0:1619677475.650073:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.650076:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.650077:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677475.650078:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498879] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.650080:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.650081:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.650081:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677475.650081:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919063fd6000] (21)++
00000400:00000200:10.0:1619677475.650084:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.650085:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.0.65@o2ib3
00000800:00000200:10.0:1619677475.650086:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:18.0:1619677475.650087:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919063fd6000] (22)--
00000800:00000200:10.0:1619677475.650090:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.650091:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677475.650091:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:2.0:1619677475.650091:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.0.65@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.650093:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677475.650094:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.650095:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.650095:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677475.650096:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.650099:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000400:00000200:2.0:1619677475.650099:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:2.0:1619677475.650101:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498878] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.650105:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.650107:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.650109:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:18.0:1619677475.650111:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.650112:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.650114:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.650116:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.65@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.650118:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.650120:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:18.0:1619677475.650122:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.650124:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.650126:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677475.650126:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.650128:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.650128:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:2.0:1619677475.650129:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:18.0:1619677475.650130:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.650130:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.650132:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:2.0:1619677475.650133:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:2.0:1619677475.650134:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919063fd6000] (21)++
00000800:00000200:2.0:1619677475.650136:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919063fd6000] (22)++
00000800:00000200:2.0:1619677475.650138:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919063fd6000] (23)--
00000800:00000200:24.2:1619677475.650139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:2.0:1619677475.650139:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919063fd6000] (22)--
00000800:00000200:24.0:1619677475.650142:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:4.2:1619677475.650144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919063fd6000] (21)++
00000800:00000200:24.0:1619677475.650145:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000800:00000200:21.2:1619677475.650146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:8.0:1619677475.650146:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677475.650147:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.650149:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.650151:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919063fd6000] (22)++
00000800:00000200:24.0:1619677475.650152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:1.0:1619677475.650152:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:18.0:1619677475.650154:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919063fd6000] (23)--
00000800:00000200:1.0:1619677475.650155:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:18.0:1619677475.650156:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919063fd6000] (22)--
00000800:00000200:17.0:1619677475.650156:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:2.0:1619677475.650157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919063fd6000] (21)--
00000400:00000200:1.0:1619677475.650158:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.650159:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.650161:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677475.650162:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498877] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.650163:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.650164:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.650165:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677475.650168:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.650170:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:1.0:1619677475.650170:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.650172:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.650172:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.650173:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.650175:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:1.0:1619677475.650175:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.650175:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.650176:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.650176:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:6.0:1619677475.650178:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.650179:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.2:1619677475.650181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:6.0:1619677475.650181:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.650183:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.650184:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.650185:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498876] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.650187:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.650188:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:6.0:1619677475.650188:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.650189:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677475.650190:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.650191:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.650192:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.650192:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.650194:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.650195:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677475.650195:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.650197:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.650198:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.650199:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.650200:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.650201:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.650202:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.650204:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.650204:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.650207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.650209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.650214:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.650214:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.650216:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.650216:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.650217:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.650218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.650218:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.650219:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.650220:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.650221:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.650264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.650271:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.650274:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.650274:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.650276:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.650279:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.650281:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.650282:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498875] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.650283:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.650284:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.650285:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.650289:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.650291:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.650293:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.650294:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.650300:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.650302:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.650303:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.650303:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.650304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.650319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.650327:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.650330:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.650331:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.650331:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.650334:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.650337:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.650515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.650521:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.650524:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.650525:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.650526:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.650528:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.650529:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.650530:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498874] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.650532:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.650533:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.650534:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.650537:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.650539:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.650540:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.650541:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.650542:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.650543:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.650543:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.650545:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.650546:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.650546:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.650550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.650552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.650558:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.650559:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.650559:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.650559:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677475.650560:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.650560:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.650561:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.650562:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.650562:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677475.650563:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.650569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.650576:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.650578:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.650579:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.650579:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.650580:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.650580:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.650596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.650602:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.650604:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.650605:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.650606:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.650608:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.650609:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.650610:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498873] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.650611:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.650612:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.650613:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.650616:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.650617:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.650619:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.650620:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.650620:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.650622:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.650622:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.650623:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.650624:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.650628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.650636:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.650638:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.650639:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.650640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:21.0:1619677475.650641:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.650643:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.650645:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677475.650648:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.650651:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.650652:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.650653:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.650657:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.650658:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.650660:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498872] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.650661:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.650662:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.650663:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.650666:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.650668:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.650671:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.650672:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.650672:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.650674:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.650674:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.650675:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.650676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.650680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.650687:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.650690:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.650691:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.650691:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.650694:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.650696:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.650699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.650706:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.650708:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.650709:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.650709:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.650710:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.650711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.650722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.650730:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.650733:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.650735:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.650737:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.650738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.650739:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.650740:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498871] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.650742:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.650743:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.650744:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.650747:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.650749:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.650752:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.650753:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.650753:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.650755:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.650756:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.650757:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.650757:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.650761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.650769:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:4.2:1619677475.650771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.650773:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.650774:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.650776:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.650776:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.650777:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.650778:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:20.0:1619677475.650778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:21.0:1619677475.650780:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.650780:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.650783:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.650785:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.650786:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498870] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.650787:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.650788:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.650789:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.650793:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.650794:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.650796:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.650797:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.650798:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.650800:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.650800:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.650801:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.650801:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.650816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.650824:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.650826:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.650827:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.650827:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.650829:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.650831:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.650877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.650885:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.650888:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.650890:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.650892:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.650893:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.650895:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.650896:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498869] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.650897:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.650898:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.650899:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.650902:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.650904:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.650906:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.650907:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.650907:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.650909:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.650910:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.650911:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.650912:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.650913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.650915:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.650918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.650921:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.650924:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.650925:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.650925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.650927:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.650927:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.650929:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:24.0:1619677475.650929:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.650930:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.650931:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677475.651334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.651341:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.651344:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.651344:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.651347:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.651349:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.651351:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.651352:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498868] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.651354:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.651355:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.651356:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.651359:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.651360:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.651363:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.651363:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.651364:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.651366:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.651366:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.651367:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.651369:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.651369:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677475.651373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677475.651375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:1.0:1619677475.651381:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677475.651382:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.651384:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:1.0:1619677475.651384:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.651385:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.651385:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:1.0:1619677475.651385:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.651386:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:1.0:1619677475.651387:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.651390:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.651489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.651495:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.651497:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.651498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.651500:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.651502:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.651504:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.651505:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498867] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.651506:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.651507:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.651508:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.651512:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.651513:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.651516:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.651517:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.651518:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.651519:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.651520:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.651520:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.651521:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.651526:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.651534:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.651536:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.651537:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.651539:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.651540:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.651541:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.652246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.652254:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.652257:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:20.2:1619677475.652258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:31.0:1619677475.652259:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.652261:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.652261:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.652263:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.652264:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498866] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.652264:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677475.652265:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.652267:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:18.0:1619677475.652267:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.652267:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677475.652268:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:18.0:1619677475.652270:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.652272:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.652274:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:18.0:1619677475.652274:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.652275:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.652276:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498865] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.652277:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.652278:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:18.0:1619677475.652278:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.652279:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:18.0:1619677475.652279:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.652280:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:18.0:1619677475.652280:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.652281:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.652281:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.652282:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:18.0:1619677475.652284:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.652285:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:25.2:1619677475.652286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677475.652288:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.652288:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.652289:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.652291:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.652291:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:30.0:1619677475.652292:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:18.0:1619677475.652292:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:20.2:1619677475.652299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (23)++
00000800:00000200:30.0:1619677475.652300:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.652301:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677475.652301:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.652302:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.652302:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.652303:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.652304:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:30.0:1619677475.652305:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:18.0:1619677475.652306:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.652306:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.652307:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.652308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.652316:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.652318:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.652319:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.652319:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.652321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000400:00000200:2.0:1619677475.652321:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.652322:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:10.0:1619677475.652329:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677475.652332:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677475.652333:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677475.652335:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.652337:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.652339:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677475.652340:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498864] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.652342:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.652343:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.652344:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677475.652347:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.652349:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677475.652351:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.652352:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677475.652352:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677475.652354:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677475.652354:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677475.652355:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:10.0:1619677475.652357:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:10.0:1619677475.652357:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:24.2:1619677475.652360:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:4.2:1619677475.652363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:24.0:1619677475.652364:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677475.652366:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677475.652367:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.652367:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677475.652369:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.652370:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:24.0:1619677475.652371:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:26.0:1619677475.652372:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677475.652373:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677475.652374:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677475.652379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677475.652382:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.652384:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677475.652385:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677475.652386:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (23)--
00000800:00000200:8.0:1619677475.652386:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677475.652387:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677475.652508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.2:1619677475.652515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677475.652516:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.652518:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.652520:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.652520:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677475.652522:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.652523:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677475.652523:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677475.652524:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.652525:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.652526:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498863] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.652526:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677475.652527:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.652528:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.652528:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.652529:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.652529:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.652531:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498862] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.652533:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.652534:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.652535:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.652536:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677475.652537:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.652540:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.652541:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.652541:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.652542:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.652542:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677475.652543:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.652544:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.652545:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677475.652545:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.652545:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:31.0:1619677475.652546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677475.652546:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.652548:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.652548:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.652549:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.652549:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677475.652549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677475.652554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677475.652560:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.652562:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:8.0:1619677475.652563:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.652564:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.652564:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:8.0:1619677475.652564:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.652565:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.652567:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.652567:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.652567:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.652569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677475.652570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.652872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.652878:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.652880:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.652881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.652883:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.652886:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.652887:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.652888:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498861] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.652890:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.652891:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.652892:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.652895:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.652897:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.652899:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.652900:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.652901:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.652902:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.652903:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.652904:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.652904:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.652908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.652916:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.652918:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.652919:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.652921:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.652921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.652923:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.652930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.652937:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.652940:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.652941:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.652942:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.652942:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.652944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.653168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.653175:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.653179:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.653181:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.653183:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.653183:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.653185:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.653186:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498860] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.653188:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.653189:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.653190:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.653193:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.653195:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.653198:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.653199:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.653200:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.653201:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.653202:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.653204:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.653205:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.653206:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.653207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.653213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.653219:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677475.653221:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.653221:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.653222:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.653224:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.653224:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.653224:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.653224:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677475.653225:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.653226:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.653227:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.653227:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (23)++
00000800:00000200:20.0:1619677475.653228:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.653229:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.653230:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.653820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.653825:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.653827:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.653830:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.653830:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.653832:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.653833:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.653835:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498859] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.653836:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.653837:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.653838:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.653841:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.653843:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.653845:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.653846:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.653846:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.653848:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.653848:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.653849:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.653849:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.653854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.653862:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.653864:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.653865:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.653866:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.653867:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.653868:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.654418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.654424:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.654426:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.654427:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.654429:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.654431:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.654432:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.654433:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498858] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.654435:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.654435:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.654436:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.654439:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.654441:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.654443:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.654444:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.654445:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.654446:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.654447:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.654447:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.654448:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.654453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.654462:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.654464:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.654465:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.654467:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.654468:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.654469:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.654492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.654500:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.654502:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.654505:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.654507:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.654508:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.654509:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.654510:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498857] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.654512:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.654513:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.654514:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.654517:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.654518:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.654521:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.654522:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.654522:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.654524:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.654524:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.654525:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.654526:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.654530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.654537:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.654539:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.654540:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.654540:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.654542:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.654544:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.654588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.654596:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.654599:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.654600:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.654602:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.654604:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.654605:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.654606:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498856] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.654608:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.654608:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.654609:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.654612:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.654614:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.654617:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.654618:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.654618:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.654620:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.654620:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.654621:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.654621:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.654626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.654634:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.654636:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.654637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.654638:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.654640:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.654643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.654650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.654656:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.654659:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.654659:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.654660:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.654661:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.654661:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.654799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.654805:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.654808:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.654809:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.654811:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.654813:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.654815:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.654816:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498855] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.654817:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.654818:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.654819:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.654822:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.654824:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.654826:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.654826:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.654827:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.654828:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.654829:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.654829:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.654830:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.654835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.654842:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:4.2:1619677475.654844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:1.0:1619677475.654844:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.654844:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.654846:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.654846:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.654847:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677475.654851:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.654854:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.654854:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.654857:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.654859:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.654861:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.654862:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498854] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.654864:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.654864:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.654866:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.654869:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.654871:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.654874:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.654875:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.654876:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.654878:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.654878:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.654879:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.654880:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.654881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677475.654884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.654887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677475.654892:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677475.654894:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677475.654894:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.654895:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.654896:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.654897:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:20.0:1619677475.654897:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.654897:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677475.654898:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.654900:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.655560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.655568:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.655571:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.655572:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.655573:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.655576:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.655578:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.655579:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498853] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.655580:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.655581:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.655582:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.655586:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.655587:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.655590:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.655590:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.655591:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.655592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677475.655593:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.655593:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.655594:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.655594:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:30.0:1619677475.655600:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677475.655600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.655603:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.655605:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.655607:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.655608:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:22.0:1619677475.655608:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677475.655609:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.655610:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498852] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.655611:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.655611:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:30.0:1619677475.655612:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.655612:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.655612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:30.0:1619677475.655613:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:22.0:1619677475.655615:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.655617:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677475.655618:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.655620:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.655621:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.655622:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.655623:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.655624:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.655625:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.655626:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.655626:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.655630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.655637:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.655639:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.655640:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.655641:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.655643:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.655644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.655648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.655655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.655657:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.655658:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.655659:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.655659:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.655660:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.655995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.656002:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.656005:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.656006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.656007:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.656010:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.656012:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.656013:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498851] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.656014:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.656015:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.656016:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.656019:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.656020:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.656023:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.656024:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.656024:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.656026:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.656027:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.656028:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.656029:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.656029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.656032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.656035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677475.656040:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.656042:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677475.656043:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.656044:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:17.0:1619677475.656044:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.656044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.656045:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.656046:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677475.656047:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.656049:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.656184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.656190:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.656193:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.656193:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.656195:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.656197:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.656199:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.656200:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498850] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.656201:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.656202:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.656203:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.656206:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.656208:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.656211:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.656211:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.656212:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.656214:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.656214:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.656216:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.656217:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.656217:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.656220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.656222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.656227:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.656229:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.656229:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.656230:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.656231:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.656232:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.656232:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.656234:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.656235:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.656236:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.656331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.656339:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.656342:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.656344:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.656346:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.656347:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.656348:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.656349:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498849] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.656350:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.656352:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.656353:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.656356:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.656358:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.656360:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.656361:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.656362:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.656364:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.656364:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.656365:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.656366:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.656370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.656377:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.656380:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.656380:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.656381:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.656383:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.656385:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.656388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.656393:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.656396:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.656397:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.656397:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.656398:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.656398:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.656452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.656460:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.656463:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.656465:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.656467:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.656467:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.656469:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:12.0:1619677475.656470:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498848] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.656471:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.656472:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.656474:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:12.0:1619677475.656477:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.656478:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:12.0:1619677475.656481:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.656482:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:12.0:1619677475.656482:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:12.0:1619677475.656484:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:12.0:1619677475.656484:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.656485:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:12.0:1619677475.656487:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:12.0:1619677475.656487:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677475.656490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.656493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677475.656498:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.656501:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:13.0:1619677475.656501:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:18.0:1619677475.656502:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.656502:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:13.0:1619677475.656503:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677475.656504:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:18.0:1619677475.656505:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.656505:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.656506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.656509:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.656511:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.656513:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.656514:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.656514:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.656516:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.656516:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.657195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.657200:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.657202:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.657203:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.657205:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.657207:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.657209:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:21.2:1619677475.657210:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:2.0:1619677475.657210:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498847] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.657212:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.657213:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.657214:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:31.0:1619677475.657218:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677475.657218:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.657219:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677475.657221:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.657223:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677475.657223:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.657224:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677475.657224:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677475.657224:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:31.0:1619677475.657226:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.657226:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:31.0:1619677475.657227:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677475.657227:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:31.0:1619677475.657228:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498846] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677475.657228:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677475.657229:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000400:00000200:31.0:1619677475.657230:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677475.657230:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:31.0:1619677475.657231:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.657232:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:9.2:1619677475.657232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:31.0:1619677475.657234:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.2:1619677475.657235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:31.0:1619677475.657236:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.657238:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.657239:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.657239:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677475.657239:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:18.0:1619677475.657240:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:31.0:1619677475.657241:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.657242:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.657242:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:31.0:1619677475.657243:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.657243:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677475.657243:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:18.0:1619677475.657243:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677475.657244:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:10.0:1619677475.657244:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.657245:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:10.0:1619677475.657246:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.657247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:10.0:1619677475.657249:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:30.0:1619677475.657253:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.657256:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.657256:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.657258:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.657260:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.657261:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.657417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.657425:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.657428:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.657429:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.657431:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.657433:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.657435:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.657436:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498845] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.657437:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.657438:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.657439:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.657442:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.657443:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.657446:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.657447:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.657447:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.657449:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.657449:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.657450:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.657451:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.657456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.657464:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.657467:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.657468:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.657468:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.657471:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.657473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.657475:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.657482:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.657484:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.657485:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.657486:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.657486:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.657487:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.657505:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.657511:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.657513:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.657514:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.657516:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.657517:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.657519:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.657520:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498844] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.657521:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.657522:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.657523:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.657526:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.657528:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.657530:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.657531:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.657532:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.657533:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.657534:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.657534:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.657535:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.657539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.657547:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.657549:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.657550:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.657552:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.657553:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.657554:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.657556:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.657562:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.657565:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.657566:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.657568:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.657570:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.657572:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.657573:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498843] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.657575:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.657576:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.657577:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.657581:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.657582:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.657585:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.657586:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.657587:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.657589:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.657589:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.657590:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.657591:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.657595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.657602:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.657605:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.657606:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.657608:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.657609:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.657611:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.657616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.657623:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.657626:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.657627:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.657627:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.657628:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.657630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.657663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.657671:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.657674:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.657675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.657677:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.657680:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.657681:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.657682:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498842] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.657684:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.657685:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.657686:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.657689:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.657690:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.657693:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.657694:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.657695:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.657696:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.657697:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.657697:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.657698:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.657702:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.657709:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.657712:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.657713:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.657713:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.657716:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.657718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.658265:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.658273:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.658276:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.658278:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.658279:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.658280:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.658281:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.658283:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498841] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.658284:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.658285:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.658286:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.658289:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.658291:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.2:1619677475.658291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.658292:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.658293:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.658299:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.658300:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.658300:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677475.658301:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.658302:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.658302:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.658302:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.658303:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677475.658304:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.658305:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.658306:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.658307:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498840] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.658308:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.658308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:30.0:1619677475.658309:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.658310:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.658313:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.658314:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.658314:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.658315:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.658315:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.658317:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.658317:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.658317:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.658318:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.658319:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.658319:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.658319:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.658320:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.658322:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.658323:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.658324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.658325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.658330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.658332:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.658335:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.658336:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.658336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.658337:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.658338:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677475.658340:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.658341:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.658342:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.658342:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677475.658385:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.658391:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.658394:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.658396:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.658396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.658398:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.658400:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.658401:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498839] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.658402:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.658403:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.658404:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.658407:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.658409:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.658411:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.658412:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.658412:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.658414:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.658414:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.658415:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.658416:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.658417:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.658420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.658423:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.658429:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.658430:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.658431:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.658431:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677475.658432:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.658432:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.658433:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.658433:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.658434:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677475.658435:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.658441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.658449:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.658451:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.658452:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.658453:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.658454:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.658454:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.2:1619677475.658617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.658623:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.658626:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.658627:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.658629:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.658632:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.658634:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.658635:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498838] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.658636:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.658637:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.658638:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.658641:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.658643:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.658645:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.658645:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.658646:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.658648:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.658648:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.658649:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.658649:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.658654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.658661:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.658663:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.658664:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.658664:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.658665:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.658667:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.659062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.659067:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.659070:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.659071:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.659073:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.659075:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.659076:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.659077:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498837] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.659079:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.659080:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.659081:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.659084:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.659085:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.659087:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.659088:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.659089:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.659090:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.659091:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.659091:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.659092:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.659096:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.659104:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.659106:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.659107:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.659108:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.659110:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.659111:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.659118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.659126:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.659128:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.659129:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.659130:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.659130:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.659131:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.659276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.659282:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.659285:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.659286:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.659289:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.659290:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.659292:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:1.0:1619677475.659298:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498836] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.659300:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.659301:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.659302:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:1.0:1619677475.659305:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.659307:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:1.0:1619677475.659310:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.659311:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:1.0:1619677475.659312:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:1.0:1619677475.659313:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677475.659314:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.659315:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.659315:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677475.659320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.659328:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.659330:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.659331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.659332:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.659334:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.659337:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677475.659364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.659371:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.659374:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.659374:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.659376:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.659379:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.659380:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.659381:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498835] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.659382:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.659383:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.659384:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.659387:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.659388:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.659391:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.659392:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.659393:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.659394:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.659395:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.659395:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.659395:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.659400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.659408:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.659411:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.659411:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.659412:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.659414:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.659416:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.659495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.659499:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.659502:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.659504:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.659505:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.659506:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.659508:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.659509:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498834] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.659510:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.659511:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.659512:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.659516:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.659517:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.659520:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.659521:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.659521:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.659523:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.659524:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.659525:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.659525:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.659529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.659536:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.659539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.659539:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.659540:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.659542:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.659545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677475.659569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.659575:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.659578:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.659579:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.659581:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.659583:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.659584:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.659586:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498833] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.659587:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.659588:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.659589:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.659592:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.659594:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:4.2:1619677475.659594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677475.659595:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.659596:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.659597:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.659598:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.659598:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.659599:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.659600:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:10.0:1619677475.659602:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:8.2:1619677475.659604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.659605:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.659605:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.659608:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.659610:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.659611:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:1.0:1619677475.659612:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:10.0:1619677475.659613:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498832] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.659614:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.659614:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:10.0:1619677475.659615:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.659615:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.659616:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:10.0:1619677475.659616:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:20.2:1619677475.659617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:1.0:1619677475.659617:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.659618:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:10.0:1619677475.659619:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.659620:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.659622:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:10.0:1619677475.659623:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.659624:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.659625:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:10.0:1619677475.659625:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:21.0:1619677475.659627:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.659627:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:10.0:1619677475.659627:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.659627:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.659628:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677475.659629:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.659629:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:21.0:1619677475.659630:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.659632:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498831] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.659633:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677475.659633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:21.0:1619677475.659634:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.659635:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.659638:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.659640:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.659640:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.659642:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.659642:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.659642:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.659643:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.659644:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:20.0:1619677475.659644:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.659645:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.659646:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.659646:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.659647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.659648:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677475.659649:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.659651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.659658:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.659660:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.659661:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.659662:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.659663:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.659665:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.659737:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.659744:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.659747:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.659748:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.659750:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.659753:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.659753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:26.0:1619677475.659754:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677475.659756:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498830] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.659757:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.659758:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.659759:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.659761:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677475.659762:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.659763:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677475.659764:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.659765:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.659766:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.659767:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.659767:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:31.0:1619677475.659768:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677475.659768:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677475.659768:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677475.659769:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677475.659770:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498829] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677475.659770:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000400:00000200:31.0:1619677475.659771:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:26.0:1619677475.659771:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000400:00000200:31.0:1619677475.659772:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.659772:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.659772:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.659773:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.659774:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677475.659775:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.659777:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:4.2:1619677475.659778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677475.659780:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.659781:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.659781:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.659781:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.659782:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.659783:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.659783:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:30.0:1619677475.659783:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.659784:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.659785:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:10.0:1619677475.659785:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:31.0:1619677475.659786:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.659786:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.659787:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677475.659787:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:30.0:1619677475.659788:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:10.0:1619677475.659788:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677475.659789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.659790:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:21.2:1619677475.659792:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.659796:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.659799:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677475.659800:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677475.659800:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.659802:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677475.659802:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.659802:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.659803:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.659804:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.659805:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.659861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.659869:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.659871:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.659874:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.659875:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.659877:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.659879:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.659880:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498828] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.659881:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.659883:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.659883:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.659886:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.659888:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.659890:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.659891:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.659891:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.659893:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.659894:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.659894:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.659895:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.659899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.659905:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.659907:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.659908:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.659909:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.659911:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.659913:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.660518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677475.660524:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.660526:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677475.660529:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.660529:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677475.660530:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.660532:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677475.660533:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498827] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.660534:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.660535:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.660536:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677475.660539:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.660541:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.660543:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.660544:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.660545:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.660546:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.660547:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.660547:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677475.660548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677475.660553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.660561:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.660563:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.660564:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.660565:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.660567:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.660567:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.660568:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.660569:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.660569:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.660570:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.660570:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.660571:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.660572:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.660621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.660628:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.660630:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.660632:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.660633:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.660635:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.660637:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677475.660638:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498826] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.660639:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.660640:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.660641:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677475.660644:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.660646:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677475.660649:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.660649:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677475.660650:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677475.660652:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677475.660652:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.660654:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677475.660655:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677475.660655:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677475.660659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.2:1619677475.660662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:26.0:1619677475.660666:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:1.0:1619677475.660667:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:26.0:1619677475.660668:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677475.660669:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:1.0:1619677475.660669:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000400:00000200:26.0:1619677475.660670:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.660670:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.660671:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:26.0:1619677475.660672:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.660675:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.660856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.660862:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.660866:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.660866:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.660868:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.660870:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.660871:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.660873:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498825] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.660874:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.660875:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.660876:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.660879:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.660881:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.660884:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.660885:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.660885:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.660887:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.660887:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.660888:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.660888:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.660893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.660901:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.660903:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.660904:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.660906:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.660907:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.660908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.660933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.660941:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.660943:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.660944:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.660946:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.660948:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.660949:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.660951:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498824] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.660952:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.660953:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.660954:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.660957:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.660959:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.660962:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.660963:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.660964:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.660965:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.660966:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.660967:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.660967:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.660971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.660979:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.660982:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.660983:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.660983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.660985:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.660988:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.661100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.661108:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.661111:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.661113:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.661115:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.661116:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.661117:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.661118:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498823] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.661119:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.661120:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.661121:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.661124:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.661126:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.661128:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.661128:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.661129:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.661130:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.661130:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.661131:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.661132:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.661136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.661142:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.661145:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.661145:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.661147:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.661148:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.661149:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.661154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.661160:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.661163:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.661164:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.661165:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.661165:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.661166:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.661512:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.661519:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.661522:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.661523:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.661524:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.661527:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.661529:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.661530:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498822] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.661531:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.661532:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.661533:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.661536:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.661537:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.661540:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.661541:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.661542:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.661543:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.661544:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.661545:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.661545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.661550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.661558:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.661561:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.661562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.661562:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.661564:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.661567:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.661951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.661956:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.661959:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.661961:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.661962:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.661963:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.661964:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.661965:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498821] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.661967:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.661968:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.661969:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.661972:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.661974:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.661976:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.661977:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.661977:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.661979:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.661980:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.661980:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.661981:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.661985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.661992:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.661994:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.661995:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.661997:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.661997:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.661999:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.662174:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.662179:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.662182:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.662183:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.662185:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.662187:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.662188:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.662189:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498820] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.662191:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.662192:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.662193:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.662196:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.662198:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.662200:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.662201:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.662201:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.662203:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.662203:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.662205:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.662206:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.662206:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.662209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.662211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.662216:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.662217:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.662218:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.662219:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.662219:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.662219:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.662220:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.662221:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.662222:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.662223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.662476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.662484:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.662487:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.662488:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.662490:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.662493:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.662494:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.662496:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498819] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.662497:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.662498:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.662499:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.662502:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.662504:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.662506:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.662507:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.662508:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.662509:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.662510:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.662511:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.662511:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.662516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.662524:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.662527:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.662528:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.662528:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.662530:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.662533:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.662536:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.662538:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.662540:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.662541:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.662541:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.662542:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.662542:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.662584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.662590:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.662593:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.662594:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.662596:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.662598:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.662600:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.662601:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498818] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.662602:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.662603:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.662605:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.662608:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.662609:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.662612:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.662613:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.662613:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.662615:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.662616:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.662616:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.662617:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.662621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.662629:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.662632:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.662632:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.662634:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.662635:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.662636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.662718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.662726:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.662729:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.662731:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.662733:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.662734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.662735:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.662736:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498817] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.662738:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.662739:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.662740:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.662743:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.662745:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.662748:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.662749:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.662750:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.662752:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.662753:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.662754:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.662754:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.662758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.662766:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.662768:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.662769:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.662771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.662771:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.662774:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.663658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.663663:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.663665:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.663667:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.663668:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.2:1619677475.663670:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:21.0:1619677475.663673:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.663674:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.663675:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498816] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.663676:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677475.663677:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.663678:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.663679:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677475.663679:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.663680:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.663683:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.663684:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.663685:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.663686:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:6.0:1619677475.663687:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.663688:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498815] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.663689:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.663689:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.663690:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:6.0:1619677475.663690:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.663691:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.663692:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000400:00000200:6.0:1619677475.663692:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.663695:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.663696:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.663697:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.663697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:6.0:1619677475.663697:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.2:1619677475.663698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677475.663699:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.663700:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.663701:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.663702:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.663703:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.663703:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.663704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.663706:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.663708:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:25.2:1619677475.663709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677475.663709:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.663711:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.663713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.663714:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.663715:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.663718:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.663718:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.663718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000400:00000200:20.0:1619677475.663720:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.663722:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.663723:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.663724:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.663726:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.663727:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.663728:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.663729:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.663730:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677475.663827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.663834:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.663837:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.663837:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:1.0:1619677475.663840:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.663842:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.663843:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.663844:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498814] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.663846:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.663847:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.663848:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.663851:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.663853:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.663854:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.663855:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.663856:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.663857:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.663857:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.663858:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.663859:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.663863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.663870:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.663872:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.663873:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.663874:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.663874:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.663876:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.663976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.663984:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.663986:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.663987:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.663989:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.663992:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.663993:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:26.0:1619677475.663995:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498813] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.663996:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.663997:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.663998:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:26.0:1619677475.664001:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.664002:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.664004:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.664005:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.664006:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.664007:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.664007:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.664008:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.664008:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.664013:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.664019:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.664021:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.664021:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.664022:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.664023:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.664024:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677475.664031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:26.0:1619677475.664037:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.664039:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.664039:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.664040:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677475.664040:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.664041:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677475.664053:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.664060:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.664062:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.664064:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.664065:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.664066:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.664068:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.664069:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498812] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.664070:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.664071:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.664072:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.664075:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.664077:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.664079:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.664079:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.664080:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.664082:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.664082:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.664082:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.664084:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.664084:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677475.664087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.664090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677475.664094:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:26.0:1619677475.664097:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677475.664097:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.664097:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.664098:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.664099:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.664099:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:8.0:1619677475.664099:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.664100:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677475.664102:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.664228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.664237:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.664240:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.664242:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.664244:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.664245:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.664246:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.664248:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498811] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.664249:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.664250:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677475.664250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:31.0:1619677475.664251:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.664254:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.664255:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.664257:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677475.664258:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.664259:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.664259:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:31.0:1619677475.664260:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.664260:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677475.664261:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.664262:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:10.0:1619677475.664262:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.664263:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.664263:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:10.0:1619677475.664264:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.664265:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.664266:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498810] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.664268:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.664268:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.2:1619677475.664268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:10.0:1619677475.664269:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.664272:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.664274:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:21.0:1619677475.664276:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677475.664277:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.664277:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:21.0:1619677475.664278:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677475.664278:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000400:00000200:21.0:1619677475.664279:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.664280:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.664280:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:21.0:1619677475.664281:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.664281:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:20.0:1619677475.664282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.664282:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.664283:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.0:1619677475.664284:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:25.2:1619677475.664286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.664288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.664297:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.664298:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677475.664299:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.664300:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.664301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677475.664301:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.664302:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.664302:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.664304:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.664307:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.664800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.664806:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.664809:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.664810:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.664812:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.664814:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.664815:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.664817:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498809] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.664818:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.664819:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.664820:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.664823:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.664825:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.664827:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.664828:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.664828:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.664830:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.664831:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.664831:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.664832:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.664837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.664845:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.664848:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.664849:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.664849:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.664851:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.664854:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.665218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.665222:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.665225:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.665227:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.665228:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.665229:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.665231:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.665232:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498808] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.665234:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.665235:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.665236:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.665239:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.665241:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.665244:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.665245:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.665245:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.665247:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.665248:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.665249:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.665249:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.665254:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.665260:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.665263:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.665264:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.665264:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.665266:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.665269:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.665270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.665276:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.665279:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.665280:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.665280:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.2:1619677475.665281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.665281:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.665282:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.665289:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.665292:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:31.0:1619677475.665299:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.665300:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.665303:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.665305:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.665306:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498807] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.665307:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.665308:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.665309:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.665312:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.665314:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.665317:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.665318:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.665318:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.665320:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.665321:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.665322:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.665324:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.665324:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.665327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.665331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.665339:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.665339:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677475.665342:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677475.665342:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:31.0:1619677475.665343:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.665343:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.665344:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677475.665345:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.665345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.665348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.665448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.665454:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.665457:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.665458:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.665460:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.665462:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.665463:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.665464:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498806] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.665465:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.665466:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.665467:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.665470:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.665472:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.665475:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.665475:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.665476:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.665477:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.665478:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.665478:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.665479:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.665484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.665492:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.665494:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.665495:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.665497:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.665498:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.665499:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.665501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.665508:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.665511:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.665512:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.665512:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.665513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.665514:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.665551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:20.2:1619677475.665553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:31.0:1619677475.665558:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:21.0:1619677475.665558:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.665560:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677475.665561:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:21.0:1619677475.665563:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.665563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677475.665565:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.665565:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:21.0:1619677475.665565:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.665567:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.665567:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.665568:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.665569:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.665570:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.665571:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.665572:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498805] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.665573:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.665573:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:31.0:1619677475.665574:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.665576:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.665577:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.665580:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.665580:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.665582:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.665583:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.665583:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.665584:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:31.0:1619677475.665585:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.665585:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:31.0:1619677475.665586:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.665586:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.665586:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:31.0:1619677475.665587:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.665587:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:31.0:1619677475.665588:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.665589:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.665590:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.665591:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.2:1619677475.665591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:9.2:1619677475.665594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.665598:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.665600:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:24.0:1619677475.665601:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:20.0:1619677475.665601:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.665603:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:20.0:1619677475.665603:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:24.0:1619677475.665604:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.665604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.665605:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677475.665605:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.665606:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.665609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.665750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.665759:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.665761:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.665762:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.665764:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.665767:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.665768:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.665769:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498803] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.665771:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.665772:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.665773:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.665776:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.665778:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.665781:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.665781:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.665782:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.665784:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.665784:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.665785:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.665785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.665790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.665798:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.665801:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.665802:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.665802:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.665804:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.665807:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.665957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.665965:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.665968:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.665970:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.665972:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.665972:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.665973:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.665975:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498802] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.665976:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.665977:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.665978:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.665981:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.665983:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.665985:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.665986:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.665986:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.665988:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.665988:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.665989:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.665990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.665993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.666000:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.666003:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.666004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.666004:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.666005:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.666008:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.666539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.666547:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.666549:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.666550:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.666552:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.666555:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.666556:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.666557:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498801] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.666558:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.666559:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.666560:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.666563:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.666564:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.666566:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.666567:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.666568:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.666569:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.666570:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.666570:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.666571:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.666575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.666580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.666586:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.666587:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.666588:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.666590:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.666590:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.666591:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.666592:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.666593:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:8.0:1619677475.666593:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.666594:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.666596:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.666598:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.666599:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498800] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.666600:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.666601:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.666602:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.666605:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.666607:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.666610:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677475.666611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.666611:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.666611:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.666613:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.666614:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.666615:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.666616:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.666617:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.666619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.666620:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.666622:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:4.2:1619677475.666622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:31.0:1619677475.666624:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.666626:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.666627:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:6.0:1619677475.666627:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677475.666628:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.666629:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498799] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.666629:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:31.0:1619677475.666630:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.666630:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:31.0:1619677475.666631:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.666631:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677475.666631:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.666631:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.666632:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677475.666632:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.666633:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677475.666634:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.666637:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.666637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677475.666638:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.666640:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.666641:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.666641:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.666643:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.666643:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.666644:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.666645:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.666649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.666655:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.666658:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.666659:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.666660:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.666662:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.666663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.666779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.666784:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.666786:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.666787:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.666789:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.666791:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.666793:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.666794:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498798] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.666796:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.666797:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.666798:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.666801:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.666803:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.666805:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.666806:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.666807:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.666808:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.666809:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.666810:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.666810:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.666814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.666822:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.666824:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.666825:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.666827:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.666828:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.666829:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.667003:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.667010:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.667013:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.667014:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.667016:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.667018:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.667020:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677475.667021:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498797] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.667022:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.667023:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.667024:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.667028:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.667030:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677475.667032:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.667033:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677475.667034:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677475.667036:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677475.667036:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.667037:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.667037:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677475.667042:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.667050:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.667053:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677475.667053:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.667054:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.667056:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:18.0:1619677475.667059:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.667428:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.667436:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.667439:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.667441:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.667443:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.667443:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.667444:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.667446:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498796] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.667447:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.667448:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.667449:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.667452:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.667454:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.667456:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.667457:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.667457:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.667459:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.667460:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.667460:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.667461:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.667466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.2:1619677475.667469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:8.0:1619677475.667473:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.667475:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.667476:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.2:1619677475.667476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:8.0:1619677475.667476:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.667476:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.667478:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.667479:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:8.0:1619677475.667479:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.667481:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:2.0:1619677475.667481:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.667483:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:2.0:1619677475.667483:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.667485:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:30.0:1619677475.667486:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:2.0:1619677475.667486:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498795] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.667487:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.667488:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.667489:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.667490:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:2.0:1619677475.667490:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.667492:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.667494:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677475.667494:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.667495:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498794] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677475.667496:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.667497:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.667498:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677475.667499:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.667500:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:2.0:1619677475.667500:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.667500:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.667502:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.667502:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000400:00000200:30.0:1619677475.667503:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.667503:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.667504:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:30.0:1619677475.667505:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.667507:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.667508:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:9.2:1619677475.667508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.667509:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.667510:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.667511:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.667512:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.667512:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:18.0:1619677475.667515:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:25.2:1619677475.667516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677475.667517:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.667518:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.667519:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.667520:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.667522:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:18.0:1619677475.667522:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677475.667525:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.667526:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.667527:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:4.2:1619677475.667528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:31.0:1619677475.667529:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.667529:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677475.667535:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.667538:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.667539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.667540:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.667543:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.667545:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.667546:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498793] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.667547:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.667548:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.667549:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.667552:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.667553:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.667556:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.667557:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.667557:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.667559:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.667560:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.667560:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.667561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.667565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.667573:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.667576:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.667576:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.667577:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.667579:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.667582:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.667704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.667711:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.667714:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.667715:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.667717:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.667719:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.667721:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.667722:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498792] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.667723:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.667724:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.667725:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.667728:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.667730:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.667732:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.667733:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.667734:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.667735:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.667736:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.667736:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.667737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.667741:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.667749:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.667751:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.667752:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.667752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.667755:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.667757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.667759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.667766:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.667769:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.667769:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.667770:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.667771:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.667771:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.668041:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.668048:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.668051:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677475.668053:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.116@o2ib7 : GET - for me
00000800:00000200:31.0:1619677475.668055:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:30.0:1619677475.668055:0:19944:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.116@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:30.0:1619677475.668057:0:19944:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.116@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:30.0:1619677475.668059:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.668060:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:30.0:1619677475.668061:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677475.668065:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : REPLY try# 0
00000800:00000200:30.0:1619677475.668067:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.668069:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.668069:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.668070:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.668071:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.668072:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:25.2:1619677475.668077:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.668084:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.668086:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.668087:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.668088:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: REPLY: OK
00000800:00000200:30.0:1619677475.668090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.668091:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:5.2:1619677475.668110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.668116:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.668119:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.668120:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.668122:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.668124:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.668125:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.668127:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498791] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.668128:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.668129:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.668130:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.668133:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.668134:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.668136:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.668137:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.668138:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.668139:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.668140:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.668141:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.668142:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.668143:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.668146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.668148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.668155:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.668156:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.668156:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.668156:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.668156:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.668157:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.668158:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.668158:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.668159:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677475.668160:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.668163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:8.2:1619677475.668165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677475.668170:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.668173:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677475.668173:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677475.668173:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:17.0:1619677475.668176:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677475.668176:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.668177:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.668177:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.668178:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.668178:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:10.0:1619677475.668178:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.668179:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.668181:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498790] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.668182:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.668183:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.668184:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.668187:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.668189:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.668191:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.668192:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.668193:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.668194:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.668194:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.668195:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.668196:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.668200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.668206:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.668208:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.668209:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.668209:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.668211:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.668213:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.668219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.668224:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.668227:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.668228:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.668228:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.668229:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.668229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.668299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.668305:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.668308:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.668308:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.668310:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.668312:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.668314:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.668315:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498789] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.668317:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.668317:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.668319:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.668322:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.668324:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.668326:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.668327:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.668328:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.668329:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.668330:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.668331:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.668332:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.668332:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.668336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.668338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.668343:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.668344:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.668345:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.668345:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.668346:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.668346:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.668347:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.668348:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.668350:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.668350:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.669113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.669121:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.669123:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.669124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.669126:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.669129:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.669130:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.669131:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498788] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.669133:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.669133:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.669135:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.669138:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.669140:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.669142:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.669143:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.669144:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.669145:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.669146:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.669147:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.669147:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.669151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.669159:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.669162:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.669163:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.669163:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.669165:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.669168:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.669409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.669417:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.669420:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.669422:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.669424:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.669425:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.669425:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.669427:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498787] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.669428:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.669429:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.669430:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.669433:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.669435:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.669437:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.669438:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.669438:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.669440:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.669441:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.669442:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.669444:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.669444:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.669446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.669451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.669454:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.669457:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.669458:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.669458:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.669459:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.669460:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.669462:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.669462:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677475.669462:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.669463:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.2:1619677475.669587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.669592:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.669595:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.669597:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.669598:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.669600:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.669601:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.669602:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498786] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.669604:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.669605:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.669606:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.669609:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.669611:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.669613:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.669614:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.669615:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.669616:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.669617:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.669618:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.669620:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.669620:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.669623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.669626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.669632:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.669633:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.669633:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677475.669634:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.669634:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677475.669635:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.669635:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.669636:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.669637:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.0:1619677475.669638:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677475.669726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:10.0:1619677475.669734:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:26.0:1619677475.669737:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677475.669737:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000400:00000200:10.0:1619677475.669739:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.669742:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.669744:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677475.669745:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498785] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.669746:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.669747:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.669748:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677475.669751:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.669753:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677475.669755:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.669756:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677475.669757:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677475.669758:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677475.669759:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677475.669759:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677475.669760:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:25.2:1619677475.669764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.669771:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.669774:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.669774:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.669776:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.669778:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.669779:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.670058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.2:1619677475.670059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:6.0:1619677475.670064:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677475.670065:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677475.670067:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.670068:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:2.0:1619677475.670068:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.670069:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:6.0:1619677475.670070:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.670071:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:6.0:1619677475.670072:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.670073:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.670074:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677475.670075:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:6.0:1619677475.670076:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.670077:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498784] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.670078:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.670079:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.670080:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.670081:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.670082:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677475.670082:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:6.0:1619677475.670087:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677475.670088:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:6.0:1619677475.670090:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677475.670091:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:6.0:1619677475.670092:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.670093:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.670094:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677475.670094:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.670095:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:6.0:1619677475.670096:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.670096:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:2.0:1619677475.670096:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:6.0:1619677475.670097:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.670098:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:2.0:1619677475.670098:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.670099:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.670100:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.670100:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.670102:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:25.2:1619677475.670106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.0:1619677475.670109:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.670111:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.670112:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.670114:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.670114:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.670115:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:20.0:1619677475.670116:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:13.0:1619677475.670119:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.670120:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.670123:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.670125:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.670126:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.670184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.670191:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.670194:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.670195:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.670197:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.670199:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.670201:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.670202:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498782] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.670204:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.670205:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.670206:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.670209:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.670211:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.670213:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.670214:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.670215:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.670216:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.670217:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.670218:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.670219:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.670220:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677475.670222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.670225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677475.670228:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.670231:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.670231:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.670232:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677475.670232:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.670234:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.670234:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.670235:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677475.670236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677475.670236:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677475.670441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.670445:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.670448:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.670450:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.670451:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.670452:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.670454:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.670455:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498781] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.670456:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.670457:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.670458:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677475.670462:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.670464:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.670466:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.670467:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.670468:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.670469:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.670470:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.670471:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.670471:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.670476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.670484:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.670487:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.670488:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.670488:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.670490:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:18.0:1619677475.670493:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.670637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.670645:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.670648:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.670650:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.670652:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.670652:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.670654:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.670655:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498780] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.670656:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.670657:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.670658:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.670662:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.670664:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.670666:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.670667:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.670667:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.670669:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.670669:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.670671:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.670672:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.670673:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.670675:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.670679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.670683:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.670684:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.670685:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.670686:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.670687:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.670687:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.670688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.670689:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.670690:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.670691:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.671067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.671075:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.671077:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.671079:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.671081:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.671082:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.671083:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.671084:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498779] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.671085:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.671086:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.671086:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.671089:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.671091:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.671093:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.671093:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.671094:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.671095:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.671096:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.671096:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.671097:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.671101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.671109:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.671111:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.671112:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.671112:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.671114:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.671116:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.671120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.671126:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.671128:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.671129:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.671129:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.671130:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.671130:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.671206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.671214:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.671216:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.671217:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.671219:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.671221:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.671222:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.671224:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498778] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.671225:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.671226:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.671226:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.671229:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.671231:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.671233:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.671234:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.671234:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.671236:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.671236:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.671237:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.671237:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.671241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.671247:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.671250:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.671251:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.671251:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.671252:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.671254:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.671413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.671419:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.671422:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.671423:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.671425:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.671427:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.671428:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.671430:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498777] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.671431:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.671432:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.671433:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.671436:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.671438:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.671440:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.671441:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.671442:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.671443:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.671444:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.671445:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.671445:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.671450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.671458:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.671460:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.671461:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.671463:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.671463:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.671465:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.671466:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:25.2:1619677475.671467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.671474:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:10.0:1619677475.671475:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677475.671477:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.671478:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.671478:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:10.0:1619677475.671478:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.671479:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:21.0:1619677475.671479:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.671480:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:10.0:1619677475.671481:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.671483:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.671485:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677475.671486:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498776] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.671488:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.671488:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.671490:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.671493:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.671494:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677475.671497:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.671498:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677475.671499:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677475.671500:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677475.671501:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.671501:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.671502:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677475.671506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.671515:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.671518:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.671519:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.671519:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.671521:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.671524:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.671660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.671669:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.671671:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.671673:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.671676:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.671676:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.671677:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.671678:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498775] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.671680:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.671681:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.671682:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.671685:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.671686:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.671690:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.671690:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.671691:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.671693:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.671693:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.671694:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.671695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.671700:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.671708:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.671710:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.671711:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.671713:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.671713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.671716:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.671717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.671725:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.671728:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.671729:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.671730:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.671730:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.671731:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.671732:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.671735:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.671736:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.671739:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.671740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.671741:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.671743:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.671744:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498774] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.671745:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.671746:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.671747:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.671751:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.671752:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.671755:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.671756:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.671756:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.671758:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.671759:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.671759:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.671760:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.671764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.671773:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.671775:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.671777:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.671777:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.671779:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.671781:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.671828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf2000] (20)++
00000800:00000200:26.0:1619677475.671835:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf2000] (21)++
00000800:00000200:26.0:1619677475.671838:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.55@o2ib3
00000800:00000200:10.0:1619677475.671839:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf2000] (22)--
00000400:00000200:26.0:1619677475.671841:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.3.55@o2ib3 : GET - for me
00000400:00000200:26.0:1619677475.671844:0:19941:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.3.55@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:26.0:1619677475.671846:0:19941:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.3.55@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:26.0:1619677475.671848:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf2000] (21)++
00000800:00000200:26.0:1619677475.671848:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf2000] (22)--
00000400:00000200:26.0:1619677475.671850:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.3.55@o2ib3 local destination
00000400:00000200:26.0:1619677475.671854:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.3.55@o2ib3(10.51.3.55@o2ib3:10.51.3.55@o2ib3) : REPLY try# 0
00000800:00000200:26.0:1619677475.671855:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.3.55@o2ib3
00000800:00000200:26.0:1619677475.671857:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3c4599b80] -> 10.51.3.55@o2ib3 (2) version: 12
00000800:00000200:26.0:1619677475.671858:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a070bf2000] (21)++
00000800:00000200:26.0:1619677475.671858:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf2000] (22)++
00000800:00000200:26.0:1619677475.671860:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a070bf2000] (23)--
00000800:00000200:26.0:1619677475.671861:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf2000] (22)--
00000800:00000200:4.2:1619677475.671868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf2000] (21)++
00000800:00000200:10.0:1619677475.671875:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf2000] (22)++
00000800:00000200:10.0:1619677475.671877:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf2000] (23)--
00000800:00000200:26.0:1619677475.671878:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf2000] (22)--
00000400:00000200:10.0:1619677475.671878:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.671879:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.3.55@o2ib3: REPLY: OK
00000800:00000200:10.0:1619677475.671881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf2000] (21)--
00000800:00000200:4.2:1619677475.672004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.672012:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.672014:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.672016:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.672019:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.672019:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.672020:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677475.672021:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498773] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.672023:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.672024:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.672025:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677475.672028:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.672028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:12.0:1619677475.672030:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677475.672031:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.672032:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.672033:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:12.0:1619677475.672033:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677475.672034:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677475.672035:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:21.0:1619677475.672036:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:12.0:1619677475.672036:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.672036:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:21.0:1619677475.672037:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.672038:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.672040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:21.0:1619677475.672040:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.672041:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.672042:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498772] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.672044:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.672044:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.672045:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.672047:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:21.0:1619677475.672048:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:30.0:1619677475.672050:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.672050:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.672050:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.672053:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.672053:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.672054:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.672054:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.672054:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.672056:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.672057:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.0:1619677475.672057:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.672058:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.672059:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.672060:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.672062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.672065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.672069:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:6.0:1619677475.672070:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.672071:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.672072:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:6.0:1619677475.672072:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677475.672073:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:6.0:1619677475.672073:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.672074:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.672075:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.672078:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.672129:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.672137:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.672139:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.672140:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.672142:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.672145:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.672146:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.672148:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498771] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.672149:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.672150:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.672151:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.672154:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.672156:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.672159:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.672160:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.672160:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.672162:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.672163:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.672164:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.672164:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.672168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.672175:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.672178:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.672179:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.672179:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.672181:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.672184:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.672740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.672748:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.672751:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.672753:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.672756:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.672756:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.672758:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.672759:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498770] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.672760:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.672761:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.672762:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.672766:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.672767:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.672770:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.672771:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.672772:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.672774:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.672774:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.672775:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.672776:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.672781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.672789:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.672792:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.672793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.672794:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.672795:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.672797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.672883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.672889:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.672892:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.672893:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.672895:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.672897:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.672898:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.672899:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498769] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.672901:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.672902:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.672903:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.672906:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.672908:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.672911:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.672912:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.672912:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.672914:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.672914:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.672915:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.672916:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.672920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.672928:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:5.2:1619677475.672929:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.0:1619677475.672931:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.672932:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.672934:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.672934:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.672935:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.0:1619677475.672936:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:1.0:1619677475.672938:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.672939:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.672940:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.672943:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.672944:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.672945:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498768] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.672946:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.672947:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.672948:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.672950:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.672952:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.672953:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.672954:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.672954:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.672955:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.672956:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.672956:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.672957:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:21.2:1619677475.672961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.672967:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.672970:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.672971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.2:1619677475.672972:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:6.0:1619677475.672972:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.672974:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.672976:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.672977:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498767] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.672978:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.672979:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677475.672980:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:6.0:1619677475.672981:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677475.672982:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.672982:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.672983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:6.0:1619677475.672984:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677475.672985:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.672986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677475.672987:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.672990:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.672990:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.672991:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.672993:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.672993:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.672994:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.672995:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.672999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.673007:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.673009:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.673010:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.673011:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.673012:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.673013:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.673028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.673036:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.673039:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.673041:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.673043:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.673044:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.673046:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.673047:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498766] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.673049:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.673050:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.673051:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.673055:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.673057:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.673059:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.673060:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.673060:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.673062:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.673063:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.673063:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.673064:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.673068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.673074:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.673077:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.673078:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.673078:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.673080:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.673082:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.673499:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.673507:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.673509:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.673511:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.673512:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.673515:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.673516:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.673517:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498765] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.673519:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.673519:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.673520:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.673523:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.673525:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.673528:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.673528:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.673529:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.673531:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.673531:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.673532:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.673532:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.673537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.673545:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.673548:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:1.0:1619677475.673549:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.673550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.673551:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.673554:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.673557:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.673565:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.673568:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.673568:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.673569:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.673570:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.673570:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.674023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.674030:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.674033:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.674034:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.674036:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.674038:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.674039:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.674040:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498764] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.674042:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.674043:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.674043:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.674047:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.674048:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.674051:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.674052:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.674052:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.674054:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.674054:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.674055:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.674055:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.674061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.674069:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.674072:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.674073:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.674073:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.674076:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.674078:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.674080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.674087:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.674090:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.674090:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.674091:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.674092:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.674092:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.674203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.674210:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.674213:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.674214:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.674216:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.674218:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.674219:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.674221:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498763] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.674222:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.674223:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.674224:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.674227:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.674229:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.674231:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.674231:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.674232:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.674233:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.674234:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.674235:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.674235:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.674239:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.674245:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.674247:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.674248:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.674248:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.674250:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.674252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.674736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677475.674740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.674744:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.674746:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.674747:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677475.674749:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:31.0:1619677475.674750:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.674750:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677475.674752:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677475.674752:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.674753:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.674754:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677475.674754:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.674755:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.674756:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.674757:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498762] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.674758:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.674760:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.674760:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.674761:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.674762:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677475.674763:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.674768:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.674768:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.674771:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677475.674771:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.674773:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.674774:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:22.0:1619677475.674774:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.674775:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677475.674775:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.674776:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.674777:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.674777:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.674778:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677475.674778:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.674778:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.674779:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.674779:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.674780:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.674781:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:9.2:1619677475.674783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:25.2:1619677475.674784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.674786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677475.674791:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:6.0:1619677475.674791:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:24.0:1619677475.674793:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677475.674793:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.674793:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.674794:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:24.0:1619677475.674795:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.674795:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:8.0:1619677475.674795:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:6.0:1619677475.674795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.674796:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.674798:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.674798:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.674799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:24.0:1619677475.674800:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677475.674800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.675006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.675009:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.675012:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.675014:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.675015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.675016:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.675018:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.675019:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498760] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.675021:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.675022:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.675023:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.675026:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.675028:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.675030:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.675031:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.675032:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.675033:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.675034:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.675035:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.675035:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.675039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.675046:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.675049:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.675050:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.675050:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.675052:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.675054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.675431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.675436:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.675439:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.675441:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.675442:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.675444:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.675445:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.675446:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498759] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.675447:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.675448:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.675449:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.675452:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.675454:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.675455:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.675456:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.675457:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.675458:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.675459:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.675460:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.675460:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.675465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.675472:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.675475:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.675476:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.675477:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.675478:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.675480:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.675484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.2:1619677475.675485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.675493:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.675493:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.675496:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:30.0:1619677475.675497:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.675497:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.675498:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.675499:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.675499:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:30.0:1619677475.675500:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.675502:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.675502:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.675503:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.675505:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498758] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.675506:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.675507:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.675508:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.675511:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.675513:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.675515:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.675515:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.675516:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.675517:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.675518:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.675519:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.675520:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.675524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.675531:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.675533:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.675534:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.675536:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.675537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.675538:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.675543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.675550:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.675553:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.675554:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.675554:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.675555:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.675556:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.675794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.675802:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.675805:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.675806:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.675808:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.675810:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.675812:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.675813:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498757] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.675815:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.675816:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.675817:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.675820:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.675822:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.675825:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.675826:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.675826:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.675828:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.675829:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.675830:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.675831:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.675832:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.675834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677475.675837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677475.675841:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:2.0:1619677475.675843:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677475.675844:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.675845:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.675845:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.675845:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.675846:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677475.675847:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.675848:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.675850:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.675853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.675860:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.675863:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.675864:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.675864:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.675865:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.675866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.676030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.676039:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.676041:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.676042:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.676044:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.676046:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.676047:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.676049:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498756] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.676050:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.676051:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.676052:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.676055:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.676057:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.676060:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.676061:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.676062:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.676063:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.676064:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.676064:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.676065:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.676070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.676078:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.676080:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.676081:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.676082:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.676084:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.676087:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.676094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.676100:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.676103:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.676104:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.676104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.676105:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.676106:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.676110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.676116:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.676118:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.676119:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.676121:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.676123:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.676124:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.676125:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498755] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.676127:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.676127:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.676129:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.676132:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.676134:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.676136:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.676137:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.676138:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.676139:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.676139:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.676140:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.676140:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.676145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.676153:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.676155:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.676156:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.676157:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.676158:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.676159:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.676175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.676181:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.676184:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.676185:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.676187:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.676188:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.676190:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:1.0:1619677475.676191:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498754] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.676193:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.676194:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.676195:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:1.0:1619677475.676198:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.676200:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:1.0:1619677475.676203:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.676204:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:1.0:1619677475.676205:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:1.0:1619677475.676206:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:1.0:1619677475.676207:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.676208:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.676208:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:9.2:1619677475.676212:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.676219:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.676222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.676222:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.676223:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.676225:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.676229:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.676545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.676554:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.676557:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.676559:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.676561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.676561:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.676562:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.676563:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498753] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.676565:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.676566:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.676567:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.676571:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.676572:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.676574:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.676575:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.676576:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.676577:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.676578:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.676579:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.676580:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.676581:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.676583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.676586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.676591:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.676592:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.676593:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.676594:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677475.676595:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.676595:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.676596:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.676596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.676597:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.676598:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677475.676802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.676809:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.676812:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.676813:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.676815:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.676818:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.676819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:18.0:1619677475.676819:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677475.676820:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498752] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.676822:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.676823:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.676824:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:30.0:1619677475.676827:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:18.0:1619677475.676827:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.676829:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:30.0:1619677475.676830:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.676832:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.676833:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.676834:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:18.0:1619677475.676834:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677475.676834:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:30.0:1619677475.676835:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.676836:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:18.0:1619677475.676836:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677475.676837:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000400:00000200:30.0:1619677475.676838:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498751] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.676838:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:30.0:1619677475.676839:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:18.0:1619677475.676839:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000400:00000200:30.0:1619677475.676840:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.676841:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:9.2:1619677475.676842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:30.0:1619677475.676844:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.676846:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.676848:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.676849:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.676849:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677475.676849:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:30.0:1619677475.676851:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677475.676851:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:30.0:1619677475.676852:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.676853:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000400:00000200:26.0:1619677475.676853:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.676853:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:30.0:1619677475.676854:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.676855:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:26.0:1619677475.676855:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:9.2:1619677475.676857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677475.676858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.676860:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:9.2:1619677475.676861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677475.676864:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.676866:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.676867:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.676867:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.676868:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:10.0:1619677475.676868:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:8.0:1619677475.676869:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.676870:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:31.0:1619677475.676871:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:26.0:1619677475.676871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.676871:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:8.0:1619677475.676871:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.676872:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:10.0:1619677475.676872:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:31.0:1619677475.676873:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:10.0:1619677475.676873:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677475.676996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.677002:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.677005:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.677006:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.677008:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.677011:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.677013:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.677014:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498750] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.677015:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.677016:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.677017:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.677020:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.677022:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.677024:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.677025:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.677026:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.677027:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.677028:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.677029:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.677029:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.677033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.677041:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.677043:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.677043:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.677044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.677045:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.677047:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.677057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.677064:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.677067:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.677068:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.677068:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.677068:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.677069:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.677280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.677288:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.677291:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.677292:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.677299:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677475.677301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:24.0:1619677475.677303:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.677305:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.677306:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498749] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.677308:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.677308:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:24.0:1619677475.677309:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.677309:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.677312:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000400:00000200:24.0:1619677475.677313:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.677313:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:24.0:1619677475.677315:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:17.0:1619677475.677316:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.677318:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.677319:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.677319:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000400:00000200:17.0:1619677475.677319:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.677320:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:24.0:1619677475.677321:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.677321:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:17.0:1619677475.677321:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498748] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.677322:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:17.0:1619677475.677323:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.677323:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.677324:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.677324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677475.677325:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.2:1619677475.677327:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:17.0:1619677475.677328:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.677330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677475.677331:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.677334:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.677334:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.677335:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.677337:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.677337:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.677338:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:8.0:1619677475.677338:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.677338:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.677339:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.677339:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.677340:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677475.677341:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.677341:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:1.0:1619677475.677341:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.677342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.0:1619677475.677342:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:1.0:1619677475.677344:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.677345:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.677353:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.677355:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.677356:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.677357:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.677358:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.677359:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.677436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.677444:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.677447:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.677449:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.677451:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.677451:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.677453:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.677454:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498747] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.677455:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.677456:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.677457:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.677460:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.677462:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.677464:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.677465:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.677465:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.677467:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.677468:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.677469:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.677469:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.677473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.677480:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.677483:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.677484:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.677485:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.677487:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.677487:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.2:1619677475.677701:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.677708:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.677711:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.677712:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.677714:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.677717:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.677718:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.677719:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498746] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.677720:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.677721:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.677722:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.677725:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.677727:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.677729:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.677730:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.677731:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.677732:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.677733:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.677733:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.677734:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.677738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.677746:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.677748:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.677749:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.677750:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.677752:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.677753:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.677757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.677764:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.677767:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.677768:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.677768:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.677769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.677770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.677945:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.677950:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.677952:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.677954:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.677955:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.677957:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.677958:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.677960:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498745] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.677961:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.677962:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.677963:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.677966:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.677968:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.677971:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.677972:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.677973:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.677975:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.677975:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.677976:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.677977:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.677981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.677984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:8.0:1619677475.677988:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.677990:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.677991:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.677991:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.677992:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:8.0:1619677475.677994:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.677995:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.677996:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.0:1619677475.677997:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677475.677998:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.678000:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.678001:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000400:00000200:26.0:1619677475.678002:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.678003:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498744] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.678004:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.678005:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.678006:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:24.0:1619677475.678007:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000400:00000200:26.0:1619677475.678009:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677475.678009:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.678010:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.678010:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677475.678011:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:24.0:1619677475.678011:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.678012:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677475.678014:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.678015:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.678016:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.678017:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.678018:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.678019:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677475.678020:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.678021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677475.678024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.678027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677475.678033:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677475.678034:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677475.678035:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.678036:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.678037:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:10.0:1619677475.678037:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.678037:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.678038:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677475.678039:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.678041:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.678061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.678066:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.678068:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.678070:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.678071:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.678073:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.678075:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.678076:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498743] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.678077:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.678078:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.678079:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.678082:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.678084:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.678086:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.678087:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.678087:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.678089:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.678089:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.678090:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.678091:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.678095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.678103:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.678105:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.678106:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.678107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.678108:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.678111:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.678112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.678120:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.678122:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.678123:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.678124:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.678124:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.678125:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.678563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.678571:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.678574:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.678576:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.678578:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.678579:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.678579:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.678581:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498742] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.678583:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.678584:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.678585:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.678588:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.678590:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.678593:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.678594:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.678595:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.678597:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.678598:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.678599:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.678599:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.678604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.678612:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.678615:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.678615:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.678617:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.678618:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.678620:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.678688:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.678696:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.678699:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.678702:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.678703:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.678705:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.678706:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.678708:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498741] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.678709:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.678710:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.678711:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.678714:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.678715:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.678717:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.678718:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.678718:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.678720:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.678721:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.678722:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.678722:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.678726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.678733:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.678735:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.678736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.678736:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.678739:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.678741:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.678855:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.678864:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.678866:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.678867:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.678869:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.678872:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.678873:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.678874:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498740] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.678875:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.678876:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.678877:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.678880:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.678882:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.678885:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.678885:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.678886:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.678888:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.678888:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.678889:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.678889:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.678894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.678902:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.678904:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.678905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.678906:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.678908:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.678911:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.678953:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:24.0:1619677475.678960:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677475.678963:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:8.0:1619677475.678964:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:24.0:1619677475.678966:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.678969:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.678970:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.678971:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498739] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.678973:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.678974:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.678974:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.678978:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.678979:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.678982:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.678983:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.678983:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.678983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.678986:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.678986:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677475.678987:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677475.678988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:24.0:1619677475.678989:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.678990:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677475.678992:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677475.678992:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:24.0:1619677475.678996:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.678996:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.678997:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.678998:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498738] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.678999:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.679000:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.679000:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:24.0:1619677475.679001:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.679002:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:24.0:1619677475.679003:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.679003:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.679004:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.679005:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000400:00000200:1.0:1619677475.679006:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.679008:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.679009:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.679009:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.679009:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677475.679011:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.679012:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.679013:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.679013:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.679017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.679025:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.679028:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.679029:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.679029:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.679031:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.679033:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.679113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.679120:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.679123:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.679124:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.679126:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.679129:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.679130:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.679131:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498737] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.679132:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.679134:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.679134:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.679138:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.679139:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.679141:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.679142:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.679143:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.679144:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.679145:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.679146:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.679147:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.679148:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677475.679150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.679153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677475.679157:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.679159:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.679160:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.679160:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677475.679160:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.679162:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677475.679162:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.679163:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.679164:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677475.679164:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.679172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.679178:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.679181:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.679182:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.679182:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.679183:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.679183:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.679213:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.679219:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.679221:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.679223:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.679224:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.679225:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.679227:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.679228:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498736] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.679230:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.679230:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.679232:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.679235:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.679236:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.679238:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.679239:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.679240:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.679241:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.679242:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.679242:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.679243:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.679247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.679255:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.679257:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.679258:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.679258:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.679260:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.679262:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.679489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.679494:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.679497:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.679498:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.679499:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.679501:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.679503:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.679504:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498735] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.679505:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.679506:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.679507:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.679510:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.679512:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.679514:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.679515:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.679515:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.679517:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.679517:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.679518:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.679519:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.679524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.679532:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.679534:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.679535:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.679537:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.679538:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.679539:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.679548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.679553:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.679555:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.679557:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.679558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.679560:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.679562:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.679563:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498734] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.679564:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.679565:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.679567:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677475.679570:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.679572:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.679574:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.679575:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.679576:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.679577:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.679578:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.679579:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.679579:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.679584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.679591:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.679595:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.679596:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.679596:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.679598:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.679600:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.679679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677475.679688:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.679691:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.679693:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.679695:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.679696:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.679697:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677475.679698:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498733] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.679700:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.679701:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.679702:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.679705:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.679707:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.679710:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.679711:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677475.679711:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.679713:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677475.679714:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.679715:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677475.679715:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.679719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.679727:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.679729:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.679730:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.679730:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.679732:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.679736:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.680820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.680828:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.680831:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.680833:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677475.680836:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.680837:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:13.0:1619677475.680837:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:12.0:1619677475.680837:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.680839:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498732] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.680841:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.680842:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.680843:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:31.0:1619677475.680845:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:13.0:1619677475.680846:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.680847:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:13.0:1619677475.680847:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000400:00000200:31.0:1619677475.680850:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.680850:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.680851:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677475.680851:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:30.0:1619677475.680852:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.680853:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.680853:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677475.680853:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000400:00000200:31.0:1619677475.680854:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:13.0:1619677475.680854:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000400:00000200:31.0:1619677475.680855:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498731] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677475.680855:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000400:00000200:31.0:1619677475.680857:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.680857:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.680858:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:9.2:1619677475.680859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:31.0:1619677475.680861:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.680863:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.680865:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.680866:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677475.680866:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:21.2:1619677475.680866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.680867:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.680868:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.680869:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.680869:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:26.0:1619677475.680869:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:31.0:1619677475.680870:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:26.0:1619677475.680870:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.680870:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.680872:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.680872:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677475.680874:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677475.680874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677475.680875:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.680875:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.680878:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.680880:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.680881:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.0:1619677475.680881:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:22.0:1619677475.680882:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498730] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.680883:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.680883:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.680884:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:22.0:1619677475.680884:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.680884:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.680885:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677475.680888:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.680889:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.680889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677475.680892:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.680894:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.680895:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.680896:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.680897:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.680898:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.680899:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.680900:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.680900:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.680903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.680905:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.680910:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.680912:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.680912:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.680913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677475.680913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:6.0:1619677475.680914:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.680915:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.680916:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.680918:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.680918:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.680920:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677475.680921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:10.0:1619677475.680923:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.680924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.680925:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.680927:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.680929:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.680930:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498729] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.680932:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.680933:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.680934:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.680937:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.680938:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.680942:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.680942:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.680943:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.680945:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.680945:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.680946:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.680946:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.680951:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.680958:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.680961:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.680962:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.680962:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.680964:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.680967:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.680995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.681002:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.681005:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.681007:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.681009:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.681009:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.681011:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.681012:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498728] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.681013:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.681014:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.681015:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.681018:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.681020:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.681021:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.681022:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.681022:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.681024:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.681024:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.681025:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.681025:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.681029:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.681036:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.681038:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.681039:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.681041:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.681042:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.681043:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.681151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.681157:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.681159:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.681161:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.681162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.681164:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.681165:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.681167:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498727] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.681168:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.681169:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.681170:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.681173:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.681175:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.681178:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.681178:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.681179:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.681181:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.681181:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.681182:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.681184:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.681184:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.681187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.681190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.681197:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.681198:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.681198:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:21.0:1619677475.681199:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.681199:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677475.681200:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.681200:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.681201:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.681202:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.681203:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:5.2:1619677475.681661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.681667:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.681669:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.681670:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.681672:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.681674:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.681675:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.681676:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498726] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.681679:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.681680:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.681681:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.681683:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.681684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:1.0:1619677475.681685:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.681686:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.681687:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.681688:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.681689:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.681690:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.681691:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.681691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677475.681692:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.681695:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677475.681696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:30.0:1619677475.681697:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.681699:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.681700:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.681700:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.681702:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498725] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.681703:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.681704:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677475.681704:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:30.0:1619677475.681705:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:17.0:1619677475.681706:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.681707:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.681708:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:30.0:1619677475.681710:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677475.681710:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.681711:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677475.681712:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.681714:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.681715:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.681716:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.681717:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.681718:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.681719:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.681721:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.681722:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.681724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.681727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.681732:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.681733:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.681734:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.681736:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.681736:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.681736:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.681737:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.681737:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.681738:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.681739:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.681742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.681749:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.681751:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.681752:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.681753:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.681753:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.681754:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.682442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.682447:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.682450:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.682451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.682453:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.682455:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.682456:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.682458:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498724] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.682459:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.682460:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.682461:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.682464:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.682466:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.682468:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.682469:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.682470:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.682472:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.682472:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.682473:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.682473:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.682478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.682487:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.682489:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.682490:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.682492:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.682493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.682494:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.682572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:21.2:1619677475.682579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:18.0:1619677475.682579:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.682582:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.682582:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.682585:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.682586:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.682587:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:18.0:1619677475.682588:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677475.682589:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498723] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.682591:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:18.0:1619677475.682591:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.682592:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.682593:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677475.682594:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.682595:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.682596:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.682597:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.682598:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:18.0:1619677475.682599:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000400:00000200:30.0:1619677475.682600:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498722] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.682601:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.682602:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.682604:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:18.0:1619677475.682604:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.682605:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677475.682605:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677475.682607:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:30.0:1619677475.682608:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.682608:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.682609:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:30.0:1619677475.682610:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:18.0:1619677475.682610:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:30.0:1619677475.682612:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.682613:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.682613:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:9.2:1619677475.682613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:30.0:1619677475.682615:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.682616:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.682617:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.682618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677475.682620:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:25.2:1619677475.682621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677475.682623:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.682624:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.682624:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.682626:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.682628:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:26.0:1619677475.682629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:31.0:1619677475.682630:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.682631:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.682633:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.682634:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.682635:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.682757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.682765:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.682767:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.682769:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.682771:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.682771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.682772:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.682773:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498721] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.682774:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.682775:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.682776:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.682779:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.682780:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.682783:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.682783:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.682784:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.682785:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.682786:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.682786:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.682787:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.682791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.682798:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.682800:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.682801:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.682802:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.682803:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.682805:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.682868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.682875:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.682877:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.682878:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.682879:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.682882:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.682883:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.682884:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498720] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.682886:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.682887:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.682888:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.682890:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.682892:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.682895:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.682896:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.682896:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.682898:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.682899:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.682900:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.682900:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.682904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.682912:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.682915:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.682915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.682916:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.682918:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.682921:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.683319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.683325:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.683327:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.683329:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.683331:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.683334:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.683335:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.683337:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498719] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.683338:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.683339:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.683340:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.683343:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.683345:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.683347:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.683348:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.683348:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.683350:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.683351:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:21.2:1619677475.683352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677475.683352:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.683352:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.683357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:30.0:1619677475.683360:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.683363:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.683365:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.683366:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:31.0:1619677475.683367:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.683367:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.683368:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:30.0:1619677475.683369:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:4.2:1619677475.683369:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:1.0:1619677475.683369:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.683370:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498718] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.683370:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:30.0:1619677475.683372:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.683372:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:30.0:1619677475.683373:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.683374:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:1.0:1619677475.683375:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.683376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677475.683377:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677475.683378:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.683379:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.683379:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.683380:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.683382:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.683383:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.683384:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.683384:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:17.0:1619677475.683385:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000400:00000200:10.0:1619677475.683385:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.683386:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:10.0:1619677475.683386:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:30.0:1619677475.683387:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.683388:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.683388:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:17.0:1619677475.683388:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677475.683388:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498717] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.683388:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.683389:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.683390:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000400:00000200:10.0:1619677475.683390:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.683391:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:10.0:1619677475.683391:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.683392:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.2:1619677475.683392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:10.0:1619677475.683395:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.683396:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.683399:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.683400:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:20.0:1619677475.683401:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677475.683401:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.683402:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:20.0:1619677475.683403:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677475.683403:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000400:00000200:20.0:1619677475.683404:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.683404:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.683405:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000400:00000200:20.0:1619677475.683406:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.683406:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.0:1619677475.683407:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.683409:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.683410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:4.2:1619677475.683411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:21.0:1619677475.683418:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.683419:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:25.2:1619677475.683419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.683421:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:21.0:1619677475.683421:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.683422:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:21.0:1619677475.683422:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.683422:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677475.683422:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.0:1619677475.683423:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.683424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677475.683427:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.683429:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.683430:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.683430:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.683433:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.683435:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.683838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.683846:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.683848:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.683849:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.683851:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.683854:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.683855:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.683856:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498716] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.683858:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.683859:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.683859:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.683862:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.683864:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.683866:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.683866:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.683867:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.683869:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.683869:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.683870:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.683871:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.683875:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.683882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:12.0:1619677475.683884:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.683887:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.683889:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.683889:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:12.0:1619677475.683891:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677475.683892:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:13.0:1619677475.683892:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.683892:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:12.0:1619677475.683894:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:10.0:1619677475.683894:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.683897:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.683898:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.683899:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498715] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.683900:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.683901:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.683902:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.683905:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.683907:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.683909:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.683909:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.683910:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.683911:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.683911:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.683912:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.683913:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.683916:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.683923:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.683925:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.683926:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.683927:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.683928:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.683930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.684204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.684209:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.684212:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.684213:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.684215:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.684217:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.684218:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.684220:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498714] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.684221:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.684222:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.684223:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.684226:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.684228:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.684231:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.684232:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.684232:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.684234:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.684234:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.684235:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.684236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.684240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.684248:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.684250:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.684251:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.684253:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.684253:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.684255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.684260:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.684268:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.684270:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.684271:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.684272:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.684273:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.684273:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.2:1619677475.684370:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.684377:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.684379:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.684380:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.684383:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.684385:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.684387:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.684388:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498713] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.684389:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.684390:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.684391:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.684394:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.684396:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.684398:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.684399:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.684399:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.684401:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.684402:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.684403:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.684404:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.684404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677475.684405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.684408:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:9.2:1619677475.684408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.684411:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:21.0:1619677475.684414:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.684415:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:2.0:1619677475.684416:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:20.0:1619677475.684417:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677475.684418:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.684418:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.684419:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.684419:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000400:00000200:20.0:1619677475.684420:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.684420:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:20.0:1619677475.684421:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:18.0:1619677475.684421:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.684421:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.684422:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:20.0:1619677475.684423:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498712] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:18.0:1619677475.684423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000400:00000200:20.0:1619677475.684424:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.684425:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.684426:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.684430:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.684432:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.684434:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.684435:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.684435:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.684437:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.684438:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.684438:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.684439:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.684444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.684452:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.684455:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.684456:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.684456:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.684458:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.684461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.684525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.684534:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.684536:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.684539:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.684541:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.684542:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.684544:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.684545:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498711] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.684546:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.684547:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.684548:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.684552:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.684554:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.684557:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.684558:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.684559:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.684560:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.684561:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.684562:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.684563:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.684567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.684574:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.684577:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.684578:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.684579:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.684580:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.684582:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.685150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.685157:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.685160:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.685163:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.685164:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.685166:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.685167:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.685169:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498710] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.685170:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.685171:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.685172:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.685175:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.685177:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.685179:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.685180:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.685180:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.685182:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.685182:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.685184:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.685186:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.685186:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677475.685192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:9.2:1619677475.685198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.685200:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.685202:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.685203:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.685204:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:8.0:1619677475.685205:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.685207:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.685208:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:8.0:1619677475.685208:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.685209:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.685210:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (23)++
00000400:00000200:8.0:1619677475.685210:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.685211:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677475.685212:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:8.0:1619677475.685212:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.685213:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.685268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.685271:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.685274:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.685276:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.685277:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.685279:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.685280:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.685282:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498709] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.685283:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.685284:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.685285:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.685288:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.685290:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.685293:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.685294:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.685302:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.685303:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.685304:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.685305:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.685306:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.685307:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.685310:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.685315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:8.0:1619677475.685317:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.685320:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.685320:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677475.685321:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:8.0:1619677475.685321:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.685323:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:8.0:1619677475.685323:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.685324:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.685325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:8.0:1619677475.685326:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.685453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.685459:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.685461:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.685463:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.685464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.685466:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.685467:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.685469:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498708] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.2:1619677475.685469:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e2feb400] (20)++
00000400:00000200:21.0:1619677475.685470:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.685471:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.685472:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.685472:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e2feb400] (21)++
00000800:00000200:20.0:1619677475.685473:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.59@o2ib3
00000400:00000200:21.0:1619677475.685477:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000400:00000200:20.0:1619677475.685477:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.59@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.685479:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.685480:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.685481:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.0:1619677475.685482:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.685483:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:20.0:1619677475.685483:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498707] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.685484:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:20.0:1619677475.685484:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.685485:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.685486:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:20.0:1619677475.685486:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:20.0:1619677475.685489:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.59@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.685490:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.685491:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.685491:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:20.0:1619677475.685491:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:9.2:1619677475.685491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.685492:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e2feb400] (22)--
00000800:00000200:20.0:1619677475.685493:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.685494:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.685495:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.685496:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.685497:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e2feb400] (21)++
00000800:00000200:24.0:1619677475.685498:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.685498:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197e2feb400] (22)++
00000800:00000200:20.0:1619677475.685499:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e2feb400] (23)--
00000800:00000200:24.0:1619677475.685500:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677475.685500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e2feb400] (22)--
00000400:00000200:24.0:1619677475.685502:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.685502:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:24.0:1619677475.685504:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:20.2:1619677475.685506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e2feb400] (21)++
00000800:00000200:8.0:1619677475.685507:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.685508:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677475.685510:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.685511:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e2feb400] (22)++
00000800:00000200:20.0:1619677475.685511:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.685512:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.685513:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.59@o2ib3
00000400:00000200:20.0:1619677475.685514:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.685516:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.59@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.685516:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.685517:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:21.0:1619677475.685518:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.685519:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e2feb400] (23)++
00000800:00000200:20.0:1619677475.685519:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197e2feb400] (24)--
00000400:00000200:21.0:1619677475.685520:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:20.0:1619677475.685520:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e2feb400] (23)--
00000400:00000200:21.0:1619677475.685521:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498706] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.685521:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e2feb400] (22)++
00000400:00000200:21.0:1619677475.685523:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.685523:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.59@o2ib3
00000400:00000200:21.0:1619677475.685524:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.685525:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:20.0:1619677475.685528:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.59@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.685529:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.59@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677475.685530:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.685531:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000400:00000200:20.0:1619677475.685532:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.685533:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498705] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.685535:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.685536:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.685537:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.685541:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.685542:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677475.685542:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:20.0:1619677475.685542:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.59@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.685543:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.685544:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677475.685545:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e2feb400] (23)++
00000800:00000200:21.0:1619677475.685546:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e2feb400] (24)--
00000800:00000200:21.0:1619677475.685547:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e2feb400] (23)--
00000800:00000200:20.0:1619677475.685547:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.685547:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.685548:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e2feb400] (22)--
00000800:00000200:20.0:1619677475.685548:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:25.2:1619677475.685550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:20.0:1619677475.685550:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.685550:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e2feb400] (21)++
00000800:00000200:20.0:1619677475.685551:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e2feb400] (22)--
00000800:00000200:20.0:1619677475.685552:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e2feb400] (21)--
00000800:00000200:9.2:1619677475.685555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:13.0:1619677475.685558:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677475.685560:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677475.685561:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.685562:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:13.0:1619677475.685563:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.685564:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.685565:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:12.0:1619677475.685565:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:8.0:1619677475.685565:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.685566:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:8.0:1619677475.685568:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.685571:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.685573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197e2feb400] (20)++
00000800:00000200:21.2:1619677475.685578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:21.0:1619677475.685582:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197e2feb400] (21)++
00000800:00000200:21.0:1619677475.685585:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.59@o2ib3
00000800:00000200:6.0:1619677475.685585:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677475.685587:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.59@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.685587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e2feb400] (22)--
00000800:00000200:22.0:1619677475.685588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.685588:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:21.0:1619677475.685590:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.685591:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:6.0:1619677475.685591:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677475.685591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd6c00] (20)++
00000400:00000200:21.0:1619677475.685593:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.685593:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.685594:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:21.0:1619677475.685595:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.685595:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498704] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.685596:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.685597:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:6.0:1619677475.685597:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.685598:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.685599:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd6c00] (21)++
00000400:00000200:6.0:1619677475.685599:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.685602:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.1@o2ib3
00000400:00000200:21.0:1619677475.685602:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.59@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000400:00000200:6.0:1619677475.685603:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.685605:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.1@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.685605:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677475.685606:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd6c00] (22)--
00000800:00000200:6.0:1619677475.685606:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:31.0:1619677475.685608:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.685608:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.685609:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000800:00000200:21.0:1619677475.685609:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:6.0:1619677475.685609:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.685610:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:6.0:1619677475.685610:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:31.0:1619677475.685611:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96498702] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.685611:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.685612:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.685612:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:31.0:1619677475.685613:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.685613:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197e2feb400] (21)++
00000800:00000200:21.0:1619677475.685613:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197e2feb400] (22)--
00000800:00000200:6.0:1619677475.685613:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.685613:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677475.685614:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000800:00000200:21.0:1619677475.685614:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197e2feb400] (21)--
00000800:00000200:6.0:1619677475.685614:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.685615:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:31.0:1619677475.685618:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677475.685618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:25.2:1619677475.685619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.685620:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:31.0:1619677475.685623:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.685624:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:31.0:1619677475.685625:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677475.685626:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:31.0:1619677475.685627:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:20.0:1619677475.685627:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677475.685628:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86fbd6c00] (21)++
00000800:00000200:31.0:1619677475.685629:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86fbd6c00] (22)++
00000800:00000200:26.0:1619677475.685629:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:20.0:1619677475.685629:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:26.0:1619677475.685630:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.685630:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.685630:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:31.0:1619677475.685631:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86fbd6c00] (23)--
00000800:00000200:31.0:1619677475.685631:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd6c00] (22)--
00000800:00000200:24.2:1619677475.685633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:21.0:1619677475.685633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:26.0:1619677475.685634:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:20.0:1619677475.685634:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.685636:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:4.2:1619677475.685637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd6c00] (21)++
00000800:00000200:26.0:1619677475.685638:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:24.0:1619677475.685638:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677475.685639:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.685639:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677475.685641:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.685641:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:24.0:1619677475.685644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:30.0:1619677475.685645:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd6c00] (22)++
00000800:00000200:30.0:1619677475.685647:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86fbd6c00] (23)--
00000800:00000200:30.0:1619677475.685648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd6c00] (22)--
00000800:00000200:30.0:1619677475.685649:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd6c00] (21)--
00000800:00000200:4.2:1619677475.685761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.685769:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.685771:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.685772:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.685774:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.685775:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.685777:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.685778:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498701] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.685779:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.685780:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.685781:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.685785:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.685786:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.685790:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.685790:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.685791:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.685792:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.685793:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.685794:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.685794:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.685799:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.685806:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.685809:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.685810:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.685810:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.685812:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.685815:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.685818:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.685825:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.685828:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.685828:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.685829:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.685830:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.685831:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.685888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.685893:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.685896:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.685898:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.685898:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.685901:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.685903:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677475.685904:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498700] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.685905:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.685906:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.685907:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.685911:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.685912:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.685915:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.685916:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.685917:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.685918:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.685919:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.685921:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.685922:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.685922:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677475.685924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.685927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677475.685931:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677475.685931:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:30.0:1619677475.685934:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677475.685934:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:30.0:1619677475.685935:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.685935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.685935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677475.685937:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.685938:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.685940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:5.2:1619677475.686030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.686037:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.686040:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.686040:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.686042:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.686044:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.686045:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.686047:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498699] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.686048:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.686049:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.686050:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.686053:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.686055:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.686057:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.686057:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.686058:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.686060:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.686060:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.686062:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.686063:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.686063:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.686066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.686068:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.686075:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.686075:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.686076:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.686076:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.686076:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.686077:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677475.686077:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.686078:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.686079:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.686079:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.686087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.686095:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.686098:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.686100:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.686101:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.686102:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.686104:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.686105:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498698] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.686106:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.686107:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.686109:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.686112:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.686113:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.686116:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.686116:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.686117:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.686118:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.686119:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.686120:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.686122:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.686122:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.686124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.686128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.686132:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.686134:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.686135:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.686136:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.686137:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.686138:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.686138:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.686139:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.686140:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.686141:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.686401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.686407:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.686410:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.686411:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.686412:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.686414:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.686415:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.686416:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498697] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.686417:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.686418:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.686419:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.686422:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.686424:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.686426:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.686427:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.686428:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.686429:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.686430:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.686431:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.686432:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.686432:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.686436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.686438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:20.0:1619677475.686444:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.686444:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.686445:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.686446:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:20.0:1619677475.686447:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.686447:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.686447:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:20.0:1619677475.686449:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.686451:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.686452:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.686861:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.686869:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:20.2:1619677475.686870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:24.0:1619677475.686872:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.686872:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.686874:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.686875:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.686877:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:8.0:1619677475.686877:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.686879:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.686880:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.686880:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:8.0:1619677475.686881:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498696] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.686882:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.686883:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.686883:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.686884:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.686885:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:21.2:1619677475.686887:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:8.0:1619677475.686888:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.686890:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.686891:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498695] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.686892:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.686893:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.686893:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.686894:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:8.0:1619677475.686894:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.686894:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.686895:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.686896:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.686896:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.686897:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:21.0:1619677475.686898:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.686898:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677475.686899:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.0:1619677475.686901:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:31.0:1619677475.686902:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.686903:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.686903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677475.686904:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.686904:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.686904:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.686905:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:31.0:1619677475.686906:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.686907:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96498694] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.686907:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.686907:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.686908:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.686908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:31.0:1619677475.686909:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.686909:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.686911:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677475.686911:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.2:1619677475.686912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:31.0:1619677475.686914:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.686914:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.686915:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.686915:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.686916:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:22.0:1619677475.686917:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.686918:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.686919:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.686920:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677475.686920:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.686920:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:31.0:1619677475.686922:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.686922:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677475.686922:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:31.0:1619677475.686923:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000400:00000200:20.0:1619677475.686923:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.686924:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:20.0:1619677475.686925:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.686926:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.686927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677475.686928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.686934:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.686936:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.686937:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.686937:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.686939:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.686942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.686947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.686953:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.686956:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.686957:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.686957:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.686958:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.686958:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.687113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.687119:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.687121:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.687122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.687124:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.687126:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.687127:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.687129:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498693] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.687130:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.687131:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.687132:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.687135:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.687136:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.2:1619677475.687139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:6.0:1619677475.687139:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.687140:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.687140:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.687142:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.687142:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.687143:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.687144:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.687144:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677475.687147:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677475.687148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677475.687150:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.2:1619677475.687150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677475.687152:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.687154:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.687154:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.687155:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:30.0:1619677475.687156:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.687157:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498692] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.687157:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.687157:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:30.0:1619677475.687158:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.687158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:30.0:1619677475.687159:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.687159:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.687159:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:30.0:1619677475.687160:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:21.0:1619677475.687160:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.687162:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:21.0:1619677475.687163:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.687165:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.687165:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.687167:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.687169:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.687170:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.687171:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.687172:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.687173:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.687174:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.687175:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.687178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.687185:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.687187:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.687188:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.687189:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.687191:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.687192:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.687850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.687857:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.687860:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.687861:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.687863:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.687866:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.687867:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.687869:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498691] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.687870:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.687871:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.687872:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:4.2:1619677475.687873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000400:00000200:24.0:1619677475.687875:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.687876:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.687879:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.687880:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677475.687881:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:24.0:1619677475.687881:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.687883:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.687883:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677475.687884:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:24.0:1619677475.687884:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.687885:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:10.0:1619677475.687885:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:24.0:1619677475.687886:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:26.0:1619677475.687887:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.687889:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.687889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:26.0:1619677475.687890:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677475.687891:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498690] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.687892:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677475.687892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677475.687893:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.687894:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677475.687897:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.687897:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677475.687898:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:8.0:1619677475.687899:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677475.687900:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:26.0:1619677475.687901:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:17.0:1619677475.687901:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.687901:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:1.0:1619677475.687901:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677475.687902:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:8.0:1619677475.687902:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677475.687903:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:24.0:1619677475.687903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:26.0:1619677475.687904:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000400:00000200:17.0:1619677475.687904:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.687905:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.687905:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.687906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:17.0:1619677475.687907:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:9.2:1619677475.687911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.687919:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.687922:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.687923:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.687923:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.687926:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.687928:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677475.688271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.688278:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.688280:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.688281:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.688283:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.688285:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.688286:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.688287:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498689] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.688288:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.688289:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.688290:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.688298:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.688300:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.688303:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.688303:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.688304:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.688306:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.688306:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.688307:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.688308:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.688312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.688320:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.688323:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.688324:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.688326:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.688327:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.688328:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.688332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677475.688338:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.688341:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.688342:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.688343:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.688343:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.688344:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.688412:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.688420:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.688422:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.688423:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.688425:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.688428:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.688429:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.688431:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498688] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.688432:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.688433:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.688434:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.688436:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.688438:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.688441:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.688442:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.688443:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.688444:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.688445:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.688446:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.688446:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.688461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.688469:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.688472:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.688473:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.688473:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.688475:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.688478:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.688632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.688640:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.688643:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.688644:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.688645:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.688648:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.688649:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.688650:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498687] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.688651:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.688652:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.688653:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.688656:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.688657:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.688659:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.688659:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.688660:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.688661:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.688662:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.688663:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.688664:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.688665:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677475.688667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.688670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677475.688674:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.688676:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:10.0:1619677475.688677:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677475.688677:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.688678:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.688679:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:8.0:1619677475.688679:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.688680:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.688681:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.0:1619677475.688682:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.688857:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.688863:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.688865:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.688866:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.688868:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.688870:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.688871:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.688873:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498686] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.688874:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.688875:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.688876:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.688879:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.688880:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.688883:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.688884:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.688884:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.688886:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.688886:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.688887:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.688888:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.688892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.688900:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.688902:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.688903:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.688904:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.688906:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.688907:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.688995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.689003:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.689006:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.689008:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.689011:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.689011:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.689012:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.689013:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498685] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.689015:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.689016:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.689017:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.689020:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.689022:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.689024:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.689025:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.689026:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.689027:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.689028:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.689029:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.689030:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.689043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.689050:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.689052:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.689053:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.689053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.689055:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.689057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.689088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.689096:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.689099:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.689101:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.689103:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.689103:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.689105:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.689106:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498684] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.689107:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.689108:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.689109:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.689112:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.689114:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.689116:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.689117:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.689118:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.689119:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.689120:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.689121:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.689121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.689125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.689131:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.689133:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.689134:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.689135:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.689136:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.689138:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.689523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.689530:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.689533:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.689535:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.689537:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.689537:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.689538:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:12.0:1619677475.689539:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498683] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.689540:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.689541:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.689542:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:12.0:1619677475.689545:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.689547:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:12.0:1619677475.689549:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.689550:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:12.0:1619677475.689550:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:12.0:1619677475.689552:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677475.689552:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.689553:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.689554:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677475.689558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.689566:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.689569:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.689570:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.689571:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.689573:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.689574:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.690087:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677475.690095:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.690098:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677475.690100:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.690102:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.690102:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677475.690104:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:31.0:1619677475.690105:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498682] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.690106:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.690107:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.690108:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677475.690111:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.690112:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:31.0:1619677475.690115:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.690116:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677475.690116:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677475.690118:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677475.690119:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.690119:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677475.690120:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:8.2:1619677475.690125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.690134:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.690136:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.690137:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.690138:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.690140:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.690143:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.690144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.690151:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.690154:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.690155:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.690155:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.690156:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.690157:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.690194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.690201:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.690203:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.690204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.690206:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.690209:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.690210:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.690211:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498681] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.690213:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.690214:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.690215:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.690218:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.690220:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.690222:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.690223:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.690224:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.690225:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.690226:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.690226:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.690226:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.690231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.690240:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.0:1619677475.690244:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.690246:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677475.690247:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:21.0:1619677475.690247:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.690249:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.690249:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:30.0:1619677475.690250:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.690251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.690251:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.690252:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000400:00000200:30.0:1619677475.690253:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.690253:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.690253:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.690254:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677475.690254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677475.690256:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.690257:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.690259:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498680] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.690260:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.690261:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.690262:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.690266:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.690267:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.690269:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.690270:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.690271:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.690272:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.690273:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.690274:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.690274:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677475.690277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:25.2:1619677475.690278:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677475.690283:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677475.690285:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.0:1619677475.690285:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:31.0:1619677475.690287:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:21.0:1619677475.690287:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.690288:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.690288:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677475.690289:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.690290:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.690291:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:21.0:1619677475.690291:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677475.690292:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:21.0:1619677475.690292:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498679] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.690295:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.2:1619677475.690298:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:30.0:1619677475.690302:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677475.690303:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.690304:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000400:00000200:21.0:1619677475.690304:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.690305:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000400:00000200:21.0:1619677475.690305:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:30.0:1619677475.690306:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.690307:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:22.0:1619677475.690307:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677475.690307:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:30.0:1619677475.690308:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.690309:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:22.0:1619677475.690310:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.690310:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:5.2:1619677475.690311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.0:1619677475.690312:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:22.0:1619677475.690313:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.690315:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:21.0:1619677475.690315:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.690316:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498678] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.690316:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:22.0:1619677475.690317:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.690317:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.690317:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:22.0:1619677475.690318:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.690318:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:22.0:1619677475.690319:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000800:00000200:21.0:1619677475.690319:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677475.690319:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:21.0:1619677475.690320:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.690320:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.690321:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.690323:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.690324:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677475.690325:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:1.0:1619677475.690325:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.690326:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000400:00000200:1.0:1619677475.690327:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:22.0:1619677475.690329:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000400:00000200:1.0:1619677475.690329:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:32, s:-96498677] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.690330:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:22.0:1619677475.690331:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000400:00000200:1.0:1619677475.690331:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.690332:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.690333:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:22.0:1619677475.690333:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.690334:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677475.690334:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:17.0:1619677475.690334:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:1.0:1619677475.690334:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.690335:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.690336:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.690338:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:1.0:1619677475.690338:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677475.690339:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677475.690340:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.690342:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.690343:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677475.690343:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.690343:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.690344:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.690346:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:2.0:1619677475.690347:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:1.0:1619677475.690347:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.690348:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.690348:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:2.0:1619677475.690349:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.690351:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.690351:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.2:1619677475.690352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:2.0:1619677475.690353:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.690356:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677475.690357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677475.690359:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.690361:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.690362:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.690363:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.690364:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.690365:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677475.690366:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:18.0:1619677475.690367:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:2.0:1619677475.690368:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.690369:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.690370:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.690370:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.690611:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.690619:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.690622:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.690623:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.690625:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.690627:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.690629:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.690630:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498676] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.690631:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.690632:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.690633:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.690636:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.690638:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.690641:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.690641:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.690642:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.690644:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.690644:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.690645:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.690645:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.690650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.690657:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.690660:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.690661:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.690661:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.690664:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.690666:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.690941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.690946:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.690949:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.690950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.690952:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.690954:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.690956:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.690957:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498675] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.690959:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.690960:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.690961:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.690964:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.690966:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.690968:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.690968:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.690969:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.690971:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.690971:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.690972:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.690973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.690977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.690984:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.690986:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.690987:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.690987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.690989:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.690990:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.690996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.691004:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.691006:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.691007:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.691007:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.691007:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.691008:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.691054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.691062:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.691064:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.691067:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.691068:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.691068:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.691070:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.691071:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498674] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.691072:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.691074:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.691075:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.691078:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.691079:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.691083:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.691084:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.691084:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.691086:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.691086:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.691088:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.691089:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.691090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.691092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.691095:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.691100:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.691102:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677475.691103:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:20.0:1619677475.691103:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.691105:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:20.0:1619677475.691105:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.691106:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.691106:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.691107:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.0:1619677475.691108:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.691163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.691170:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.691173:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.691175:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.691177:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.691178:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.691178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:30.0:1619677475.691180:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.691181:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498673] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.691182:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.691183:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.691183:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:30.0:1619677475.691184:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.691186:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:20.0:1619677475.691188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.691189:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.691190:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.691192:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.691192:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.691193:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:30.0:1619677475.691195:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.691195:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:21.0:1619677475.691195:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498672] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.691196:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:21.0:1619677475.691196:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.691197:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:21.0:1619677475.691197:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.691198:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:21.0:1619677475.691198:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.691199:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.691199:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:21.0:1619677475.691201:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.691203:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:9.2:1619677475.691203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.691206:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.691207:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.691207:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.691209:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:24.0:1619677475.691210:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677475.691210:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.691211:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.691211:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:24.0:1619677475.691212:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.691213:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.691213:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:25.2:1619677475.691215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:24.0:1619677475.691215:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.691218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.691222:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.691225:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.691226:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.691227:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.691228:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.691230:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.691631:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.691638:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.691641:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.691643:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.691645:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.691646:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.691647:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.691648:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498671] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.691649:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.691650:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.691651:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.691654:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.691656:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.691658:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.691659:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.691659:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.691661:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.691661:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.691662:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.691663:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.691667:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.691675:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.691677:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.691677:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.691679:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.691681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.691682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.691730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.691736:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.691739:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.691740:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.691742:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.691744:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.691746:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.691747:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498670] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.691748:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.691749:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.691750:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.691754:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.691756:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.691758:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.691759:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.691759:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.691761:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.691761:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.691762:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.691763:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.691767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.691774:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.691776:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.691777:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.691779:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.691780:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.691782:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.692280:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.692288:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.692290:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.692291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.692297:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.692300:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.692301:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.692303:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498669] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.692304:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.692305:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.692306:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.692310:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.692311:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.692314:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.692315:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.692316:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.692317:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.692318:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.692319:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.692319:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.692323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.692332:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.692335:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.692336:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.692336:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.692338:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.692341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.692736:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:17.0:1619677475.692742:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.692744:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:1.0:1619677475.692745:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.692747:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.692748:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.692750:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:17.0:1619677475.692751:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498668] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.692752:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.692753:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.692754:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:17.0:1619677475.692757:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.692759:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:17.0:1619677475.692762:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.692763:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.2:1619677475.692764:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677475.692764:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:17.0:1619677475.692766:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:17.0:1619677475.692766:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.692767:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677475.692767:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:31.0:1619677475.692772:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:25.2:1619677475.692772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677475.692775:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.692777:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.692779:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.692779:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.692780:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:13.0:1619677475.692780:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:31.0:1619677475.692781:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498667] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:13.0:1619677475.692782:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:31.0:1619677475.692783:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.692783:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.692784:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.692785:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:13.0:1619677475.692785:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.692787:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677475.692788:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:31.0:1619677475.692789:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.692791:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.692793:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.692794:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.692795:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.692796:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.692797:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.692798:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.692798:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.692802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.692809:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.692811:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.692812:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.692812:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.692814:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.692816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.692820:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.692826:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.692828:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.692829:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.692829:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.692830:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.692831:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.692833:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677475.692839:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.692842:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677475.692842:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000400:00000200:1.0:1619677475.692845:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.692847:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.692849:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677475.692850:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498666] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.692851:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.692852:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.692853:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677475.692857:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.692859:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.692861:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.692862:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.692863:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.692864:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.692865:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.692866:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:1.0:1619677475.692866:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:25.2:1619677475.692870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.692878:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.692881:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.692882:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.692882:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.692884:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.692886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.693233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.693240:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.693243:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.693244:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.693245:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.693248:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.693249:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.693250:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498665] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.693252:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.693253:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.693254:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.693257:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.693259:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.693262:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.693263:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.693263:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.693265:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.693265:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.693266:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.693267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.693271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.693280:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.693282:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.693283:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.693284:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.693286:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.693288:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.693344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.693353:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.693355:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.693356:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.693358:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.693361:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.693362:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.693363:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498664] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.693365:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.693366:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.693366:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.693369:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.693371:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.693373:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.693374:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.693374:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.693376:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.693376:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.693377:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.693377:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.693382:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.693389:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.693392:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.693393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.693393:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.693395:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.693398:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.693419:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.693426:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.693428:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.693429:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.693430:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.693433:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.693434:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.693435:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498663] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.693436:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.693437:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.693438:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.693440:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.693442:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.693445:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.693445:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.693446:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.693447:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.693448:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.693449:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.693449:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.693454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.693462:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.693464:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.693465:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.693466:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.693468:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.693470:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.693474:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:25.2:1619677475.693478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.693483:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.693485:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.693487:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000400:00000200:6.0:1619677475.693487:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.693488:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.693489:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.693490:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.693490:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.693491:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.693491:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:22.0:1619677475.693492:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.693493:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:6.0:1619677475.693493:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498662] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.693495:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.693496:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.693497:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.693500:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.693502:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.693505:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.693506:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.693506:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.693508:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.693508:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.693509:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.693510:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.693511:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.693514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.693516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.693521:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.693523:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.693524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.693524:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.2:1619677475.693524:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.0:1619677475.693528:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.693530:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677475.693531:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:21.0:1619677475.693531:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.693533:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.693534:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:20.0:1619677475.693534:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.693535:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:30.0:1619677475.693537:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.693539:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.693539:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.693540:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.693541:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498661] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.693542:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.693544:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.693545:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.2:1619677475.693547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000400:00000200:30.0:1619677475.693548:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.693549:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.693552:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.693552:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.693553:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.693555:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.693555:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:18.0:1619677475.693555:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:30.0:1619677475.693557:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.693558:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:18.0:1619677475.693558:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.693558:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:30.0:1619677475.693559:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:18.0:1619677475.693561:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.693563:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.693564:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:21.2:1619677475.693565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:18.0:1619677475.693566:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498660] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.693567:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.693568:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.693569:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:9.2:1619677475.693570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:18.0:1619677475.693572:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.693573:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:18.0:1619677475.693574:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.693575:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.693576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.693576:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677475.693576:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:18.0:1619677475.693577:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.693578:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.693579:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:18.0:1619677475.693579:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:24.0:1619677475.693580:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.693580:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:18.0:1619677475.693581:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.693582:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.693582:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.693583:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.693583:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:24.0:1619677475.693585:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.693587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.693593:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.693595:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.693596:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.693596:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.693598:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.693601:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.693996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.694001:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.694004:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.694005:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.694006:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.694009:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.694011:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.694012:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498659] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.694013:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.694014:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.694015:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.694019:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.694020:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.694023:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.694024:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.694024:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.694026:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.694027:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.694027:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.694028:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.694032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.694039:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.694042:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.694042:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.694044:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.694045:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.694047:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.694054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.694061:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.694064:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.694065:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.694065:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.694066:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.694067:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.694071:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed962600] (20)++
00000800:00000200:24.0:1619677475.694079:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed962600] (21)++
00000800:00000200:24.0:1619677475.694082:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.62@o2ib3
00000800:00000200:8.0:1619677475.694083:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed962600] (22)--
00000400:00000200:24.0:1619677475.694084:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.1.62@o2ib3 : GET - for me
00000400:00000200:24.0:1619677475.694087:0:19938:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.1.62@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:24.0:1619677475.694089:0:19938:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.1.62@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:24.0:1619677475.694091:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed962600] (21)++
00000800:00000200:24.0:1619677475.694093:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed962600] (22)++
00000800:00000200:24.0:1619677475.694095:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed962600] (23)--
00000400:00000200:24.0:1619677475.694096:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.1.62@o2ib3 local destination
00000400:00000200:24.0:1619677475.694100:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.1.62@o2ib3(10.51.1.62@o2ib3:10.51.1.62@o2ib3) : REPLY try# 0
00000800:00000200:4.2:1619677475.694101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed962600] (22)++
00000800:00000200:24.0:1619677475.694102:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.1.62@o2ib3
00000800:00000200:24.0:1619677475.694104:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919fedd18900] -> 10.51.1.62@o2ib3 (2) version: 12
00000800:00000200:24.0:1619677475.694105:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919fed962600] (23)++
00000800:00000200:24.0:1619677475.694106:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed962600] (24)++
00000800:00000200:24.0:1619677475.694107:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919fed962600] (25)--
00000800:00000200:24.0:1619677475.694108:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed962600] (24)--
00000800:00000200:24.0:1619677475.694109:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed962600] (23)++
00000800:00000200:24.0:1619677475.694110:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed962600] (24)--
00000800:00000200:24.0:1619677475.694111:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed962600] (23)--
00000800:00000200:24.0:1619677475.694112:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed962600] (22)++
00000800:00000200:24.0:1619677475.694113:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed962600] (23)--
00000400:00000200:24.0:1619677475.694113:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.694114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed962600] (22)--
00000400:00000200:24.0:1619677475.694115:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.1.62@o2ib3: REPLY: OK
00000800:00000200:24.0:1619677475.694117:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed962600] (21)--
00000800:00000200:20.2:1619677475.694538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.694545:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.694548:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.694548:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.694551:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.694554:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.694555:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:18.0:1619677475.694557:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498658] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.694558:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.694559:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.694560:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:18.0:1619677475.694563:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.694565:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:18.0:1619677475.694569:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.694570:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:18.0:1619677475.694570:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:18.0:1619677475.694572:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:18.0:1619677475.694573:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.694574:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677475.694575:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677475.694576:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:20.2:1619677475.694581:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.694586:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:2.0:1619677475.694589:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:2.0:1619677475.694589:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:9.2:1619677475.694590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:2.0:1619677475.694590:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:10.0:1619677475.694597:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.694600:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677475.694600:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.694601:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.694603:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.694606:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.694854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.694860:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.694863:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.694863:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.694866:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.694868:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.694869:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.694870:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498657] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.694871:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.694873:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.694873:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.694877:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.694878:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.694881:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.694882:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.694882:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.694884:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.694884:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.694885:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.694885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.694890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.694897:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.694900:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.694901:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.694902:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.694903:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.694905:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.695070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.695078:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.695081:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.695083:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:5.2:1619677475.695083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000400:00000200:31.0:1619677475.695085:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.695086:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.695087:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.695088:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498656] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.695089:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.695089:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000400:00000200:31.0:1619677475.695090:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.695091:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677475.695092:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.695093:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:31.0:1619677475.695095:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.695096:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.695098:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:1.0:1619677475.695098:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.695100:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:31.0:1619677475.695101:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.695102:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.695102:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:1.0:1619677475.695102:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498655] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.695103:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.695104:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.695104:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:1.0:1619677475.695104:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.695105:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677475.695105:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677475.695106:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:1.0:1619677475.695108:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:25.2:1619677475.695109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:1.0:1619677475.695110:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.695112:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.695112:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.695113:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.695115:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.695115:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.695115:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:30.0:1619677475.695116:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:1.0:1619677475.695116:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:30.0:1619677475.695119:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.695120:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.695121:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:8.2:1619677475.695121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:31.0:1619677475.695123:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.695124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:17.0:1619677475.695129:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.695131:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.695132:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.695132:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.695134:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.695135:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.695163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677475.695169:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.695172:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677475.695172:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677475.695175:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.695178:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.695179:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:18.0:1619677475.695181:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498654] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.695182:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.695183:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.695184:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:18.0:1619677475.695187:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.695189:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:18.0:1619677475.695192:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.695193:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:18.0:1619677475.695193:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:18.0:1619677475.695195:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677475.695196:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.695197:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677475.695197:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:25.2:1619677475.695211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.695219:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.695221:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.695222:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.695222:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.695225:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.695227:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.695237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.695244:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.695247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.695247:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.695248:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.695249:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.695249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:23.0:1619677475.695303:0:525:0:(o2iblnd_cb.c:3309:kiblnd_cm_callback()) Ignore TIMEWAIT_EXIT event
00000800:00000200:4.2:1619677475.695312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86fbd6c00] (20)++
00000800:00000200:31.0:1619677475.695321:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86fbd6c00] (21)++
00000800:00000200:31.0:1619677475.695323:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.1@o2ib3
00000400:00000200:31.0:1619677475.695326:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.1@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.695328:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.695328:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd6c00] (22)--
00000400:00000200:31.0:1619677475.695329:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:31.0:1619677475.695330:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498653] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.695332:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.695333:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.695334:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:31.0:1619677475.695337:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.1@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.695339:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:31.0:1619677475.695342:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.695342:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:31.0:1619677475.695343:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:31.0:1619677475.695344:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:31.0:1619677475.695345:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86fbd6c00] (21)++
00000800:00000200:31.0:1619677475.695346:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86fbd6c00] (22)--
00000800:00000200:31.0:1619677475.695347:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86fbd6c00] (21)--
00000800:00000200:24.2:1619677475.695351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.695354:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677475.695357:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677475.695358:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.695358:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677475.695359:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.695362:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677475.695524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:1.0:1619677475.695530:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.695533:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:17.0:1619677475.695534:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:1.0:1619677475.695536:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:1.0:1619677475.695538:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.695540:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:1.0:1619677475.695541:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498652] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.695542:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.695543:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.695544:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:1.0:1619677475.695548:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:1.0:1619677475.695549:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:1.0:1619677475.695552:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.695553:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:1.0:1619677475.695554:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.695555:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.695556:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:1.0:1619677475.695557:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:1.0:1619677475.695558:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:1.0:1619677475.695559:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:9.2:1619677475.695562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.695569:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.695571:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.695572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.695573:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.695575:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.695578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.695579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.695584:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.695587:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.695588:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.695588:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.695589:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.695589:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.695603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.695610:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.695613:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.695614:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.695616:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.695618:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.695620:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.695621:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498651] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.695622:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.695623:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.695624:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.695628:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.695628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:26.0:1619677475.695629:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.695632:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.695633:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.695633:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:17.0:1619677475.695633:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)++
00000800:00000200:26.0:1619677475.695635:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.695635:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:17.0:1619677475.695635:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc1200] (23)--
00000800:00000200:26.0:1619677475.695636:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:17.0:1619677475.695636:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:26.0:1619677475.695637:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:1.0:1619677475.695637:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:25.2:1619677475.695641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.695649:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.695651:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.695652:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.695653:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.695656:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.695656:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.695755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.695763:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.695765:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.695767:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.695769:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.695770:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.695771:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.695772:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498650] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.695773:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.695774:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.695775:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.695778:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.695780:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.695783:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.695784:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.695785:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.695787:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.695787:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.695789:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:20.2:1619677475.695789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:30.0:1619677475.695790:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.695791:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.695793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.695798:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.695801:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.695803:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.695803:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.695805:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.695806:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:20.0:1619677475.695808:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.695810:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.695810:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.695810:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.695810:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.695811:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498649] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.695812:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:21.0:1619677475.695812:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.695813:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:20.0:1619677475.695813:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.695814:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:20.0:1619677475.695814:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.695815:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677475.695815:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:31.0:1619677475.695816:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677475.695817:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677475.695818:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.695819:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.695819:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.695820:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.695820:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.695821:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.695822:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.695822:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:31.0:1619677475.695823:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.695823:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.695823:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.695824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.0:1619677475.695824:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.695825:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:20.0:1619677475.695825:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:31.0:1619677475.695826:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498648] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.695826:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:31.0:1619677475.695827:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.695827:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:31.0:1619677475.695828:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.695828:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677475.695829:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.695829:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.695831:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.695831:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:31.0:1619677475.695832:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.695834:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.2:1619677475.695834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.695836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:31.0:1619677475.695837:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.695837:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.695838:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.695840:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.695840:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677475.695841:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:31.0:1619677475.695842:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.695842:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:21.0:1619677475.695842:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.695844:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677475.695844:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677475.695845:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677475.695845:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:9.2:1619677475.695845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:1.0:1619677475.695846:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.695847:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677475.695847:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.695850:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.695852:0:19938:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677475.695857:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.695859:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:1.0:1619677475.695859:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677475.695860:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.695860:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.695862:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.695863:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:8.0:1619677475.695863:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.695864:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677475.695865:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:1.0:1619677475.695865:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.695865:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.695955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.695962:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.695965:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.695966:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.695968:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.695970:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.695972:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.695973:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498647] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.695974:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.695975:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.695976:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.695980:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.695982:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.695983:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:10.0:1619677475.695985:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.695985:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.695986:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.695987:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.695988:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.695988:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:17.0:1619677475.695989:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:10.0:1619677475.695989:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:17.0:1619677475.695992:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677475.695993:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:25.2:1619677475.695994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:17.0:1619677475.695995:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677475.695997:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.695998:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677475.696000:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498646] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.696001:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.696002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:17.0:1619677475.696002:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.696003:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.696004:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.696005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.696006:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.696006:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000400:00000200:22.0:1619677475.696008:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.696008:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.696010:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677475.696010:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.696011:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677475.696012:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.696013:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.696014:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.696015:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677475.696015:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:25.2:1619677475.696019:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.696027:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.696029:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.696030:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.696030:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.696032:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.696035:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.696259:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.696267:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.696270:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.696272:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.696274:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.696275:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.696276:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.696277:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498645] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.696279:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.696280:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.696281:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.696284:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.696286:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.696288:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.696289:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.696289:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.696291:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.696291:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.696292:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.696293:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.696299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.696307:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.696309:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.696310:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.696312:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.696313:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.696315:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.696404:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.696409:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.696412:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.696413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.696415:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.696417:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.696419:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.696420:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498644] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.696421:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.696422:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.696423:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.696427:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.696428:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.696431:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.696431:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.696432:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.696434:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.696434:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.696435:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.696436:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.696440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.696448:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.696450:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.696451:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.696453:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.696454:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.696455:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.696458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.696465:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.696467:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.696468:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.696469:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.696470:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.696470:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677475.697021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:30.0:1619677475.697028:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.697031:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:30.0:1619677475.697033:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.697035:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:30.0:1619677475.697035:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.697037:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:30.0:1619677475.697038:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498643] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.697040:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.697040:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.697042:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:30.0:1619677475.697045:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.697047:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:30.0:1619677475.697050:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.697051:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:30.0:1619677475.697052:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.697053:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:30.0:1619677475.697054:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:30.0:1619677475.697056:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871380600] (22)++
00000800:00000200:30.0:1619677475.697057:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (23)--
00000800:00000200:30.0:1619677475.697058:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:9.2:1619677475.697060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.697063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (21)++
00000800:00000200:2.0:1619677475.697067:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:21.2:1619677475.697069:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677475.697070:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.697071:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.697072:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.697073:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.697074:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (22)++
00000800:00000200:31.0:1619677475.697075:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871380600] (23)--
00000800:00000200:31.0:1619677475.697076:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000800:00000200:2.0:1619677475.697077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:31.0:1619677475.697078:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.697079:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:30.0:1619677475.697079:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:30.0:1619677475.697080:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.697081:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.697084:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.697086:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.697087:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498642] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.697088:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.697089:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.697091:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.697094:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.697096:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.697099:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.697100:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.697100:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.697102:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.697102:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.697103:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.697104:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.697108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.697116:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.697118:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.697119:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.697121:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.697122:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.697124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.697523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.697531:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.697534:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.697537:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.697539:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.697540:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.697541:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.697543:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498641] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.697544:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.697545:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.697546:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.697549:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.697550:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.697553:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.697554:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.697554:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.697556:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.697556:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.697557:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.697558:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.697562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.697569:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.697571:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.697572:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.697573:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.697574:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.697577:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.697680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.697687:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.697690:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.697690:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.697692:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.697694:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.697696:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.697697:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498640] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.697699:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.697700:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.697701:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.697704:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.697705:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.697709:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.697710:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.697710:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.697712:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.697713:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.697713:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.697714:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.697718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.697726:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.697728:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.697729:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.697730:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.697732:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.697735:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.697950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.697956:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.697959:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.697960:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.697962:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.697965:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.697966:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.697967:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498639] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.697969:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.697970:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.697971:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.697974:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.697976:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.697977:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.697978:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.697979:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.697980:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.697980:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.697982:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677475.697983:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677475.697983:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677475.697987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.697989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.697995:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.697996:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.697996:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.697996:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677475.697997:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677475.697997:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.697998:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.697998:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.697999:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:17.0:1619677475.698000:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.698079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.698084:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.698087:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.698088:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.698090:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.698092:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.698094:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:22.0:1619677475.698095:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498638] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.698096:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.698097:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.698098:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677475.698102:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.698104:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677475.698107:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.698108:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677475.698108:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:22.0:1619677475.698110:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677475.698111:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.698111:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.698112:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677475.698116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.698119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:13.0:1619677475.698124:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677475.698126:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:13.0:1619677475.698126:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.698128:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.698129:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:13.0:1619677475.698130:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.698130:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:12.0:1619677475.698131:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:26.0:1619677475.698132:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.698133:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:26.0:1619677475.698135:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.698136:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.698137:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498637] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677475.698138:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000400:00000200:26.0:1619677475.698139:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.698140:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.698141:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.698144:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.698144:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:26.0:1619677475.698145:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:21.0:1619677475.698146:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:26.0:1619677475.698148:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.698149:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:21.0:1619677475.698149:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.698149:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:26.0:1619677475.698150:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.698151:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.698152:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:21.0:1619677475.698152:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.698153:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:21.2:1619677475.698153:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:26.0:1619677475.698154:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.698155:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:21.0:1619677475.698157:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:9.2:1619677475.698157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:21.0:1619677475.698158:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498636] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:6.0:1619677475.698158:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677475.698159:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.698160:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677475.698160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000400:00000200:21.0:1619677475.698161:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:6.0:1619677475.698161:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.698162:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677475.698163:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.698165:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.698166:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.698166:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.698167:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.698167:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677475.698167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.698168:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.698169:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:10.0:1619677475.698169:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677475.698169:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.698170:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677475.698170:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:24.0:1619677475.698171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677475.698171:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.0:1619677475.698172:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000400:00000200:6.0:1619677475.698172:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498635] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.698173:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.698173:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677475.698173:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.698174:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.698175:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000400:00000200:6.0:1619677475.698175:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.698179:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.698180:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.698180:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.698181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.698181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677475.698181:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.698183:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.698184:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.698184:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.698186:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.698186:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.698187:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.698188:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:18.0:1619677475.698190:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:25.2:1619677475.698192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.698192:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.698193:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.698194:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.698196:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.698198:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677475.698200:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.698202:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.698203:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.698204:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.698205:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.698207:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.698509:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871380600] (20)++
00000800:00000200:31.0:1619677475.698517:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.698520:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.39@o2ib3
00000400:00000200:31.0:1619677475.698522:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.39@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.698525:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.698525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (22)--
00000400:00000200:31.0:1619677475.698527:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677475.698528:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498634] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.698530:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.698530:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.698532:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:31.0:1619677475.698536:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.698537:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.698541:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.698541:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677475.698542:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:31.0:1619677475.698544:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677475.698544:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871380600] (21)++
00000800:00000200:31.0:1619677475.698545:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871380600] (22)--
00000800:00000200:31.0:1619677475.698546:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871380600] (21)--
00000800:00000200:9.2:1619677475.698550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.698558:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.698561:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.698562:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.698562:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.698565:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.698568:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.698674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.698681:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.698684:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.698685:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.698687:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.698690:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.698691:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.698692:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498633] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.698694:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.698695:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.698695:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.698698:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.698700:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.698703:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.698704:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.698704:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.698706:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.698706:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.698707:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.698708:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.698709:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.698711:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.698714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.698719:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.698721:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.698722:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.698723:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677475.698723:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.698723:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.698724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.698725:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:1.0:1619677475.698726:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.698729:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.698730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.698738:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.698740:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.698741:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.698741:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.698742:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.698743:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.699165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.699171:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.699173:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.699174:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.699176:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.699177:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.699179:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.699180:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498632] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.699181:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.699182:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.699183:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.699186:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.699188:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.699191:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.699192:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.699192:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.699194:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.699194:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.699195:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.699196:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.699200:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.699208:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.699211:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.699212:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.699213:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.699214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.699216:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.699222:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.699229:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.699232:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.699233:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.699234:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.699234:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.699236:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.699506:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677475.699510:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.699514:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.699516:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677475.699517:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.699520:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.699520:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.699520:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677475.699522:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677475.699523:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.699524:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.699526:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677475.699526:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.699527:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677475.699528:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.699529:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498631] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.699530:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.699531:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.699532:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.699532:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.699533:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677475.699534:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.699540:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.699541:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.699543:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.699544:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677475.699546:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.699547:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.699548:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:22.0:1619677475.699548:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677475.699549:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677475.699549:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677475.699551:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677475.699551:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.699552:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.699552:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677475.699553:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677475.699553:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:30.0:1619677475.699554:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.699554:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.699555:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.699558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:9.2:1619677475.699558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.699560:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.0:1619677475.699565:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:6.0:1619677475.699565:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.699567:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.0:1619677475.699568:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:6.0:1619677475.699568:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:24.0:1619677475.699569:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677475.699569:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.699569:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.699570:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.699570:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.699570:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.699573:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.699573:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.699573:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.699575:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677475.699576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677475.699660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.699668:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.699670:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.699671:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.699673:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.699675:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.699677:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.699678:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498629] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.699680:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.699681:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.699682:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.699686:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.699687:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.699689:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.699690:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.699691:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.699692:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.699693:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.699693:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.699694:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.699698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.699706:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.699708:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.699709:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.699710:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.699711:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.699712:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.700368:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.700376:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.700378:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.700379:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.700381:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.700384:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.700385:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.700387:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498628] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.700388:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.700389:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.700390:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.700393:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.700395:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.700398:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.700399:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.700400:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.700401:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.700402:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.700403:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677475.700404:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.700404:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677475.700408:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.700410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677475.700417:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:10.0:1619677475.700417:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:21.0:1619677475.700419:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:10.0:1619677475.700419:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.700420:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:21.0:1619677475.700421:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.700421:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:21.0:1619677475.700422:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.700423:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.700425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.700493:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.700499:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.700502:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.700503:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.700505:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.700507:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.700508:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.700510:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498627] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.700511:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.700512:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.700513:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.700516:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.700518:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.700519:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.700520:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.700521:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.700523:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.700523:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.700524:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.700525:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.700529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.700537:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.700539:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.700540:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.700540:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.700541:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.700543:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.700548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.700556:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.700558:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.700559:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.700559:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.700559:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.700560:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.700637:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.700645:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.700648:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.700650:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.700652:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.700652:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.700654:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.700655:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498626] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.700656:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.700657:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.700658:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.700661:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.700661:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:31.0:1619677475.700663:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:20.2:1619677475.700664:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:31.0:1619677475.700665:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.700665:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.700666:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.700667:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.700668:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.700669:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:24.0:1619677475.700669:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:31.0:1619677475.700670:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:2.0:1619677475.700670:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677475.700672:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:25.2:1619677475.700673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:18.0:1619677475.700673:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677475.700673:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:2.0:1619677475.700673:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:24.0:1619677475.700676:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.700676:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:24.0:1619677475.700679:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.700679:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.700680:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:24.0:1619677475.700681:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677475.700681:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:30.0:1619677475.700682:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.700683:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.700683:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.700683:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498625] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.700685:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:24.0:1619677475.700685:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.700685:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.700686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:24.0:1619677475.700686:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.700687:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:2.0:1619677475.700687:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.700688:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677475.700688:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:25.2:1619677475.700690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000400:00000200:24.0:1619677475.700693:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677475.700694:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:24.0:1619677475.700696:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.700697:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:2.0:1619677475.700697:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.700699:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.700699:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.700700:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:24.0:1619677475.700700:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677475.700700:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:31.0:1619677475.700701:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:24.0:1619677475.700701:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.700701:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677475.700702:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:24.0:1619677475.700702:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.700703:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:24.0:1619677475.700703:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:2.0:1619677475.700703:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:30.0:1619677475.700704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:24.0:1619677475.700704:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:2.0:1619677475.700704:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:24.0:1619677475.700706:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677475.700706:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:2.0:1619677475.700706:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.700707:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.700707:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.700708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:8.2:1619677475.700709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.700712:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:20.2:1619677475.700713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:6.0:1619677475.700716:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677475.700718:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:6.0:1619677475.700719:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.700720:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:18.0:1619677475.700720:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677475.700720:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:8.0:1619677475.700720:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:18.0:1619677475.700721:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:17.0:1619677475.700721:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.700721:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.700722:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:1.0:1619677475.700722:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.700723:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677475.700723:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:2.0:1619677475.700723:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:24.0:1619677475.700724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000400:00000200:17.0:1619677475.700724:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:6.0:1619677475.700724:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.700726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.700727:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:6.0:1619677475.700729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.700731:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.700733:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.700734:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.700734:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.700735:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.700736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.700888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.700896:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.700898:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.700899:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.700901:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.700904:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.700905:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.700907:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498623] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.700909:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.700910:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.700911:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.700914:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.700916:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.700919:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.700920:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.700920:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.700922:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.700923:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.700923:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.700924:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.700928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.700936:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.700938:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.700939:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.700939:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.700942:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.700944:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.700966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.700969:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.700971:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.700974:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.700974:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.700976:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.700978:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.700979:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498622] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.700981:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.700982:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.700983:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.700986:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.700988:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.700991:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.700991:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.700992:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.700994:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.700995:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.700995:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.700996:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.701000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.701008:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.701010:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.701011:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.701013:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.701014:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.701015:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.701031:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.701037:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.701039:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.701040:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.701042:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.701044:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.701045:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.701047:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498621] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.701048:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.701049:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.701050:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.701053:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.701055:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.701057:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.701058:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.701058:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.701060:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.701061:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.701061:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.701062:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.701067:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.701074:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.701076:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.701077:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.701079:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.701079:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.701081:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.701191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.701198:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.701201:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.701203:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.701205:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.701206:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.701207:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.701209:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498620] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.701210:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.701211:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.701212:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.701215:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.701217:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.701219:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.701220:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.701221:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.701222:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.701223:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.701225:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.701226:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.701226:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.701228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.701232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.701235:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.701237:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.701238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.701238:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.701240:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.701240:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.701242:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.701243:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677475.701243:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.701244:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.701246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.2:1619677475.701248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677475.701252:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.701254:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.701255:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.701255:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.701255:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.701256:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.701256:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.701258:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.701260:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.701262:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.701262:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.701263:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.701264:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498619] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.701266:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.701267:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.701268:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.701271:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.701272:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.701274:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.701275:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.701275:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.701276:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.701277:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.701278:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.701279:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.701280:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.701282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.701285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.701290:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.701291:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.701292:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.701294:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677475.701301:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.701301:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.701302:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.701303:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.701304:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.701305:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.701775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.701783:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.701786:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.701788:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.701789:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.701790:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.701791:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.701792:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498618] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.701793:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.701794:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.701795:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.701797:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.701799:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.701801:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.701802:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.701802:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.701804:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.701804:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.701805:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.701805:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.701810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.701817:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.701819:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.701820:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.701820:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.701822:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.701824:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.701828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.701834:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.701837:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.701838:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.701838:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.701839:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.701839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.702106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677475.702113:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677475.702116:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677475.702117:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677475.702119:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.702121:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.702123:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677475.702124:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498617] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.702125:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.702126:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.702127:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677475.702130:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.702131:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677475.702134:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.702134:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677475.702135:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677475.702136:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677475.702136:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677475.702137:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677475.702138:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677475.702142:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.702145:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677475.702148:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677475.702149:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.702149:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677475.702150:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.702152:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677475.702199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.702207:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.702209:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.702211:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.702213:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.702213:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.702214:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.702216:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498616] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.702217:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.702218:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.702218:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.702221:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.702223:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.702225:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.702225:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.702226:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.702227:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.702228:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.702229:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.702229:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.702233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.702240:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.702242:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.702243:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.702244:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.702246:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.702247:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.702498:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.702504:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.702506:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.702508:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.702509:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.702510:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.702512:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.702513:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498615] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.702515:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.702516:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.702517:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.702520:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.702522:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.702524:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.702525:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.702525:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.702527:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.702528:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.702529:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.702529:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.702534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.702543:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.702545:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.702546:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.702547:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.702549:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.702551:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.702811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.702817:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.702819:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.702821:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.702822:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.702825:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.702826:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.702827:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498614] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.702829:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.702830:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.702831:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.702834:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.702836:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.702838:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:4.2:1619677475.702838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:22.0:1619677475.702839:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.702840:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.702841:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.702842:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.702842:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.702843:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:8.0:1619677475.702845:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:25.2:1619677475.702848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677475.702848:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.702848:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.702849:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.702851:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.702852:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.702854:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.702854:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.702855:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000400:00000200:8.0:1619677475.702856:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:20.0:1619677475.702858:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.702858:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.702858:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498613] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.702859:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.702860:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.702860:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.702861:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677475.702861:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.702863:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:20.0:1619677475.702863:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:24.0:1619677475.702865:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498612] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.702866:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677475.702867:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.702867:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.702868:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.702868:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.702872:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677475.702873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000400:00000200:24.0:1619677475.702873:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.702873:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.702874:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.702874:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.702876:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.702877:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.702877:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (23)++
00000800:00000200:24.0:1619677475.702878:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.702878:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (24)--
00000800:00000200:24.0:1619677475.702879:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.702879:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.702880:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.702881:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.702881:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677475.702881:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677475.702882:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.702882:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.702883:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.0:1619677475.702883:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.702884:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.702885:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.702885:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.702886:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.702887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.702891:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.702894:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.702895:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.702895:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:1.0:1619677475.702895:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.702897:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.702897:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.702898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.702899:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.702900:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:6.0:1619677475.702901:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.702904:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.702997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.703005:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.703008:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.703009:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.703011:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.703013:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.703015:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.703016:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498611] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.703017:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.703018:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.703019:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.703022:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.703024:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.703027:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.703028:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.703029:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.703031:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.703032:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.703033:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:18.0:1619677475.703033:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677475.703037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.703045:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.703047:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.703048:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.703050:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.703052:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.703053:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.703127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.703132:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.703134:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.703137:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.703137:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.703139:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.703141:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.703142:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498610] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.703143:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.703144:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.703145:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.703148:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.703150:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.703152:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.703153:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.703153:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.703155:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.703155:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.703156:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.703157:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.703161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.703168:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.703170:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.703171:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.703173:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.703174:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.703175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.703242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.703246:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:30.0:1619677475.703250:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.2:1619677475.703252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.703253:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:26.0:1619677475.703254:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:30.0:1619677475.703255:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.703256:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.703256:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.703257:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677475.703258:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.703258:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:31.0:1619677475.703259:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677475.703260:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498609] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.703260:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.703261:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.703261:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.703262:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.703263:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.703263:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:30.0:1619677475.703264:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.703264:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498608] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.703266:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.703266:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.703267:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.703268:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.703268:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.703268:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677475.703270:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677475.703270:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.703271:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96498607] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.703272:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.703273:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.703273:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.703274:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677475.703274:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:30.0:1619677475.703277:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.703278:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.703278:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.703278:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.703278:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.703278:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.703279:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677475.703280:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.703280:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.703281:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:26.0:1619677475.703281:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.703281:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677475.703282:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.703282:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.703282:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677475.703282:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:31.0:1619677475.703283:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677475.703283:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:31.0:1619677475.703284:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.703284:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.703285:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.703286:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.703286:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.703286:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.703287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.703287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:9.2:1619677475.703291:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677475.703301:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.703304:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:8.0:1619677475.703304:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:22.0:1619677475.703305:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.703305:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:8.0:1619677475.703306:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:22.0:1619677475.703307:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.703307:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.703309:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:22.0:1619677475.703309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:24.0:1619677475.703310:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.703310:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:24.0:1619677475.703311:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.703312:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.703312:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.703314:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.703314:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:8.0:1619677475.703315:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.703321:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.703328:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.703331:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.703332:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.703334:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.703336:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.703338:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.703339:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498606] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.703340:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.703341:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.703342:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.703345:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.703347:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.703349:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.703349:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.703350:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.703351:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.703352:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.703352:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.703353:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.703357:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.703363:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.703367:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.703368:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.703369:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.703370:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.703372:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677475.703978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.703984:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.703987:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.703989:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.703990:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.703992:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.703993:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.703995:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498605] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.703996:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.703997:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.703998:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.704000:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.704002:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.704004:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.704005:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.704005:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.704007:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.704007:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.704008:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.704009:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.704010:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.704014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.704016:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.704022:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.704023:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.704023:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.704024:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.704024:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.704025:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.704025:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:17.0:1619677475.704026:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.704026:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.704028:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.704032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.704039:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.704042:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.704043:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.704043:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.704043:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.704044:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.704204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.704210:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.704214:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.704215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.704217:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.704219:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.704220:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.704221:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498604] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.704222:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.704224:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.704225:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.704228:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.704230:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.704232:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.704233:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.704234:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.704235:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.704236:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.704237:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.704237:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.704242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.704250:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.704252:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.704253:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.704255:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.704255:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.704257:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.704767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.704775:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.704778:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.704779:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.704781:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.704784:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.704785:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.704787:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498603] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.704788:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.704789:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.704790:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.704794:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.704796:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.704798:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.704799:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.704800:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.704802:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.704802:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.704803:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.704803:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.704808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.704817:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.704819:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.704820:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.704820:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.704823:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.704825:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.704909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.704918:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.704921:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.704923:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.704925:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.704925:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.704927:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.704928:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498602] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.704929:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.704930:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.704931:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.704934:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.704936:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.704938:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.704939:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.704939:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.704941:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.704941:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.704942:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.704943:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.704947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.704953:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.704956:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.704957:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.704957:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.704959:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.704961:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.705352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.705360:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.705363:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677475.705365:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677475.705365:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.705368:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.705369:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677475.705371:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498601] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.705372:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.705373:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.705374:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677475.705377:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.705378:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677475.705381:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.705382:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677475.705382:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677475.705384:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677475.705384:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.705385:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.705385:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677475.705390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.705399:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.705402:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677475.705403:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.705404:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.705405:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.705409:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.705752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.705760:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.705762:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.705763:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.705765:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.705767:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.705769:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.705770:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498600] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.705772:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.705773:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.705773:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.705776:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.705778:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.705781:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.705782:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.705782:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.705784:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.705784:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.705785:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.705785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.705790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.705793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.705798:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.705801:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.705801:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.705802:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:22.0:1619677475.705802:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.705804:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.705805:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.705807:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.705807:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:31.0:1619677475.705809:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.705810:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677475.705810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.705812:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498599] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.705813:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.705814:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.705815:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.705818:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.705819:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.705821:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.705822:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.705822:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.705824:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.705824:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.705826:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.705827:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.705828:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.705829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.705834:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.705838:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.705839:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.705840:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.705842:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.705842:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.705843:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.705843:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.705844:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.705845:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.705845:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.705846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:6.0:1619677475.705850:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.705853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.705853:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.705856:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.705858:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.705859:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.705861:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498598] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.705862:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.705863:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.705864:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.705867:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.705869:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.705871:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.705872:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.705873:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.705874:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.705875:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.705876:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.705877:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.705877:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.705880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.705882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.705887:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.705888:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.705889:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.705889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.705890:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.705890:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.705891:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.705893:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.705894:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.705895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.706206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.706211:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.706213:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.706216:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.706216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.706218:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.706219:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.706221:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498597] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.706222:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.706223:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.706224:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.706227:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.706229:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.706231:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.706232:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.706233:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.706234:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.706235:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.706236:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.706236:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.706241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.706249:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.706251:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.706252:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.706253:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.706254:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.706256:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.706256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677475.706263:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.706266:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677475.706267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677475.706269:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.706272:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.706273:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.706275:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498596] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.706276:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.706277:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.706278:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.706281:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.706283:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.706286:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.706287:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.706288:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.706289:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.706290:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.706291:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677475.706291:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:25.2:1619677475.706297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.706305:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.706308:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.706309:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.706309:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.706311:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.706314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.706318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.706319:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.706321:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.706322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.706323:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.706324:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.706324:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.706335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.706342:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.706344:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.706345:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.706347:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.706350:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.706351:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.706352:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498595] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.706354:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.706355:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.706356:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.706359:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.706361:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.706364:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.706365:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.706366:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.706367:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.706368:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.706369:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.706369:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.706373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.706381:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.706384:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.706385:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.706386:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.706390:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.706391:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.706392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:20.2:1619677475.706396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:13.0:1619677475.706399:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.0:1619677475.706402:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:13.0:1619677475.706402:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.706403:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.706404:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.706404:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:21.0:1619677475.706405:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000800:00000200:13.0:1619677475.706405:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:21.0:1619677475.706407:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.706408:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677475.706410:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.706411:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.706413:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498594] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.706414:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.706415:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.706416:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.706419:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.706420:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.706423:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.706424:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.706424:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.706426:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.706426:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.706427:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197ae708600] (22)++
00000800:00000200:21.0:1619677475.706429:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (23)--
00000800:00000200:21.0:1619677475.706429:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:8.2:1619677475.706432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.706435:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.706441:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.706442:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.706443:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.706443:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (22)++
00000400:00000200:21.0:1619677475.706444:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.706444:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197ae708600] (23)--
00000800:00000200:20.0:1619677475.706445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677475.706446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.706447:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:20.0:1619677475.706447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.706451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.706458:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.706461:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.706462:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.706463:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.706464:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.706465:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.706550:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.706558:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.706561:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.706563:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.706565:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.706565:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.706566:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.706568:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498593] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.706569:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.706570:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.706571:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.706574:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.706576:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.706578:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.706578:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.706579:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.706581:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.706581:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.706582:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.706583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.706587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.706593:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.706596:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.706597:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.706598:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.706600:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.706601:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.706604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.706611:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.706613:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.706614:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.706615:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.706615:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.706616:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.706891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.706899:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.706901:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.706903:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.706905:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.706905:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.706906:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.706907:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498592] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.706908:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.706909:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.706910:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.706913:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.706914:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.706917:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.706917:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.706918:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.706919:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.706920:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.706921:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.706921:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.706925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.706932:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.706934:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.706935:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.706936:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.706937:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.706939:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.707250:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.707258:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.707261:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.707262:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.707264:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.707267:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.707268:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677475.707270:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498591] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.707271:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.707272:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.707273:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677475.707275:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.707277:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.707279:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.707279:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677475.707280:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.707281:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.707282:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.707282:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.707283:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.707287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.707293:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.707298:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677475.707301:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677475.707302:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.707303:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.707304:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.707603:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.707609:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.707612:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.707613:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.707615:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.707617:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.707619:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.707620:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498590] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.707621:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.707622:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.707623:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.707626:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.707628:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.707630:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.707631:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.707632:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.707633:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.707634:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.707634:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.707635:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.707640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.707648:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.707651:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.707651:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.707653:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.707654:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.707655:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.707676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.707682:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.707685:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:20.2:1619677475.707686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:17.0:1619677475.707686:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.707688:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.707690:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.707691:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:1.0:1619677475.707691:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.707692:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498589] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.707693:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:1.0:1619677475.707694:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.707695:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.707696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.707697:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.707697:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.707700:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.707702:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.707702:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.707704:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.707705:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498588] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.707705:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.707705:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:21.0:1619677475.707706:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.707706:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:21.0:1619677475.707707:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.707708:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.707708:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.707708:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.707709:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.707710:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000400:00000200:21.0:1619677475.707712:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.707713:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:8.2:1619677475.707714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.707716:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.707717:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.707718:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (23)++
00000800:00000200:21.0:1619677475.707719:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (24)--
00000800:00000200:21.0:1619677475.707720:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.707721:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.707721:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:21.0:1619677475.707722:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.707723:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.707723:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:20.0:1619677475.707723:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.707724:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)++
00000800:00000200:21.0:1619677475.707725:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (24)--
00000400:00000200:21.0:1619677475.707725:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.707725:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.707727:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.2:1619677475.707727:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.707731:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.707732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.707733:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.707734:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.707736:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.707737:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.707738:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:4.2:1619677475.707798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:13.0:1619677475.707806:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.707809:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.707811:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677475.707813:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.707813:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.707815:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:13.0:1619677475.707816:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498587] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.707818:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.707819:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.707820:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:13.0:1619677475.707823:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.707825:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:13.0:1619677475.707827:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.707828:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:13.0:1619677475.707828:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:13.0:1619677475.707830:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:13.0:1619677475.707831:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.707832:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677475.707833:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677475.707834:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:25.2:1619677475.707836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.707839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:31.0:1619677475.707843:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.707846:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.707847:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.707847:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000400:00000200:31.0:1619677475.707848:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.707849:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000800:00000200:31.0:1619677475.707850:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:12.0:1619677475.707850:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:30.0:1619677475.707851:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:12.0:1619677475.707851:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:21.2:1619677475.708056:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.708063:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677475.708064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:31.0:1619677475.708066:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.708068:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.708070:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.708070:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.708071:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:26.0:1619677475.708071:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677475.708072:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498586] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.708073:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.708074:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677475.708074:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.708075:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:31.0:1619677475.708076:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.708077:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.708079:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.708079:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.708080:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:31.0:1619677475.708081:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.708081:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498585] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.708083:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.708083:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.708084:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.708085:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:26.0:1619677475.708085:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.708086:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.708088:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000400:00000200:26.0:1619677475.708088:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.708089:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.708089:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:26.0:1619677475.708089:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.708090:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:26.0:1619677475.708092:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.708092:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.708093:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.708094:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.2:1619677475.708094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.708095:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.708096:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.708097:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.708098:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677475.708100:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677475.708103:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:4.2:1619677475.708104:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677475.708106:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:8.0:1619677475.708106:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:20.0:1619677475.708107:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.708108:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.708109:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.708109:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.708110:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.708110:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.708112:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:10.0:1619677475.708112:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:8.0:1619677475.708112:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.708114:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:8.0:1619677475.708114:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677475.708115:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.708116:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:4.2:1619677475.708192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.708198:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.708201:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.708202:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.708204:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.708206:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.708208:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.708209:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498584] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.708210:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.708211:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.708212:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.708215:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.708217:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:4.2:1619677475.708218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677475.708220:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.708220:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.708221:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.708222:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.708223:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677475.708224:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677475.708224:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.708225:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.708226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677475.708227:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.708228:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.708229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677475.708230:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677475.708231:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677475.708232:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.708233:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.708234:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498583] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.708235:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.708236:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:6.0:1619677475.708236:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:26.0:1619677475.708237:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:8.0:1619677475.708238:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.708239:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:26.0:1619677475.708240:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.708240:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.708240:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:6.0:1619677475.708240:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.708241:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:8.0:1619677475.708241:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.708242:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677475.708242:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.708244:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.708245:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.708245:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:26.0:1619677475.708246:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.708247:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.708248:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.708249:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.708250:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.708251:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.708253:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.708256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.708261:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.708263:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677475.708263:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:22.0:1619677475.708264:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.708264:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.708265:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677475.708266:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.708266:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.708267:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677475.708269:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.708559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.708568:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.708571:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.708573:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.708575:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.708576:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.708578:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.708579:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498582] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.708580:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.708581:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.708582:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.708585:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.708587:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.708589:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.708590:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.708590:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.708592:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.708593:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.708594:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.708596:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.708596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.708599:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.708602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.708606:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.708608:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.708609:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.708609:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.708610:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.708611:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.708612:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.708613:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677475.708613:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.708614:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.708616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.708622:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.708624:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.708625:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.708626:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.708626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.708627:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677475.708684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.708690:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.708693:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.708694:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.708696:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.708698:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.708700:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.708701:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498581] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.708702:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.708703:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.708704:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.708707:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.708709:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.708711:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.708712:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.708712:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.708713:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.708714:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.708714:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.708715:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.708720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.708727:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.708729:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.708730:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.708731:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.708732:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.708732:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.708942:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.708948:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.708950:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.708951:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.708953:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.708955:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.708956:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.708958:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498580] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.708959:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.708960:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.708961:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.708965:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.708966:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.708969:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.708970:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.708971:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.708972:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.708973:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.708974:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.708974:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.708978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.708987:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.708989:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.708990:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.708991:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.708993:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.708994:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.708997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.709004:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.709007:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.709008:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.709009:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.709009:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.709011:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.709232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.709240:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.709242:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.709243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.709245:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.709248:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.709249:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677475.709251:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498579] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.709252:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.709253:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.709254:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677475.709258:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.709260:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677475.709263:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.709263:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677475.709264:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.709266:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.709266:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.709267:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.709268:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.709269:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:4.2:1619677475.709274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.709281:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:8.2:1619677475.709282:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677475.709283:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.709284:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.709285:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677475.709290:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.709294:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.709301:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.709302:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.709304:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.709306:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677475.709444:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.709450:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.709453:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.709454:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.709457:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.709459:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.709460:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.709462:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498578] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.709463:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.709464:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.709465:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.709468:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.709470:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.709472:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.709473:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.709474:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.709476:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.709476:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.709477:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.709478:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.709483:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.709490:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.709492:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.709493:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.709493:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.709495:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.709496:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.709679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:1.0:1619677475.709685:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.709688:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.709689:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.709691:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.709693:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.709694:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:1.0:1619677475.709696:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498577] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.709697:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.709698:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.709699:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:1.0:1619677475.709702:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.709703:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:1.0:1619677475.709706:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.709707:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:1.0:1619677475.709707:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:1.0:1619677475.709709:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:1.0:1619677475.709710:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.709710:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.709711:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677475.709715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.709723:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.709726:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.709727:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.709727:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.709729:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.709731:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.709895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:2.0:1619677475.709900:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.709903:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.709904:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:2.0:1619677475.709906:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.709908:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.709909:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677475.709911:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498576] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.709912:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.709913:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.709914:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:2.0:1619677475.709917:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.709919:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677475.709922:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.709923:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677475.709923:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.709925:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:2.0:1619677475.709926:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.709927:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.709927:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:25.2:1619677475.709931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.709938:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.709941:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.709942:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.709942:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.709944:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.709946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.710257:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.710265:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.710267:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.710269:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.710272:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.710272:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.710273:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.710274:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498575] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.710275:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.710276:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.710277:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.710281:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.710282:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.710284:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.710285:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.710286:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.710287:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.710288:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.710289:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.710289:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.710293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.710300:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.710303:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.710304:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.710305:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.710307:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.710308:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.710515:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.710520:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.710523:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.710524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.710526:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.710528:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.710529:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.710531:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498574] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.710532:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.710533:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.710534:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.710537:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.710539:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.710542:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.710543:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.710543:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.710545:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.710545:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.710546:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.710547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.710551:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.710559:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.710561:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.710562:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.710564:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.710565:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.710566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.710643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.710651:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.710654:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.710656:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.710657:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.710658:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.710659:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.710660:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498573] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.710661:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.710663:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.710664:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.710667:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.710668:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.710671:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.710671:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.710672:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.710674:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.710674:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.710675:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.710676:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.710689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:9.2:1619677475.710690:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677475.710697:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:8.0:1619677475.710697:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.710699:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:26.0:1619677475.710700:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:24.0:1619677475.710700:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.710700:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.710701:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.710702:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.710703:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677475.710705:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.710705:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677475.710707:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.710708:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498572] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.710709:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.710710:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.710711:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.710714:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.710716:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.710718:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.710719:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.710720:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.710722:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.710722:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.710723:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.710723:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.710728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.710735:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.710738:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.710739:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.710739:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.710741:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.710744:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.710747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.710754:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.710756:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.710757:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.710758:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.710759:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.710759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.711004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.711011:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.711013:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.711014:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.711016:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.711019:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.711021:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.711022:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498571] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.711023:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.711024:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.711025:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.711028:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.711030:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.711032:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.711033:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.711034:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.711035:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.711036:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.711036:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.711037:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.711043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.711052:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.711055:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.711056:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.711056:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.711058:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.711061:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.711064:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.711072:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.711074:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.711075:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.711075:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.711076:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.711077:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.711122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677475.711129:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.711132:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677475.711133:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677475.711134:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677475.711137:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.711138:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677475.711140:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498570] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.711141:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.711142:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.711142:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677475.711145:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677475.711146:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.711147:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.711148:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.711148:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.711150:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.711150:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.711151:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677475.711152:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:9.2:1619677475.711155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.711161:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.711163:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.711163:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.711165:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.711165:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.711167:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.711243:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.711249:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.711251:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.711253:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.711254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.711255:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.711256:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.711257:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498569] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.711259:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.711260:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.711261:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.711263:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.711265:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.711267:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.711268:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.711268:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.711270:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.711270:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.711272:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.711273:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.711274:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677475.711276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.711279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.711285:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.711286:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.711286:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.711287:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000400:00000200:20.0:1619677475.711287:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.711288:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.711288:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.711289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.711290:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677475.711291:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.711367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.711376:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.711379:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.711381:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.711383:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.711383:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.711384:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.711386:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498568] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.711387:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.711388:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.711389:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.711392:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.711394:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.711396:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.711396:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.711397:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.711398:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.711399:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.711400:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.711402:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.711402:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.711405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.711409:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.711413:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.711415:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.711415:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.711417:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.711417:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.711418:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.711419:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.711420:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.711421:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.711422:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:20.2:1619677475.711497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.711502:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.711505:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.711507:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.711507:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.711509:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.711511:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.711512:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498567] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.711513:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.711514:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.711515:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.711518:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.711520:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.711522:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.711523:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.711524:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.711525:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.711526:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.711527:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.711527:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.711532:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.711539:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.711542:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.711542:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.711544:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.711545:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.711546:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.711552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.711560:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.711563:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.711563:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.711564:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.711565:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.711566:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.2:1619677475.711613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677475.711621:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.711623:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677475.711626:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.711627:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677475.711628:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.711629:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677475.711630:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498566] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.711631:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.711632:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.711634:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677475.711637:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.711639:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677475.711642:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.711643:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677475.711643:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677475.711645:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677475.711646:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.711648:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87b3e5800] (22)++
00000800:00000200:30.0:1619677475.711649:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (23)--
00000800:00000200:30.0:1619677475.711650:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:9.2:1619677475.711651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:20.2:1619677475.711655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:18.0:1619677475.711660:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.711663:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677475.711663:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:31.0:1619677475.711664:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)++
00000400:00000200:18.0:1619677475.711664:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.711666:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87b3e5800] (23)--
00000400:00000200:18.0:1619677475.711666:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.711667:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677475.711668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:18.0:1619677475.711670:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.712227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:20.2:1619677475.712230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:10.0:1619677475.712235:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677475.712235:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.712238:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:26.0:1619677475.712239:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:18.0:1619677475.712239:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677475.712239:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:2.0:1619677475.712242:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.712243:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.712244:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.712245:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.712246:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.712247:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.712248:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.712249:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498565] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.712249:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.712250:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.712251:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677475.712252:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.712254:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.712256:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.712256:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.712259:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000400:00000200:10.0:1619677475.712260:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.712262:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.712263:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.712263:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.712264:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677475.712265:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.712266:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.712266:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677475.712267:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.712267:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:10.0:1619677475.712268:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.712268:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.712268:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:10.0:1619677475.712269:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.712270:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.712271:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.712272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.712279:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.712281:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.712282:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.712284:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.712285:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.712287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.712298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.712307:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.712310:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.712311:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.712313:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.712314:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.712316:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.712334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.712340:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.712343:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.712344:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.712346:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.712348:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.712350:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.712351:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498563] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.712352:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.712353:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.712354:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.712358:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.712360:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.712362:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.712363:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.712364:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.712365:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.712366:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.712367:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.712368:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.712369:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.712372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.712374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.712380:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.712381:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.712381:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.712382:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.712383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.712383:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.712384:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.712386:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.712387:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.712388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.712643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.712651:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.712654:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.712657:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.712659:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.712659:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.712661:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.712662:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498562] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.712663:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.712664:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.712665:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.712669:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.712670:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.712673:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.712674:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.712674:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.712676:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.712677:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.712677:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.712678:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.712682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.712689:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.712692:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.712693:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.712693:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.712695:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.712697:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.712755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.712762:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.712765:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.712766:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.712768:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.712770:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.712771:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.712773:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498561] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.712774:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.712775:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.712776:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.712779:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.712780:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.712782:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.712783:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.712784:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.712785:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.712786:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.712786:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.712787:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.712791:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.712797:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.712799:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.712800:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.712800:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.712802:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.712804:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.712810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.712816:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.712819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.712819:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.712820:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.712821:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.712821:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.713039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.713045:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.713048:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.713049:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.713050:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.713053:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.713055:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.713056:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498560] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.713057:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.713058:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677475.713058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:24.0:1619677475.713059:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.713062:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.713063:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.713065:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677475.713066:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.713067:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.713067:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.713067:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.713068:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:24.0:1619677475.713069:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.713070:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.713070:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:10.0:1619677475.713070:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.713071:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:10.0:1619677475.713072:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.713073:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.713074:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498559] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.713075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:10.0:1619677475.713075:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.713076:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.713077:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.713080:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.713081:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.713083:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:5.2:1619677475.713083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:10.0:1619677475.713084:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.713085:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.713085:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.713086:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:10.0:1619677475.713086:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.713087:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.713087:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.713088:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.713088:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:17.0:1619677475.713089:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:10.0:1619677475.713089:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.713089:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.713092:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:6.0:1619677475.713092:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.713093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:1.0:1619677475.713093:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.713095:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.713097:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.713098:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.713100:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498558] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.713101:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000400:00000200:17.0:1619677475.713101:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.713102:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.713103:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:17.0:1619677475.713103:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677475.713104:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.713104:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.713107:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.713107:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.713109:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:17.0:1619677475.713110:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.713112:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.713112:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.713113:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.713115:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.713115:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.713116:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.713116:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.713121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.713128:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.713130:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.713131:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.713132:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.713133:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.713134:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677475.713362:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.713365:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.713368:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.713370:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.713371:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.713372:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.713374:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.713375:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498557] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.713376:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.713378:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.713378:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.713382:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.713384:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.713386:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.713387:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.713388:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.713389:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.713390:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.713391:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.713391:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.713396:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.713404:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.713406:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.713408:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.713408:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.713410:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.713412:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.713416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:4.2:1619677475.713416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677475.713423:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:1.0:1619677475.713424:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.713426:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:1.0:1619677475.713426:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.713427:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:17.0:1619677475.713428:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.713428:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:26.0:1619677475.713429:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.713429:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.713429:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:26.0:1619677475.713432:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.713434:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:26.0:1619677475.713435:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498556] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.713436:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.713437:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.713438:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:26.0:1619677475.713442:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.713444:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:26.0:1619677475.713446:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.713447:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:26.0:1619677475.713447:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:26.0:1619677475.713449:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:26.0:1619677475.713449:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677475.713450:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677475.713450:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677475.713454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.713458:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677475.713460:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677475.713461:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.713462:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677475.713463:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.713465:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:24.2:1619677475.713476:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (20)++
00000800:00000200:24.0:1619677475.713479:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.713481:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.54@o2ib7
00000800:00000200:24.0:1619677475.713482:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e42d9c400] (22)++
00000800:00000200:8.0:1619677475.713482:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677475.713483:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:24.0:1619677475.713483:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:21.2:1619677475.713535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.713543:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.713546:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.713548:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.713550:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.713551:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.713552:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.713553:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498555] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.713554:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.713555:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.713556:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.713559:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.713560:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.713563:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.713564:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.713565:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.713567:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.713567:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.713568:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.713569:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.713573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.713581:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.713583:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.713584:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.713586:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.713587:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.713589:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.713591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.713598:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.713600:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.713601:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.713602:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.713603:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.713604:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.713733:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.713740:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.713743:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.713743:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.713745:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.713747:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.713749:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.713750:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498554] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.713752:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.713752:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.713754:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.713757:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.713758:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.713761:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.713761:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.713762:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.713764:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.713764:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.713765:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.713765:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.713771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.713778:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.713780:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.713781:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.713783:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.713784:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.713785:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.714501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677475.714507:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.714510:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677475.714511:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677475.714513:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.714516:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.714518:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677475.714519:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498553] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.714520:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.714521:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.714523:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677475.714526:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.714528:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677475.714531:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.714532:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677475.714532:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677475.714534:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:18.0:1619677475.714535:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.714536:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677475.714537:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:18.0:1619677475.714538:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677475.714541:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.2:1619677475.714544:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:10.0:1619677475.714548:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:2.0:1619677475.714549:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:10.0:1619677475.714551:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:2.0:1619677475.714551:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:26.0:1619677475.714552:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:2.0:1619677475.714552:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:10.0:1619677475.714553:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.714554:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000400:00000200:10.0:1619677475.714555:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.714559:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677475.714563:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:30.0:1619677475.714571:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.714574:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:30.0:1619677475.714577:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.714579:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:30.0:1619677475.714579:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.714581:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:30.0:1619677475.714582:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498552] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.714583:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.714584:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.714585:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:30.0:1619677475.714588:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.714590:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:30.0:1619677475.714592:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.714593:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:30.0:1619677475.714594:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:30.0:1619677475.714595:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:30.0:1619677475.714596:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:30.0:1619677475.714597:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:30.0:1619677475.714597:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677475.714601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.714607:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.714610:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.714611:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.714611:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.714612:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.714615:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677475.714620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.2:1619677475.714625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:10.0:1619677475.714626:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.714628:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.714629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.714629:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.714630:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.714631:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:31.0:1619677475.714633:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.714635:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.714637:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.714639:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.714639:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.714641:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.714642:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498551] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.714643:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.714644:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.714645:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.714648:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.714649:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.714651:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.714652:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.714653:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.714654:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.714655:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.714656:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.714656:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.714660:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.714667:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.714669:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.714670:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.714671:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.714672:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.714674:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.714678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.714683:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.714686:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.714687:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.714687:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.714688:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.714688:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.714933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.714941:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.714943:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.714945:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.714947:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.714948:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.714949:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.714950:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498550] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.714951:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.714952:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.714953:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.714956:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.714958:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.714960:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.714961:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.714961:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.714962:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.714963:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.714964:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.714964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.714968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.714975:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.714977:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.714978:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.714979:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.714981:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.714982:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.715182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:22.0:1619677475.715188:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.715191:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:6.0:1619677475.715192:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:22.0:1619677475.715194:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.715197:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.715198:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:22.0:1619677475.715199:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498549] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.715201:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.715202:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.715203:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677475.715206:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.715208:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677475.715211:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.715211:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677475.715212:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677475.715214:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:22.0:1619677475.715214:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.715215:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:22.0:1619677475.715215:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:8.2:1619677475.715220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.715228:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.715230:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.715231:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.715231:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.715233:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.715236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.715418:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.715424:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.715427:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.715428:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.715429:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.715432:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.715433:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.715434:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498548] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.715435:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.715436:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.715437:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.715439:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.715441:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.715443:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.715444:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.715444:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.715446:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.715446:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.715447:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.715447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.715452:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.715460:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.715463:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.715463:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.715465:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.715466:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.715467:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.715471:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.715478:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.715480:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.715481:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.715482:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.715483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.715484:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.715588:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.715596:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.715599:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.715599:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.715601:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.715604:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.715605:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.715606:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498547] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.715608:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.715609:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.715610:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.715613:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.715614:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.715617:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.715618:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.715619:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.715620:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.715621:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.715621:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.715622:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.715626:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.715634:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.715637:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.715637:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.715638:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.715638:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000400:00000200:22.0:1619677475.715640:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.715643:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.715644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:31.0:1619677475.715646:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.715649:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:10.0:1619677475.715650:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:31.0:1619677475.715651:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.715653:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.715653:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:10.0:1619677475.715653:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.715654:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:31.0:1619677475.715655:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.715656:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498546] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.715657:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.715658:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.715659:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.715659:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:10.0:1619677475.715659:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.715661:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677475.715662:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498545] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.715663:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.715664:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.715665:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:10.0:1619677475.715665:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.715666:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000800:00000200:31.0:1619677475.715667:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.715668:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.715669:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:10.0:1619677475.715669:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.715670:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.715671:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:10.0:1619677475.715671:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:31.0:1619677475.715672:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:31.0:1619677475.715672:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:10.0:1619677475.715672:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.715673:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677475.715674:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.715675:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.715675:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.715676:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.715676:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.715676:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:9.2:1619677475.715680:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:8.0:1619677475.715683:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:26.0:1619677475.715686:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:8.0:1619677475.715686:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.715687:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.715687:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.715688:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:8.0:1619677475.715688:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677475.715689:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.715690:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:26.0:1619677475.715691:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.715691:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677475.715693:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.715876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.715883:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.715886:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.715887:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.715889:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.715891:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.715892:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.715893:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498544] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.715895:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.715896:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.715897:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.715900:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.715902:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.715905:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.715906:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.715906:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.715908:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.715908:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.715909:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.715910:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.715914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.715922:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.715924:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.715925:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.715927:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.715927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.715929:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.716057:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.716064:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.716067:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.716070:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.716072:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.716072:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.716074:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.716075:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498543] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.716076:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.716077:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.716078:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.716082:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.716083:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.716085:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.716086:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.716087:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.716088:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.716089:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.716090:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.716090:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.716094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.716101:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.716103:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.716104:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.716106:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.716107:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.716108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.716118:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.716124:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.716127:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.716128:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.716128:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.716129:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.716130:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.716182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.716189:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.716192:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.716193:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.716194:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.716197:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.716198:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.716199:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498542] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.716201:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.716201:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.716202:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.716205:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.716207:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.716209:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.716210:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.716211:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.716212:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.716213:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.716214:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.716214:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.716218:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.2:1619677475.716221:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:17.0:1619677475.716226:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:31.0:1619677475.716228:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:17.0:1619677475.716229:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.716230:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:31.0:1619677475.716231:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:17.0:1619677475.716231:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.716233:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.716234:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.716235:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.716236:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.716236:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:31.0:1619677475.716237:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.716239:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498541] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.716240:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.716241:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.716242:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.716245:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.716247:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.716249:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.716250:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.716251:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.716252:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.716253:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.716254:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.716255:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.716256:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.716258:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.716262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.716264:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.716267:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.716267:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.716268:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.716269:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.716270:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.716272:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677475.716272:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.716273:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.716274:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.716714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.716722:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.716725:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.716725:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:10.0:1619677475.716727:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.716729:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.716730:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.716731:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498540] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.716733:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.716734:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.716734:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.716737:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.716739:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.716741:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.716742:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.716743:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.716744:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.716745:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.716746:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:20.2:1619677475.716748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:10.0:1619677475.716748:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.716748:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.716751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.716754:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:4.2:1619677475.716754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:20.0:1619677475.716757:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.716760:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.716760:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.716760:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.716761:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000400:00000200:20.0:1619677475.716762:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.716763:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:20.0:1619677475.716763:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677475.716763:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677475.716764:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677475.716764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:20.0:1619677475.716765:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498539] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.716765:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.716765:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.716766:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.716767:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.716767:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677475.716768:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:6.0:1619677475.716770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:20.0:1619677475.716772:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.716774:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.716776:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.716777:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.716778:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.716779:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.716780:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.716781:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.716781:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.716786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.716794:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.716796:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.716797:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.716798:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.716799:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.716800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.717224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.717231:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.717234:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.717235:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.717237:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.717239:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.717241:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677475.717242:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498538] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.717244:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.717245:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.717246:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677475.717249:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.717251:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.2:1619677475.717252:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:26.0:1619677475.717254:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.717255:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677475.717255:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.717257:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.717258:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:22.0:1619677475.717258:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:26.0:1619677475.717259:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.717259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:22.0:1619677475.717261:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.717262:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.717263:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677475.717263:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:22.0:1619677475.717265:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.717267:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.717268:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498537] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.717269:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.717270:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.717271:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:1.0:1619677475.717271:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.717274:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:22.0:1619677475.717275:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.717275:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.717275:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.717277:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:1.0:1619677475.717278:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.717279:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.717280:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.717281:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677475.717281:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677475.717282:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.717283:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.717283:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.717284:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.717288:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.717296:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.717298:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.717299:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.717301:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.717302:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.717303:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.717329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.717335:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.2:1619677475.717336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:17.0:1619677475.717338:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.717338:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.717340:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.717342:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.717343:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:30.0:1619677475.717344:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:17.0:1619677475.717345:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498536] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.717346:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.717347:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:17.0:1619677475.717347:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.717348:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.717350:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.717352:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:17.0:1619677475.717352:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.717353:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.717354:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.717355:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:17.0:1619677475.717356:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.717357:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498535] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:17.0:1619677475.717357:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.717358:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:30.0:1619677475.717359:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.717359:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677475.717360:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.717360:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000400:00000200:30.0:1619677475.717361:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:17.0:1619677475.717361:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.717361:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:30.0:1619677475.717364:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.717366:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 984 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.2:1619677475.717366:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.717369:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.717370:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677475.717371:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677475.717372:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.717373:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:1.0:1619677475.717373:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:30.0:1619677475.717374:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:1.0:1619677475.717375:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:30.0:1619677475.717376:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.717376:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:1.0:1619677475.717376:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.717377:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.717378:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.717379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:1.0:1619677475.717380:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.717383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.717390:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677475.717391:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.717393:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677475.717394:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:21.0:1619677475.717394:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.717395:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.717395:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.717396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.717397:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677475.717398:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.717987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.717995:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.717997:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.717998:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.718000:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.718003:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.718004:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.718005:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498534] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.718006:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.718007:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.718008:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.718011:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.718013:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.718016:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.718016:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.718017:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.718019:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.718019:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.718020:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.718020:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.718025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.718032:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.718035:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.718036:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.718036:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.718039:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.718042:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.718045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.718052:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.718055:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:6.0:1619677475.718055:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.718056:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.718057:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.718057:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.718112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.718119:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.718122:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.718125:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.718126:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.718128:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.718129:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.718130:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498533] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.718132:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.718133:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.718133:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.718136:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.718138:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.718140:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.718141:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.718141:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.718143:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.718144:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.718145:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.718145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.718149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.718155:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.718158:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.718159:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.718159:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.718161:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.718162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.718285:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.718292:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.718297:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.718301:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677475.718303:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.718305:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.718307:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.718308:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498532] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.718309:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.718310:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.718311:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.718314:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.718315:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.718318:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.718319:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.718319:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.718321:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.718322:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.718322:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.718323:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677475.718328:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.718337:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.718339:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.718340:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.718342:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.718343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.718345:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.718376:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.718383:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.718386:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.718386:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.718388:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.718391:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.718392:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.718394:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498531] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.718395:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.718396:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.718397:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.718400:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.718401:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.718404:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.718405:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.718406:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.718407:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.718408:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.718409:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.718410:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.718411:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.718414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.718416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:6.0:1619677475.718422:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:24.0:1619677475.718424:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.718425:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677475.718426:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677475.718426:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.718426:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.718427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.718428:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677475.718429:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.718430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.718433:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.718434:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.718436:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.718437:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.718438:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.718438:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.718439:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.718593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.718598:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.718600:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.718601:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.718603:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.718605:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.718607:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.718608:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498530] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.718609:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.718610:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.718611:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.718614:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.718616:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.718618:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.718619:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.718620:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.718622:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.718622:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.718623:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.718624:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.718625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.718628:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.718630:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.718635:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.718637:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.718637:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.718638:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.718638:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.718639:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.718640:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.718641:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.718642:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.718643:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.718709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.718716:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.718719:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.718719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.718722:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.718725:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.718726:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.718728:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498529] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.718729:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.718730:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.718731:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.718734:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.718736:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.718738:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.718739:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.718740:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.718741:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.718742:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.718743:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.718743:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.718748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.718757:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.718759:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.718760:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.718760:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.718762:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.718765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.719165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.719168:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.719170:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:21.0:1619677475.719173:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.719173:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.719175:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.719176:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.719178:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498528] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.719179:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.719180:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.719181:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.719184:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.719186:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.719188:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.719189:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.719189:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.719191:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.719191:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.719192:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.719194:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.719194:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.719197:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.719199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:1.0:1619677475.719204:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.719205:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.719207:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:1.0:1619677475.719207:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677475.719208:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:1.0:1619677475.719208:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.719209:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.719209:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.719210:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.719213:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.719216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.719224:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.719226:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.719227:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.719228:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.719228:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.719229:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.719344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.719352:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.719355:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.719357:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.719359:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.719360:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.719362:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.719363:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498527] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.719365:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.719366:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.719367:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.719370:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.719372:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.719376:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.719377:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.719377:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.719379:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.719380:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.719381:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.719381:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.719386:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.719395:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.719397:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.719398:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.719400:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.719401:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.719403:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.719618:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.719621:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.719624:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.719626:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.719627:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.719628:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.719629:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677475.719631:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498526] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.719632:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.719633:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.719634:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677475.719638:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.719639:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677475.719642:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.719643:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677475.719643:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677475.719645:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677475.719646:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.719646:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.719647:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677475.719652:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.719659:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.719662:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.719663:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.719665:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.719665:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.719668:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.719778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.719785:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.719788:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.719790:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.719792:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.719792:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.719794:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.719795:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498525] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.719796:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.719797:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.719798:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.719801:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.719802:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.719805:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.719806:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.719806:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.719808:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.719809:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.719810:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.719810:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.719814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.719820:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.719823:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.719824:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.719824:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.719826:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.719828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.719832:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.719838:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.719840:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.719841:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.719841:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.719842:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.719843:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.719895:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.719903:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.719905:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.719906:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.719909:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.719911:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.719912:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677475.719913:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498524] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.719915:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.719915:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.719916:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677475.719919:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.719921:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677475.719923:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.719924:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677475.719924:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677475.719926:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677475.719926:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.719927:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.719928:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.719932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.719938:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.719941:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.719941:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.719943:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.719945:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677475.719945:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:25.2:1619677475.719952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.719959:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.719962:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.719963:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.719963:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.719964:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.719965:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.720172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.720179:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.720181:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.720182:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.720185:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.720187:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.720188:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.720190:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498523] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.720191:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.720192:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.720193:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.720196:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.720198:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.720201:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.720201:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.720202:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.720203:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.720204:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.720204:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.720205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.720209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.720217:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.720219:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.720220:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.720222:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.720223:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.720224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.720229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.720236:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.720239:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.720240:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.720240:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.720241:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.720242:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.720390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.720398:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.720401:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.720401:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.720403:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.720406:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.720408:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.720409:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498522] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.720410:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.720411:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.720412:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.2:1619677475.720413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000400:00000200:26.0:1619677475.720416:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.720417:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677475.720419:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:26.0:1619677475.720420:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.720421:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.720421:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.720422:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:26.0:1619677475.720423:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.720423:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:18.0:1619677475.720423:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:26.0:1619677475.720424:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.720425:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:2.0:1619677475.720425:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.720428:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.720429:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:25.2:1619677475.720430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:2.0:1619677475.720430:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498521] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.720432:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.720433:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.720434:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.720437:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.720438:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:2.0:1619677475.720439:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677475.720441:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:2.0:1619677475.720441:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.720441:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:22.0:1619677475.720442:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.720442:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.720442:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:6.0:1619677475.720444:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.720444:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.720444:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.720446:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:6.0:1619677475.720447:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677475.720447:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:2.0:1619677475.720447:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:20.2:1619677475.720453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:9.2:1619677475.720457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.720462:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.720464:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.720465:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.720466:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.720466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:18.0:1619677475.720467:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.720467:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.720468:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.720470:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.720471:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.720497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.720506:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.720509:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.720511:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.720513:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.720513:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.720515:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.720516:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498520] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.720517:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.720518:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.720519:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.720522:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.720524:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.720526:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.720527:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.720527:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.720529:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.720529:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.720530:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.720531:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.720534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.720541:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.720544:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.720544:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.720546:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.720548:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.720549:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.721093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.721101:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.721103:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.721105:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.721107:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.721108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.721108:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.721109:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498519] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.721111:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.721112:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.721112:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.721115:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.721116:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.721118:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.721119:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.721119:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.721120:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.721121:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.721122:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.721122:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.721126:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.721133:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.721135:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.721136:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.721137:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.721139:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.721140:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.721743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.721751:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.721754:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.721756:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.721758:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.721759:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677475.721759:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.721760:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498518] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.721761:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.721762:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.721763:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.721766:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.721767:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.721770:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.721770:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.721771:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.721772:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.721773:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.721774:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.721774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.721779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.721786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.721789:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.721791:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.721792:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.721794:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:8.0:1619677475.721795:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.721796:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.721797:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.721798:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.721798:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:8.0:1619677475.721799:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.721801:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.721803:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.721804:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498517] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.721805:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.721806:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.721807:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.721810:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.721811:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.721814:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.721815:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.721816:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.721817:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.721818:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.721818:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.721819:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.721823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.2:1619677475.721828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:4.2:1619677475.721828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:1.0:1619677475.721833:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.721834:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.721835:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.721836:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:10.0:1619677475.721836:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:1.0:1619677475.721836:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.721838:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.721839:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.721839:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:17.0:1619677475.721840:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.721841:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.721842:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.721842:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.721842:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677475.721843:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.721844:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:10.0:1619677475.721844:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.721846:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.721846:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.721847:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498516] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.721848:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.721849:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.721849:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.721850:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:10.0:1619677475.721851:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.721853:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677475.721855:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.721858:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:10.0:1619677475.721858:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.721860:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.721861:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:10.0:1619677475.721861:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677475.721862:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.721863:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:10.0:1619677475.721863:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.721864:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:10.0:1619677475.721864:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677475.721865:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:10.0:1619677475.721865:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677475.721866:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:10.0:1619677475.721866:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.721866:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:17.0:1619677475.721867:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:10.0:1619677475.721867:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.721868:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677475.721869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.721872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:9.2:1619677475.721872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:20.0:1619677475.721878:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.721878:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:8.0:1619677475.721879:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.0:1619677475.721880:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:1.0:1619677475.721880:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000400:00000200:20.0:1619677475.721881:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.721882:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:1.0:1619677475.721882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.721882:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:24.0:1619677475.721883:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.721883:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.721884:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.721884:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.721885:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.721886:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.0:1619677475.721888:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.722006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.722012:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.722015:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.722015:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.722017:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.722020:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.722021:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.722022:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498514] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.722024:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.722025:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.722026:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.722029:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.722031:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.722034:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.722034:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.722035:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.722037:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.722037:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.722038:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.722038:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.722043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.722051:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.722053:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.722054:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.722056:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.722056:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.722058:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.722194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.722200:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.722202:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.722203:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.722205:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.722207:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.722209:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.722210:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498513] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.722212:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.722213:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.722214:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.722216:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.722218:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.722220:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.722221:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.722221:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.722223:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.722223:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.722224:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.722225:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.722225:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.722229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.722232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.722238:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.722238:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.722239:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.722239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.722239:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.722240:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:1.0:1619677475.722240:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.722241:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.722242:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.722243:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.722843:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.722851:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.722853:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.722854:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.722856:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.722858:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.722860:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.722861:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498512] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.722862:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.722863:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.722865:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.722868:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.722870:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.722873:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.722874:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.722874:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.722876:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.722876:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.722877:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.722877:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.722882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.722891:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.722893:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.722894:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.722895:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.722897:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.722899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.723192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.723199:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.723202:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.723203:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.723205:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.723208:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.723209:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.723210:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498511] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.723212:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.723213:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.723214:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.723217:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.723218:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.723221:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.723222:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.723222:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.723224:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.723225:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.723225:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.723226:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.723241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.723248:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.723251:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.723251:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.723252:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.723254:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.723257:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.723338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.723344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.723346:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.723347:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.723349:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.723351:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.723352:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.723353:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498510] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.723355:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.723356:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.723357:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.723359:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.723361:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.723364:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.723364:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.723365:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.723367:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.723367:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.723368:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.723369:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.723374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.723382:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.723384:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.723385:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.723386:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.723388:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.723389:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.723449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.723457:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.723460:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.723462:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.723465:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.723465:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.723466:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.723468:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498509] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.723469:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.723470:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.723471:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.723474:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.723476:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.723478:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.723479:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.723479:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.723481:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.723482:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.723483:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.723485:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.723485:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.723487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.723491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.723494:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.723496:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.723497:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.723497:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.723499:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.723500:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:8.0:1619677475.723501:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.723502:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.723503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.723503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.723938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.723946:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.723949:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.723950:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.723951:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.723954:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.723956:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677475.723957:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498508] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.723958:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.723959:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.723960:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.723963:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.723964:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677475.723967:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.723968:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677475.723969:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677475.723970:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677475.723971:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.723972:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.723973:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.723974:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:9.2:1619677475.723977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677475.723979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:2.0:1619677475.723986:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:26.0:1619677475.723987:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677475.723989:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.723990:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:2.0:1619677475.723990:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:26.0:1619677475.723991:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:2.0:1619677475.723992:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.723993:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.723995:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.723999:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.724649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87affcc00] (20)++
00000800:00000200:31.0:1619677475.724657:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87affcc00] (21)++
00000800:00000200:31.0:1619677475.724660:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.59@o2ib3
00000400:00000200:31.0:1619677475.724661:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.2.59@o2ib3 : GET - for me
00000400:00000200:31.0:1619677475.724664:0:19945:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.2.59@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:30.0:1619677475.724664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87affcc00] (22)--
00000400:00000200:31.0:1619677475.724666:0:19945:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.2.59@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:31.0:1619677475.724668:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87affcc00] (21)++
00000800:00000200:31.0:1619677475.724668:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87affcc00] (22)--
00000400:00000200:31.0:1619677475.724670:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.2.59@o2ib3 local destination
00000400:00000200:31.0:1619677475.724674:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.2.59@o2ib3(10.51.2.59@o2ib3:10.51.2.59@o2ib3) : REPLY try# 0
00000800:00000200:31.0:1619677475.724675:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.2.59@o2ib3
00000800:00000200:31.0:1619677475.724678:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a867b82400] -> 10.51.2.59@o2ib3 (2) version: 12
00000800:00000200:31.0:1619677475.724678:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a87affcc00] (21)++
00000800:00000200:31.0:1619677475.724679:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87affcc00] (22)++
00000800:00000200:31.0:1619677475.724681:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a87affcc00] (23)--
00000800:00000200:31.0:1619677475.724681:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87affcc00] (22)--
00000800:00000200:4.2:1619677475.724687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87affcc00] (21)++
00000800:00000200:30.0:1619677475.724696:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87affcc00] (22)++
00000800:00000200:30.0:1619677475.724698:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87affcc00] (23)--
00000400:00000200:30.0:1619677475.724699:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.724701:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.2.59@o2ib3: REPLY: OK
00000800:00000200:31.0:1619677475.724703:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87affcc00] (22)--
00000800:00000200:30.0:1619677475.724704:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87affcc00] (21)--
00000800:00000200:21.2:1619677475.724746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.724753:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.724756:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.724758:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.724760:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.724760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.724761:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.724763:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498507] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.724764:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.724765:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.724766:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.724769:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.724770:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.724773:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.724774:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.724775:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.724777:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.724777:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.724779:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.724780:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.724781:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.724783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.724786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.724794:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.724795:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.724797:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677475.724798:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.724799:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:21.0:1619677475.724799:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.724800:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:21.0:1619677475.724800:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.724800:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.724803:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.724880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.2:1619677475.724883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:22.0:1619677475.724886:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.2:1619677475.724886:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:22.0:1619677475.724889:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:20.0:1619677475.724889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677475.724889:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.724892:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.724892:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:18.0:1619677475.724893:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:20.0:1619677475.724894:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.724895:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.724895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:18.0:1619677475.724895:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000400:00000200:22.0:1619677475.724896:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677475.724896:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:22.0:1619677475.724897:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498506] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.724897:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.724899:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.724899:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:18.0:1619677475.724899:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:22.0:1619677475.724900:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.724901:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:20.0:1619677475.724901:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498505] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.724901:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.724902:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.724903:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:20.0:1619677475.724904:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.724904:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498504] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.724905:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:18.0:1619677475.724907:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.724908:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.724910:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.724910:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:22.0:1619677475.724911:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.724914:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:22.0:1619677475.724915:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.724915:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.724917:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.724917:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:22.0:1619677475.724918:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.724918:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677475.724919:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.724919:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677475.724920:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.724921:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.724921:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:18.0:1619677475.724921:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.724921:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:22.0:1619677475.724922:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.724922:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:18.0:1619677475.724922:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.0:1619677475.724923:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677475.724924:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.724924:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:18.0:1619677475.724924:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:22.0:1619677475.724925:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:18.0:1619677475.724925:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:22.0:1619677475.724926:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677475.724926:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:18.0:1619677475.724927:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:8.2:1619677475.724927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:25.2:1619677475.724928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.724928:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:21.2:1619677475.724931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677475.724933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.2:1619677475.724933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:17.0:1619677475.724936:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:6.0:1619677475.724937:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.724938:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:17.0:1619677475.724938:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:2.0:1619677475.724938:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:17.0:1619677475.724939:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.724939:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.724940:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:1.0:1619677475.724940:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.724941:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:17.0:1619677475.724941:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.724941:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:6.0:1619677475.724941:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:2.0:1619677475.724941:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:21.0:1619677475.724942:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.724942:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.724943:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:21.0:1619677475.724944:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.724944:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:17.0:1619677475.724944:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:13.0:1619677475.724944:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.724945:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.724946:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:13.0:1619677475.724947:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.724949:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.724951:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.724958:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:10.0:1619677475.724965:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677475.724968:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:26.0:1619677475.724969:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:10.0:1619677475.724971:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.724974:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.724975:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:10.0:1619677475.724976:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498503] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.724977:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.724979:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.724980:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:10.0:1619677475.724983:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.724985:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:10.0:1619677475.724987:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.724988:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:10.0:1619677475.724989:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:10.0:1619677475.724990:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:10.0:1619677475.724991:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:10.0:1619677475.724991:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:10.0:1619677475.724992:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:24.2:1619677475.724996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.724999:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677475.725001:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677475.725002:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.725002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677475.725004:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.725006:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677475.725157:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.725161:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.725163:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.725165:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.725166:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.725168:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.725169:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.725171:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498502] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.725172:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.725173:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.725174:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.725178:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.725179:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.725182:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.725183:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.725184:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.725185:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.725186:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.725187:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.725188:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.725191:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.725198:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.725200:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.725201:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.725201:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.725204:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.725206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.725230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.725238:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.725240:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.725241:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.725243:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.725246:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.725247:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.725248:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498501] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.725250:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.725251:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.725252:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.725255:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.725256:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.725259:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.725260:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.725261:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.725262:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.725263:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.725263:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.725264:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.725269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.725276:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.725279:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.725280:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.725280:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.725282:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.725285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.725334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.725342:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.725345:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.725348:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.725350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.725351:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.725352:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.725353:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498500] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.725354:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.725355:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.725356:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.725359:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.725361:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.725363:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.725363:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.725364:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.725365:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.725366:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.725367:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.725367:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.725371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.725378:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.725381:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.725381:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.725383:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.725385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.725386:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.725566:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.725573:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.725576:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.725578:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.725580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.725580:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.725581:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.725583:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498499] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.725584:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.725585:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.725585:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.725589:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.725590:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.725592:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.725593:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.725593:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.725595:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.725595:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.725596:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.725596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.725600:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.725607:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.725609:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.725610:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.725610:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.725612:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.725614:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.725619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.725625:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.725627:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.725628:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.725628:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.725629:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.725629:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.725880:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.725888:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.725890:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.725892:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.725894:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.725895:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677475.725895:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.725897:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498498] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.725898:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.725899:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.725900:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.725903:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.725904:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.725906:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.725906:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.725907:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.725909:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.725909:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.725910:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.725911:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.725914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.725921:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.725923:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.725924:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.725925:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.725927:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.725928:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.725932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.725938:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.725941:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.725942:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.725942:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.725943:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.725944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677475.726446:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.726453:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.726456:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.726457:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.726459:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.726460:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.726462:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.726463:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498497] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.726464:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.726465:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.726466:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.726469:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.726471:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.726472:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.726473:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.726474:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.726475:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.726476:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.726477:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.726477:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.726482:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.726490:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.726492:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.726493:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.726494:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.726494:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.726496:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.726579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.726587:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.726590:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.726591:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.726593:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.726596:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.726597:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.726598:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498496] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.726599:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.726600:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.726601:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.726604:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.726606:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.726608:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.726609:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.726609:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.726611:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.726611:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.726612:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.726614:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.726614:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677475.726617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.726620:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677475.726623:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.726626:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.726626:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.726627:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:21.2:1619677475.726627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:8.0:1619677475.726627:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.726628:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.726629:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.726630:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:24.0:1619677475.726630:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677475.726631:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677475.726633:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.726636:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.726636:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.726638:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.726641:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.726642:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.726643:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498495] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.726644:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.726646:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.726646:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.726650:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.726651:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.726654:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.726655:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.726655:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.726657:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.726658:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.726658:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.726659:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.726663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.726671:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.726673:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.726674:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.726676:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.726677:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.726678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.726678:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.726679:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.726680:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.726681:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (23)++
00000800:00000200:21.0:1619677475.726682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.726683:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.0:1619677475.726683:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677475.726912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.726916:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.726918:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.726920:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.726921:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.726923:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.726924:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.726925:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498494] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.726926:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.726927:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.726928:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.726931:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.726933:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.726935:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.726936:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.726937:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.726938:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.726939:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.726939:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.726940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.726944:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.726952:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.726954:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.726955:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.726956:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.726957:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.726958:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.726967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.726975:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.726977:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.726978:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.726979:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.726980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.726981:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.726981:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.726989:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.726992:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.726993:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.726995:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.726997:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.726999:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:10.0:1619677475.727000:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498493] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.727002:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.727003:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.727004:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:10.0:1619677475.727007:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.727008:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:10.0:1619677475.727011:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.727012:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:10.0:1619677475.727013:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:10.0:1619677475.727014:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:10.0:1619677475.727015:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.727016:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:10.0:1619677475.727017:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.727018:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:8.2:1619677475.727021:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.727023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:1.0:1619677475.727029:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:26.0:1619677475.727031:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:1.0:1619677475.727031:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.727032:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677475.727033:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000400:00000200:1.0:1619677475.727033:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.727034:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.727034:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000400:00000200:1.0:1619677475.727035:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.727038:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.727039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:17.0:1619677475.727047:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.727049:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.727050:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.727050:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.727051:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.727052:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.727177:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:21.2:1619677475.727182:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.727185:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.727188:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.727189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677475.727190:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677475.727191:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.727193:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:8.0:1619677475.727193:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.727195:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:30.0:1619677475.727196:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.727197:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.727197:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498492] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.727198:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677475.727198:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.727199:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.727200:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498491] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.727200:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.727200:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:30.0:1619677475.727201:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.727202:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.727204:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:8.0:1619677475.727204:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.727207:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.727207:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677475.727210:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.727211:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.727212:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.727212:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:30.0:1619677475.727213:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.727213:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.727213:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.727214:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:30.0:1619677475.727215:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.727215:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.727215:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:30.0:1619677475.727216:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677475.727216:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.727216:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:30.0:1619677475.727217:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.727220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.2:1619677475.727220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.727227:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677475.727228:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.727230:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.727231:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.727231:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:1.0:1619677475.727231:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.727232:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.727232:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.727233:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:1.0:1619677475.727235:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.727236:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:1.0:1619677475.727238:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.727244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.727252:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.727255:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.727257:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.727259:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.727259:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.727261:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.727262:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498490] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.727264:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.727265:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.727266:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.727269:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.727271:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.727273:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.727274:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.727274:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.727276:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.727276:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.727277:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.727278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.727281:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.727288:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.727290:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.727291:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.727291:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.727298:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.727300:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.727773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.727781:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.727784:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.727785:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.727787:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.727789:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.727791:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.727792:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498489] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.727793:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.727794:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.727795:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.727798:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.727800:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.727802:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.727803:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.727804:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.727805:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.727806:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.727807:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.727808:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.727808:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.727812:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.727814:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.727820:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.727821:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:22.0:1619677475.727823:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677475.727823:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:22.0:1619677475.727824:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.727824:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:6.0:1619677475.727824:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677475.727825:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:22.0:1619677475.727826:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.727829:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.727844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.727849:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.727852:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.727854:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.727854:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.727856:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.727857:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.727859:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498488] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.727860:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.727861:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.727862:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.727865:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.727867:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.727869:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.727870:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.727871:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.727872:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.727873:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.727874:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.727874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.727878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.727885:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.727888:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.727889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.727889:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.727891:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.727893:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.727896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.727903:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.727905:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.727906:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.727907:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.727908:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.727908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.727954:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.727959:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.727961:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.727962:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.727964:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.727966:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.727967:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.727968:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498487] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.727969:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.727970:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.727971:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.727974:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.727976:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.727978:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.727979:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.727980:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.727981:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.727982:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.727982:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.727983:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.727987:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.727995:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.727997:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.727998:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.728000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.728000:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.728002:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.728083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.728090:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.728093:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.728094:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.728095:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.728099:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.728100:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.728102:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498486] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.728103:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.728104:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.728105:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.728108:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.728110:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.728113:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.728113:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.728114:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.728116:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.728116:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.728117:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.728118:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.728119:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:25.2:1619677475.728121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.728124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:22.0:1619677475.728129:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.728131:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:8.0:1619677475.728131:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:6.0:1619677475.728132:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.728133:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.728133:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.728134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:22.0:1619677475.728135:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.728135:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.728138:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.728274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.728281:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.728283:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.728284:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.728286:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.728289:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.728290:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677475.728292:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498485] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.728298:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.728299:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.728300:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677475.728303:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.728305:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677475.728307:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.728308:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677475.728308:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:10.0:1619677475.728310:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677475.728311:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.728311:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.728312:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.728316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.728324:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.728326:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677475.728327:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.728328:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.728330:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677475.728331:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:20.2:1619677475.728545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.728553:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.728556:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.728556:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.728559:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.728562:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.728563:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.728565:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498484] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.728566:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.728567:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.728568:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.728571:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.728573:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.728575:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.728576:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.728576:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.728578:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.728579:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.728580:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.728580:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.728584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.728592:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.728594:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.728595:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.728595:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.728597:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.728598:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.728900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.728908:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.728911:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.728912:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.728914:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.728917:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.728918:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.728919:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498483] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.728920:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.728921:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.728922:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.728925:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.728927:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.728929:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.728930:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.728931:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.728932:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.728933:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.728933:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.728934:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.728938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.728944:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.728947:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.728947:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.728949:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.728950:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.728952:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.729028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.729035:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.729037:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.729038:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.729040:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.729042:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.729043:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.729044:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498482] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.729045:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.729046:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.729047:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.729050:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.729052:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.729054:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.729055:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.729056:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.729057:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.729057:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.729058:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.729058:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.729063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.729071:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.729073:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.729074:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.729076:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.729077:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.729078:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.729122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.729131:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.729134:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.729136:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.729138:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.729139:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.729140:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.729141:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498481] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.729143:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.729144:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.729145:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.729148:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.729149:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.729152:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.729153:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.729153:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.729155:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.729155:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.729156:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.729157:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.729160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.729168:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.729170:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.729171:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.729171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.729173:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.729175:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.729248:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.729255:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.729258:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.729259:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.729261:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.729263:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.729264:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:10.0:1619677475.729265:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498480] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.729266:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.729267:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.729268:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:10.0:1619677475.729271:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.729273:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:10.0:1619677475.729275:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.729276:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:10.0:1619677475.729277:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:10.0:1619677475.729278:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677475.729279:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.729279:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.729280:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:25.2:1619677475.729284:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677475.729290:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677475.729300:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677475.729300:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:13.0:1619677475.729301:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.729302:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.729305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677475.729307:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:13.0:1619677475.729309:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:13.0:1619677475.729310:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:13.0:1619677475.729311:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:13.0:1619677475.729312:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677475.729312:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:13.0:1619677475.729313:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:21.2:1619677475.729828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.729837:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.729840:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.729842:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.729844:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.729845:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:30.0:1619677475.729845:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.729847:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498479] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.729848:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.729849:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.729849:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.729853:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.729854:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.729856:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.729857:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.729857:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.729859:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.729859:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.729861:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.729862:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.729863:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.729864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.729869:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.729873:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:21.2:1619677475.729873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.729875:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.729875:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.729877:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.729877:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.729878:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.729878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.729879:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677475.729879:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.729880:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.729881:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677475.729882:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.729883:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.729885:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.729887:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.729889:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.729890:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498478] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.729891:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.729892:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.729893:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.729897:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.729899:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.729901:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.729902:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.729903:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.729904:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.729905:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.729906:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.729907:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.729908:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.729910:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.729913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.729918:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.729918:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.729920:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.729920:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.729921:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.2:1619677475.729922:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:21.0:1619677475.729924:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.729924:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.729925:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.729927:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.729928:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677475.729929:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:18.0:1619677475.729931:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.729932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.729934:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.729937:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.729938:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.729939:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498477] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.729941:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.729942:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.729943:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.729946:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.729948:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.729951:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.729952:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.729953:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.729955:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.729955:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.729956:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.729957:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.729960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.729967:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.729970:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.729971:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.729973:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.729974:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.729976:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.730120:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.730128:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.730131:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.730132:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.730134:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.730136:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.730137:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.730138:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498476] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.730140:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.730141:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.730142:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.730145:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.730146:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.730149:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.730150:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.730151:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.730152:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.730153:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.730154:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.730154:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.730158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.730166:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.730169:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.730170:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.730170:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.730172:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.730175:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.730429:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.730438:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.730441:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.730443:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.730444:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.730446:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.730446:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.730447:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498475] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.730449:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.730450:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.730450:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.730453:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.730455:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.730458:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.730459:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.730459:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.730461:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.730462:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.730463:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.730463:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.730468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.730477:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.730479:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.730480:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.730482:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.730483:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.730485:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.730487:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:20.0:1619677475.730495:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.730497:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:20.0:1619677475.730498:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.730499:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:20.0:1619677475.730500:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.730501:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:20.2:1619677475.730556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.730559:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.730561:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.730564:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.730565:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.730566:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.730567:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.730569:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498474] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.730570:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.730571:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.730572:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.730576:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.730578:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.730580:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.730581:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.730582:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.730583:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.730584:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.730585:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.730585:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.730590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.730598:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.730601:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.730602:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.730602:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.730604:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.730606:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.730612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.730619:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.730622:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.730623:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.730623:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.730624:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.730624:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.730766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.730771:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.730774:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.730775:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.730777:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.730779:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.730780:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.730782:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498473] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.730783:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.730784:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.730785:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.730788:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.730790:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.730792:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.730793:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.730794:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.730796:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.730796:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.730797:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.730797:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.730802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.730810:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.730812:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.730813:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.730814:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.730815:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.730817:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.730819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.730827:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.730829:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.730830:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.730831:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.730832:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.730833:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.730842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.730850:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.730853:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.730855:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.730856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.730858:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.730859:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.730861:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498472] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.730862:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.730863:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.730864:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.730868:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.730869:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.730872:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.730873:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.730874:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.730875:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.730876:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.730878:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.730879:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.730880:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.730881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.730885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.730888:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.730890:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.730891:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.730892:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.730893:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.730893:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.730895:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:8.0:1619677475.730895:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.730896:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.730897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.731025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.731028:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.731031:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.731033:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.731034:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.731036:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.731037:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.731039:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498471] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.731040:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.731041:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.731042:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.731045:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.731047:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.731050:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.731051:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.731052:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.731053:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.731054:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.731056:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.731057:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.731057:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.731059:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.731063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:24.0:1619677475.731065:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.731068:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677475.731068:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000400:00000200:24.0:1619677475.731069:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.731069:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.731070:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000400:00000200:24.0:1619677475.731071:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.731071:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.731072:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:24.0:1619677475.731074:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.731094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.731100:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.731103:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.731103:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.731104:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.731105:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.731106:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.731230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.731236:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.731240:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.731240:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.731242:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.731244:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.731245:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.731246:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498470] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.731248:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.731249:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.731250:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.731253:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.731254:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.731257:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.731258:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.731258:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.731260:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.731260:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.731261:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.731262:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.731266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.731274:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.731276:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.731277:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.731279:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.731280:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.731282:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.731320:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.731322:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:30.0:1619677475.731328:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.731330:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677475.731331:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:10.0:1619677475.731332:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:30.0:1619677475.731333:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.731334:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677475.731336:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.731336:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.731336:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.731338:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.731338:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.731339:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.731340:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.731341:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.731341:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498469] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.731342:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.731343:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.731343:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.731344:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.731346:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.731348:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.731349:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.731350:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.731351:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.731353:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.731354:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.731354:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.731355:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.731355:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.731356:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.731357:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.731357:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.731358:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.731358:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:30.0:1619677475.731359:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:10.0:1619677475.731359:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:30.0:1619677475.731360:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:10.0:1619677475.731360:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.731363:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.2:1619677475.731364:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:24.0:1619677475.731370:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:20.0:1619677475.731372:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:24.0:1619677475.731373:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.731374:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.731374:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.731375:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.731375:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.731376:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.731377:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.731378:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.0:1619677475.731378:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.731381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:9.2:1619677475.731383:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.731390:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.731393:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.731393:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.731394:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.731395:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.731395:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.731760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.731767:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.731769:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.731770:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.731772:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.731775:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.731776:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.731777:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498467] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.731779:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.731780:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.731781:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.731784:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.731786:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.731788:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.731789:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.731790:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.731792:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.731792:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.731793:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.731793:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.731798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.731806:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.731808:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.731809:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.731811:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.731811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.731813:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.732128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.732135:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.732138:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.732139:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.732141:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.732143:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.732145:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.732146:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498466] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.732147:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.732148:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.732149:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.732153:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.732154:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:4.2:1619677475.732155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:26.0:1619677475.732157:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.732158:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.732159:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.732161:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.732161:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.732161:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.732162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:24.0:1619677475.732162:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.732164:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:24.0:1619677475.732167:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677475.732167:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:24.0:1619677475.732170:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.732171:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:8.0:1619677475.732171:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.732173:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498465] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.732174:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.732175:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.732175:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000400:00000200:24.0:1619677475.732176:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.732178:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:24.0:1619677475.732179:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677475.732179:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.732180:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.732181:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.732181:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:24.0:1619677475.732184:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.732184:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:24.0:1619677475.732185:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.732186:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.732187:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.732188:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.732189:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.732189:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.732193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.732201:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.732204:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.732205:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.732205:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.732207:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.732210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.732586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.732595:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.732598:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.732600:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.732602:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.732603:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.732604:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.732606:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498464] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.732607:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.732608:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.732609:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.732612:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.732614:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.732616:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.732616:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.732617:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.732618:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.732619:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.732620:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.732620:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.732621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:9.2:1619677475.732625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677475.732627:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.732629:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.732630:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.732632:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.732632:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:22.0:1619677475.732635:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.732635:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.732636:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:22.0:1619677475.732636:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:8.0:1619677475.732636:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.732637:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498463] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.732638:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.732639:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.732640:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.732641:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677475.732641:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:22.0:1619677475.732645:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.732647:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.732649:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.732650:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.732651:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.732653:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.732653:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.732654:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.732654:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.732659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.732667:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.732669:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.732670:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.732672:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.732673:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.732674:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.732739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.732747:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.732750:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.732751:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.732753:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.732755:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.732756:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.732758:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498462] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.732759:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.732760:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.732761:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.732765:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.732766:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.732769:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.732770:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.732771:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.732773:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.732773:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.732774:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.732774:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.732779:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.732786:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.732789:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.732790:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.732790:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.732792:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.732795:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.732842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.732850:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.732853:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.732855:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.732856:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.732858:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.732859:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.732861:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498461] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.732862:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.732863:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.732863:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.732866:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.732868:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.732870:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.732871:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.732871:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.732873:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.732874:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.732875:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.732875:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.732879:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.732885:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.732888:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.732889:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.732889:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.732891:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.732893:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677475.733116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.733123:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.733125:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.733126:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.733129:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.733131:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.733132:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.733133:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498460] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.733135:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.733136:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.733137:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.733140:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.733141:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.733143:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.733144:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.733144:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.733146:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.733146:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:4.2:1619677475.733147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677475.733147:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.733147:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.733154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677475.733159:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:17.0:1619677475.733161:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:24.0:1619677475.733162:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.733162:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:17.0:1619677475.733163:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.733164:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.733165:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.733165:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.733166:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.733167:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:8.0:1619677475.733168:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.733170:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.733171:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498459] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.733172:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.733174:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.733174:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.733178:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.733180:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.733183:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.733184:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.733184:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.733186:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.733187:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.733187:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.733188:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.733192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.733199:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.733202:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.733203:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.733203:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.733205:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.733208:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.733211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.733218:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.733220:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.733221:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.733221:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.733222:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.733223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.733454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.733460:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.733462:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.733463:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.733465:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.733467:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.733468:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.733469:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498458] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.733471:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.733472:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677475.733472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000400:00000200:22.0:1619677475.733473:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.733476:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.733478:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.733480:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.733481:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:22.0:1619677475.733481:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.733482:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.733483:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.733484:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000800:00000200:22.0:1619677475.733484:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.733485:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:31.0:1619677475.733486:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.733486:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.733487:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677475.733488:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.733488:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677475.733490:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:25.2:1619677475.733490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.733491:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498457] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.2:1619677475.733491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677475.733493:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.733494:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.733495:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.733497:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.733497:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000400:00000200:31.0:1619677475.733498:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.733498:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.733499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677475.733500:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.733500:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.733500:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.733500:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.733502:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.733504:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.733504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.733505:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:31.0:1619677475.733505:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.733505:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.733507:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:31.0:1619677475.733508:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.733509:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677475.733509:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:9.2:1619677475.733514:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.733523:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.733527:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.733529:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.733530:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.733533:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:9.2:1619677475.733537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.733541:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.733543:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.733546:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.733547:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.733548:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.733549:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.733550:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.734025:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677475.734032:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677475.734035:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677475.734036:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677475.734038:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.734041:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.734043:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677475.734044:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498456] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.734045:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.734047:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.734047:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677475.734051:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.734053:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677475.734054:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.734055:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677475.734056:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677475.734057:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677475.734058:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677475.734059:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a070bf0c00] (22)++
00000800:00000200:26.0:1619677475.734060:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:26.0:1619677475.734061:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:25.2:1619677475.734063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677475.734066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:30.0:1619677475.734070:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.734072:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.734073:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.734074:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)++
00000400:00000200:30.0:1619677475.734075:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.734076:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a070bf0c00] (23)--
00000800:00000200:31.0:1619677475.734077:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:10.0:1619677475.734077:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:30.0:1619677475.734078:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:10.0:1619677475.734078:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:4.2:1619677475.734268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.734273:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.734276:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.734278:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.734278:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.734281:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.734282:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.734284:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498455] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.734285:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.734286:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.734287:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.734290:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.734292:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.734301:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.734302:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.734302:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.734304:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.734305:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.734306:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.734306:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.734311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.734317:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.734320:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.734321:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.734321:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.734323:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.734326:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.734464:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.734472:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.734475:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.734477:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.734479:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.734480:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.734482:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.734483:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498454] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.734484:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.734485:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.734486:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:21.2:1619677475.734486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:31.0:1619677475.734489:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.734491:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:22.0:1619677475.734492:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.734494:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.734495:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:22.0:1619677475.734495:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:31.0:1619677475.734496:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677475.734496:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677475.734497:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.734498:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.734498:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000400:00000200:22.0:1619677475.734498:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.734499:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:22.0:1619677475.734501:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.734502:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:9.2:1619677475.734502:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:22.0:1619677475.734503:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498453] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.734504:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.734506:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.734507:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.0:1619677475.734509:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:22.0:1619677475.734510:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.734511:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.734512:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677475.734512:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:8.0:1619677475.734512:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.734514:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.734515:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:8.0:1619677475.734515:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.734516:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.734517:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677475.734517:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:22.0:1619677475.734518:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.734518:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.734519:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.734524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.734531:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.734533:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.734534:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.734536:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.734537:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.734538:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.734542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.734550:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.734553:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.734553:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.734554:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.734555:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.734556:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.734583:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.734588:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.734590:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.734592:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.734593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.734595:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.734597:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.734598:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498452] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.734600:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.734601:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.734602:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677475.734605:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.734607:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.734610:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.734610:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.734611:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.734613:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.734613:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.734614:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.734615:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.734619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.734623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:18.0:1619677475.734627:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.734629:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:26.0:1619677475.734630:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:18.0:1619677475.734630:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.734631:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:26.0:1619677475.734633:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:21.2:1619677475.734633:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:18.0:1619677475.734633:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.734634:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.734636:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.734636:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:26.0:1619677475.734638:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.734640:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:26.0:1619677475.734640:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677475.734641:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498451] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.734642:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:30.0:1619677475.734643:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:26.0:1619677475.734644:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.734645:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.734646:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.734647:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.734648:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.734649:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677475.734649:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.734650:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:30.0:1619677475.734651:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498450] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.734652:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.734653:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:26.0:1619677475.734653:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.734654:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:26.0:1619677475.734654:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677475.734655:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:30.0:1619677475.734657:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.734657:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.734657:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677475.734658:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677475.734658:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.734659:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:30.0:1619677475.734660:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.734660:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:30.0:1619677475.734661:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.734661:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.734662:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.734663:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677475.734663:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:30.0:1619677475.734664:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:4.2:1619677475.734665:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:30.0:1619677475.734666:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.734666:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.734668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:1.0:1619677475.734671:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.2:1619677475.734672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:10.0:1619677475.734672:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:20.2:1619677475.734674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:10.0:1619677475.734674:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:1.0:1619677475.734674:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.734675:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677475.734675:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:1.0:1619677475.734675:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.734676:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:20.0:1619677475.734676:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:31.0:1619677475.734677:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:1.0:1619677475.734677:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.734679:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:20.0:1619677475.734679:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:31.0:1619677475.734680:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.734680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677475.734681:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.734681:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.734682:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.734682:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:20.0:1619677475.734682:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.734683:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.734684:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.734685:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:20.0:1619677475.734685:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.734686:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:20.0:1619677475.734687:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:20.0:1619677475.734688:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498449] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.734689:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.734690:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.734692:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:20.0:1619677475.734695:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.734697:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:20.0:1619677475.734700:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.734701:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:20.0:1619677475.734701:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:20.0:1619677475.734703:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:20.0:1619677475.734704:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.734705:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677475.734706:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:9.2:1619677475.734709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.734716:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.734718:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:26.0:1619677475.734719:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000400:00000200:10.0:1619677475.734720:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.734722:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.734725:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:21.2:1619677475.735338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.735344:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.735347:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.735348:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.735350:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.735352:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.735353:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.735354:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498448] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.735356:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.735356:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.735357:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:21.2:1619677475.735358:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:6.0:1619677475.735360:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.735362:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.735364:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.735365:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.735365:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.735366:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677475.735367:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.735367:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.735368:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677475.735369:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:6.0:1619677475.735369:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:31.0:1619677475.735371:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.735372:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.735373:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.735374:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:25.2:1619677475.735374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.735375:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498447] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.735376:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.735377:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.735378:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.735382:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.735382:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677475.735384:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.735384:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.735385:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.735386:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.735387:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000400:00000200:21.0:1619677475.735387:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.735388:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:20.0:1619677475.735388:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.735389:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.735390:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.735390:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.735391:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.735392:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.735395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.735402:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.735404:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.735405:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.735407:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.735408:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.735409:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.735414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.735421:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.735423:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.735424:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.735425:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.735425:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.735426:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.735577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.735585:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.735588:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.735588:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.735590:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.735592:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.735594:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.735595:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498446] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.735596:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.735598:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.735598:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.735602:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.735604:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.735607:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.735607:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.735608:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.735610:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.735610:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.735611:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.735611:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.735623:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.735631:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.735634:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.735634:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.735635:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.735637:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.735640:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.736026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:2.0:1619677475.736032:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.736035:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:18.0:1619677475.736036:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.736038:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.736040:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.736041:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.736043:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498445] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.736044:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.736045:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.736046:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.736049:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.736051:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.736052:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.736053:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.736054:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.736056:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.736056:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.736057:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:2.0:1619677475.736058:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.736062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.736069:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.736071:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.736072:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.736072:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.736074:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.736075:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.736203:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.736209:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.736212:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.736212:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.736214:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.736216:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.736217:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.736218:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498444] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.736220:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.736221:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.736222:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.736224:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.736226:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.736228:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.736229:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.736230:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.736232:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.736232:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.736233:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.736233:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.736238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.736245:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.736247:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.736248:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.736250:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.736251:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.736252:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.736261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.736267:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.736269:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.736271:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.736272:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.736275:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.736276:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.736278:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498443] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.736279:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.736280:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.736281:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.736284:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.736286:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.736288:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.736289:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.736290:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.736291:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.736292:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.2:1619677475.736299:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.736299:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.736300:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:17.0:1619677475.736308:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.736310:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.736311:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.736311:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.736314:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.736316:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.736440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.736448:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.736450:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.736451:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.736453:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.736456:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.736457:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.736458:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498442] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.736460:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.736461:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.736462:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.736464:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.736466:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.736469:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.736470:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.736470:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.736472:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.736472:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.736474:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.736475:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.736475:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.736479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.736481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.736487:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.736488:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.736489:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677475.736490:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.736491:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:22.0:1619677475.736491:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.736491:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:26.0:1619677475.736493:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.736493:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.736496:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.736715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.736723:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.736726:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.736726:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.736728:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.736731:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.736732:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677475.736733:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498441] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.736734:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.736735:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.736736:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.736739:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.736741:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677475.736743:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.736744:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677475.736745:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677475.736746:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677475.736747:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.736748:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.736748:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677475.736752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.736761:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.736763:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677475.736764:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.736765:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.736767:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.736770:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677475.736773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677475.736780:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.736783:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677475.736784:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.736784:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.736785:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.736785:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.736939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.736947:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.736950:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.736952:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.736954:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.736954:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.736956:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.736957:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498440] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.736958:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.736959:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.736960:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.736963:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.736964:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.736966:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.736967:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.736967:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:9.2:1619677475.736968:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:31.0:1619677475.736969:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.736970:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.736971:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.736971:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:26.0:1619677475.736974:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:9.2:1619677475.736975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:26.0:1619677475.736977:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677475.736978:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.736980:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.106@o2ib7 : GET - for me
00000800:00000200:24.0:1619677475.736981:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:26.0:1619677475.736983:0:19941:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.106@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:24.0:1619677475.736983:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:26.0:1619677475.736984:0:19941:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.106@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000400:00000200:24.0:1619677475.736984:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.736985:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677475.736986:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.736986:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:24.0:1619677475.736986:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:26.0:1619677475.736988:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.106@o2ib7 local destination
00000800:00000200:24.0:1619677475.736988:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:26.0:1619677475.736991:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : REPLY try# 0
00000800:00000200:26.0:1619677475.736993:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.736994:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:9.2:1619677475.736994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677475.736995:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.736995:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.736997:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.736997:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677475.737001:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:9.2:1619677475.737002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677475.737003:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.737004:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.737004:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.737005:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.737006:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677475.737008:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.737010:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.737011:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677475.737011:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.737012:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: REPLY: OK
00000800:00000200:10.0:1619677475.737014:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.737164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.737172:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.737174:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.737176:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.737179:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.737179:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.737180:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.737181:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498439] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.737183:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.737184:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.737185:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.737188:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.737189:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.737193:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.737193:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.737194:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.737196:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.737196:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.737197:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.737198:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.737202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.737210:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.737212:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.737213:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.737215:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.737215:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.737218:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.737237:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:4.2:1619677475.737241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:6.0:1619677475.737242:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.737245:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.737246:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.737247:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.737248:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:6.0:1619677475.737250:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.737251:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:6.0:1619677475.737251:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.0:1619677475.737252:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:6.0:1619677475.737253:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498438] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.737254:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.737255:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.737255:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.737257:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:24.0:1619677475.737258:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.737260:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677475.737260:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677475.737261:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498437] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.737263:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.737263:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:24.0:1619677475.737264:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.737265:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:6.0:1619677475.737266:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.737267:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.737267:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:24.0:1619677475.737269:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.737269:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.737269:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677475.737271:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677475.737271:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.737272:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.737272:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:24.0:1619677475.737274:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677475.737275:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677475.737275:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.737275:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.737277:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.2:1619677475.737277:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677475.737278:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.737278:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.737279:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:22.0:1619677475.737282:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.737283:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.2:1619677475.737283:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:22.0:1619677475.737284:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.737285:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.737285:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.737286:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.737286:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.737287:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.737289:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.737289:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:1.0:1619677475.737291:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.737299:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.737301:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.737302:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.737304:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.737307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.737410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.737415:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.737417:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.737420:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.737420:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.737422:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.737424:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.737425:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498436] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.737427:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.737428:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.737429:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.737432:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.737435:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:5.2:1619677475.737436:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:21.0:1619677475.737438:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.737438:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.737439:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.737441:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.737442:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.737442:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677475.737442:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:21.0:1619677475.737443:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.737444:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.737445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.737447:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677475.737448:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:17.0:1619677475.737449:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.737451:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.737452:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498435] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.737454:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.737455:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.737456:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.737456:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.737457:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.737458:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.737460:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:1.0:1619677475.737460:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.737462:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:4.2:1619677475.737463:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:1.0:1619677475.737463:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.737464:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.737464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:17.0:1619677475.737465:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.737466:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.737467:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.737468:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.737469:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.737469:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:26.0:1619677475.737471:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.737473:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:8.2:1619677475.737473:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677475.737474:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.737476:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.737479:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.737480:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:1.0:1619677475.737480:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:26.0:1619677475.737482:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498434] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.737482:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:26.0:1619677475.737483:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.737483:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.737484:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:17.0:1619677475.737484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:26.0:1619677475.737485:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.737485:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:26.0:1619677475.737489:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.737491:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.737491:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.737492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677475.737494:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.737494:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.737495:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:26.0:1619677475.737497:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:26.0:1619677475.737498:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.737498:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.737499:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677475.737499:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.737503:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.737503:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:17.0:1619677475.737504:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (23)++
00000800:00000200:17.0:1619677475.737504:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (25)--
00000800:00000200:1.0:1619677475.737504:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (25)--
00000800:00000200:17.0:1619677475.737505:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.737505:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.737506:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.737507:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.737510:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.737552:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677475.737559:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:24.0:1619677475.737562:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677475.737562:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000400:00000200:8.0:1619677475.737565:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.737568:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.737569:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:8.0:1619677475.737571:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498433] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.737572:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.737573:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.737574:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:8.0:1619677475.737577:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.737579:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:8.0:1619677475.737581:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.737582:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:8.0:1619677475.737582:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:8.0:1619677475.737584:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:8.0:1619677475.737584:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677475.737586:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919fed9d3400] (22)++
00000800:00000200:8.0:1619677475.737587:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (23)--
00000800:00000200:8.0:1619677475.737588:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:25.2:1619677475.737590:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.737593:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (21)++
00000800:00000200:31.0:1619677475.737598:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.737600:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:24.0:1619677475.737600:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)++
00000400:00000200:31.0:1619677475.737601:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.737602:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919fed9d3400] (23)--
00000400:00000200:31.0:1619677475.737603:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.737603:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:24.0:1619677475.737604:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:31.0:1619677475.737605:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677475.737605:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:4.2:1619677475.737836:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.737843:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.737845:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.737846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.737848:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.737851:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.737852:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.737853:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498432] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.737854:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.737855:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.737856:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.737859:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.737860:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.737863:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.737864:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.737864:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.737866:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.737867:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.737867:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.737867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.737872:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.737881:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.737883:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.737884:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.737885:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.737887:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.737890:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.737927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.737933:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.737936:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.737937:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.737937:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.737938:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.737939:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.738017:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677475.738024:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:24.0:1619677475.738028:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677475.738028:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000400:00000200:8.0:1619677475.738030:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.738033:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.738034:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:8.0:1619677475.738036:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498431] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.738037:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.738038:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.738039:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:8.0:1619677475.738042:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.738043:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:8.0:1619677475.738045:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.738046:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:8.0:1619677475.738047:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:8.0:1619677475.738048:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:8.0:1619677475.738049:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.738049:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677475.738050:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:9.2:1619677475.738054:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.738060:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.738062:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.738063:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.738063:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.738065:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.738067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:20.2:1619677475.738072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:18.0:1619677475.738079:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.738081:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.738082:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.738084:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.738087:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.738088:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:18.0:1619677475.738089:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498430] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.738091:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.738092:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.738093:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:18.0:1619677475.738096:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.738098:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:18.0:1619677475.738098:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:18.0:1619677475.738101:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.738102:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:18.0:1619677475.738103:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:18.0:1619677475.738104:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:6.0:1619677475.738104:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:18.0:1619677475.738105:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.738106:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:22.0:1619677475.738107:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677475.738107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:6.0:1619677475.738107:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.738109:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677475.738110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000400:00000200:6.0:1619677475.738111:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.738113:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.738114:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498429] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.738115:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.738116:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:6.0:1619677475.738116:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.738118:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.738119:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.738120:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677475.738120:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.738121:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:10.0:1619677475.738123:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.738123:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:10.0:1619677475.738126:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:6.0:1619677475.738126:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.738127:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.738128:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.738130:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.738130:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.738131:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.738131:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.738136:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.738143:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.738145:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.738146:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.738148:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.738149:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.738150:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.738151:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.738151:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.738152:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.738152:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.738153:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.738155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.738162:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.738164:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.738166:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.738167:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.738170:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.738171:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.738173:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498428] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.738174:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.738175:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.738176:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.738180:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.738181:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:21.2:1619677475.738183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:24.0:1619677475.738185:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.738185:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.738186:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.738188:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.738188:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.738189:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.738190:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:30.0:1619677475.738191:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.738191:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:30.0:1619677475.738194:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.2:1619677475.738194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:30.0:1619677475.738196:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:4.2:1619677475.738196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:31.0:1619677475.738198:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.738199:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.738200:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.738201:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498427] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.738202:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.738202:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:30.0:1619677475.738203:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.738204:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.738204:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677475.738205:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.738205:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.738206:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.738206:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.738206:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677475.738207:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.738207:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677475.738208:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.738209:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.738211:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.738211:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:30.0:1619677475.738212:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.738213:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.738214:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.738215:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677475.738215:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:30.0:1619677475.738216:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.738216:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.738220:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:1.0:1619677475.738223:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.738226:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.738226:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:1.0:1619677475.738226:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.738227:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.738228:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.738228:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:24.0:1619677475.738229:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:8.0:1619677475.738229:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.738230:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.738232:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.738235:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.738699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.738707:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.738710:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.738712:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.738714:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.738714:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.738715:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.738716:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498426] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.738718:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.738719:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.738720:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.738722:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.738724:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.738726:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.738727:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.738727:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.738729:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.738729:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.738731:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.738732:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.738733:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.2:1619677475.738738:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.738746:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:9.2:1619677475.738746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.738749:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.738749:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.738750:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:24.0:1619677475.738752:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.738755:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.738756:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.738756:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.738757:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.738760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.738850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677475.738856:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.738859:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677475.738860:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677475.738862:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.738865:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.738866:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.738868:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498425] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.738869:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.738870:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.738871:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.738874:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.738876:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.738878:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.738878:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.738879:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.738880:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.738881:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.738882:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677475.738882:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677475.738886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.738894:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.738896:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.738897:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.738897:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.738898:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.738900:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.738966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.738972:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.738975:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.738976:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.738978:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.738980:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.738981:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.738982:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498424] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.738984:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.738985:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.738986:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.738989:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.738990:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.738993:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.738993:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.738994:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.738996:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.738996:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.738997:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.738997:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.739002:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.739010:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.739012:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.739013:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.739014:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.739016:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.739016:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.739172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.739177:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.739180:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.739182:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.739182:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.739185:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.739186:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.739187:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498423] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.739189:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.739189:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.739191:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.739194:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.739196:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.739198:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.739199:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.739200:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.739202:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.739202:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.739203:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.739204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.739208:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.739215:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.739217:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.739218:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.739219:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.739221:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.739223:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.739316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.739323:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.739326:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.739326:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.739328:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.739330:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.739332:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.739333:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498422] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.739334:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.739335:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.739337:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.739340:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.739342:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.739344:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.739345:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.739346:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.739348:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.739348:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.739349:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.739350:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.739355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.739363:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.739365:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.739366:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.739368:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.739369:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.739371:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.739388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.739396:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.739399:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.739401:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.739404:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.739404:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.739405:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.739407:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498421] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.739408:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.739409:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.739410:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.739413:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.739415:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.739417:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.739418:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.739419:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.739420:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.739421:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.739422:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.739424:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.739424:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.739427:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.739430:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.739434:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.739436:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.739437:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.739438:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.739438:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.739438:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.739439:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.739440:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.739441:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.739442:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.739992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.739998:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.740001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.740001:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.740003:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.740006:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.740007:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.740008:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498420] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.740010:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.740011:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.740012:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.740015:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.740017:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.740019:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.740020:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.740020:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.740022:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.740023:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.740023:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.740023:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.740028:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.740036:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.740038:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.740039:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.740041:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.740041:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.740043:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.740122:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.740129:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.740132:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.740134:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.740136:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.740137:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.740138:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.740140:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498419] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.740141:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.740142:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.740143:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.740146:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.740148:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.740150:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.740151:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.740151:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.740153:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.740154:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.740154:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.740155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.740159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.740165:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.740168:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.740168:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.740170:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.740172:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.740173:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.740568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.740576:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.740578:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.740580:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.740581:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.740584:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.740585:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.740586:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498418] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.740588:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.740588:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.740589:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.740592:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.740594:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.740596:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.740597:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.740597:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.740599:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.740599:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.740600:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.740600:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.740605:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.740612:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.740614:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.740615:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.740616:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.740617:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.740619:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.740715:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.740724:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.740726:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.740728:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.740730:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.740731:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677475.740731:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.740733:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498417] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.740734:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.740735:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.740736:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.740738:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.740740:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.740742:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.740743:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.740743:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.740745:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.740745:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.740746:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.740747:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.740750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.740757:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.740759:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.740760:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.740760:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.740762:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.740764:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.740838:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.740844:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.740847:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.740848:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.740849:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.740852:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.740853:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.740855:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498416] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.740856:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.740857:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.740858:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.740861:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.740862:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.740865:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.740866:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.740866:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.740868:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.740868:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.740870:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.740871:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.740871:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.740874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.740877:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.740881:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.740882:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.740883:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.740884:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.740884:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.740885:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.740885:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.740886:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.740888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.740889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.740930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.740935:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.740937:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.740940:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.740940:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.740942:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.740944:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677475.740945:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498415] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.740946:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.740947:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.740948:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.740952:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.740954:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.740957:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.740958:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.740958:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.740960:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.740961:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.740962:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.740963:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:21.0:1619677475.740964:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677475.740966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.740969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:30.0:1619677475.740973:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:20.0:1619677475.740973:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.740975:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:30.0:1619677475.740976:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:20.0:1619677475.740976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.740976:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:30.0:1619677475.740977:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.740979:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.740980:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.740982:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.741110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.741117:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.741120:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.741121:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.741123:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.741125:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.741127:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.741128:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498414] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.741129:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.741130:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.741131:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.741134:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.741136:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.741139:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.741139:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.741140:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.741142:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.741142:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.741143:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.741143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677475.741148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.741156:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.741158:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.741159:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.741161:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.741161:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:20.0:1619677475.741164:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.741169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.741177:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.741180:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.741181:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.741182:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.741182:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.741183:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.741316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.741324:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.741327:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.741327:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.741329:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.741332:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.741333:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.741334:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498413] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.741335:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.741337:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.741337:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.741341:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.741342:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.741344:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.741345:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.741345:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.741347:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.741347:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.741348:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.741348:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.741353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.741360:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.741362:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.741363:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.741363:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.741365:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.741367:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.741372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.741378:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.741380:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.741381:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.741381:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.741382:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.741382:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.741489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677475.741495:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.741498:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677475.741500:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677475.741500:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.741503:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.741504:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.741505:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498412] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.741507:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.741508:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.741509:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.741511:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.741513:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.741514:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.741515:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.741516:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.741517:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.741518:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.741519:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677475.741519:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:9.2:1619677475.741524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.741531:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.741533:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.741534:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.741534:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.741535:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.741536:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.741548:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.741555:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.741557:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.741558:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.741558:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.741559:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.741559:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.741694:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.741700:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.741703:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.741704:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.741706:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.741708:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.741709:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.741711:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498411] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.741712:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.741713:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.741714:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.741717:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.741719:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.741721:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.741722:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.741723:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.741724:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.741725:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.741725:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.741726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.741730:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.741738:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.741740:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.741741:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.741742:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.741744:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.741745:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.741751:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.741758:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.741761:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.741762:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.741762:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.741763:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.741764:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.742229:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.742235:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.742238:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.742239:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.742241:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.742243:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.742245:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.742246:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498410] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.742247:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.742248:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.742250:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.742253:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.742255:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.742257:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.742257:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.742258:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.742260:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.742260:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.742261:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.742262:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.742263:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.742266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.742268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.742275:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.742275:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.742276:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.742276:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.742276:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.742277:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.742277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677475.742278:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.742279:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.742280:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.742315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.742324:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.742327:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.742329:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.742331:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.742331:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.742333:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.742334:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498409] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.742335:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.742336:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.742337:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.742341:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.742343:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.742345:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.742346:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.742346:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.742348:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.742349:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.742350:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.742350:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.742354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.742361:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.742364:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.742365:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.742365:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.742367:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.742369:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.742372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.742378:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.742381:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.742382:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.742382:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.742382:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.742383:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.742413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:4.2:1619677475.742420:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:1.0:1619677475.742420:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.742422:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:17.0:1619677475.742423:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:1.0:1619677475.742425:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.742427:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:1.0:1619677475.742427:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.742428:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:24.0:1619677475.742430:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.742430:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677475.742430:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498408] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.742431:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.742433:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.742434:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677475.742435:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.742438:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.742440:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:24.0:1619677475.742441:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.742443:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:24.0:1619677475.742444:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:1.0:1619677475.742444:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000400:00000200:24.0:1619677475.742445:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498407] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.742445:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000400:00000200:24.0:1619677475.742447:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:1.0:1619677475.742447:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:1.0:1619677475.742447:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:1.0:1619677475.742447:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:24.0:1619677475.742448:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.742448:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000400:00000200:24.0:1619677475.742449:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.742452:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:9.2:1619677475.742453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.742454:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.742458:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.742459:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.742459:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.742460:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.742461:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.742462:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.742462:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.742463:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.742463:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.742464:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:24.0:1619677475.742465:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.742465:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.2:1619677475.742467:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.742472:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:17.0:1619677475.742475:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.742478:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.742479:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.742479:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.742482:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.742484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.742714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.742720:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.742723:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.742724:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.742726:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.742728:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.742729:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.742731:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498406] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.742732:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.742733:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.742734:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.742737:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.742739:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.742741:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.742742:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.742743:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.742744:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.742745:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.742745:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.742746:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.742750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.742758:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.742760:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.742761:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.742763:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.742764:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.742765:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.742835:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.742843:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.742846:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.742846:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677475.742848:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.742851:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.742852:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.742853:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498405] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.742854:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.742855:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.742856:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.742859:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.742861:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.742864:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.742864:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.742865:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.742867:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.742867:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.742868:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.742868:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.742873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.742880:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.742883:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.742884:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.742884:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.742886:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.742889:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.743119:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.743125:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.743128:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.743129:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.743131:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.743133:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.743135:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.743136:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498404] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.743137:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.743138:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.743139:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.743142:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.743143:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.743146:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.2:1619677475.743147:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:1.0:1619677475.743147:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.743147:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.743149:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.743149:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.743150:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.743150:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:18.0:1619677475.743155:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:8.2:1619677475.743155:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:18.0:1619677475.743157:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.743158:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.743160:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.743162:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.743163:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000400:00000200:18.0:1619677475.743163:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:20.0:1619677475.743165:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:18.0:1619677475.743165:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498403] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.743166:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.743166:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.743167:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.743168:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.743169:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.743169:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.743171:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:18.0:1619677475.743171:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.743174:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.743175:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.743176:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.743177:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.743179:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.743179:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.743180:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.743181:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.743185:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.743192:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.743194:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.743195:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.743195:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.743197:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.743198:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.743261:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.743267:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.743270:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.743271:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.743272:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.743274:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.743275:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.743276:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498402] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.743277:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.743278:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.743279:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.743282:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.743283:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.743286:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.743287:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.743288:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.743290:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.743290:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.743290:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.743291:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.743298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.743307:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.743310:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.743312:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.743314:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.743315:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.743318:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.743323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:13.0:1619677475.743330:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.743333:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:13.0:1619677475.743334:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.743335:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:13.0:1619677475.743335:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.743336:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.743580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.743585:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.743588:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.743589:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.743591:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.743593:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.743594:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.743595:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498401] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.743597:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.743598:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.743599:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.743602:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.743604:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.743607:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.743608:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.743608:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.743610:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.743610:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.743611:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.743612:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.743616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.743624:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.743626:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.743627:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.743628:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.743629:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.743630:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.743658:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.743661:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.743663:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.743665:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.743666:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.743668:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.743670:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.743671:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498400] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.743672:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.743673:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.743674:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.743677:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.743679:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.743681:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.743682:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.2:1619677475.743683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677475.743683:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.743685:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.743685:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.743686:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.743687:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.743688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:30.0:1619677475.743691:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.2:1619677475.743691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.743693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:30.0:1619677475.743694:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.743696:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.743698:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.743698:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:31.0:1619677475.743699:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.743700:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:21.0:1619677475.743700:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677475.743700:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:30.0:1619677475.743701:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498399] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.743701:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:30.0:1619677475.743702:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:21.0:1619677475.743702:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.743702:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:30.0:1619677475.743703:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.743704:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677475.743704:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.743704:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.743706:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.743708:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.743709:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677475.743710:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.743713:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.743713:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.743714:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.743716:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.743716:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.743717:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.743718:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.743721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.743728:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.743730:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.743731:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.743732:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.743734:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.743736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.744297:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.744305:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.744308:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.744309:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.744312:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.744312:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.744314:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.744315:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498398] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.744316:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.744317:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.744318:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.744321:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.744322:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.744324:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.744325:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.744326:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.744327:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.744328:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.744329:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.744329:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.744334:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.744341:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.744343:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.744344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.744344:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.744346:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.744348:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.744399:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.744406:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.744409:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.744411:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.744413:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.744414:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.744415:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.744416:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498397] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.744417:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.744418:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.744419:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.744422:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.744424:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.744426:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.744426:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.744427:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.744428:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.744429:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.744430:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.744430:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.744442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:25.2:1619677475.744445:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677475.744447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.744450:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:31.0:1619677475.744451:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:6.0:1619677475.744451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.744453:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.744454:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.744454:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.744455:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.744456:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.744457:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:30.0:1619677475.744458:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:22.0:1619677475.744458:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498396] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.744459:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:22.0:1619677475.744459:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.744460:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.744461:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.744465:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.744467:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.744469:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.744470:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.744470:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.744472:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.744473:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.744474:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.744475:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.744475:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.744478:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.744480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.744486:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.744488:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.744488:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.744488:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.744489:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.744490:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.744491:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.744492:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.744493:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.744495:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.744535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.744543:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.744545:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.744546:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.744548:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.744551:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.744552:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.744554:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498395] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.744555:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.744556:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.744557:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.744560:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.744562:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.744565:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.744566:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.744566:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.744568:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.744568:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.744569:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.744570:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.744574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.744582:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.744585:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.744586:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.744586:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.744588:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.744591:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.744856:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a070bf0c00] (20)++
00000800:00000200:26.0:1619677475.744863:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677475.744866:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.4@o2ib3
00000800:00000200:10.0:1619677475.744867:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (22)--
00000400:00000200:26.0:1619677475.744868:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.4@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.744871:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.744873:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:26.0:1619677475.744874:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498394] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.744875:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.744876:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.744877:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:26.0:1619677475.744880:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.4@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.744882:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:26.0:1619677475.744885:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.744885:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:26.0:1619677475.744886:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:26.0:1619677475.744888:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:26.0:1619677475.744888:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a070bf0c00] (21)++
00000800:00000200:26.0:1619677475.744889:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a070bf0c00] (22)--
00000800:00000200:26.0:1619677475.744890:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a070bf0c00] (21)--
00000800:00000200:25.2:1619677475.744894:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.744901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.0:1619677475.744901:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.744904:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.744905:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.744907:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.744908:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.744909:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:20.0:1619677475.744909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.744912:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677475.744912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000400:00000200:30.0:1619677475.744914:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.744916:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.744916:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.744917:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.744919:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498393] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.744919:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:30.0:1619677475.744920:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.744921:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.744922:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:21.0:1619677475.744922:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.744923:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.744924:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.744925:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:30.0:1619677475.744926:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.744926:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.744927:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.744929:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.744930:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.744930:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.744932:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.744932:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.744934:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.744935:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.744936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.744937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.744941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.744946:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.744948:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.744948:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.744950:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.744950:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.744951:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.744951:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.744952:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.744953:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.744954:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.744959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.744966:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.744969:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.744970:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.744970:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.744971:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.744972:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.745401:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.745408:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.745411:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.745412:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.745414:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.745416:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.745418:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.745419:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498392] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.745420:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.745421:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.745422:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.745425:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.745426:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.745429:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.745430:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.745430:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.745432:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.745432:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.745433:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.745433:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.745439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.745447:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.745450:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.745451:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.745451:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.745453:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.745456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.745465:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.745471:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.745473:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.745474:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.745476:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.745477:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.745479:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.745480:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498391] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.745481:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.745482:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.745483:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.745486:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.745488:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.745490:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.745491:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.745492:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.745494:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.745494:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.745495:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.745495:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.745500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.745500:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.745509:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.745509:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.745511:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:30.0:1619677475.745512:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:21.0:1619677475.745512:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.745514:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.745515:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.745515:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.745516:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.745517:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.745517:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.745519:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.745520:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498390] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.745521:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.745522:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.745523:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.745527:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.745529:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.745532:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.745533:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.745534:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.745536:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.745536:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.745537:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.745538:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.745542:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.745550:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.745552:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.745553:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.745555:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.745556:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.745558:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.745577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.745583:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.745586:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.745586:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.745588:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.745591:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.745592:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:22.0:1619677475.745593:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498389] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.745595:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.745596:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.745596:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:22.0:1619677475.745600:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.745601:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:22.0:1619677475.745604:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.745605:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:22.0:1619677475.745606:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:22.0:1619677475.745607:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:22.0:1619677475.745608:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.745609:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc6f800] (22)++
00000800:00000200:22.0:1619677475.745610:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (23)--
00000800:00000200:22.0:1619677475.745611:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:9.2:1619677475.745613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.2:1619677475.745616:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (21)++
00000800:00000200:26.0:1619677475.745620:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:6.0:1619677475.745621:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)++
00000800:00000200:26.0:1619677475.745623:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:6.0:1619677475.745623:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc6f800] (23)--
00000400:00000200:26.0:1619677475.745624:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.745624:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:6.0:1619677475.745624:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.745625:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000400:00000200:26.0:1619677475.745626:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.745629:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677475.745682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.745689:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.745692:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.745692:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:10.0:1619677475.745694:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.745696:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.745698:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.745699:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498388] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.745700:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.745701:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.745702:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.745706:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.745708:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:10.0:1619677475.745710:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.745711:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.745712:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.745713:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.745714:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.745715:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.745716:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.745717:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:8.2:1619677475.745719:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:4.2:1619677475.745722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:21.0:1619677475.745726:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:26.0:1619677475.745729:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:21.0:1619677475.745729:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.745730:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.745731:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:21.0:1619677475.745731:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:26.0:1619677475.745732:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:20.0:1619677475.745732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.745733:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:21.0:1619677475.745734:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.745986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.745994:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.745996:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.745999:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.746000:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.746002:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.746003:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.746004:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498387] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.746005:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.746006:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.746008:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.746011:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.746012:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.746014:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.746015:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.746016:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.746017:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.746018:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.746019:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.746021:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.746021:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.746023:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.746027:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.746030:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.746032:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.746033:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.746033:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.746035:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.746035:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.746037:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.746038:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677475.746038:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.746039:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.746132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.746139:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.746142:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.746143:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.746145:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.746147:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.746148:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.746150:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498386] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.746151:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.746152:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.746153:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.746156:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.746157:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.746159:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.746160:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.746161:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.746162:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.746163:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.746164:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.746165:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.746166:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677475.746168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.746171:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677475.746174:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.746176:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.746177:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.746178:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:8.0:1619677475.746178:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.746179:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.746180:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:24.0:1619677475.746181:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:10.0:1619677475.746181:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.746182:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:21.2:1619677475.746312:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.746318:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.746321:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.746321:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.746323:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.746326:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.746327:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.746329:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498385] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.746330:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.746331:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.746332:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.746335:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.746337:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.746340:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.746340:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.746341:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.746343:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.746343:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.746344:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.746344:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.746350:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.746357:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.746360:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.746361:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.746362:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.746364:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.746364:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.746587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.746593:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.746596:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.746597:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.746599:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.746601:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.746602:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.746603:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498384] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.746605:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.746606:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.746607:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.746610:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.746612:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.746614:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.746615:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.746615:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.746617:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.746617:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.746618:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.746618:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.746624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.746631:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.746633:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.746634:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.746635:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.746636:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.746637:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.746643:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.746650:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.746653:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.746654:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.746654:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.746654:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.746655:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.747108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.747113:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.747116:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.747117:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.747118:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.747119:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.747120:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.747121:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498383] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.747123:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.747123:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.747124:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.747127:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.747128:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.747130:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.747131:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.747131:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.747133:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.747133:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.747134:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:1.0:1619677475.747135:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.747136:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.747139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.747141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.747146:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.747148:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:21.0:1619677475.747149:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:17.0:1619677475.747149:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.747150:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677475.747151:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.747152:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.747154:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.747154:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.747156:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.747163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.2:1619677475.747169:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.0:1619677475.747170:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.747173:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677475.747174:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.747174:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.747174:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.747175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.747176:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:22.0:1619677475.747177:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.747178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.747180:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.747182:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.747184:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.747184:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498382] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.747186:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.747187:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.747188:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.747191:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.747193:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.747195:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.747196:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.747196:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.747198:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.747198:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.747199:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.747199:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.747204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.747211:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.747213:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.747214:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.747216:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.747216:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.747217:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.747504:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.747507:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.747510:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.747512:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.747513:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.747515:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.747516:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.747518:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498381] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.747519:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.747520:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.747521:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.747524:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.747526:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.747528:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.747529:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.747529:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.747531:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.747532:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.747532:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.747533:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.747538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.747546:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.747549:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.747550:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.747550:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.747552:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:25.2:1619677475.747554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.747558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.747559:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.747561:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.747562:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.747562:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.747563:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.747564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.747886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.747894:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.747896:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.747897:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.747899:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.747902:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.747903:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.747904:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498380] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.747905:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.747907:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.747907:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.747911:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.747913:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.747916:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.747917:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.747917:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.747919:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.747919:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.747920:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.747921:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.747922:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:4.2:1619677475.747927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:25.2:1619677475.747935:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.747935:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.747936:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.747937:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.747938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677475.747943:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.747946:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.747946:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.747947:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.747949:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.747952:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.748040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.748046:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.748049:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.748050:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.748053:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.748056:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.748057:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.748059:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498379] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.748060:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.748061:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.748062:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.748066:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.748068:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.748071:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.748071:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.748072:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.748074:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.748075:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.748076:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.748077:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.748077:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.748080:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.2:1619677475.748083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:30.0:1619677475.748087:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677475.748089:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:30.0:1619677475.748090:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.748091:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.748091:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.748091:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.748092:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:30.0:1619677475.748093:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.748094:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.748096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.748105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.748112:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.748115:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.748116:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.748116:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.748117:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.748119:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.748146:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.748151:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.748154:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.748155:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.748157:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.748159:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.748160:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.748161:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498378] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.748163:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.748164:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.748165:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.748168:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.748169:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.748172:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.748173:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.748173:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.748175:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.748175:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.748177:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.748178:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.748178:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.748181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.748184:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.748189:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.748191:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.748191:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.748191:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.748192:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.748193:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.748193:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.748195:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.748196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.748197:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.748199:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:25.2:1619677475.748202:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:8.0:1619677475.748207:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.748209:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.748210:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677475.748210:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.748212:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000400:00000200:8.0:1619677475.748212:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.748213:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.748214:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.748214:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677475.748214:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.748215:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:8.0:1619677475.748216:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.748218:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498377] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.748219:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.748220:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.748221:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.748224:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.748226:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.748229:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.748230:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.748230:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.748232:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.748233:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.748233:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.748234:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.748238:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.748246:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.748248:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.748249:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.748250:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.748252:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.748255:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.748272:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.748280:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.748283:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.748285:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.748287:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.748288:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.748289:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.748291:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498376] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.748292:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.748298:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.748298:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.748302:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.748304:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.748306:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.748306:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.748307:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.748309:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.748309:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.748310:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.748311:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.748315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.748322:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.748324:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.748325:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.748325:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.748327:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.748329:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.748332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.748339:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.748341:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.748342:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.748342:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.748343:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.748343:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.748914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.748922:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.748925:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.748927:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.748929:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.748929:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.748931:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.748932:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498375] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.748933:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.748934:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.748935:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.748938:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.748939:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.748941:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.748942:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.748942:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.748944:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.748945:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.748946:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.748946:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.748950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.748957:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.748959:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.748960:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.748961:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.748963:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.748964:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.749007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.749013:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.749015:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.749016:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.749018:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.749020:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.749022:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.749023:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498374] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.749024:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.749025:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.749026:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.749030:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.749032:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.749034:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.749035:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.749035:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.749037:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.749037:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.749038:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.749039:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.749043:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.749051:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.749053:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.749054:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.749056:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.749057:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.749059:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.749144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677475.749150:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.749153:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677475.749153:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000400:00000200:6.0:1619677475.749155:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.749157:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.749159:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:6.0:1619677475.749160:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498373] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.749161:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.749162:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.749163:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:6.0:1619677475.749166:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.749167:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677475.749169:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.749170:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:6.0:1619677475.749170:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:6.0:1619677475.749172:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:6.0:1619677475.749172:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.749173:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (22)--
00000800:00000200:6.0:1619677475.749173:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000800:00000200:9.2:1619677475.749178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.749186:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.749188:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.749189:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.749190:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.749191:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.749194:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.749528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.749536:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.749539:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.749541:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.749543:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.749544:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.749544:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.749546:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498372] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.749547:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.749548:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.749549:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.749552:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.749554:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.749556:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.749557:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.749557:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.749559:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.749559:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.749560:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.749561:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.749565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.749573:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.749575:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.749576:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.749578:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.749579:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.749580:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.749654:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.749660:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.749663:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.749665:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.749665:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.749668:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.749670:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.749671:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498371] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.749672:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.749673:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.749674:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.749678:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.749679:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.749682:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.749683:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.749683:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.749685:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.749686:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.749686:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.749687:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.749692:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.749701:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.749703:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.749704:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.749704:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.749706:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.749709:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.749844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.749850:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.749852:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.749853:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.749855:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.749858:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.749859:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.749861:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498370] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.749862:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.749863:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.749864:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.749867:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.749869:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.749871:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.749872:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.749873:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.749875:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.749875:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.749875:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.749876:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.749881:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.749888:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.749890:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.749891:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.749892:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.749893:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.749895:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.749966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:18.0:1619677475.749973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.749976:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:2.0:1619677475.749976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:18.0:1619677475.749978:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.749981:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.749982:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.749983:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498369] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.749985:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.749986:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.749987:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.749991:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.749992:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.749996:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.749996:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.749997:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.749999:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.749999:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:21.2:1619677475.750000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:18.0:1619677475.750001:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (22)--
00000800:00000200:18.0:1619677475.750001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:25.2:1619677475.750005:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.750009:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.750011:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:31.0:1619677475.750013:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000400:00000200:30.0:1619677475.750013:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.750014:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.750015:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.750015:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.750017:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.750018:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000400:00000200:30.0:1619677475.750018:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498368] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.750020:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.750021:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.750022:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.750025:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:30.0:1619677475.750025:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.750027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.750027:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677475.750027:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.750029:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.750030:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.750030:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.750032:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.750033:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.750034:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.750034:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.750038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.750044:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.750047:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.750048:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.750048:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.750050:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.750052:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.750152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.750160:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.750162:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.750164:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.750165:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.750168:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.750169:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.750170:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498367] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.750172:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.750173:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.750173:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.750176:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.750178:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.750180:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.750181:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.750181:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.750183:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.750183:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.750184:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.750184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.750194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.750201:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.750203:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.750204:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.750204:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.750205:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.750207:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.750329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.750337:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:4.2:1619677475.750338:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:31.0:1619677475.750339:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.750341:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.750343:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.750344:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.750344:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000400:00000200:31.0:1619677475.750345:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.750346:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498366] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.750347:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:31.0:1619677475.750348:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.750349:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.750349:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:31.0:1619677475.750350:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:21.0:1619677475.750350:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:21.0:1619677475.750352:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.750353:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.750354:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000800:00000200:31.0:1619677475.750355:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:21.0:1619677475.750355:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498365] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.750356:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.750358:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.750358:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.750359:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.750359:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677475.750359:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.750360:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.750361:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.750362:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:21.0:1619677475.750362:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : ACK try# 0
00000800:00000200:31.0:1619677475.750364:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:21.0:1619677475.750364:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.750365:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:21.0:1619677475.750366:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.750367:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.750368:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677475.750370:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.750374:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.750375:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.750376:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.750376:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677475.750377:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.750381:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:9.2:1619677475.750381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:30.0:1619677475.750383:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.750384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.750386:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:30.0:1619677475.750386:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.750387:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.750388:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:30.0:1619677475.750388:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.750388:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:30.0:1619677475.750389:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.750391:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:30.0:1619677475.750392:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:18.0:1619677475.750392:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.750392:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.750394:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.750397:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.750828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.750833:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.750836:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.750837:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.750839:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.750841:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.750843:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.750844:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498364] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.750845:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.750846:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.750847:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.750851:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.750852:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.750855:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.750856:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.750856:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.750858:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.750858:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.750859:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.750860:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.750864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.750871:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.750873:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.750874:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.750876:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.750877:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.750878:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.751055:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.751063:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.751066:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.751067:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.751069:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.751071:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.751072:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.751074:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498363] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.751075:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.751076:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.751077:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.751081:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.751083:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.751085:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.751086:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.751087:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.751088:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.751089:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.751090:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.751090:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.751105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.751113:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.751115:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.751116:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.751117:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.751119:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.751122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.751403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.751408:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.751411:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.751413:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.751414:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.751415:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.751417:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.751418:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498362] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.751419:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.751420:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.751421:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.751424:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.751426:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.751428:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.751429:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.751430:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.751431:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.751432:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.751433:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.751434:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.751435:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:25.2:1619677475.751438:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.751440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.751444:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:20.0:1619677475.751446:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:6.0:1619677475.751446:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.751447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.751447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:6.0:1619677475.751448:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.751449:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.751450:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.751451:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.751453:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.751454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.751460:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.751463:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.751464:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.751466:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.751468:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.751469:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.751470:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498361] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.751472:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.751473:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.751474:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.751477:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.751479:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.751480:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.751481:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.751482:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.751483:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.751483:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.751484:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.751484:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.751490:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.751498:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.751499:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.751500:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.751501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.751502:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.751503:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.751743:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.751749:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.751751:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.751752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.751754:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.751756:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.751757:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.751759:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498360] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.751760:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.751761:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.751762:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.751765:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.751767:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.751769:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.751770:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.751771:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.751772:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.751773:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.751774:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.751775:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.751776:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.751778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.751781:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.751786:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.751786:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.751788:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.751788:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:20.0:1619677475.751789:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.751789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.751790:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.751790:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.751792:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.751793:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.751798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.751805:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.751808:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.751809:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.751809:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.751810:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.751811:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.751829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.751838:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.751840:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.751843:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.751845:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.751845:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.751847:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.751848:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498359] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.751850:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.751851:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.751852:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.751855:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.751857:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.751861:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.751861:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.751862:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.751864:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.751864:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.751865:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.751866:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.751870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.751878:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.751880:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.751881:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.751883:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.751883:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.751886:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.752567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc0000] (20)++
00000800:00000200:30.0:1619677475.752576:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)++
00000800:00000200:30.0:1619677475.752579:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.3.49@o2ib3
00000400:00000200:30.0:1619677475.752581:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.54@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.49@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.752583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000400:00000200:30.0:1619677475.752584:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.752586:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.54@o2ib7 local destination
00000400:00000200:30.0:1619677475.752587:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498358] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.752588:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.752589:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.752590:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.54@o2ib7
00000400:00000200:30.0:1619677475.752593:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.54@o2ib7(10.0.10.54@o2ib7:10.0.10.54@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.752595:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.54@o2ib7
00000800:00000200:30.0:1619677475.752598:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f100] -> 10.0.10.54@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.752598:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e42d9c400] (20)++
00000800:00000200:30.0:1619677475.752599:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e42d9c400] (21)++
00000800:00000200:30.0:1619677475.752601:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e42d9c400] (22)--
00000800:00000200:30.0:1619677475.752601:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc0000] (21)++
00000800:00000200:30.0:1619677475.752602:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc0000] (22)--
00000800:00000200:30.0:1619677475.752603:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)--
00000800:00000200:24.2:1619677475.752607:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e42d9c400] (21)++
00000800:00000200:24.0:1619677475.752611:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)++
00000800:00000200:24.0:1619677475.752613:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e42d9c400] (23)--
00000400:00000200:24.0:1619677475.752614:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.752614:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (22)--
00000400:00000200:24.0:1619677475.752616:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.54@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.752618:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e42d9c400] (21)--
00000800:00000200:4.2:1619677475.752642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc0000] (20)++
00000800:00000200:21.2:1619677475.752650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.752651:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)++
00000800:00000200:31.0:1619677475.752654:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.49@o2ib3
00000400:00000200:31.0:1619677475.752655:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.49@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.752656:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677475.752657:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.752658:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000400:00000200:31.0:1619677475.752659:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:22.0:1619677475.752659:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.752660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677475.752661:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498357] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.752663:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.752663:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.752664:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.752665:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:22.0:1619677475.752665:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.752668:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.752670:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:22.0:1619677475.752671:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.752672:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498356] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.752674:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.752674:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.752675:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:22.0:1619677475.752675:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.752676:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:22.0:1619677475.752676:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.752677:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.752678:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc0000] (21)++
00000800:00000200:31.0:1619677475.752679:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc0000] (22)--
00000800:00000200:4.2:1619677475.752679:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc0000] (22)++
00000400:00000200:22.0:1619677475.752680:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.752681:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.752682:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000800:00000200:31.0:1619677475.752683:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)++
00000800:00000200:25.2:1619677475.752684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.752684:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.752685:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.752685:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.752686:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.49@o2ib3
00000800:00000200:22.0:1619677475.752687:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677475.752688:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.49@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.752688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000800:00000200:22.0:1619677475.752688:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.752689:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:31.0:1619677475.752690:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.752690:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:31.0:1619677475.752691:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:22.0:1619677475.752692:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677475.752693:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96498355] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.752693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.752693:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:31.0:1619677475.752694:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:6.0:1619677475.752694:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677475.752695:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.752695:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.752696:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:22.0:1619677475.752697:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.752700:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.752700:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.0:1619677475.752700:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:31.0:1619677475.752702:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.752702:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.752703:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.752705:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.752705:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:20.0:1619677475.752705:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.752706:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.752706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:25.2:1619677475.752707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.752707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:31.0:1619677475.752708:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:31.0:1619677475.752708:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc0000] (21)++
00000800:00000200:31.0:1619677475.752710:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc0000] (22)--
00000800:00000200:31.0:1619677475.752710:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)--
00000800:00000200:8.2:1619677475.752714:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677475.752715:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.752717:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.752718:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.752719:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.752720:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.752720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:17.0:1619677475.752722:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:4.2:1619677475.752724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc0000] (20)++
00000800:00000200:17.0:1619677475.752725:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.752726:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.752726:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.752728:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.752731:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:30.0:1619677475.752732:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)++
00000800:00000200:30.0:1619677475.752736:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.49@o2ib3
00000400:00000200:30.0:1619677475.752738:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.49@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.752740:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000400:00000200:30.0:1619677475.752741:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.752742:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.752743:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498354] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.752745:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.752746:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.752747:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:4.2:1619677475.752748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc0000] (21)++
00000400:00000200:30.0:1619677475.752750:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.752751:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.752753:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.752753:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.752754:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.752755:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.752756:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)++
00000800:00000200:30.0:1619677475.752756:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc0000] (22)++
00000800:00000200:31.0:1619677475.752757:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.49@o2ib3
00000800:00000200:30.0:1619677475.752758:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc0000] (24)++
00000400:00000200:31.0:1619677475.752759:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.49@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.752759:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc0000] (25)--
00000800:00000200:30.0:1619677475.752760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (24)--
00000400:00000200:31.0:1619677475.752761:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:25.2:1619677475.752761:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.752762:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (23)--
00000400:00000200:31.0:1619677475.752763:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:30.0:1619677475.752763:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:31.0:1619677475.752764:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498353] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.752764:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.752764:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.752765:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.752766:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.752766:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.752767:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:4.2:1619677475.752767:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc0000] (22)++
00000800:00000200:30.0:1619677475.752769:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:31.0:1619677475.752771:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.752771:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.752772:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677475.752772:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc0000] (23)++
00000800:00000200:30.0:1619677475.752773:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc0000] (24)--
00000800:00000200:30.0:1619677475.752773:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (23)--
00000800:00000200:30.0:1619677475.752774:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000800:00000200:31.0:1619677475.752775:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.752776:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:31.0:1619677475.752777:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:31.0:1619677475.752778:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:31.0:1619677475.752779:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc0000] (21)++
00000800:00000200:31.0:1619677475.752780:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc0000] (22)--
00000800:00000200:31.0:1619677475.752780:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)--
00000800:00000200:8.2:1619677475.752784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:25.2:1619677475.752788:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:21.0:1619677475.752792:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.752794:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:30.0:1619677475.752795:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:21.0:1619677475.752796:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.752797:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.752797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:30.0:1619677475.752798:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.752799:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.752799:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.752800:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.0:1619677475.752800:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:30.0:1619677475.752801:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.752825:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc0000] (20)++
00000800:00000200:30.0:1619677475.752832:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)++
00000800:00000200:30.0:1619677475.752835:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.49@o2ib3
00000400:00000200:30.0:1619677475.752837:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.49@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.752840:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000400:00000200:30.0:1619677475.752840:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.752841:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:30.0:1619677475.752843:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498352] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.752844:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.752845:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.752846:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:30.0:1619677475.752849:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.752851:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:30.0:1619677475.752854:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.752855:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:30.0:1619677475.752855:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:30.0:1619677475.752857:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:30.0:1619677475.752857:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc0000] (21)++
00000800:00000200:30.0:1619677475.752858:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc0000] (22)--
00000800:00000200:30.0:1619677475.752859:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)--
00000800:00000200:8.2:1619677475.752862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:4.2:1619677475.752870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc0000] (20)++
00000800:00000200:20.0:1619677475.752871:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.752875:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.752876:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.752878:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)++
00000800:00000200:21.0:1619677475.752878:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.2:1619677475.752879:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:31.0:1619677475.752881:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.49@o2ib3
00000400:00000200:20.0:1619677475.752882:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.752883:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.49@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.752885:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000800:00000200:20.0:1619677475.752885:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:18.0:1619677475.752886:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000400:00000200:31.0:1619677475.752887:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.752888:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000400:00000200:31.0:1619677475.752889:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000800:00000200:2.0:1619677475.752889:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:31.0:1619677475.752890:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498351] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.752892:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.752892:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.752893:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.752894:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.752895:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.752896:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:31.0:1619677475.752898:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000400:00000200:18.0:1619677475.752898:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498350] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.752899:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.752900:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.752901:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.752902:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.752904:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.752905:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:31.0:1619677475.752905:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000400:00000200:18.0:1619677475.752905:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.752907:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:31.0:1619677475.752907:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc0000] (21)++
00000800:00000200:18.0:1619677475.752908:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:31.0:1619677475.752909:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc0000] (22)--
00000800:00000200:31.0:1619677475.752909:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)--
00000800:00000200:18.0:1619677475.752910:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.752910:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.752911:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:25.2:1619677475.752913:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.752913:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.752913:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.752915:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.752916:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.752917:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.752919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:13.0:1619677475.752920:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:20.2:1619677475.752922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:13.0:1619677475.752923:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.752923:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.752925:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.752927:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.752927:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:13.0:1619677475.752928:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:2.0:1619677475.752929:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.752930:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000400:00000200:2.0:1619677475.752930:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.752931:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.752932:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.752933:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.752933:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.752934:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:2.0:1619677475.752934:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.752938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:9.2:1619677475.752939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677475.752945:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677475.752947:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677475.752948:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.752949:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677475.752949:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.752950:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.752951:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.752951:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000400:00000200:8.0:1619677475.752952:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677475.752952:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:8.0:1619677475.752954:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.752956:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.752957:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498349] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.752959:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.752960:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.752961:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:4.2:1619677475.752961:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc0000] (20)++
00000400:00000200:8.0:1619677475.752963:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.752965:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.752968:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)++
00000800:00000200:8.0:1619677475.752968:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.752969:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.752970:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.752971:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.49@o2ib3
00000800:00000200:8.0:1619677475.752971:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.752972:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677475.752973:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.49@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.752973:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.752974:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:31.0:1619677475.752975:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000400:00000200:30.0:1619677475.752975:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.752975:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:30.0:1619677475.752977:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:25.2:1619677475.752977:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.752978:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498348] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.752979:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.752980:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:4.2:1619677475.752980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677475.752981:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:30.0:1619677475.752984:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.752985:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:22.0:1619677475.752985:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.752987:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.752987:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:30.0:1619677475.752988:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:30.0:1619677475.752988:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:22.0:1619677475.752988:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:24.0:1619677475.752989:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000400:00000200:22.0:1619677475.752989:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.752989:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.752990:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:30.0:1619677475.752990:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc0000] (21)++
00000800:00000200:24.0:1619677475.752990:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.752991:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:22.0:1619677475.752991:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.752992:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc0000] (22)++
00000800:00000200:30.0:1619677475.752993:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc0000] (23)--
00000800:00000200:22.0:1619677475.752993:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.752994:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000800:00000200:25.2:1619677475.752995:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:4.2:1619677475.752999:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc0000] (21)++
00000800:00000200:12.0:1619677475.753002:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.753004:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:4.2:1619677475.753004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:12.0:1619677475.753005:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.753007:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)++
00000400:00000200:12.0:1619677475.753007:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.753008:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677475.753009:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:31.0:1619677475.753010:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc0000] (23)--
00000800:00000200:31.0:1619677475.753010:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000800:00000200:31.0:1619677475.753011:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)++
00000800:00000200:26.0:1619677475.753011:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:31.0:1619677475.753013:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.49@o2ib3
00000800:00000200:30.0:1619677475.753014:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000800:00000200:26.0:1619677475.753014:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.753014:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:31.0:1619677475.753016:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.49@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.753017:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.753018:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.753019:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677475.753020:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.753021:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.753021:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:31.0:1619677475.753022:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.753022:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498347] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.753023:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.753024:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677475.753024:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.753025:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.753026:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:31.0:1619677475.753028:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.753030:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:26.0:1619677475.753030:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.753032:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.753032:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:31.0:1619677475.753033:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.753033:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.753035:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.753035:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc0000] (21)++
00000800:00000200:26.0:1619677475.753035:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.753036:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc0000] (22)--
00000800:00000200:26.0:1619677475.753036:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.753036:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:31.0:1619677475.753037:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)--
00000800:00000200:26.0:1619677475.753038:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.753038:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.753039:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.753039:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.753040:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:25.2:1619677475.753044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677475.753046:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.753049:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.753050:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.753050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc0000] (20)++
00000400:00000200:30.0:1619677475.753051:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.753051:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:30.0:1619677475.753054:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677475.753054:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:6.0:1619677475.753054:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:31.0:1619677475.753055:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:6.0:1619677475.753055:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.753056:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)++
00000400:00000200:6.0:1619677475.753057:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.753059:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.49@o2ib3
00000800:00000200:6.0:1619677475.753060:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:30.0:1619677475.753061:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.49@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.753062:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.753063:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (22)--
00000400:00000200:30.0:1619677475.753064:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:30.0:1619677475.753065:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498345] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.753066:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.753067:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.753068:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:30.0:1619677475.753071:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.49@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.753073:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.753075:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.753075:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.753076:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.753078:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.753078:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc0000] (21)++
00000800:00000200:30.0:1619677475.753079:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc0000] (22)--
00000800:00000200:30.0:1619677475.753080:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc0000] (21)--
00000800:00000200:25.2:1619677475.753083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.753090:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.753092:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.753093:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.753095:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.753096:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.753097:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.753304:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.753312:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.753314:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.753315:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.753317:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.753319:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.753321:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.753322:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498344] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.753323:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.753324:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.753325:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.753328:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.753329:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.753331:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.753332:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.753332:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.753334:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.753334:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.753335:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.753335:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.753340:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.753347:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.753349:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.753350:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.753351:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.753353:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.753354:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.753727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.753733:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.753736:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.753737:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.753738:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.753741:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.753742:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.753743:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498343] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.753745:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.753746:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.753747:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.753750:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.753752:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.753754:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.753755:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.753756:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.753757:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.753758:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.753758:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.753759:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.753763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.753771:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.753773:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.753774:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.753775:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.753777:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.753778:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.753862:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.753868:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.753871:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.753872:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.753874:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.753876:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.753878:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.753879:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498342] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.753881:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.753882:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.753883:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.753887:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.753889:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.753891:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.753892:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.753892:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.753894:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.753895:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.753896:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.753896:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.753901:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.753910:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.753912:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.753913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.753913:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.753915:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.753918:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.754050:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.754054:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.754056:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.754058:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.754059:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.754060:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.754062:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.754063:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498341] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.754065:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.754066:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.754067:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.754070:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.754072:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.754075:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.754076:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.754076:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.754078:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.754079:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.754080:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.754080:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.754084:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.754091:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.754094:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.754094:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.754095:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.754097:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.754100:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.754128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.754136:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.754139:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.754141:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.754144:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.754144:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.754145:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.754147:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498340] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.754148:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.754149:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.754150:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.754153:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.754155:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.754157:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.754158:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.754158:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.754160:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.754160:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.754161:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.754162:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.754165:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.754172:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.754174:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.754175:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.754176:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.754178:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.754179:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.754336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (20)++
00000800:00000200:2.0:1619677475.754342:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.754345:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.72@o2ib3
00000800:00000200:18.0:1619677475.754346:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000400:00000200:2.0:1619677475.754348:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.72@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.754350:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.754351:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:2.0:1619677475.754353:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498339] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.754354:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.754355:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.754356:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:2.0:1619677475.754358:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.72@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.754360:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:2.0:1619677475.754362:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.754362:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:2.0:1619677475.754363:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.754364:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.754365:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba0c00] (21)++
00000800:00000200:2.0:1619677475.754366:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919066ba0c00] (22)++
00000800:00000200:2.0:1619677475.754367:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677475.754367:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:9.2:1619677475.754371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.2:1619677475.754374:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba0c00] (21)++
00000800:00000200:18.0:1619677475.754380:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.754381:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)++
00000800:00000200:18.0:1619677475.754382:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.754382:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.754382:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919066ba0c00] (23)--
00000800:00000200:2.0:1619677475.754382:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (22)--
00000800:00000200:2.0:1619677475.754383:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.754384:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.754384:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba0c00] (21)--
00000800:00000200:18.0:1619677475.754385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.754651:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.754657:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.754659:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.754660:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.754662:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.754664:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.754666:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.754667:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498338] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.754668:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.754669:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.754670:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.754673:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.754675:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.754677:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.754678:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.754679:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.754680:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.754681:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.754681:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.754682:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.754686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.754694:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.754696:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.754697:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.754699:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.754700:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.2:1619677475.754702:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.0:1619677475.754706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:30.0:1619677475.754710:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.754713:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.754715:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.754717:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.754717:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.754719:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.754720:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498337] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.754721:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.754722:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.754724:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.754727:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.754729:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.754731:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.754732:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.754732:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.754734:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.754734:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.754735:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.754736:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.754739:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.754746:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.754749:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.754749:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.754751:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.754752:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.754753:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.755189:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.755197:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.755200:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.755201:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.755203:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.755205:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.755206:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.755207:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498336] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.755209:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.755210:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.755210:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.755214:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.755215:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.755218:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.755219:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.755219:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.755221:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.755221:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.755222:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.755222:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.755227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.755234:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.755237:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.755238:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.755238:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.755240:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.755243:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.755543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.755548:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.755551:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.755552:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.755554:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.755555:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.755556:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.755557:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498335] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.755559:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.755560:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.755561:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.755564:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.755566:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.755568:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.755569:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.755569:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.755571:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.755571:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.755573:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.755574:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.755574:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.755578:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.755580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.755585:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.755585:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.755586:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.755587:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.755588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.755588:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.755589:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.755590:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.755591:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.755593:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.755593:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.755594:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.755594:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.755595:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.755596:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.755659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.755662:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.755664:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.755666:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.755667:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.755669:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.755670:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.755671:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498334] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.755673:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.755674:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.755675:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.755678:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.755680:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.755683:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.755684:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.755685:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.755686:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.755687:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.755687:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.755688:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.755693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.755702:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.755705:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.2:1619677475.755706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000400:00000200:17.0:1619677475.755706:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.755706:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.755708:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.755710:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.755713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:31.0:1619677475.755714:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.755717:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.755719:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.755720:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.755721:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:1.0:1619677475.755721:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:31.0:1619677475.755722:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.755723:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498333] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:1.0:1619677475.755724:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000400:00000200:31.0:1619677475.755725:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.755725:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.755725:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:31.0:1619677475.755726:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:1.0:1619677475.755726:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:31.0:1619677475.755727:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:1.0:1619677475.755727:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:31.0:1619677475.755731:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.755732:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.755735:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.755736:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.755736:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.755738:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.755739:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.755740:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.755742:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.755742:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.755744:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.755748:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.755750:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.755753:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.755753:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.755753:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.755755:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.755756:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:24.0:1619677475.755757:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.755758:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.755759:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.755760:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.755765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.755771:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.755774:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.755775:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.755775:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.755776:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.755776:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.756038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.756046:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.756049:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.756050:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.756052:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.756054:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.756055:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.756057:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498332] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.756058:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.756059:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.756060:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.756063:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.756065:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.756067:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.756067:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.756068:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.756069:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.756070:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.756071:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.756072:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.756073:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:9.2:1619677475.756074:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:4.2:1619677475.756078:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677475.756081:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.756083:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.756084:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.756084:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.756085:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.756086:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:24.0:1619677475.756087:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677475.756088:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.756088:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.756090:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:5.2:1619677475.756318:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.756324:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.756327:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.756328:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.756330:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.756331:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.756331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000400:00000200:17.0:1619677475.756333:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.756334:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498331] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.756336:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.756337:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.756338:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.756339:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000400:00000200:17.0:1619677475.756341:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.756341:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.756342:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:17.0:1619677475.756343:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.756345:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677475.756345:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:17.0:1619677475.756346:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.756346:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:10.0:1619677475.756347:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.756348:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.756348:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.756349:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.756349:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000400:00000200:10.0:1619677475.756349:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677475.756350:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498330] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.756351:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.756352:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.756353:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:8.2:1619677475.756354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:10.0:1619677475.756357:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.756358:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.756361:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.756362:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.756362:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:10.0:1619677475.756363:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (24)++
00000800:00000200:10.0:1619677475.756366:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (25)--
00000800:00000200:10.0:1619677475.756366:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:17.0:1619677475.756367:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (24)--
00000800:00000200:10.0:1619677475.756367:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.756367:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:1.0:1619677475.756367:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (24)--
00000400:00000200:1.0:1619677475.756368:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.756370:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:8.2:1619677475.756371:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.756374:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.756376:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.756378:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.756379:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.756379:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.756380:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.756382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.756604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.756611:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.756613:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.756614:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.756616:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.756619:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.756620:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.756621:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498329] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.756622:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.756624:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.756624:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.756628:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.756629:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.756632:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.756633:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.756633:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.756635:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.756635:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.756636:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.756636:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.756642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.756650:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.756652:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.756653:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.756654:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.756656:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.756657:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.757497:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.2:1619677475.757503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.757505:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.757508:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:22.0:1619677475.757509:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:30.0:1619677475.757511:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.757512:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:31.0:1619677475.757513:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.757513:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.757513:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:30.0:1619677475.757515:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:22.0:1619677475.757515:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.757517:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498328] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.757518:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.757518:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.757519:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.757520:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:30.0:1619677475.757521:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:22.0:1619677475.757522:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498327] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.757523:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.757524:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.757526:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:30.0:1619677475.757528:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.757530:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.757531:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.757533:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.757533:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.757534:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.757535:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677475.757535:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.757536:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:22.0:1619677475.757536:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:30.0:1619677475.757537:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.757537:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:30.0:1619677475.757538:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677475.757538:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:30.0:1619677475.757539:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:22.0:1619677475.757539:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.757540:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.757541:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:9.2:1619677475.757543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:25.2:1619677475.757545:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677475.757552:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.757554:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:8.0:1619677475.757554:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.757555:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.757556:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.757556:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.757557:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.757558:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.757559:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.757559:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.757560:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.0:1619677475.757561:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.757713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.757720:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.757723:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.757724:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.757726:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.757728:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.757730:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.757731:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498326] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.757733:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.757734:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.757735:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.757739:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.757740:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.757743:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.757744:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.757745:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.757746:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.757747:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.757748:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.757748:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.757753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.757760:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.757763:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.757764:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.757764:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.757766:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.757769:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.757824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919066ba3a00] (20)++
00000800:00000200:2.0:1619677475.757831:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919066ba3a00] (21)++
00000800:00000200:2.0:1619677475.757834:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.51.5.39@o2ib3
00000800:00000200:18.0:1619677475.757835:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba3a00] (22)--
00000400:00000200:2.0:1619677475.757838:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.39@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.757840:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.757842:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:2.0:1619677475.757843:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498325] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.757845:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.757846:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.757847:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677475.757850:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.39@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.757852:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:2.0:1619677475.757856:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.757857:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:2.0:1619677475.757857:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.757859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:2.0:1619677475.757859:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:2.0:1619677475.757860:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919066ba3a00] (21)++
00000800:00000200:2.0:1619677475.757861:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919066ba3a00] (22)--
00000800:00000200:2.0:1619677475.757861:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919066ba3a00] (21)--
00000800:00000200:31.0:1619677475.757867:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.757870:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.757873:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.757874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.757875:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.757876:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:25.2:1619677475.757876:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677475.757877:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498324] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.757879:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.757880:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.757880:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.757882:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:31.0:1619677475.757883:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.757884:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.757885:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.757885:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.757886:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.757888:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.757888:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.757889:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.757889:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.757890:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.757891:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.757892:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.757892:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.757893:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.757896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:25.2:1619677475.757902:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:8.0:1619677475.757903:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.757905:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.757906:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.757906:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.757908:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:8.0:1619677475.757908:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.757910:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.757911:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.757912:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.757912:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.757913:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.757914:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.758045:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.758051:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.758054:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.758055:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.758057:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.758059:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.758060:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.758061:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498323] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.758063:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.758064:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.758065:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.758068:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.758069:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.758072:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.758073:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.758073:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.758075:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.758076:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.758077:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.758077:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.758081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.758090:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.758092:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.758093:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.758094:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.758095:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.758097:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.758193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.758200:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.758203:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.758204:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.758206:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.758209:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.758211:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.758212:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498322] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.758213:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.758214:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.758215:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.758219:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.758220:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.758223:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.758224:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.758225:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.758226:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.758227:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.758228:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.758228:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.758232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.758240:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.758243:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.758244:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.758244:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.758246:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.758249:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.758305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.758312:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.758315:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.758316:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.758316:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.758317:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.758318:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.758405:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.758410:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.758412:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.758413:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.758415:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.758417:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.758418:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.758419:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498321] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.758421:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.758422:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.758423:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.758426:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.758428:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.758430:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.758431:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.758431:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.758433:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.758433:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.758434:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.758434:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.758439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.758447:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.758449:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.758450:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.758452:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.758453:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.758454:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.758691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.758696:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.758698:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.758701:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.758701:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.758703:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.758704:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:4.2:1619677475.758704:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:21.0:1619677475.758706:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498320] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.758707:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.758708:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.758709:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:24.0:1619677475.758712:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:21.0:1619677475.758712:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.758714:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:24.0:1619677475.758715:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:21.0:1619677475.758716:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.758716:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:21.0:1619677475.758717:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000400:00000200:24.0:1619677475.758718:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.758718:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.758720:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.758720:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:24.0:1619677475.758721:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.758721:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000400:00000200:24.0:1619677475.758722:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677475.758722:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:24.0:1619677475.758724:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498319] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.758725:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.2:1619677475.758726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:24.0:1619677475.758727:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.758728:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.758732:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.758734:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.758734:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.758736:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000800:00000200:24.0:1619677475.758737:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000400:00000200:20.0:1619677475.758737:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.758738:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.758738:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:20.0:1619677475.758739:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.758740:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.758740:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:21.0:1619677475.758740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:24.0:1619677475.758741:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:20.0:1619677475.758741:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:24.0:1619677475.758742:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.758745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:25.2:1619677475.758746:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.758753:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:6.0:1619677475.758754:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:21.0:1619677475.758756:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677475.758757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677475.758757:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.758757:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:6.0:1619677475.758757:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:21.0:1619677475.758758:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:6.0:1619677475.758758:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.758759:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000400:00000200:6.0:1619677475.758760:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.758763:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.758765:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.758772:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.758775:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.758775:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.758776:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.758777:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.758778:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.758802:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.758809:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.758812:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.758813:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.758815:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.758818:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.758819:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677475.758821:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498318] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.758822:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.758823:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.758824:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.758827:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.758829:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677475.758832:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.758832:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677475.758833:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677475.758835:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677475.758835:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.758836:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.758836:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677475.758841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.758849:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.758852:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.758853:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.758853:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.758856:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.758859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:20.2:1619677475.758884:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677475.758889:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.758892:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677475.758894:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.758894:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677475.758897:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.758898:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.758899:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498317] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.758901:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.758902:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.758903:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.758906:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.758907:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.758910:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.758911:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.758911:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.758913:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.758913:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.758915:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919460fda200] (22)++
00000800:00000200:21.0:1619677475.758916:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (23)--
00000800:00000200:21.0:1619677475.758916:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:25.2:1619677475.758919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.2:1619677475.758921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (21)++
00000800:00000200:20.0:1619677475.758924:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (22)++
00000800:00000200:20.0:1619677475.758926:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919460fda200] (23)--
00000800:00000200:6.0:1619677475.758926:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.758927:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000800:00000200:20.0:1619677475.758928:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:6.0:1619677475.758928:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.758929:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.758930:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.758931:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.758934:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.758940:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.758948:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.758951:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.758953:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.758955:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.758956:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.758957:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.758958:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498316] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.758959:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.758960:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.758961:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.758964:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.758966:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.758969:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.758970:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.758971:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.758973:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.758973:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.758974:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.758975:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.758979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.758987:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.758989:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.758990:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.758992:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:20.0:1619677475.758992:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.758995:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.758997:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.759004:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.759007:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.759008:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.759008:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.759009:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.759010:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.759413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.759418:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.759421:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.759422:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.759424:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.759427:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.759428:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677475.759429:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498315] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.759431:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.759432:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.759433:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677475.759436:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.759438:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677475.759440:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.759441:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677475.759441:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677475.759443:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677475.759444:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.759444:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.759445:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677475.759450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.759458:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.759461:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.759462:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.759462:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.759464:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.759466:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.759517:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.759523:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.759526:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.759526:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.759528:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:21.2:1619677475.759529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:22.0:1619677475.759531:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.759532:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.759533:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498314] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.759535:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.759536:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:31.0:1619677475.759537:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:22.0:1619677475.759537:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.759540:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:22.0:1619677475.759541:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:31.0:1619677475.759543:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:22.0:1619677475.759543:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:30.0:1619677475.759545:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:22.0:1619677475.759545:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:31.0:1619677475.759546:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.759546:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000400:00000200:31.0:1619677475.759547:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:22.0:1619677475.759547:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.759548:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498313] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.759548:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.759549:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:31.0:1619677475.759550:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.759550:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:31.0:1619677475.759551:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:22.0:1619677475.759551:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000400:00000200:31.0:1619677475.759552:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:22.0:1619677475.759552:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:4.2:1619677475.759553:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:25.2:1619677475.759554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:31.0:1619677475.759555:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.2:1619677475.759556:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:31.0:1619677475.759557:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.759559:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.759560:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.759561:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.759561:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.759562:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.759562:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000800:00000200:6.0:1619677475.759562:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:31.0:1619677475.759563:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.759563:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000400:00000200:30.0:1619677475.759564:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.759564:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.759564:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:31.0:1619677475.759565:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:6.0:1619677475.759565:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:31.0:1619677475.759566:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000400:00000200:30.0:1619677475.759566:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.759566:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:31.0:1619677475.759567:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:21.0:1619677475.759567:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.759568:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.759568:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:9.2:1619677475.759568:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:21.0:1619677475.759569:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.759569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:30.0:1619677475.759570:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96498312] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.759571:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.759572:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677475.759572:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000400:00000200:30.0:1619677475.759573:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:25.2:1619677475.759574:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:8.0:1619677475.759575:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:30.0:1619677475.759576:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.759577:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:8.0:1619677475.759577:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677475.759578:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.759578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.759578:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000400:00000200:8.0:1619677475.759578:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.759580:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.759581:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.759581:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.759581:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.759581:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.759581:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:8.0:1619677475.759581:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.759582:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.759582:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.759582:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.759583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:30.0:1619677475.759583:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677475.759583:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.0:1619677475.759583:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677475.759584:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (23)--
00000800:00000200:30.0:1619677475.759585:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.759586:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677475.759586:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.759586:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.759592:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.759595:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.759596:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (24)++
00000800:00000200:8.0:1619677475.759596:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)++
00000800:00000200:24.0:1619677475.759597:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (25)--
00000800:00000200:8.0:1619677475.759597:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (24)--
00000800:00000200:24.0:1619677475.759598:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.759598:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.759599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.759599:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.759602:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.759641:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.759647:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.759650:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.759651:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.759653:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.759656:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.759657:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.759658:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498311] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.759660:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.759660:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.759662:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.759665:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.759667:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.759668:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.759669:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.759670:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.759672:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.759672:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.759673:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.759674:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.759678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.759685:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.759687:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.759687:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.759688:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.759689:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.759691:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.759762:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.759769:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.759773:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.759775:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.759776:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.759777:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.759778:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.759780:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498310] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.759781:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.759782:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.759783:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.759786:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.759788:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.759790:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.759791:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.759792:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.759793:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.759794:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.759795:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.759795:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.759810:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.759816:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.759819:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.759820:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.759821:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.759823:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.759824:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:5.2:1619677475.759960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677475.759966:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677475.759969:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677475.759969:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677475.759971:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677475.759974:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.759975:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:17.0:1619677475.759977:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498309] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.759978:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.759979:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.759980:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:17.0:1619677475.759984:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.759985:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:17.0:1619677475.759988:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.759989:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:17.0:1619677475.759990:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:17.0:1619677475.759992:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:17.0:1619677475.759992:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677475.759993:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677475.759994:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677475.759998:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:26.0:1619677475.760005:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.760008:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000800:00000200:10.0:1619677475.760008:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.760009:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.760011:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: ACK: OK
00000800:00000200:26.0:1619677475.760015:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:9.2:1619677475.760038:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677475.760044:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.760047:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.114@o2ib7
00000800:00000200:26.0:1619677475.760048:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.760048:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:10.0:1619677475.760049:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.760050:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677475.760117:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.760124:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.760126:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.760127:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.760129:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.760131:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.760132:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.760133:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498308] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.760134:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.760136:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.760137:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.760140:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.760142:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.760144:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.760145:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.760146:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.760147:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.760148:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.760148:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.760149:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.760153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.760159:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.760161:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.760162:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.760163:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.760164:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.760166:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.760425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.760431:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.760433:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.760434:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.760436:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.760438:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.760439:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.760441:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498307] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.760442:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.760443:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.760444:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.760447:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.760449:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.760451:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.760452:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.760453:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.760454:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.760455:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.760455:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.760456:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.760461:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.760469:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.760471:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.760472:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.760474:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.760476:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.760476:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.760632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.760638:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.760641:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.760642:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.760644:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.760647:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.760648:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.760649:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498306] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.760650:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.760651:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.760652:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.760655:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.760657:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.760659:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.760659:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.760660:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.760662:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.760662:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.760664:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.760665:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.760665:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.760668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.760671:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.760677:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.760678:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.760678:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000400:00000200:17.0:1619677475.760679:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.760679:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.760680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.760681:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.760681:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.760682:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677475.760682:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677475.761105:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.761112:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.761115:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.761116:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.761118:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.761120:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.761121:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.761122:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498305] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.761124:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.761125:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.761126:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.761129:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.761131:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.761132:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.761133:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.761134:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.761135:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.761136:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.761136:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.761137:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.761141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.761148:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.761150:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.761151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.761151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.761152:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.761154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.761160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.761166:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.761168:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.761169:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.761169:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.761170:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.761170:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.761319:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.761325:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.761328:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.761329:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.761331:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.761333:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.761334:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.761335:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498304] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.761336:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.761337:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.761338:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.761341:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.761343:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.761345:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.761346:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.761347:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.761348:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.761349:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.761350:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.761350:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.761355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.761363:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.761365:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.761366:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.761368:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.761369:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.761370:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.761491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.761499:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.761502:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.761505:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.761507:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.761507:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.761508:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.761509:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498303] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.761511:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.761512:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.761513:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.761516:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.761518:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.761520:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.761521:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.761522:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.761523:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.761524:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.761525:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.761526:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.761529:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.761536:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.761539:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.761540:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.761540:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.761542:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.761544:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:20.2:1619677475.761635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.761641:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.761644:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.761645:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.761646:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.761648:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.761650:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:18.0:1619677475.761651:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498302] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.761652:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.761653:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.761654:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.761658:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.761659:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:18.0:1619677475.761662:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.761663:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.761664:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:18.0:1619677475.761666:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:18.0:1619677475.761667:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.761668:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.761668:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.761672:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.761679:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.761682:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.761683:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.761685:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.761686:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.761688:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.762355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.762361:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.762364:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.762364:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.762366:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.762369:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.762370:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.762372:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498301] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.762373:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.762374:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.762375:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.762378:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.762380:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.762382:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.762383:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.762383:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.762385:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.762386:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.762386:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.762386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.762392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.762400:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.762402:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.762403:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.762404:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.762406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.762407:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.762449:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.762454:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.762457:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.762459:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.762459:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.762461:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.762463:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.762464:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498300] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.762465:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.762466:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.762467:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.762471:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.762472:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.762475:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.762475:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.762476:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.762477:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.762478:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.762479:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.762480:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.762484:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.762491:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.762494:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.762495:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.762495:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.762497:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.762499:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.762883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:22.0:1619677475.762888:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.762890:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.762891:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.762893:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.762895:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.762897:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677475.762898:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498299] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.762899:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.762900:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.762901:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677475.762904:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.762906:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.762909:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.762910:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677475.762911:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677475.762913:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677475.762913:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.762914:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.762914:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:9.2:1619677475.762919:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.762926:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.762929:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.762930:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.762930:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.762932:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.762935:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.762952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.762960:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.762963:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.762965:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.762967:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.762968:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.762969:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.762970:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498298] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.762971:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.762972:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.762973:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.762977:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.762979:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.762981:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.762982:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.762982:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.762984:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.762984:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.762986:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.762987:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.762988:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.762989:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.762993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:8.0:1619677475.762995:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.762997:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:8.0:1619677475.762998:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.762999:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.763000:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.763001:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677475.763002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:30.0:1619677475.763004:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.763005:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.763006:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.763011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.763018:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.763021:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.763022:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.763023:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.763026:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.763027:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.763028:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498297] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.763030:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.763031:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.763032:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.763035:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.763036:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.763039:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.763040:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.763040:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.763042:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.763043:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.763043:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.763044:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.763048:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.763056:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.763058:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.763059:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.763060:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.763062:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.763064:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.763179:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:21.2:1619677475.763186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:26.0:1619677475.763187:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.763190:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.763191:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.763192:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.763193:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:26.0:1619677475.763195:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.763196:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:26.0:1619677475.763196:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.763198:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498296] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.763199:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.763199:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:31.0:1619677475.763200:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:26.0:1619677475.763200:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.763201:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.763201:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:30.0:1619677475.763202:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.763204:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498295] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.763205:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.763205:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.763206:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.763207:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:26.0:1619677475.763207:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.763210:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:30.0:1619677475.763211:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.763211:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.763211:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:30.0:1619677475.763212:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677475.763213:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.763213:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.763214:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:30.0:1619677475.763215:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.763215:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:26.0:1619677475.763215:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:30.0:1619677475.763216:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.763217:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.763218:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.763219:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.763219:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.763219:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:9.2:1619677475.763223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:22.0:1619677475.763227:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.763229:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:8.0:1619677475.763230:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:6.0:1619677475.763230:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.763231:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.763232:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.763233:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:22.0:1619677475.763233:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.763234:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.763235:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:8.0:1619677475.763236:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.763238:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.763270:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.763275:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.763278:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.763278:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.763280:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.763282:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.763284:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.763285:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498294] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.763287:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.763288:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.763289:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.763292:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.763300:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.763302:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.763303:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.763304:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.763305:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.763306:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.763307:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.763308:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.763309:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.763313:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.763314:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.763320:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.763320:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.763322:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.763322:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.763322:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.763323:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.763323:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.763325:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.763326:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.763327:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.763331:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.763338:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.763341:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.763341:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.763342:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.763343:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.763344:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.763530:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.763538:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.763541:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.763542:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.763544:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.763547:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.763548:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:10.0:1619677475.763549:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498293] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.763551:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.763552:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.763553:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:10.0:1619677475.763556:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.763557:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:10.0:1619677475.763560:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.763561:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:10.0:1619677475.763562:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:10.0:1619677475.763563:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:10.0:1619677475.763564:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.763565:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.763565:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677475.763569:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.763578:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.763580:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:18.0:1619677475.763581:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.763582:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.763584:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: ACK: OK
00000800:00000200:2.0:1619677475.763587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.763687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.763692:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.763694:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.763697:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.763697:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.763699:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.763701:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:21.0:1619677475.763702:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498292] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.763703:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.763704:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.763705:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:21.0:1619677475.763708:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.763710:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:21.0:1619677475.763712:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.763713:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:21.0:1619677475.763714:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:21.0:1619677475.763715:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:21.0:1619677475.763716:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.763717:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.763718:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.763722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.763729:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.763732:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.763733:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.763733:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.763735:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.763737:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.763815:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.763823:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.763826:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.763829:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.763831:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.763831:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.763833:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.763834:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498291] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.763835:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.763836:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.763837:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.763841:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.763843:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.763845:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.763846:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.763846:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.763848:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.763849:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.763850:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.763850:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.763854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.763861:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.763863:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.763864:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.763865:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.763867:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.763869:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.764149:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.764154:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.764157:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.764158:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.764160:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.764163:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.764164:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.764165:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498290] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.764167:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.764168:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.764169:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.764172:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.764174:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.764177:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.764177:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.764178:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.764180:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.764180:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.764181:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.764181:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.764187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.764195:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.764197:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.764198:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.764199:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.764200:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.764202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.764456:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.764464:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.764467:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.764470:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.764472:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.764472:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.764473:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.764475:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498289] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.764476:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.764477:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.764478:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.764482:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.764483:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.764485:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.764486:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.764487:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.764488:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.764488:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.764489:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.764490:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.764494:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.764501:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.764504:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.764505:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.764506:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.764508:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.764509:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.764979:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.764987:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.764990:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.764991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.764992:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.764995:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.764996:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.764997:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498288] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.764999:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.765000:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.765001:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.765004:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.765005:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.765007:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.765008:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.765008:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.765010:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.765010:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.765011:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.765011:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.765015:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.765022:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.765025:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.765026:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.765026:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.765027:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.765029:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.765037:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.765042:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.765045:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.765045:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.765046:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.765046:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.765047:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.765106:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.765114:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.765117:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.765119:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.765121:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.765121:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.765122:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.765123:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498287] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.765125:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.765125:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.765126:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.765129:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.765131:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.765133:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.765134:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.765134:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.765136:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.765136:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.765137:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.765137:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.765141:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.765148:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.765150:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.765151:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.765151:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.765152:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.765154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.765163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.765169:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.765172:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.765173:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.765173:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.765174:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.765174:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.765205:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.765210:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.765213:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.765214:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.765216:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.765218:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.765219:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.765220:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498286] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.765222:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.765223:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.765224:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.765227:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.765229:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.765231:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.765232:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.765233:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.765234:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.765235:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.765235:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.765236:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.765240:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.765248:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.765250:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.765251:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.765253:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.765254:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.765255:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.765268:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.765276:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.765279:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.765281:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.765283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.765284:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.765285:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.765287:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498285] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.765288:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.765289:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.765290:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.765298:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.765300:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.765302:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.765303:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.765303:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.765305:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.765305:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.765306:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.765307:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.765311:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.765318:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.765321:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.765321:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.765323:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.765325:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.765326:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.765337:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.765345:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.765347:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.765349:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.765350:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.765353:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.765354:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.765356:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498284] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.765357:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.765358:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.765359:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.765362:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.765363:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.765366:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.765367:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.765368:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.765369:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.765370:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.765371:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.765372:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.765373:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.765375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.765378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.765385:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:17.0:1619677475.765385:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.765387:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.765388:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:17.0:1619677475.765388:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.765389:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.765389:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.765390:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:17.0:1619677475.765391:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.765394:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.765398:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.765405:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.765408:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.765409:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.765409:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.765410:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.765410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:5.2:1619677475.765547:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.765552:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.765555:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.765556:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.765558:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.765560:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.765561:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.765562:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498283] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.765563:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.765564:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.765565:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.765568:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.765569:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.765571:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.765572:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.765573:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.765574:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.765574:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.765575:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.765575:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.765580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.765587:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.765590:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.765590:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.765591:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.765592:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.765593:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.765601:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.765609:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.765611:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.765612:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.765612:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.765613:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.765613:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.765709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.765716:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.765718:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.765719:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.765721:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.765723:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.765725:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:10.0:1619677475.765726:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498282] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.765727:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.765728:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.765729:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:10.0:1619677475.765732:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.765734:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:10.0:1619677475.765736:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.765737:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677475.765738:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.765739:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.765739:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.765740:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.765741:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:9.2:1619677475.765745:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.765751:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.765753:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:26.0:1619677475.765753:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.765754:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:26.0:1619677475.765755:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.765757:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:9.2:1619677475.765769:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:10.0:1619677475.765775:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.765778:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:10.0:1619677475.765778:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.765779:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.765779:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677475.765779:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677475.765914:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.765921:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.765923:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.765924:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.765925:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.765927:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.765928:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.765929:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498281] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.765930:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.765931:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.765932:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.765935:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.765936:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.765939:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.765939:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.765940:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.765941:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.765942:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.765943:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.765944:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.765945:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.765948:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.765950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.765955:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.765957:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.765958:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.765959:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.765959:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.765960:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.765960:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.765961:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.765961:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.765964:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.765967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.765974:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.765977:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.765978:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.765978:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.765979:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.765979:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.766153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.766158:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.766160:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.766161:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.766163:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.766166:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.766167:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.766168:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498280] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.766169:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.766170:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.766171:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.766174:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.766175:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.766178:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.766178:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.766179:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.766181:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.766181:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.766182:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.766182:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.766187:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.766195:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.766197:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.766198:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.766199:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.766201:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.766202:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.766709:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.766717:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.766720:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.766723:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.766725:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.766725:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.766726:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.766727:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498279] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.766729:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.766730:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.766731:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.766734:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.766736:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.766738:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.766739:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.766739:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.766741:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.766742:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.766743:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:30.0:1619677475.766745:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.766745:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.766747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.766752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.766755:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.766758:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.766759:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.766759:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.766760:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.766761:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.766763:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.766763:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:8.0:1619677475.766763:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.766764:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.766904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.766910:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.766912:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.766914:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.766915:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.766917:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.766919:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.766920:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498278] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.766921:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.766922:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.766923:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:21.0:1619677475.766927:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.766928:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.0:1619677475.766930:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.766931:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.766932:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.766934:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.766934:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.766935:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:21.0:1619677475.766935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.766950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.766957:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.766960:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.766961:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.766961:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.766963:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.766965:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.767036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.767041:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.767044:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.767044:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.767046:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.767049:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.767050:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.767051:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498277] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.767052:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.767053:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.767054:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.767057:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.767059:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.767061:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.767062:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.767062:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.767064:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.767064:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.767065:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.767066:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.767067:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.767070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.767072:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:20.0:1619677475.767077:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.767077:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.767079:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.767079:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.767079:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.767080:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.767080:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.767082:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.767083:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.767084:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.767090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.767098:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.767100:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.767101:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.767102:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.767102:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.767103:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.767124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.767130:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.767133:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.767134:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.767135:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.767138:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.767140:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.767141:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498276] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.767143:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.767144:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.767145:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.767148:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.767149:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.767152:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.767153:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.767153:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.767155:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.767156:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.767156:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.767157:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.767161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.767169:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.767171:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.767172:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.767172:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.767174:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:1.0:1619677475.767177:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.767536:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.767542:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.767545:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.767546:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.767547:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.767550:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.767551:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.767553:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498275] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.767554:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.767555:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.767555:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.767558:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.767559:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.767560:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.767561:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.767561:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.767563:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.767563:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.767564:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.767565:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.767570:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.767577:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.767579:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.767580:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.767581:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.767582:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.767583:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.767592:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.767600:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.767602:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.767603:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.767603:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:2.0:1619677475.767603:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.767604:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.767682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc6000] (20)++
00000800:00000200:22.0:1619677475.767688:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc6000] (21)++
00000800:00000200:22.0:1619677475.767691:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.47@o2ib3
00000800:00000200:6.0:1619677475.767692:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6000] (22)--
00000400:00000200:22.0:1619677475.767694:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.47@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.767696:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.767698:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:22.0:1619677475.767699:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498274] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.767701:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.767702:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.767703:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:22.0:1619677475.767705:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.47@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.767707:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 2432 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:22.0:1619677475.767711:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.767711:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:22.0:1619677475.767712:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:22.0:1619677475.767714:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:22.0:1619677475.767714:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919874bc6000] (21)++
00000800:00000200:22.0:1619677475.767715:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919874bc6000] (22)++
00000800:00000200:22.0:1619677475.767716:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919874bc6000] (23)--
00000800:00000200:22.0:1619677475.767717:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6000] (22)--
00000800:00000200:9.2:1619677475.767720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.767722:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919874bc6000] (21)++
00000800:00000200:24.0:1619677475.767727:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677475.767727:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919874bc6000] (22)++
00000800:00000200:6.0:1619677475.767729:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919874bc6000] (23)--
00000800:00000200:6.0:1619677475.767729:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6000] (22)--
00000800:00000200:24.0:1619677475.767730:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:6.0:1619677475.767730:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919874bc6000] (21)--
00000400:00000200:24.0:1619677475.767731:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.767731:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.767734:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.767736:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.767839:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.767845:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.767847:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.767849:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.767850:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.767852:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.767854:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.767855:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498273] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.767856:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.767857:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.767858:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.767861:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.767863:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.767865:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.767866:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.767867:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.767868:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.767869:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.767869:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.767870:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.767874:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.767882:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.767885:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.767885:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.767887:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.767888:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.767889:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.768062:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.768067:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.768070:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.768070:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.768073:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.768076:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.768077:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.768078:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498272] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.768079:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.768080:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.768081:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.768084:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.768086:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.768088:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.768089:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.768089:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.768091:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.768091:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.768092:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.768092:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.768097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.768104:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.768106:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.768107:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.768109:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.768110:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.768112:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.768400:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.768407:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.768410:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.768411:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.768413:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.768415:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.768417:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.768418:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498271] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.768419:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.768420:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.768421:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.768425:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.768426:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.768429:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.768430:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.768431:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.768432:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.768433:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.768433:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.768434:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.768439:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.768447:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.768450:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.768451:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.768451:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.768453:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.768456:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.768580:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.768587:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.768590:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.768590:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.768592:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.768594:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.768596:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.768597:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498270] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.768598:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.768599:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.768600:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.768604:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.768605:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.768608:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.768609:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.768610:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.768611:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.768611:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.768612:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:17.0:1619677475.768613:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.768617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.768625:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.768627:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.768628:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.768630:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.768631:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.768633:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.768734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.768742:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.768745:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.768747:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.768749:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.768750:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.768751:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.768752:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498269] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.768754:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.768755:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.768756:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.768759:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.768761:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.768764:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.768764:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.768765:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.768767:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.768767:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.768769:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.768769:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.768772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.768779:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.768781:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.768782:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.768783:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.768784:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.768786:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.768864:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.768869:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.768872:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:21.0:1619677475.768874:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.768874:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.768876:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.768878:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:21.0:1619677475.768879:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498268] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.768880:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.768882:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.768883:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:21.0:1619677475.768886:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.768888:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:21.0:1619677475.768891:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.768891:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:21.0:1619677475.768892:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.0:1619677475.768894:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:21.0:1619677475.768894:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.768895:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.768897:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.768897:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:9.2:1619677475.768899:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.768903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.768906:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:8.0:1619677475.768906:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:20.0:1619677475.768908:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:8.0:1619677475.768908:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.768909:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (23)--
00000800:00000200:20.0:1619677475.768909:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:8.0:1619677475.768910:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.768911:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:8.0:1619677475.768912:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.768915:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.768996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.769002:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.769005:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.769005:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.769007:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.769010:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.769011:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.769012:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498267] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.769013:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.769014:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.769015:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.769019:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.769021:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.769023:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.769024:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.769024:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.769026:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.769026:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.769027:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.769027:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.769032:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.769039:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.769041:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.769042:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.769044:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.769045:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.769046:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.769154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.769161:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.769164:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.769167:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.769168:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.769169:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.769170:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.769172:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498266] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.769173:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.769174:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.769175:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.769178:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.769180:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.769183:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.769184:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.769185:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.769186:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.769187:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.769189:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.769190:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.769191:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.2:1619677475.769192:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.2:1619677475.769198:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.769204:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:31.0:1619677475.769206:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.0:1619677475.769207:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.769208:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.769209:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:21.0:1619677475.769209:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.769210:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:20.0:1619677475.769210:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.769211:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.0:1619677475.769212:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.769316:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.769320:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.769323:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000800:00000200:21.0:1619677475.769325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.769325:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000400:00000200:20.0:1619677475.769328:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.769329:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:20.0:1619677475.769330:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498265] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.769332:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.769333:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.769334:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:20.0:1619677475.769337:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.769339:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677475.769342:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.769342:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:20.0:1619677475.769343:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:20.0:1619677475.769345:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:20.0:1619677475.769345:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.769347:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.769348:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677475.769349:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:25.2:1619677475.769351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.769354:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:31.0:1619677475.769359:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:21.0:1619677475.769359:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:21.0:1619677475.769361:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:31.0:1619677475.769362:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:21.0:1619677475.769362:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:31.0:1619677475.769363:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.769363:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000400:00000200:31.0:1619677475.769365:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.769366:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.769368:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.769372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:30.0:1619677475.769379:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.769382:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:30.0:1619677475.769383:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.769383:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:30.0:1619677475.769384:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.769385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.769520:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.769528:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.769531:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.769532:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.769533:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:10.0:1619677475.769536:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.769538:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:10.0:1619677475.769539:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498264] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.769540:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.769541:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.769542:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:10.0:1619677475.769545:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:10.0:1619677475.769547:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:10.0:1619677475.769549:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.769550:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:10.0:1619677475.769551:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.769552:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.769553:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.769554:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:10.0:1619677475.769555:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.769555:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.769558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.769561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677475.769566:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.769568:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:26.0:1619677475.769569:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:12.0:1619677475.769569:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.769570:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:26.0:1619677475.769571:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.769571:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.769572:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677475.769572:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.769574:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.769742:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.769750:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.769752:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.769753:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.769755:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.769757:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.769759:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.769760:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498263] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.769761:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.769762:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.769763:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.769765:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.769766:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.769767:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.769770:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.769770:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.769771:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.769773:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.769774:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.769774:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.769775:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.769775:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.769775:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:10.0:1619677475.769776:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.769777:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677475.769778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:26.0:1619677475.769779:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.769781:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.769781:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498262] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.769783:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.769783:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.769784:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:24.0:1619677475.769785:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:26.0:1619677475.769787:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.769787:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:26.0:1619677475.769788:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:24.0:1619677475.769788:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.769789:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.769790:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.769791:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.769792:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.769792:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:24.0:1619677475.769792:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677475.769794:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.769795:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.769796:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.769797:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.769797:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.769800:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.769803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.769808:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:10.0:1619677475.769810:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.769811:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.769812:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.769812:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000400:00000200:6.0:1619677475.769812:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.769813:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.769814:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.769814:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.769817:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.769873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.769878:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.769881:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.769881:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.769883:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.769885:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.769887:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.769888:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498261] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.769889:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.769890:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.769891:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.769895:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.769896:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.769899:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.769899:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.769900:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.769902:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.769902:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.769903:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.769903:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.769908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.769915:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.769918:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.769918:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.769920:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.769921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.769922:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.770379:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.770385:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.770388:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.770389:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.770390:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.770392:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.770394:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.770395:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498260] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.770396:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.770397:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.770399:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.770402:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.770404:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.770406:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.770406:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.770407:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.770408:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.770409:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.770409:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.770410:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.770415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.770423:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.770425:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.770426:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.770427:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.770428:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.770429:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.770831:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.770839:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.770842:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.770844:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.770846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:4.2:1619677475.770846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000400:00000200:31.0:1619677475.770847:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.770849:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.770850:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498259] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.770851:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.770852:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.770853:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:10.0:1619677475.770854:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:31.0:1619677475.770856:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.770857:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:31.0:1619677475.770858:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:26.0:1619677475.770858:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:31.0:1619677475.770860:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000400:00000200:10.0:1619677475.770860:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.770861:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.770862:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.770863:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000400:00000200:10.0:1619677475.770863:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.770864:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:10.0:1619677475.770864:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:31.0:1619677475.770865:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.770865:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:10.0:1619677475.770865:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498258] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.770867:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.770867:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.770868:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:9.2:1619677475.770870:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:10.0:1619677475.770872:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.770874:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.770876:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.770877:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.770877:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:10.0:1619677475.770878:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.770879:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.770879:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.770880:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677475.770880:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000400:00000200:8.0:1619677475.770880:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.770881:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.770881:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:8.0:1619677475.770882:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.770884:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:25.2:1619677475.770896:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.770898:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:9.2:1619677475.770900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:6.0:1619677475.770904:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.770905:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.770906:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:6.0:1619677475.770906:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.770907:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.770908:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:6.0:1619677475.770908:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.770909:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.770909:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.770910:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000400:00000200:6.0:1619677475.770910:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.770911:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.770911:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:31.0:1619677475.770912:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.770912:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.770913:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677475.770914:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.770916:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.770917:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498257] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.770918:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.770920:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.770921:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:21.2:1619677475.770922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:30.0:1619677475.770924:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.770926:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.770928:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:30.0:1619677475.770929:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.770930:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.770930:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:22.0:1619677475.770930:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.770931:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:30.0:1619677475.770932:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.770932:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.770933:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677475.770933:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.770934:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:22.0:1619677475.770935:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.770936:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.770937:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498256] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.2:1619677475.770938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677475.770939:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.770940:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.770941:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.770944:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.770945:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.770946:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:22.0:1619677475.770948:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.770949:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.770949:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:22.0:1619677475.770950:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:20.0:1619677475.770950:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.770951:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.770951:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.770952:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.770952:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:22.0:1619677475.770953:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.770954:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.770954:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.770954:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:25.2:1619677475.770957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.770959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.770965:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.770966:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.770967:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.770967:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.770968:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.770968:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.770969:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.770971:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.770971:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.770973:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.770976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.770983:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.770986:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.770987:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.770987:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.770988:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.770989:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.771152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.771156:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.771158:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.771161:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.771162:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.771163:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.771165:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:20.0:1619677475.771166:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498255] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.771167:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.771168:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.771169:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:20.0:1619677475.771172:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.771174:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:20.0:1619677475.771176:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.771177:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:20.0:1619677475.771178:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.771179:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.771180:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.771181:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.771181:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:25.2:1619677475.771196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.771203:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.771206:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.771207:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.771207:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.771209:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.771211:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.771216:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.771223:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.771226:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.771227:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.771227:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.771228:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.771229:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.771824:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.771829:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.771831:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.771832:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.771834:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.771836:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.771837:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.771839:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498254] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.771840:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.771841:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.771842:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.771845:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.771847:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.771849:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.771850:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.771851:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.771852:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.771853:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.771853:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.771854:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.771859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.771867:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.771869:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.771870:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.771872:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.771873:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.771874:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.771938:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.771946:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.771949:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.771950:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.771952:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.771954:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.771956:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.771957:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498253] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.771959:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.771960:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.771961:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.771964:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.771966:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.771969:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.771970:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.771970:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.771972:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:21.2:1619677475.771973:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:8.0:1619677475.771973:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.771973:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.771974:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.771978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677475.771982:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.771985:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:1.0:1619677475.771986:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:31.0:1619677475.771987:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.771989:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.771989:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:30.0:1619677475.771990:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:17.0:1619677475.771990:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.771990:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.771991:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.771992:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498252] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.771993:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.771994:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.771995:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.771996:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677475.771996:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:31.0:1619677475.772000:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.772001:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.772003:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.772004:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.772004:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.772006:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.772006:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.772007:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.772008:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.772011:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.772018:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.772021:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.772021:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.772023:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.772024:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.772025:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.772030:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.772036:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.772039:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.772040:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.772040:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.772041:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.772042:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.772527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.772533:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.772535:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.772537:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.772538:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.772540:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.772542:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.772543:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498251] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.772544:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.772545:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.772546:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.772550:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.772552:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.772555:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.772555:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.772556:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.772558:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.772559:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.772560:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:2.0:1619677475.772561:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:2.0:1619677475.772562:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:25.2:1619677475.772565:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.2:1619677475.772567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:20.2:1619677475.772567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:31.0:1619677475.772574:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:18.0:1619677475.772574:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:31.0:1619677475.772575:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.772576:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.772576:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:18.0:1619677475.772576:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.772577:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.772577:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:31.0:1619677475.772578:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:30.0:1619677475.772578:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.772580:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.772582:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000400:00000200:30.0:1619677475.772582:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.772583:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:31.0:1619677475.772584:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.772584:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.772585:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498250] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.772586:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.772587:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.772588:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.772592:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.772594:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.772596:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.772597:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.772597:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.772599:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.772599:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.772600:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.772601:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.772604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.772611:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.772614:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.772614:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.772616:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.772618:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.772618:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:21.2:1619677475.772657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.772663:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.772666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.772666:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.772668:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.772671:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.772673:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.772674:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498249] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.772675:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.772676:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.772677:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.772680:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.772682:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.772684:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.772685:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.772685:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.772687:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.772688:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.772688:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.772689:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.772693:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.772700:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.772703:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.772703:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.772705:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.772706:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.772707:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.772871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.772878:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.772881:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.772883:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.772885:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.772886:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.772887:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:30.0:1619677475.772889:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498248] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.772890:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.772891:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.772892:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:30.0:1619677475.772896:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.772897:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.772900:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.772901:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:30.0:1619677475.772901:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.772903:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.772903:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.772904:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.772905:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.772909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.772916:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.772918:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.772919:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.772919:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.772921:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.772923:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.773681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.773687:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.773689:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.773690:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.773692:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.773695:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.773696:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.773697:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498247] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.773699:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.773700:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.773701:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.773704:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.773706:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.773708:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.773709:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.773709:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.773711:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.773712:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.773712:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.773713:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.773718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.773726:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.773728:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.773729:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.773731:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.773732:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.773733:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.774225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.774233:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.774236:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.774237:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.774239:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.774242:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.774244:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:26.0:1619677475.774245:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498246] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.774246:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.774247:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.774248:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:26.0:1619677475.774251:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.774253:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677475.774255:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.774256:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677475.774256:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677475.774258:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677475.774258:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.774259:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a06f393000] (22)++
00000800:00000200:26.0:1619677475.774260:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (23)--
00000800:00000200:26.0:1619677475.774260:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000800:00000200:25.2:1619677475.774264:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:4.2:1619677475.774266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (21)++
00000800:00000200:31.0:1619677475.774272:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:10.0:1619677475.774273:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (22)++
00000800:00000200:31.0:1619677475.774274:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.774274:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.774275:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a06f393000] (23)--
00000800:00000200:10.0:1619677475.774275:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:31.0:1619677475.774276:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.774277:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:31.0:1619677475.774278:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.774279:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.774303:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.774311:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.774313:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.774314:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.774315:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.774318:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.774320:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.774321:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498245] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.774322:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.774323:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.774324:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.774327:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.774328:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.774331:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.774332:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.774333:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.774334:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.774335:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.774336:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.774336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.774341:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.774349:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.774352:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.774352:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.774353:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.774355:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.774358:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.774491:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.774499:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.774502:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.774503:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.774504:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.774507:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.774508:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.774509:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498244] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.774511:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.774511:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.774512:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.774515:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.774517:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.774519:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.774519:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.774520:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.774522:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.774522:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.774523:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.774523:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.774527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.774534:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.774536:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.774537:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.774538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.774539:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.774542:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.774639:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.774646:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.774648:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.774649:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.774651:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.774653:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.774654:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.774656:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498243] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.774657:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.774658:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.774658:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.774661:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.774662:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.774665:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.774666:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.774666:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.774668:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.774668:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.774669:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.774670:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.774674:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.774681:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.774684:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.774685:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.774685:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.774687:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.774690:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:5.2:1619677475.774750:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.774756:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.774759:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.774760:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.774761:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.774763:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.774764:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.774765:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498242] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.774766:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.774767:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.774768:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.774771:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.774773:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.774774:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.774775:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.774776:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.774777:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.774777:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.774778:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.774778:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.774783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.774790:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.774792:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.774793:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.774794:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.774794:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.774795:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.774890:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.774893:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.774896:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.774898:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.774899:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.774900:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.774902:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.774903:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498241] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.774904:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.774905:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.774906:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.774910:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.774912:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.774914:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.774915:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.774916:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.774917:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.774918:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.774919:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.774919:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.774923:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.774930:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.774933:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.774933:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.774934:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.774936:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:20.2:1619677475.774939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:8.0:1619677475.774939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:18.0:1619677475.774946:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.774948:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:2.0:1619677475.774949:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:18.0:1619677475.774951:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.774954:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.774955:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:18.0:1619677475.774956:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498240] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.774958:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.774959:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.774960:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.774963:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.774964:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.774967:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.774968:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:18.0:1619677475.774968:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.774970:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.774970:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:18.0:1619677475.774971:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:18.0:1619677475.774972:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:18.0:1619677475.774973:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:8.2:1619677475.774976:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:20.2:1619677475.774978:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.774983:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)++
00000800:00000200:17.0:1619677475.774984:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677475.774985:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907dbe3c00] (23)--
00000800:00000200:17.0:1619677475.774986:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:2.0:1619677475.774986:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:17.0:1619677475.774987:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.774987:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:1.0:1619677475.774988:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.774989:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.774992:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:20.2:1619677475.775079:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.775085:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.775088:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.775088:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.775090:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.775093:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.775094:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.775095:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498239] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.775097:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.775097:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.775098:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.775101:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.775102:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.775103:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.775104:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.775104:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.775106:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.775106:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.775107:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.775107:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.775112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.775119:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.775121:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.775121:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.775123:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.775123:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.775124:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.2:1619677475.775373:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.775379:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.775381:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.775383:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.775384:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.775386:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.775388:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.775389:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498238] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.775390:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.775391:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.775392:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.775396:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.775398:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.775399:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.775400:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.775401:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.775402:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.775403:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.775404:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.775405:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.775406:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.775410:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.2:1619677475.775414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.775420:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.775422:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.775422:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.775422:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.775423:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000400:00000200:21.0:1619677475.775424:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.775424:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.775425:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.775426:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:20.0:1619677475.775426:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.775747:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.775754:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.775756:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.775757:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.775759:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.775761:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.775762:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.775763:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498237] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.775765:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.775766:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.775767:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.775769:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.775771:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.775773:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.775774:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.775775:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.775776:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.775777:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.775778:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.775779:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.775779:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.775783:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.775784:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.775789:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.775790:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.775791:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.775792:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.775793:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:20.0:1619677475.775793:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.775793:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.775795:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.775796:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.775797:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.775801:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.2:1619677475.775803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:21.0:1619677475.775810:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:31.0:1619677475.775811:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:21.0:1619677475.775813:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:31.0:1619677475.775814:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:21.0:1619677475.775814:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.775814:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.775815:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:31.0:1619677475.775816:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.775816:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.775818:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.775818:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.775820:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.775821:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498236] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.775822:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.775823:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.775824:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.775828:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.775830:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.775832:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.775832:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.775833:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.775834:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.775835:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.775836:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.775838:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.775838:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.775840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.775844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.775849:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.775851:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.775851:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.775852:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.775853:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:30.0:1619677475.775853:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.775854:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.775854:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.775855:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.775856:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.775941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.775950:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.775952:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.775953:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.775955:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.775957:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.775959:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.775960:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498235] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.775961:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.775962:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.775963:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.775966:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.775968:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.775970:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.775971:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.775972:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.775973:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.775974:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.775975:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.775976:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.775977:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.775980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.775982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.775987:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.775990:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.775990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.775991:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.775991:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677475.775991:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.775992:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677475.775993:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.775994:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677475.775997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.776065:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.776073:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.776075:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.776077:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.776079:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.776080:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.776081:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.776082:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498234] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.776084:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.776085:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.776085:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.776088:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.776090:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.776092:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.776093:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.776094:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.776095:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.776096:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.776097:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.776098:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.776099:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.776101:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.776107:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.776108:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.776110:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.776111:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.776111:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.776113:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.776114:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:24.0:1619677475.776115:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.776117:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.776118:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.776119:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.776441:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.776448:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.776451:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.776452:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.776454:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.776456:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.776458:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.776459:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498233] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.776460:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.776461:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.776462:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.776465:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.776466:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.776470:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.776470:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.776471:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.776473:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.776473:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.776474:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.776474:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677475.776480:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.776488:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.776490:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.776491:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.776493:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.776494:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.776496:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.776503:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.776511:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.776514:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.776516:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.776518:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.776519:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.776520:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.776521:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498232] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.776522:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.776523:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.776524:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.776528:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.776529:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.776532:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.776533:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.776534:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.776535:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.776536:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.776537:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.776537:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.776537:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.2:1619677475.776543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:8.0:1619677475.776548:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:20.0:1619677475.776551:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:8.0:1619677475.776551:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677475.776552:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000800:00000200:20.0:1619677475.776553:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:8.0:1619677475.776553:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:20.0:1619677475.776554:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.776556:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.776556:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.776557:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:8.2:1619677475.776559:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.776563:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:8.0:1619677475.776563:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000800:00000200:20.0:1619677475.776564:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000400:00000200:8.0:1619677475.776564:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498231] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.776566:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000400:00000200:8.0:1619677475.776566:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.776567:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000400:00000200:8.0:1619677475.776567:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.776568:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.776568:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000400:00000200:8.0:1619677475.776568:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.776569:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000400:00000200:8.0:1619677475.776571:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.776573:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677475.776576:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.776577:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677475.776577:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677475.776579:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677475.776579:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677475.776580:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677475.776581:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:8.2:1619677475.776596:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.776604:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.776607:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.776607:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.776608:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.776610:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.776613:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.776917:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.776925:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.776928:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.776930:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.776932:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.776933:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.776935:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.776936:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498230] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.776937:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.776938:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.776939:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.776942:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.776944:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.776946:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.776946:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.776947:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.776948:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.776949:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.776950:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.776950:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.776955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.776961:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.776964:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.776965:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.776966:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.776968:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.776969:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.776975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.776982:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.776984:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.776985:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.776985:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.776986:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.776987:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.777172:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.777178:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.777181:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.777182:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.777184:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.777186:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.777187:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.777189:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498229] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.777190:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.777191:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.777192:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.777195:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.777197:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.777200:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.777200:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.777201:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.777203:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.777203:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.777204:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.777204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.777209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.777217:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.777219:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.777219:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.777221:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.777222:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.777224:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.777649:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.777657:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.777660:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.777662:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.777664:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.777664:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.777665:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:31.0:1619677475.777666:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498228] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.777668:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.777668:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.777669:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:31.0:1619677475.777673:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.777674:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:31.0:1619677475.777676:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.777677:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:31.0:1619677475.777678:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:31.0:1619677475.777680:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:31.0:1619677475.777680:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.777681:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.777682:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.777686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.777693:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.777696:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.777697:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.777697:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.777699:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.777701:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:9.2:1619677475.777706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:8.0:1619677475.777712:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.777714:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.777715:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.777715:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.777716:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.777717:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.778090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.778094:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.778097:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.778099:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.778100:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.778101:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.778103:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.778104:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498227] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.778106:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.778107:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.778108:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.778111:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.778112:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.778115:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.778116:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.778116:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.778118:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.778119:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.778120:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.778121:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.778124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.778130:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.778133:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.778134:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.778134:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.778136:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:24.0:1619677475.778138:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:9.2:1619677475.778144:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.778150:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.778152:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.778153:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.778153:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.778154:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.778154:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.778417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.778423:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.778425:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.778428:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.778428:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.778430:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.778432:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677475.778433:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498226] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.778434:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.778435:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.778436:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.778439:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.778441:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.778444:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.778444:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.778445:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.778447:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.778448:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.778449:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.778449:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677475.778454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.778462:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.778465:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.778466:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.778467:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.778469:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.778471:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.778485:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.778493:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.778495:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.778496:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.778498:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.778501:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.778502:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:24.0:1619677475.778504:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498225] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.778505:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.778506:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.778507:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:24.0:1619677475.778510:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.778512:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:24.0:1619677475.778515:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.778516:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:24.0:1619677475.778516:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:24.0:1619677475.778518:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:24.0:1619677475.778519:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.778520:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.778521:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.778521:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677475.778524:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:4.2:1619677475.778527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:18.0:1619677475.778532:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.778535:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:8.0:1619677475.778535:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:18.0:1619677475.778536:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.778536:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677475.778537:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.778538:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677475.778539:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.778539:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677475.778543:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.778698:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:31.0:1619677475.778706:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.778709:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:31.0:1619677475.778711:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.778713:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:31.0:1619677475.778714:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.778715:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:31.0:1619677475.778717:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498224] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.778718:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.778719:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.778720:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:31.0:1619677475.778723:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.778725:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.778727:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.778728:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.778728:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.778730:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.778730:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:31.0:1619677475.778732:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677475.778733:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (23)--
00000800:00000200:31.0:1619677475.778734:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:25.2:1619677475.778735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:4.2:1619677475.778740:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.778744:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.778746:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.778746:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.778748:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)++
00000800:00000200:31.0:1619677475.778748:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a87afc2e00] (23)--
00000400:00000200:30.0:1619677475.778748:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.778749:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677475.778750:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.778751:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:30.0:1619677475.778752:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:21.2:1619677475.779052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.779058:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.779061:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.779061:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.779064:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.779066:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.779067:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.779068:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498223] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.779070:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.779071:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.779072:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.779075:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.779076:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.779079:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.779080:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.779080:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.779082:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.779082:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.779083:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.779083:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.779088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.779095:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.779097:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.779098:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.779100:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.779101:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.779103:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.779289:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.779298:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.779301:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.779302:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.779304:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.779307:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.779308:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.779310:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498222] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.779311:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.779312:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.779313:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.779316:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.779318:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.779320:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.779321:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.779322:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.779323:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.779323:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.779324:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.779324:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.779329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.779336:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.779339:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.779340:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.779340:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.779342:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.779344:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.779417:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.779424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.779426:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.779427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.779428:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.779431:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.779431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000400:00000200:8.0:1619677475.779433:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.779433:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498221] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.779435:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.779435:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.779436:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.779438:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:8.0:1619677475.779439:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.779441:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:8.0:1619677475.779441:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.779442:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.779443:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.779444:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.779444:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.779445:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677475.779446:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.779447:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000800:00000200:8.0:1619677475.779447:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.779447:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:26.0:1619677475.779448:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498220] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677475.779448:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.779448:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:26.0:1619677475.779449:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.779450:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.779451:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000800:00000200:25.2:1619677475.779453:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:26.0:1619677475.779454:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.779455:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.779458:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.779459:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.779459:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.779460:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.779461:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.779462:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.779462:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.779463:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:22.0:1619677475.779463:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.779464:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.779464:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:25.2:1619677475.779467:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000400:00000200:22.0:1619677475.779467:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.779471:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677475.779472:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.779474:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.779475:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.779475:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.779477:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.779479:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.779492:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:6.0:1619677475.779498:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.779501:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.779502:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.779502:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.779503:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.779504:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.779525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.779533:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.779536:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.779538:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.779540:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.779540:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.779542:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.779543:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498219] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.779545:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.779546:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.779547:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.779550:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.779551:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.779553:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.779554:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.779555:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.779556:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.779557:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.779558:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.779558:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.779562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.779568:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.779571:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.779572:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.779572:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.779574:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.779576:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:5.2:1619677475.779670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.779677:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.779679:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.779680:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.779682:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.779684:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.779685:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.779686:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498218] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.779688:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.779688:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.779689:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.779693:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.779694:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.779696:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.779697:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.779698:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.779699:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.779699:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.779701:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.779702:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.779702:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.779705:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.779708:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.779714:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.779715:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.779715:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.779715:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.779716:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.779716:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.779717:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:17.0:1619677475.779718:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.779718:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.779719:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.779724:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.779732:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.779734:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.779735:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.779735:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.779736:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.779737:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.780006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:1.0:1619677475.780012:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.780014:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:17.0:1619677475.780015:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:1.0:1619677475.780016:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.780018:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.780020:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:1.0:1619677475.780021:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498217] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.780022:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.780023:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.780023:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:1.0:1619677475.780026:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.780027:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:1.0:1619677475.780030:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.780030:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:1.0:1619677475.780031:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:1.0:1619677475.780032:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:1.0:1619677475.780033:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.780033:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.780034:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:8.2:1619677475.780039:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.780048:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.780050:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.780051:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.780052:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.780053:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.780055:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.780060:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.2:1619677475.780063:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.780069:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:20.0:1619677475.780070:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:2.0:1619677475.780071:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:20.0:1619677475.780072:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.780073:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:18.0:1619677475.780073:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:20.0:1619677475.780074:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:21.0:1619677475.780075:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:20.0:1619677475.780075:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:2.0:1619677475.780075:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.780077:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.780078:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:2.0:1619677475.780080:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498216] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.780081:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.780082:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.780083:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:2.0:1619677475.780086:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.780088:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:2.0:1619677475.780091:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.780091:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:2.0:1619677475.780092:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:2.0:1619677475.780094:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:2.0:1619677475.780094:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.780095:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.780096:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.780111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:13.0:1619677475.780120:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:13.0:1619677475.780123:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.780124:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.780127:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:12.0:1619677475.780128:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.780131:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:25.2:1619677475.780139:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:12.0:1619677475.780146:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.780149:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:12.0:1619677475.780150:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.780150:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:12.0:1619677475.780151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.780151:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.780256:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.780261:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.780264:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.780265:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.780267:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.780269:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.780271:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:17.0:1619677475.780273:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498215] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.780274:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.780275:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.780276:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:17.0:1619677475.780280:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.780281:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:17.0:1619677475.780284:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.780285:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:17.0:1619677475.780286:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:17.0:1619677475.780287:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:17.0:1619677475.780287:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.780289:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce2bc0c00] (22)++
00000800:00000200:17.0:1619677475.780290:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:17.0:1619677475.780290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:9.2:1619677475.780293:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.2:1619677475.780298:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:10.0:1619677475.780301:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.780304:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:10.0:1619677475.780304:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.780305:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.780305:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)++
00000400:00000200:10.0:1619677475.780307:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.780307:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce2bc0c00] (23)--
00000800:00000200:1.0:1619677475.780307:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:1.0:1619677475.780308:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:10.0:1619677475.780310:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.780726:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.780732:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.780735:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.780736:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.780738:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.780740:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.780741:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.780743:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498214] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.780744:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.780745:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.780746:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.780749:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.780751:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.780754:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.780755:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.780756:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.780757:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.780758:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.780758:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.780758:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.780763:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.780771:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.780773:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.780774:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.780775:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.780777:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.780778:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.781093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.781101:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.781104:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.781106:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.781108:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.781109:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.781110:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.781111:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498213] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.781113:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.781114:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.781115:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.781118:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.781120:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.781122:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.781123:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.781124:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.781125:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.781126:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.781127:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.781129:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.781129:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.781131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.781135:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.781139:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.781141:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.781141:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.781143:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.781143:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.781145:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.781145:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.781146:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.781147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.781148:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.781153:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.781160:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.781162:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.781164:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.781166:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.781166:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.781167:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.781168:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498212] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.781169:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.781170:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.781171:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.781174:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.781176:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.781178:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.781178:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.781179:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.781180:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.781181:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.781182:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.781182:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.781186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.781194:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.781196:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.781197:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.781197:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.781199:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.781201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.781323:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.781330:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:4.2:1619677475.781332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.781333:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:26.0:1619677475.781335:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677475.781336:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.781337:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000400:00000200:26.0:1619677475.781338:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.781338:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000400:00000200:26.0:1619677475.781340:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:10.0:1619677475.781340:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.781341:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498211] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.781343:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.781344:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.781344:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.781347:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.781348:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.781349:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000400:00000200:10.0:1619677475.781350:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498210] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:26.0:1619677475.781351:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:10.0:1619677475.781351:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.781352:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.781353:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.781356:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.781357:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000400:00000200:10.0:1619677475.781357:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.781358:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:10.0:1619677475.781359:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.781360:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.781361:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.781361:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:10.0:1619677475.781361:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.781362:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:10.0:1619677475.781362:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.781363:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.781363:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.781365:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.781366:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.781367:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.781367:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:8.2:1619677475.781367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:25.2:1619677475.781372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.0:1619677475.781375:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.781377:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.781378:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.781380:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.781380:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.781381:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:22.0:1619677475.781383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:21.0:1619677475.781383:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:6.0:1619677475.781383:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.781384:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.781387:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:25.2:1619677475.781390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.781393:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.781395:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.781397:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.781398:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.781398:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.781399:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.781399:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.781413:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.781415:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.781418:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.781420:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.781421:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.781423:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.781424:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.781426:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498209] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.781427:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.2:1619677475.781429:0:19934:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aa13a600] (20)++
00000400:00000200:20.0:1619677475.781432:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.781433:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.781436:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aa13a600] (21)++
00000400:00000200:20.0:1619677475.781436:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.781437:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.15@o2ib3
00000800:00000200:20.0:1619677475.781438:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.781440:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3) <- 10.51.14.15@o2ib3 : GET - for me
00000800:00000200:20.0:1619677475.781440:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.781441:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.781442:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:21.0:1619677475.781443:0:19935:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.51.14.15@o2ib3 of length 272 into portal 0 MB=0x8000000000000000
00000800:00000200:20.0:1619677475.781444:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.781444:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000400:00000200:21.0:1619677475.781445:0:19935:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.51.14.15@o2ib3 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:20.0:1619677475.781445:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.781446:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aa13a600] (22)++
00000800:00000200:20.0:1619677475.781446:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:21.0:1619677475.781447:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aa13a600] (23)--
00000800:00000200:20.0:1619677475.781447:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aa13a600] (23)--
00000400:00000200:21.0:1619677475.781449:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.51.0.116@o2ib3 to MR:  10.51.14.15@o2ib3 local destination
00000800:00000200:8.2:1619677475.781450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:21.0:1619677475.781453:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.0.116@o2ib3(10.51.0.116@o2ib3:10.51.0.116@o2ib3) -> 10.51.14.15@o2ib3(10.51.14.15@o2ib3:10.51.14.15@o2ib3) : REPLY try# 0
00000800:00000200:21.0:1619677475.781455:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.51.14.15@o2ib3
00000800:00000200:17.0:1619677475.781457:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.781458:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9197eee63200] -> 10.51.14.15@o2ib3 (2) version: 12
00000800:00000200:21.0:1619677475.781458:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197aa13a600] (21)++
00000800:00000200:21.0:1619677475.781459:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aa13a600] (22)++
00000800:00000200:21.0:1619677475.781460:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197aa13a600] (23)--
00000800:00000200:17.0:1619677475.781460:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677475.781461:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aa13a600] (22)--
00000400:00000200:17.0:1619677475.781461:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.781461:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.781463:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.781466:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.2:1619677475.781468:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aa13a600] (21)++
00000800:00000200:20.0:1619677475.781471:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aa13a600] (22)++
00000800:00000200:20.0:1619677475.781473:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aa13a600] (23)--
00000400:00000200:20.0:1619677475.781474:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.781476:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.51.0.116@o2ib3->10.51.14.15@o2ib3: REPLY: OK
00000800:00000200:21.0:1619677475.781477:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aa13a600] (22)--
00000800:00000200:20.0:1619677475.781479:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aa13a600] (21)--
00000800:00000200:4.2:1619677475.781644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:26.0:1619677475.781651:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.781654:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.781655:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.781657:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.781660:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.781662:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:26.0:1619677475.781663:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498208] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.781665:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.781666:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.781667:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:26.0:1619677475.781670:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.781672:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:26.0:1619677475.781675:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.781676:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:26.0:1619677475.781676:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:26.0:1619677475.781678:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:26.0:1619677475.781678:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:26.0:1619677475.781679:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.781679:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.781684:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.781691:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.781694:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.781695:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.781695:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.781698:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.781700:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.781758:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.781766:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.781769:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.781771:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.781773:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.781774:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.781775:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.781776:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498207] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.781778:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.781779:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.781779:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.781783:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.781784:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.781786:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.781787:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.781788:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.781789:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.781790:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.781791:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.781791:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.781795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.781801:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.781804:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.781805:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.781805:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.781807:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.781809:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:20.2:1619677475.781966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.781973:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.781976:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.781976:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.781979:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.781982:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.781983:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.781984:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498206] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.781985:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.781987:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.781988:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.781991:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.781992:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.781994:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.781995:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.781996:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.781997:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.781998:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.782000:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907d39de00] (22)++
00000800:00000200:18.0:1619677475.782001:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.782001:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000800:00000200:9.2:1619677475.782004:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.2:1619677475.782006:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (21)++
00000800:00000200:2.0:1619677475.782012:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.782013:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (22)++
00000800:00000200:2.0:1619677475.782013:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.782014:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907d39de00] (23)--
00000800:00000200:18.0:1619677475.782014:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:2.0:1619677475.782014:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.782015:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.782016:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.782016:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:18.0:1619677475.782017:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:20.2:1619677475.782093:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87b3e5800] (20)++
00000800:00000200:31.0:1619677475.782101:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.782104:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.70@o2ib3
00000400:00000200:31.0:1619677475.782106:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.70@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.782108:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (22)--
00000400:00000200:31.0:1619677475.782109:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.782110:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:31.0:1619677475.782112:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498205] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.782113:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.782114:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.782115:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:31.0:1619677475.782118:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.70@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.782120:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:31.0:1619677475.782122:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.782123:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.2:1619677475.782123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:31.0:1619677475.782124:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:31.0:1619677475.782125:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:31.0:1619677475.782126:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87b3e5800] (21)++
00000800:00000200:31.0:1619677475.782127:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87b3e5800] (22)--
00000800:00000200:31.0:1619677475.782127:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87b3e5800] (21)--
00000800:00000200:22.0:1619677475.782129:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:25.2:1619677475.782131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:22.0:1619677475.782132:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.782133:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.782135:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.782137:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.782138:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:12.0:1619677475.782138:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:22.0:1619677475.782139:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498204] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.782141:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:12.0:1619677475.782141:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000400:00000200:12.0:1619677475.782141:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.782142:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.782143:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:12.0:1619677475.782144:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.782146:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.782146:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677475.782147:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:22.0:1619677475.782148:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.782151:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.782152:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.782153:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.782155:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.782155:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.782156:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.782157:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.782158:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.782161:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.782163:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.782168:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:21.0:1619677475.782170:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.782170:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.782171:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.782172:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:6.0:1619677475.782172:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:21.0:1619677475.782173:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.782175:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.782175:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.782177:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.782180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.782187:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.782190:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.782191:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.782191:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.782192:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.782193:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.782528:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.782536:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.782539:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.782542:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.782544:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.782544:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.782545:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.782547:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498203] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.782548:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.782549:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.782550:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.782554:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.782556:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.782558:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.782559:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.782559:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.782561:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.782562:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.782562:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.782563:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:9.2:1619677475.782567:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.782574:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.782577:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.782578:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.782578:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.782580:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.782582:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.782947:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.782953:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.782956:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.782956:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.782959:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.782961:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.782963:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677475.782964:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498202] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.782965:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.782966:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.782967:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677475.782971:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.782973:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.782975:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.782976:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677475.782977:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.782978:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.782979:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.782979:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.782980:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677475.782985:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.782993:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.782995:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.782996:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.782996:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.782998:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.783001:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.783114:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.783121:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.783124:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.783126:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.783128:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.783129:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.783130:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.783131:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498201] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.783133:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.783134:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.783135:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.783138:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.783139:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.783142:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.783143:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.783143:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.783145:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.783145:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.783146:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.783147:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.783150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.783157:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.783159:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.783160:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.783160:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.783162:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.783164:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.783365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:10.0:1619677475.783372:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.783375:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:26.0:1619677475.783376:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:10.0:1619677475.783378:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.783381:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.783382:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:10.0:1619677475.783383:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498200] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.783384:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.783385:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.783386:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:10.0:1619677475.783389:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.783391:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.783394:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.783394:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:10.0:1619677475.783395:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:10.0:1619677475.783397:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677475.783397:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:10.0:1619677475.783398:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:10.0:1619677475.783398:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:8.2:1619677475.783403:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.783411:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.783414:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.783415:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.783415:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.783417:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.783420:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.783472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.783479:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.783482:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.783483:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.783484:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.783487:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.783488:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.783490:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498199] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.783491:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.783492:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.783493:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.783495:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.783497:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.783499:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.783499:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.783500:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.783501:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.783502:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.783502:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.783503:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.783507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.783514:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.783516:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.783517:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.783517:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.783518:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000400:00000200:8.0:1619677475.783518:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.783520:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:17.0:1619677475.783524:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.783527:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:9.2:1619677475.783527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677475.783528:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.783529:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.783531:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.783532:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:17.0:1619677475.783533:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:17.0:1619677475.783534:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498198] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.783535:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677475.783536:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:8.0:1619677475.783536:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.783537:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:17.0:1619677475.783537:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.783538:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.783538:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:17.0:1619677475.783538:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:17.0:1619677475.783541:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.783543:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:17.0:1619677475.783545:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.783546:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:17.0:1619677475.783547:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:17.0:1619677475.783548:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:17.0:1619677475.783549:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.783550:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:17.0:1619677475.783550:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677475.783555:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.783563:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.783565:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:2.0:1619677475.783566:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.783567:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.783569:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.783571:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677475.783575:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:18.0:1619677475.783582:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.783584:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677475.783585:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.783585:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.783586:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:18.0:1619677475.783587:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.783720:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.783728:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.783730:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.783731:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.783733:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.783736:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.783737:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.783738:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498197] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.783740:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.783741:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.783742:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.783745:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.783747:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.783750:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.783751:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.783752:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.783753:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.783754:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.783755:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.783755:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.783760:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.783767:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.783770:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.783771:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.783771:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.783773:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.783776:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.783920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.783926:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.783928:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.783929:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.783931:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.783934:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.783935:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.783936:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498196] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.783938:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.783939:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.783940:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.783943:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.783945:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.783947:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.783947:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.783948:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.783949:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:1.0:1619677475.783949:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.783950:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.783951:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.783951:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:22.0:1619677475.783954:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:8.2:1619677475.783956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:22.0:1619677475.783957:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.783958:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.783959:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.783962:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.783963:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:17.0:1619677475.783963:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:22.0:1619677475.783964:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498195] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.783965:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.783965:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:22.0:1619677475.783966:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.783966:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.783967:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:22.0:1619677475.783968:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:17.0:1619677475.783969:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.783971:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000400:00000200:22.0:1619677475.783972:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.783974:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:8.2:1619677475.783975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:22.0:1619677475.783976:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.783977:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.783978:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.783980:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.783980:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.783981:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.783981:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:1.0:1619677475.783982:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.783985:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:25.2:1619677475.783986:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:17.0:1619677475.783986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.783986:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.783988:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.783988:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:20.0:1619677475.783993:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.783996:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.783996:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.783998:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.784000:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.784001:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.784204:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.784212:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.784215:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.784218:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.784220:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.784220:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.784222:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.784224:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498194] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.784225:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.784226:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.784227:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.784230:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.784232:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.784236:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.784237:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.784238:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.784239:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.784240:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.784241:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.784241:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.784245:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.784253:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.784255:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.784256:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.784258:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.784258:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.784261:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.784355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.784363:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.784366:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.784366:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677475.784369:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.784371:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.784372:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.784374:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498193] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.784375:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.784376:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.784377:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.784380:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.784382:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.784384:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.784385:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.784386:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.784387:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.784388:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.784388:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.784389:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.784394:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.784402:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.784405:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.784406:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.784406:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.784409:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.784412:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.784622:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.784628:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.784631:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.784632:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.784634:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.784636:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.784638:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.784639:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498192] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.784640:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.784641:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.784642:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:2.0:1619677475.784646:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.784647:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:2.0:1619677475.784650:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.784650:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.784651:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:2.0:1619677475.784653:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.784653:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.784654:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.784655:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:8.2:1619677475.784659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.784668:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.784671:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.784672:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.784672:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.784674:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000800:00000200:17.0:1619677475.784677:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:8.2:1619677475.784682:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.784689:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.784692:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.784693:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.784693:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.784694:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.784695:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.784852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.784859:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.784862:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.784863:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.784865:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.784868:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.784870:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.784871:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498191] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.784872:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.784873:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.784874:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.784878:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.784879:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.784882:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:21.2:1619677475.784882:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:24.0:1619677475.784883:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.784884:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.784885:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.784886:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.784886:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.784887:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:31.0:1619677475.784890:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.2:1619677475.784891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677475.784893:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.784895:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.784897:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.784897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:17.0:1619677475.784898:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000400:00000200:31.0:1619677475.784899:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.784900:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498190] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.784901:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.784901:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:31.0:1619677475.784902:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.784902:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.784902:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:31.0:1619677475.784904:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.784904:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:31.0:1619677475.784907:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.784909:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.784910:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677475.784911:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.784912:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.784913:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.784914:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.784915:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.784916:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.784918:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.784919:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.784920:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.784924:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:8.0:1619677475.784927:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.784929:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.784930:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.784930:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.784932:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:8.0:1619677475.784932:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.784934:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.784935:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.784935:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.784936:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.784941:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.784946:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.784949:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.784950:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.784950:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.784950:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.784951:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.785642:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677475.785650:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.785652:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677475.785653:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677475.785654:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.785657:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.785659:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:8.0:1619677475.785659:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677475.785660:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498189] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.785661:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.785662:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.785663:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000800:00000200:22.0:1619677475.785665:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:8.0:1619677475.785666:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.785668:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:8.0:1619677475.785668:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:6.0:1619677475.785669:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.785670:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:8.0:1619677475.785671:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.785672:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000400:00000200:22.0:1619677475.785673:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.785673:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000400:00000200:22.0:1619677475.785674:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:8.0:1619677475.785674:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677475.785675:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000400:00000200:22.0:1619677475.785676:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498188] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:8.0:1619677475.785676:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677475.785676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000400:00000200:22.0:1619677475.785677:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.785678:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.785679:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:9.2:1619677475.785681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000400:00000200:22.0:1619677475.785682:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.785684:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.785687:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.785687:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.785688:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.785690:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.785690:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.785690:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677475.785690:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:22.0:1619677475.785691:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:18.0:1619677475.785693:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:18.0:1619677475.785694:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.785694:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:25.2:1619677475.785696:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677475.785696:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.785700:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:20.0:1619677475.785704:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:9.2:1619677475.785706:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.785707:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.785708:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.785710:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.785711:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.785713:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:2.0:1619677475.785713:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.785716:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.785718:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.785718:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.785719:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (22)--
00000800:00000200:2.0:1619677475.785719:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.785721:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.785727:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.785730:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.785731:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.785732:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.785734:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.785736:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:1.0:1619677475.785737:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498187] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.785738:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.785739:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.785740:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:1.0:1619677475.785744:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.785746:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:1.0:1619677475.785748:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.785749:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:1.0:1619677475.785750:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:1.0:1619677475.785751:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:1.0:1619677475.785752:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.785752:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:1.0:1619677475.785753:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:9.2:1619677475.785757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.785765:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.785767:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.785768:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.785768:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.785770:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.785772:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.785773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.785780:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.785783:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.785784:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.785786:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.785789:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.785790:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.785791:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498186] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.785793:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.785794:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.785795:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.785798:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.785800:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.785802:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.785803:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.785804:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.785805:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.785805:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.785806:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.785807:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:25.2:1619677475.785811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:21.2:1619677475.785811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:13.0:1619677475.785819:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:30.0:1619677475.785820:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:13.0:1619677475.785822:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.785822:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.785823:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:13.0:1619677475.785824:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000400:00000200:30.0:1619677475.785825:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.785826:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000400:00000200:30.0:1619677475.785827:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.785827:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:31.0:1619677475.785828:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.785829:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.785830:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498185] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.785831:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.785832:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.785833:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.785836:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.785838:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.785839:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.785840:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.785841:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.785842:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.785843:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.785844:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.785845:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.785846:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.785848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.785851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.785856:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.785857:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.785858:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.785859:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.785859:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.785860:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.785860:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.785861:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.785862:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.785863:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.785960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.785967:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.785969:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.785971:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:13.0:1619677475.785973:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.785973:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.785975:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:12.0:1619677475.785976:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498184] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.785977:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.785979:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.785979:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:12.0:1619677475.785982:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.785984:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:12.0:1619677475.785986:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.785986:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:12.0:1619677475.785987:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:12.0:1619677475.785989:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:12.0:1619677475.785989:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.785990:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.785991:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:25.2:1619677475.785994:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.786001:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.786004:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.786004:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.786006:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.786008:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.786009:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.786196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.786203:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.786206:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.786207:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.786208:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.786210:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.786212:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.786213:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498183] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.786214:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.786215:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.786216:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.786219:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.786221:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.786223:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.786224:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.786225:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.786226:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.786227:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.786228:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:10.0:1619677475.786230:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.786230:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.786233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.786236:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.786240:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:26.0:1619677475.786243:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.786243:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.786244:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.786244:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.786245:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.786246:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:6.0:1619677475.786246:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.786247:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677475.786249:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.786301:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.2:1619677475.786306:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:21.0:1619677475.786311:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677475.786312:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:21.0:1619677475.786314:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:17.0:1619677475.786315:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000400:00000200:21.0:1619677475.786316:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.786316:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:20.0:1619677475.786317:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677475.786318:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:21.0:1619677475.786319:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.786321:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.786321:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.786322:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:21.0:1619677475.786323:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.786324:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498182] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.786325:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.786326:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.786326:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.786327:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.786328:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.786329:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:21.0:1619677475.786335:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000400:00000200:17.0:1619677475.786335:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.786338:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:21.0:1619677475.786339:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.786342:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.786342:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.786343:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.786343:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:21.0:1619677475.786344:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.786344:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:21.0:1619677475.786345:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.786346:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:17.0:1619677475.786346:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:21.0:1619677475.786347:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:17.0:1619677475.786347:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:21.0:1619677475.786348:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.786348:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.786348:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:8.2:1619677475.786352:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:9.2:1619677475.786353:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.786360:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:1.0:1619677475.786360:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.786362:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.786363:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:10.0:1619677475.786363:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:1.0:1619677475.786363:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.786364:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:10.0:1619677475.786365:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.786366:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.786367:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.786368:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:10.0:1619677475.786370:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000800:00000200:4.2:1619677475.786481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.786484:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.786487:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.786489:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.786490:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.786492:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.786493:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:20.0:1619677475.786495:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498180] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.786496:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.786497:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.786498:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:20.0:1619677475.786502:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.786504:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:20.0:1619677475.786506:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.786507:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:20.0:1619677475.786508:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:20.0:1619677475.786510:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.0:1619677475.786510:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.786511:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.786511:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:9.2:1619677475.786516:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.786524:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.786526:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.786527:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.786528:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.786529:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.786532:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.786535:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.786543:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.786546:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.786549:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.786551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.786551:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.786552:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.786553:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498179] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.786555:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.786556:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.786557:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.786560:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.786562:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.786564:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.786565:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.786566:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.786567:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.786568:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.786569:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.786569:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.786573:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.786580:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.786583:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.786583:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.786585:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.786587:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.786587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.786591:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.786598:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.786601:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.786601:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.786602:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:31.0:1619677475.786602:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.786603:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.786990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.786998:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.787000:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.787002:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.787003:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.787004:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.787005:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.787006:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498178] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.787007:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.787008:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.787009:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.787012:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.787013:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.787015:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.787016:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.787017:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.787018:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.787019:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.787020:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.787020:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.787024:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.787031:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.787033:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.787034:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.787035:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.787036:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.787038:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.2:1619677475.787092:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.787098:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.787100:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.787101:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.787103:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.787105:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.787106:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.787108:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498177] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.787109:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.787110:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.787111:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.787114:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.787116:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.787118:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.787119:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.787120:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.787121:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.787122:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.787122:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.787123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.787127:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.787135:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.787137:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.787138:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.787140:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.787140:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.787142:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.787624:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a87afc2e00] (20)++
00000800:00000200:30.0:1619677475.787633:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.787636:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.57@o2ib3
00000400:00000200:30.0:1619677475.787638:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.57@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.787640:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (22)--
00000400:00000200:30.0:1619677475.787640:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.787642:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:30.0:1619677475.787644:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498176] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.787645:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.787646:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.787647:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:30.0:1619677475.787651:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.57@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.787653:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:30.0:1619677475.787656:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.787657:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:30.0:1619677475.787658:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:30.0:1619677475.787659:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:30.0:1619677475.787660:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a87afc2e00] (21)++
00000800:00000200:30.0:1619677475.787661:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a87afc2e00] (22)--
00000800:00000200:30.0:1619677475.787662:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a87afc2e00] (21)--
00000800:00000200:9.2:1619677475.787666:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.787675:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.787678:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:18.0:1619677475.787679:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.787679:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.787681:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.787684:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:4.2:1619677475.788228:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.788236:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.788239:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.788240:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.788241:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.788244:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.788245:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:26.0:1619677475.788247:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498175] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.788248:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.788249:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.788250:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:26.0:1619677475.788253:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.788254:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:26.0:1619677475.788257:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.788257:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:26.0:1619677475.788258:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:26.0:1619677475.788260:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:26.0:1619677475.788260:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.788261:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.788261:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.788276:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.788283:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.788285:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.788286:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.788286:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.788288:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.788290:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677475.788372:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfdf63800] (20)++
00000800:00000200:17.0:1619677475.788378:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677475.788381:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.23@o2ib3
00000800:00000200:1.0:1619677475.788382:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (22)--
00000400:00000200:17.0:1619677475.788384:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.23@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.788387:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.788388:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:17.0:1619677475.788390:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498174] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.788391:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.788392:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.788393:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:17.0:1619677475.788396:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.788398:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:17.0:1619677475.788401:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.788402:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:17.0:1619677475.788402:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:17.0:1619677475.788404:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:17.0:1619677475.788405:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfdf63800] (21)++
00000800:00000200:17.0:1619677475.788405:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfdf63800] (22)--
00000800:00000200:17.0:1619677475.788406:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfdf63800] (21)--
00000800:00000200:9.2:1619677475.788421:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.788427:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.788430:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.788431:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.788431:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.788433:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.788436:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.788614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.788623:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.788625:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:20.2:1619677475.788625:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:31.0:1619677475.788628:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.788628:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:31.0:1619677475.788630:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.788630:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:20.0:1619677475.788630:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:31.0:1619677475.788632:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:31.0:1619677475.788633:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498173] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.788633:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.788633:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.788634:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.788635:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.788635:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.788636:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.788637:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.788638:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498172] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.788640:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.788641:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.788642:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:20.0:1619677475.788642:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:31.0:1619677475.788645:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.788647:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.788648:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.788649:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.788649:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.788649:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:31.0:1619677475.788651:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.788651:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.788652:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:20.0:1619677475.788652:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:31.0:1619677475.788653:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:20.0:1619677475.788653:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:31.0:1619677475.788654:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:20.0:1619677475.788654:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.788655:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.788656:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.788656:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:9.2:1619677475.788657:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.2:1619677475.788662:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:8.0:1619677475.788668:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:21.0:1619677475.788670:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:24.0:1619677475.788671:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.788671:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:21.0:1619677475.788672:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:8.0:1619677475.788672:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.788673:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.788674:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000400:00000200:21.0:1619677475.788675:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.788676:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:8.0:1619677475.788676:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:21.0:1619677475.788678:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:5.2:1619677475.788753:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.788759:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.788761:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.788762:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.788765:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.788767:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.788768:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.788770:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498171] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.788771:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.788773:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.788773:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.788777:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.788778:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.788780:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.788781:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.788782:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.788783:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.788784:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.788784:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.788785:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.788789:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.788796:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.788799:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.788799:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.788800:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.788801:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.788802:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.788900:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.788906:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.788909:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.788910:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.788912:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.788914:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.788915:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.788917:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498170] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.788918:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.788919:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.788920:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.788923:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.788925:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.788928:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.788929:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.788929:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.788931:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.788931:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.788932:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.788933:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.788934:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.788937:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.788939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.788944:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.788944:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.788946:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:20.0:1619677475.788946:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.788947:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:20.0:1619677475.788947:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.788948:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.788949:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.788950:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.788952:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.788955:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.788962:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.788965:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.788966:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.788966:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.788967:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.788968:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.789178:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.789181:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:30.0:1619677475.789185:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.789188:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677475.789188:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677475.789191:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.789191:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:31.0:1619677475.789192:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:8.0:1619677475.789192:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.789193:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.789194:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.789195:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.789196:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498169] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.789197:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.789198:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.789198:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.789200:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.789200:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498168] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.789201:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:24.0:1619677475.789202:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.789203:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.789203:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.789208:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000400:00000200:24.0:1619677475.789208:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.789211:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:24.0:1619677475.789211:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.789214:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.789215:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:24.0:1619677475.789215:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.789216:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.789216:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.789216:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.789217:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.789218:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.789218:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.789219:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.789219:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677475.789219:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.789220:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:24.0:1619677475.789221:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.789221:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:9.2:1619677475.789223:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:25.2:1619677475.789224:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.789227:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.789232:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677475.789233:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:8.0:1619677475.789234:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.789235:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000400:00000200:8.0:1619677475.789235:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.789236:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:22.0:1619677475.789236:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.789236:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:8.0:1619677475.789237:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:6.0:1619677475.789237:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.789238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.789238:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:24.0:1619677475.789239:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:6.0:1619677475.789239:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.789240:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677475.789242:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.789249:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.789257:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.789259:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.789260:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.789260:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.789262:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.789262:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.789287:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.789295:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.789301:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.789305:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:10.0:1619677475.789307:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.789309:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.789310:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.789311:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498167] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.789312:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.789314:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.789314:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.789317:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.789319:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.789322:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.789323:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.789323:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.789325:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.789325:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.789327:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.789328:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.789328:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.789332:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.789335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.789339:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677475.789342:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.789342:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.789343:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.789343:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.789344:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.789345:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:6.0:1619677475.789345:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.789346:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:6.0:1619677475.789348:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.790242:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.790250:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.790253:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.790255:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.790258:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.790258:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.790259:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:31.0:1619677475.790261:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498166] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.790262:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.790263:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.790264:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:31.0:1619677475.790267:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.790268:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:31.0:1619677475.790272:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.790272:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:31.0:1619677475.790273:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:31.0:1619677475.790275:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:31.0:1619677475.790275:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.790276:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.790277:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.790308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.790317:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.790319:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.790320:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.790322:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.790322:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.790325:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.790584:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.790590:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.790593:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.790594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.790596:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.790598:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.790599:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.790601:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498165] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.790602:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.790603:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.790604:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.790607:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.790609:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.790612:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.790612:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.790613:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.790615:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.790615:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.790616:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.790616:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.790621:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.790629:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.790631:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.790632:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.790634:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.790635:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:4.2:1619677475.790635:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:20.0:1619677475.790638:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:13.0:1619677475.790644:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.790647:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:13.0:1619677475.790649:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:13.0:1619677475.790651:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.790651:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:13.0:1619677475.790653:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:13.0:1619677475.790654:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498164] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.790656:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.790657:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.790658:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:13.0:1619677475.790661:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.790662:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:13.0:1619677475.790665:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.790666:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:13.0:1619677475.790666:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:13.0:1619677475.790668:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:13.0:1619677475.790668:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.790669:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:20.2:1619677475.790670:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:13.0:1619677475.790670:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.790673:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:2.0:1619677475.790676:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.790679:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.790681:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:10.0:1619677475.790681:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000400:00000200:2.0:1619677475.790682:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:10.0:1619677475.790683:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.790684:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677475.790684:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.790684:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.790687:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000400:00000200:2.0:1619677475.790687:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.790688:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498163] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.790690:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:2.0:1619677475.790690:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.790691:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.790692:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.790695:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.790697:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.790700:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.790701:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.790702:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.790704:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.790704:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.790705:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.790706:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.790710:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.790718:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.790721:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.790722:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.790724:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.790725:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.790728:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.790735:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.790742:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.790744:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.790745:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.790746:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.790747:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.790748:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:21.2:1619677475.790854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.790862:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.790864:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.790866:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.790869:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.790869:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.790870:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.790871:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498162] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.790872:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.790873:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.2:1619677475.790873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000400:00000200:30.0:1619677475.790875:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.790877:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.790879:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.790880:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.790881:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:30.0:1619677475.790881:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.790881:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.790882:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677475.790883:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.790883:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.790884:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677475.790885:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.790886:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:31.0:1619677475.790887:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677475.790887:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:31.0:1619677475.790888:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498161] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:30.0:1619677475.790888:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.790889:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.790889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000400:00000200:31.0:1619677475.790890:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:30.0:1619677475.790890:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.790891:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.790891:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.790891:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.790892:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.790893:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:31.0:1619677475.790894:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.790894:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.790896:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:30.0:1619677475.790896:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.790897:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:31.0:1619677475.790898:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.790898:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.790899:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.790899:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:30.0:1619677475.790899:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.790900:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.790901:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.790901:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.790901:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:31.0:1619677475.790902:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.790903:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.790907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.790913:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.790916:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.790917:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.790917:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.790919:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.790921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.791075:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.791083:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.791085:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.791086:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.791088:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.791090:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.791091:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.791092:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498160] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.791094:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.791094:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.791095:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.791098:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.791100:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.791103:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.791103:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.791104:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.791105:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.791106:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.791107:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.791107:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.791112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.791119:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.791122:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.791123:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.791123:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.791125:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.791128:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.791241:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.791249:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.791252:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.791252:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000400:00000200:8.0:1619677475.791254:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.791256:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.791258:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.791259:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498159] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.791261:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.791262:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.791263:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.791266:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.791267:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.791270:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.791271:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.791271:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.791273:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.791273:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.791274:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.791275:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.791279:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.791287:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.791290:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.791291:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.791291:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.791298:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.791302:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.791534:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.791543:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.791546:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.791548:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.791551:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.791551:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.791552:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.791553:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498158] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.791554:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.791555:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.791556:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.791560:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.791561:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.791563:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.791564:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.791565:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.791566:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.791566:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.791567:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.791568:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.791572:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.791579:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.791582:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.791583:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.791584:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.791586:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.791587:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.791811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.791818:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.791821:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.791821:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.791824:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.791826:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.791827:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.791829:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498157] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.791830:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.791831:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.791832:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.791836:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.791837:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.791839:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.791840:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.791841:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.791842:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.791843:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.791844:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.791844:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.791848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.791855:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.791858:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.791858:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.791859:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.791860:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.791861:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:21.2:1619677475.791991:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.791996:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.791999:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.792000:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.792002:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.792004:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.792005:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.792007:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498156] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.792008:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.792009:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.792010:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.792013:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.792015:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.792017:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.792018:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.792019:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.792020:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.792021:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.792021:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.792022:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.792026:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.792034:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.792036:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.792037:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.792038:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.792039:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.792041:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.792160:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.792163:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.792165:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.792167:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.792168:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.792169:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.792171:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.792172:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498155] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.792174:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.792174:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.792176:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.792179:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.792181:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.792183:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.792184:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.792185:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.792186:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.792187:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.792188:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:20.0:1619677475.792189:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.792190:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:8.2:1619677475.792193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.2:1619677475.792195:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.792200:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000800:00000200:17.0:1619677475.792200:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:21.0:1619677475.792203:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:17.0:1619677475.792203:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:21.0:1619677475.792204:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:17.0:1619677475.792204:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.792204:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.792205:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000400:00000200:17.0:1619677475.792206:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.792208:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.792871:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.792880:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.792882:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.792884:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.792886:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.792887:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000800:00000200:30.0:1619677475.792887:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.792888:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498154] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.792890:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.792891:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.792891:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:31.0:1619677475.792895:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.792896:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:31.0:1619677475.792898:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.792899:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:31.0:1619677475.792900:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:31.0:1619677475.792901:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:31.0:1619677475.792902:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.792904:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.792905:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.792906:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.792908:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:21.2:1619677475.792911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:24.0:1619677475.792914:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.792916:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.792918:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.792918:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:30.0:1619677475.792919:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:24.0:1619677475.792919:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.792921:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:30.0:1619677475.792922:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.792922:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.792923:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:9.2:1619677475.792928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.792934:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.792937:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.792937:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.792937:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.792938:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.792939:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.793009:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.793017:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.793019:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.793021:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.793022:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.793025:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.793026:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.793027:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498153] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.793028:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.793029:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.793030:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.793033:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.793034:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.793036:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.793037:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.793037:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.793039:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.793039:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.793040:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.793040:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.793044:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:24.0:1619677475.793050:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.793052:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.793053:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.793053:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.793055:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.793057:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:5.2:1619677475.793058:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.793064:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:9.2:1619677475.793066:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:1.0:1619677475.793067:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.793068:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.793069:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.793071:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.793072:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:1.0:1619677475.793073:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.793074:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498152] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.793075:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.793075:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.793076:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:1.0:1619677475.793076:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.793076:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.793077:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:8.0:1619677475.793077:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:1.0:1619677475.793078:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:4.2:1619677475.793081:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000400:00000200:1.0:1619677475.793081:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.793083:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.793084:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.793085:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.793086:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:24.0:1619677475.793087:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:1.0:1619677475.793087:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.793088:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.793089:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.793089:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:24.0:1619677475.793090:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677475.793091:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677475.793093:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000800:00000200:8.2:1619677475.793094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:24.0:1619677475.793096:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.793098:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.793099:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498151] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.793101:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.793101:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000400:00000200:24.0:1619677475.793102:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.793103:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.793103:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.793104:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.793104:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.793106:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000400:00000200:24.0:1619677475.793107:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.793107:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:24.0:1619677475.793109:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.793113:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.2:1619677475.793113:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:24.0:1619677475.793114:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.793116:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.793118:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.793118:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.793119:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:24.0:1619677475.793120:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:1.0:1619677475.793120:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.793123:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:25.2:1619677475.793124:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.793124:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.793124:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.793125:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.793126:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:6.0:1619677475.793132:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.793134:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.793135:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.793136:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.793138:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.793141:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.793271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:21.0:1619677475.793277:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.793279:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:21.0:1619677475.793282:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:20.0:1619677475.793282:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:21.0:1619677475.793284:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.793286:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:21.0:1619677475.793287:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498150] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.793289:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.793290:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.793291:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:21.0:1619677475.793300:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:21.0:1619677475.793302:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:21.0:1619677475.793305:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.793305:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:21.0:1619677475.793306:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:21.0:1619677475.793308:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.793309:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:21.0:1619677475.793309:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:21.0:1619677475.793310:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:25.2:1619677475.793315:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.793322:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.793325:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.793326:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.793328:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.793330:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.793331:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.793805:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.793812:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.793815:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.793816:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.793818:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.793821:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.793823:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:8.0:1619677475.793824:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498149] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.793825:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.793826:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.793827:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:8.0:1619677475.793830:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.793832:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:8.0:1619677475.793834:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.793835:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:8.0:1619677475.793836:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:8.0:1619677475.793838:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:8.0:1619677475.793838:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.793839:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.793839:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677475.793841:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:9.2:1619677475.793844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:22.0:1619677475.793846:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.793849:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.793850:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.793852:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.793852:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000400:00000200:22.0:1619677475.793854:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.793854:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:22.0:1619677475.793855:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:2.0:1619677475.793855:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:18.0:1619677475.793856:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.793857:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498148] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.793858:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.793859:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.793860:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.793861:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677475.793861:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000400:00000200:22.0:1619677475.793864:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.793866:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.793869:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.793869:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.793870:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.793872:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.793872:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.793873:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.793873:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.793878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.793885:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.793887:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.793888:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.793890:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.793891:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.793893:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.793918:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.793926:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.793929:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.793930:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.793932:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.793935:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.793936:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.793937:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498147] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.793938:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.793939:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.793940:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.793944:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.793946:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:26.0:1619677475.793948:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.793949:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.793949:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.793951:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:26.0:1619677475.793952:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.793953:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677475.793954:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.793954:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.793957:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:4.2:1619677475.793960:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:12.0:1619677475.793964:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.793966:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.793967:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.793967:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000400:00000200:12.0:1619677475.793968:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:10.0:1619677475.793969:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:10.0:1619677475.793969:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:12.0:1619677475.793971:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:10.0:1619677475.793971:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:13.0:1619677475.793972:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.794151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.794158:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.794161:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.794162:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.794163:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.794165:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.794166:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.794168:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498146] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.794169:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.794170:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.794170:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.794173:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.794175:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.794177:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.794178:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.794179:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.794180:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.794181:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.794181:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.794182:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.794186:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.794194:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.794197:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:6.0:1619677475.794197:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.794198:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.794200:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.794203:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.2:1619677475.794450:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.794455:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.794458:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.794460:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.794461:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.794463:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.794464:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:21.0:1619677475.794466:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498145] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.794467:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.794468:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.794469:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:21.0:1619677475.794472:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.794474:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:21.0:1619677475.794476:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.794477:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:21.0:1619677475.794478:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.794479:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.794480:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.794481:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.794481:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.794486:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.794494:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.794496:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.794497:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.794499:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.794500:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.794501:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.794608:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.794616:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.794619:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.794621:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.794623:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.794624:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.794625:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.794626:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498144] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.794628:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.794629:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.794629:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.794633:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.794635:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.794638:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.794639:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.794640:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.794641:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.794642:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.794643:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.794644:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.794648:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.794657:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.794659:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.794660:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.794661:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.794663:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.794666:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.794770:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.794776:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.794779:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.794780:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.794782:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.794785:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.794787:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.794788:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498143] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.794789:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.794790:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.794791:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.794794:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.794795:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.794798:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.794799:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.794799:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.794801:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.794802:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.794802:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.794803:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.794807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.794807:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.794815:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677475.794815:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.794817:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.794818:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:22.0:1619677475.794818:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.794819:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.794821:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.794821:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.794822:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.794824:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.794824:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:30.0:1619677475.794825:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.794827:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498142] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.794828:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:25.2:1619677475.794828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000400:00000200:30.0:1619677475.794829:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.794830:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.794833:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.794835:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.794835:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.794838:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.794838:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.794838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.794839:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.794839:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677475.794839:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.794840:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.794841:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:22.0:1619677475.794841:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:30.0:1619677475.794842:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.794842:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.794843:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.794847:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.794853:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.794856:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.794857:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.794858:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.794860:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.794861:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.795501:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.795507:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.795510:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.795511:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.795512:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.795515:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.795516:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.795517:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498141] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.795519:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.795520:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.795521:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.795524:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.795526:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.795528:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.795529:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.795530:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.795532:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.795532:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.795533:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.795534:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.795535:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.795539:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.795540:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.795545:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.795547:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.795547:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:21.0:1619677475.795547:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.795548:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.795550:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.795550:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.795552:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.795553:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.795554:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.795559:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.795566:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.795568:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.795569:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.795570:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.795571:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.795572:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.795589:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.795597:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.795600:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.795602:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.795604:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.795604:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.795606:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.795607:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498140] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.795608:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.795609:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.795610:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.795614:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.795616:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.795618:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.795619:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.795620:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.795621:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.795622:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.795623:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.795623:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.795627:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.795634:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.795636:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.795637:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.795639:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.795640:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.795642:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.795647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.795654:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.795656:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.795657:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.795658:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.795658:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.795659:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.795828:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75ca00] (20)++
00000800:00000200:10.0:1619677475.795835:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677475.795838:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.14@o2ib3
00000800:00000200:26.0:1619677475.795839:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (22)--
00000400:00000200:10.0:1619677475.795841:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.14@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.795843:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.795845:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:10.0:1619677475.795846:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498139] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.795848:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.795849:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.795850:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:10.0:1619677475.795853:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.14@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.795854:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:10.0:1619677475.795856:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.795857:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:10.0:1619677475.795857:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:10.0:1619677475.795859:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:10.0:1619677475.795860:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75ca00] (21)++
00000800:00000200:10.0:1619677475.795860:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75ca00] (22)--
00000800:00000200:10.0:1619677475.795861:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75ca00] (21)--
00000800:00000200:25.2:1619677475.795865:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.795872:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.795874:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.795875:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.795876:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.795878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.795880:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.795927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.795935:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.795937:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.795938:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.795939:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.795941:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.795943:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.795944:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498138] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.795945:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.795946:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.795947:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.795950:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.795951:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.795954:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.795955:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.795955:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.795957:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.795957:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.795958:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.795959:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.795960:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.795963:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.795965:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.795970:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.795972:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.795973:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.795973:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.795974:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.795974:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:10.0:1619677475.795975:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.795976:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.795976:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.795979:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.796140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.796148:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.796151:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.796153:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.796154:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.796155:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.796156:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.796157:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498137] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.796158:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.796159:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.796160:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.796163:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.796165:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.796167:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.796168:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.796168:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.796170:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.796170:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.796172:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.796174:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.796174:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.796175:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.796180:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.796184:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.796185:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.796186:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.796188:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.796188:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.796189:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.796189:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.796190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.796190:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.796191:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:21.2:1619677475.796757:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.796766:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.796768:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.796770:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.796772:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.796773:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.796774:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.796775:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498136] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.796776:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.796777:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.796778:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.796781:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.796782:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.796785:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.796786:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.796786:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.796788:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.796788:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.796789:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.796789:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.796794:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.796803:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.796805:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.796806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.796807:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.796809:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.796811:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.796842:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.796850:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.796853:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.796855:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.796857:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.796858:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.796859:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.796860:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498135] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.796862:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.796863:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.796863:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.796866:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.796868:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.796869:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.796870:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.796871:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.796872:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.796873:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.796874:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.796874:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.796878:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.796884:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.796887:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.796888:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.796889:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.796891:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.796892:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.796956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.796961:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.796963:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.796964:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.796966:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.796968:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.796969:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.796971:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498134] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.796972:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.796973:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.796974:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.796977:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.796979:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.796981:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.796981:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.796982:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.796984:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.796984:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.796985:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.796985:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.796990:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.796997:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.797000:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.797000:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.797002:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.797003:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.797004:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.797415:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.797421:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.797424:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.797425:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.797427:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.797428:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.797430:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.797431:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498133] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.797432:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.797433:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.797435:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.797438:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.797440:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.797441:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.797442:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.797443:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.797444:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.797444:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.797446:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.797447:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:17.0:1619677475.797448:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.797451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.797454:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.797460:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.797461:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:17.0:1619677475.797461:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.797461:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.797461:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.797462:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.797463:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.797463:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.797464:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:1.0:1619677475.797465:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.797689:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.797697:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.797700:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.797701:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.797703:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.797706:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.797707:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:10.0:1619677475.797708:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498132] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.797710:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.797711:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.797712:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.797715:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.797717:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.797719:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.797720:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.797721:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.797722:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.797723:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.797723:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.797724:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:9.2:1619677475.797728:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.797735:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.797738:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:24.0:1619677475.797739:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.797739:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.797741:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.797743:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:4.2:1619677475.797782:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:24.0:1619677475.797788:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.797791:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:8.0:1619677475.797792:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:24.0:1619677475.797793:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.797796:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.797797:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:24.0:1619677475.797798:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498131] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.797800:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.797801:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.797802:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:24.0:1619677475.797805:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.797806:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.797808:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.797808:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:24.0:1619677475.797809:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.797810:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.797811:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:24.0:1619677475.797812:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677475.797813:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (23)--
00000800:00000200:24.0:1619677475.797813:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:9.2:1619677475.797816:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.797819:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.797824:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.797824:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:8.0:1619677475.797825:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.797826:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.797827:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (22)++
00000800:00000200:24.0:1619677475.797828:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919febc27600] (23)--
00000800:00000200:8.0:1619677475.797828:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:24.0:1619677475.797829:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677475.797829:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:24.0:1619677475.797830:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:4.2:1619677475.797854:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:12.0:1619677475.797862:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.797865:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000400:00000200:12.0:1619677475.797867:0:19943:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000400:00000200:12.0:1619677475.797869:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.797870:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000400:00000200:12.0:1619677475.797871:0:19943:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:12.0:1619677475.797872:0:19943:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498130] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:12.0:1619677475.797873:0:19943:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:12.0:1619677475.797874:0:19943:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:12.0:1619677475.797875:0:19943:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:12.0:1619677475.797878:0:19943:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.797880:0:19943:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:12.0:1619677475.797883:0:19943:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:12.0:1619677475.797884:0:19943:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:12.0:1619677475.797884:0:19943:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:12.0:1619677475.797886:0:19943:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:12.0:1619677475.797887:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:12.0:1619677475.797888:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.797888:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:9.2:1619677475.797892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677475.797892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.797899:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:2.0:1619677475.797901:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:8.0:1619677475.797902:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.797903:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:2.0:1619677475.797903:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000400:00000200:8.0:1619677475.797904:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:18.0:1619677475.797905:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:2.0:1619677475.797905:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.797906:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.797907:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:2.0:1619677475.797907:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000400:00000200:8.0:1619677475.797908:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498129] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.797910:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:2.0:1619677475.797910:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000400:00000200:8.0:1619677475.797911:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677475.797912:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:8.0:1619677475.797912:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.797915:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.797917:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:18.0:1619677475.797918:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:8.0:1619677475.797919:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.797920:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.797920:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:18.0:1619677475.797921:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:8.0:1619677475.797922:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.797922:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.0:1619677475.797923:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.797923:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000400:00000200:18.0:1619677475.797924:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000800:00000200:8.0:1619677475.797925:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.797925:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:18.0:1619677475.797927:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.797928:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:8.2:1619677475.797928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000400:00000200:18.0:1619677475.797929:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498128] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.797931:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677475.797931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000400:00000200:18.0:1619677475.797932:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.797933:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:18.0:1619677475.797936:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.797936:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.797937:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:18.0:1619677475.797938:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.797939:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:17.0:1619677475.797939:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:24.0:1619677475.797940:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:17.0:1619677475.797940:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.797940:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677475.797941:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.797941:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.797942:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:18.0:1619677475.797942:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (22)++
00000400:00000200:17.0:1619677475.797942:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:4.2:1619677475.797943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a06f393000] (20)++
00000800:00000200:18.0:1619677475.797944:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:18.0:1619677475.797944:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.797945:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91906ffc8a00] (22)++
00000800:00000200:17.0:1619677475.797945:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:18.0:1619677475.797947:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (23)--
00000800:00000200:18.0:1619677475.797947:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:8.2:1619677475.797950:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677475.797951:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a06f393000] (21)++
00000800:00000200:20.2:1619677475.797952:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:26.0:1619677475.797953:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.24@o2ib3
00000800:00000200:10.0:1619677475.797954:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (22)--
00000400:00000200:26.0:1619677475.797956:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.24@o2ib3 : PUT - routed
00000800:00000200:1.0:1619677475.797957:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:2.0:1619677475.797958:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)++
00000400:00000200:26.0:1619677475.797959:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.797959:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:26.0:1619677475.797960:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:2.0:1619677475.797960:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91906ffc8a00] (23)--
00000400:00000200:1.0:1619677475.797960:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.797961:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:2.0:1619677475.797961:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:26.0:1619677475.797962:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498127] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:2.0:1619677475.797962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000400:00000200:26.0:1619677475.797963:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.797963:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: ACK: OK
00000400:00000200:26.0:1619677475.797964:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.797966:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:1.0:1619677475.797966:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:26.0:1619677475.797969:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.24@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:8.2:1619677475.797970:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677475.797971:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:26.0:1619677475.797973:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.797974:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:26.0:1619677475.797975:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:26.0:1619677475.797976:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:26.0:1619677475.797977:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a06f393000] (21)++
00000800:00000200:17.0:1619677475.797977:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677475.797978:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a06f393000] (22)--
00000800:00000200:26.0:1619677475.797978:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a06f393000] (21)--
00000800:00000200:17.0:1619677475.797980:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:17.0:1619677475.797981:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.797981:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:25.2:1619677475.797982:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:17.0:1619677475.797982:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.797983:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:31.0:1619677475.797990:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.797993:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.797993:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.797995:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.797997:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.797998:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.798384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.798393:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.798396:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.798398:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.798400:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.798401:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.798402:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.798403:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498126] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.798404:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.798406:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.798406:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.798410:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.798412:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.798415:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.798416:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.798416:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.798418:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.798419:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.798420:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.798420:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.798425:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.798433:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.798436:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.798437:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.798438:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.798440:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.798443:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.798481:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.798488:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.798491:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.798492:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.798494:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.798497:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.798498:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.798499:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498125] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.798501:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.798502:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.798503:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.798506:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.798507:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.798509:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.798510:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.798511:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.798512:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.798513:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.798514:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.798514:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.798519:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.798525:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.798527:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000400:00000200:2.0:1619677475.798528:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:18.0:1619677475.798529:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.798530:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.798531:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:9.2:1619677475.798538:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.798545:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.798547:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.798548:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986000] (22)++
00000800:00000200:18.0:1619677475.798549:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.798549:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:2.0:1619677475.798549:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000800:00000200:20.2:1619677475.798613:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197ae708600] (20)++
00000800:00000200:21.0:1619677475.798619:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.798621:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.27@o2ib3
00000400:00000200:21.0:1619677475.798623:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.27@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.798624:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (22)--
00000400:00000200:21.0:1619677475.798626:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.798627:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:21.0:1619677475.798628:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498124] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.798631:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.798632:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.798633:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677475.798636:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.798638:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.798640:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.798641:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.798641:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.798643:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.798644:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197ae708600] (21)++
00000800:00000200:21.0:1619677475.798645:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197ae708600] (22)--
00000800:00000200:21.0:1619677475.798646:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197ae708600] (21)--
00000800:00000200:8.2:1619677475.798650:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.798657:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.798659:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.798660:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.798662:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.798663:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.798664:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:8.2:1619677475.798668:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.798676:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.798678:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.115@o2ib7
00000800:00000200:21.0:1619677475.798679:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.798680:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.798680:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:21.0:1619677475.798681:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:4.2:1619677475.798772:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:10.0:1619677475.798779:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:10.0:1619677475.798782:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:26.0:1619677475.798783:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:10.0:1619677475.798785:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.798787:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.798788:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:21.2:1619677475.798790:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:10.0:1619677475.798790:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498123] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.798791:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.798792:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.798793:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000800:00000200:22.0:1619677475.798795:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:10.0:1619677475.798797:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.798798:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:10.0:1619677475.798799:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:6.0:1619677475.798799:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.798801:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:10.0:1619677475.798802:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000400:00000200:22.0:1619677475.798803:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.798803:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:10.0:1619677475.798803:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677475.798805:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:10.0:1619677475.798805:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:10.0:1619677475.798805:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:22.0:1619677475.798806:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498122] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:10.0:1619677475.798806:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:22.0:1619677475.798807:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:10.0:1619677475.798807:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000400:00000200:22.0:1619677475.798808:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.798809:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:8.2:1619677475.798811:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:22.0:1619677475.798813:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.798814:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.798817:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.798818:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:20.0:1619677475.798818:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:22.0:1619677475.798819:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.798820:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.798821:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.798821:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.798821:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:22.0:1619677475.798822:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.798822:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.798823:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.798824:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.798826:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:25.2:1619677475.798827:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.798834:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.798836:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.798837:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.798839:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.798840:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.798841:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.798943:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:20.0:1619677475.798945:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.798948:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:20.0:1619677475.798950:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.798951:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:20.0:1619677475.798952:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.798953:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:20.0:1619677475.798955:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498121] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.798956:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.798957:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.798958:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:20.0:1619677475.798961:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.798963:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.798965:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.798966:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:20.0:1619677475.798967:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:20.0:1619677475.798968:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:20.0:1619677475.798969:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:20.0:1619677475.798970:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.798970:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.798975:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.798982:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.798985:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.798986:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.798986:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.798988:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.798990:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.799051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.799058:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.799061:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.799061:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.799064:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.799066:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.799067:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.799069:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498120] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.799070:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.799071:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.799072:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.799076:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.799077:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.799081:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.799081:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.799082:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.799084:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.799084:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.799085:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.799085:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.799090:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:6.0:1619677475.799097:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.799100:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.799101:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:6.0:1619677475.799101:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.799104:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.799106:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:25.2:1619677475.799110:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:22.0:1619677475.799117:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.799119:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.799120:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.799120:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.799122:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.799122:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.799686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.799695:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.799698:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.799700:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.799702:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.799703:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.799704:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:31.0:1619677475.799705:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498119] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.799707:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.799708:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.799709:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:31.0:1619677475.799712:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.799714:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:31.0:1619677475.799716:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.799717:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:31.0:1619677475.799718:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:31.0:1619677475.799719:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:31.0:1619677475.799720:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.799721:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.799723:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.799723:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:9.2:1619677475.799725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.2:1619677475.799729:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:10.0:1619677475.799733:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.799735:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:26.0:1619677475.799736:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000400:00000200:10.0:1619677475.799736:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.799737:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000400:00000200:10.0:1619677475.799738:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.799739:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.799740:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.799741:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:10.0:1619677475.799741:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:21.2:1619677475.799785:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:17.0:1619677475.799792:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.799794:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000800:00000200:1.0:1619677475.799795:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677475.799797:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:17.0:1619677475.799799:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.799801:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677475.799802:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498118] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.799804:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.799805:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.799806:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677475.799809:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : ACK try# 0
00000800:00000200:17.0:1619677475.799811:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.799813:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.799814:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677475.799814:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.799816:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.799817:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:17.0:1619677475.799817:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677475.799818:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:25.2:1619677475.799822:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.799830:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.799833:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.799834:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.799834:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.799836:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: ACK: OK
00000800:00000200:6.0:1619677475.799838:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.799840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.799847:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.799850:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.799851:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.799851:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.799853:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.799853:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.799927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:4.2:1619677475.799931:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:20.2:1619677475.799933:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907dbe3c00] (20)++
00000800:00000200:30.0:1619677475.799934:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.799937:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:8.0:1619677475.799939:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000400:00000200:30.0:1619677475.799940:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677475.799940:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:31.0:1619677475.799942:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.799942:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.799943:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000800:00000200:24.0:1619677475.799943:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.799943:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:2.0:1619677475.799943:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.54@o2ib3
00000800:00000200:18.0:1619677475.799944:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (22)--
00000400:00000200:30.0:1619677475.799946:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498117] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.799947:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.799947:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.799947:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.54@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.799948:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.799949:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:2.0:1619677475.799949:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.799950:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.799950:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.799952:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:2.0:1619677475.799953:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.799954:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498116] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.799955:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:2.0:1619677475.799955:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.799956:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.799956:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.799957:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.799957:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000800:00000200:30.0:1619677475.799958:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000400:00000200:8.0:1619677475.799959:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.799960:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.799961:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.799962:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000400:00000200:2.0:1619677475.799962:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.54@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.799963:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.799964:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000400:00000200:8.0:1619677475.799964:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.799965:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:2.0:1619677475.799965:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:30.0:1619677475.799966:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.0:1619677475.799967:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:2.0:1619677475.799968:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.799969:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:2.0:1619677475.799969:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.799970:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:25.2:1619677475.799971:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:8.0:1619677475.799971:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:2.0:1619677475.799971:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.799972:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:2.0:1619677475.799972:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907dbe3c00] (21)++
00000800:00000200:2.0:1619677475.799973:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907dbe3c00] (22)--
00000800:00000200:2.0:1619677475.799973:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907dbe3c00] (21)--
00000800:00000200:8.0:1619677475.799974:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.799974:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.799975:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:31.0:1619677475.799977:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:8.2:1619677475.799977:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:31.0:1619677475.799980:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000800:00000200:25.2:1619677475.799980:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677475.799981:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.799981:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:31.0:1619677475.799983:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.799984:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:1.0:1619677475.799984:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:31.0:1619677475.799985:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:1.0:1619677475.799987:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.799988:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677475.799988:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:1.0:1619677475.799988:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.799990:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:1.0:1619677475.799990:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.799992:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.799992:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.799993:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (20)++
00000800:00000200:1.0:1619677475.799993:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:6.0:1619677475.799994:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.799997:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.800000:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:13.0:1619677475.800001:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.800004:0:19942:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.11@o2ib3
00000800:00000200:30.0:1619677475.800006:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000400:00000200:13.0:1619677475.800006:0:19942:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.11@o2ib3 : PUT - routed
00000800:00000200:12.0:1619677475.800008:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:30.0:1619677475.800009:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.800010:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000400:00000200:13.0:1619677475.800010:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.800011:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.800011:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000400:00000200:13.0:1619677475.800011:0:19942:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000800:00000200:30.0:1619677475.800012:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000400:00000200:13.0:1619677475.800012:0:19942:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498114] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:13.0:1619677475.800014:0:19942:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:13.0:1619677475.800015:0:19942:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:13.0:1619677475.800016:0:19942:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:13.0:1619677475.800019:0:19942:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.11@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:13.0:1619677475.800021:0:19942:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:13.0:1619677475.800024:0:19942:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:13.0:1619677475.800024:0:19942:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:13.0:1619677475.800025:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:13.0:1619677475.800027:0:19942:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:13.0:1619677475.800027:0:19942:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a47f748e00] (21)++
00000800:00000200:13.0:1619677475.800029:0:19942:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a47f748e00] (22)++
00000800:00000200:13.0:1619677475.800030:0:19942:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a47f748e00] (23)--
00000800:00000200:13.0:1619677475.800031:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:9.2:1619677475.800033:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:4.2:1619677475.800036:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a47f748e00] (21)++
00000800:00000200:18.0:1619677475.800041:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.800044:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:12.0:1619677475.800044:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)++
00000800:00000200:2.0:1619677475.800044:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.800045:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.800046:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a47f748e00] (23)--
00000400:00000200:18.0:1619677475.800047:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.800047:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (22)--
00000800:00000200:12.0:1619677475.800048:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a47f748e00] (21)--
00000800:00000200:18.0:1619677475.800051:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:9.2:1619677475.800052:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (20)++
00000800:00000200:2.0:1619677475.800059:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (21)++
00000800:00000200:2.0:1619677475.800062:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.102@o2ib7
00000800:00000200:18.0:1619677475.800063:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.800063:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9193c2986800] (22)++
00000800:00000200:2.0:1619677475.800064:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9193c2986800] (22)--
00000800:00000200:2.0:1619677475.800065:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.800451:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:22.0:1619677475.800457:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.800460:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.800460:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.800462:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.800465:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.800466:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.800467:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498113] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.800468:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.800470:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.800471:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.800474:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.800476:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.800479:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.800479:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.800480:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.800482:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.800482:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.800483:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.800483:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.800489:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.800497:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.800499:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.800500:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.800502:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.800503:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.800504:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.800686:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.800694:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.800697:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.800699:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:30.0:1619677475.800701:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.800702:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.800703:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.800704:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498112] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.800705:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.800706:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.800707:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.800711:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.800712:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.800714:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.800715:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.800716:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.800717:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.800718:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.800719:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.800719:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.800723:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.800730:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.800732:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.800733:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.800734:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.800736:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.800737:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.801158:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.801165:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.801168:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.801170:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.801171:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.801172:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.801173:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.801174:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498111] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.801175:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.801176:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.801177:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.801180:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.801181:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.801184:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.801185:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.801185:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.801187:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.801188:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.801188:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.801189:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.801193:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.801201:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.801204:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.801204:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.801205:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.801207:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.801210:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.801266:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.801273:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.801276:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.801277:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.801279:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.801282:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.801283:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.801284:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498110] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.801285:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.801286:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.801287:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.801290:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.801291:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.801300:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.801301:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.801301:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.801303:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.801304:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.801304:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.801305:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.801309:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.801318:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.801320:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.801321:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.801322:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.801324:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.801326:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.801508:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.801516:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.801519:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.801522:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.801523:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.801525:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.801526:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.801527:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498109] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.801529:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.801530:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.801531:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.801533:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.801535:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.801537:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.801538:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.801538:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.801540:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.801540:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.801542:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:30.0:1619677475.801543:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.801544:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.801546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:21.2:1619677475.801549:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.801553:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:24.0:1619677475.801555:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:24.0:1619677475.801556:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.801557:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677475.801557:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.801558:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.801559:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.801560:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.801561:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:24.0:1619677475.801561:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:21.2:1619677475.801853:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.801858:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.801861:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.801862:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.801864:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.801866:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.801867:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.801869:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498108] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.801870:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.801871:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.801872:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.801875:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.801877:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.801879:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.801880:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.801881:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.801882:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.801883:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.801885:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:6.0:1619677475.801885:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.801886:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.801889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.801891:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.801896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.801896:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.801897:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.801898:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:20.0:1619677475.801898:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:22.0:1619677475.801899:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:20.0:1619677475.801899:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.801901:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.801902:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.801903:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.801907:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.801915:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.801918:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.801919:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.801919:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.801920:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.801921:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:5.2:1619677475.802267:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:17.0:1619677475.802273:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.802276:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:1.0:1619677475.802277:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.802278:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.802281:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.802282:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:17.0:1619677475.802283:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498107] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.802285:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.802285:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.802287:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:17.0:1619677475.802290:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.802292:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:17.0:1619677475.802293:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.802294:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.802300:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.802302:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.802302:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.802303:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:17.0:1619677475.802304:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.802308:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.802316:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.802319:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:1.0:1619677475.802319:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.802320:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.802321:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.802322:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.802411:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919febc27600] (20)++
00000800:00000200:8.0:1619677475.802419:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.802421:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.19@o2ib3
00000800:00000200:24.0:1619677475.802422:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (22)--
00000400:00000200:8.0:1619677475.802424:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.19@o2ib3 : ACK - routed
00000400:00000200:8.0:1619677475.802426:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.802427:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:8.0:1619677475.802429:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498106] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.802430:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.802431:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.802432:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:8.0:1619677475.802435:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.19@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:8.0:1619677475.802437:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.802438:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.802439:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.802440:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.802441:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.802441:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919febc27600] (21)++
00000800:00000200:8.0:1619677475.802442:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919febc27600] (22)--
00000800:00000200:8.0:1619677475.802442:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919febc27600] (21)--
00000800:00000200:4.2:1619677475.802457:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:9.2:1619677475.802458:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:24.0:1619677475.802464:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:10.0:1619677475.802464:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:24.0:1619677475.802466:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.802466:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.802467:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.802468:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:8.0:1619677475.802468:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:24.0:1619677475.802469:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000400:00000200:10.0:1619677475.802470:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:24.0:1619677475.802471:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000400:00000200:10.0:1619677475.802473:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.802474:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.802475:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498105] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.802476:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.802477:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.802478:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.802481:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.802483:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.802486:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.802486:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.802487:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.802489:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.802489:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.802490:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.802490:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.802495:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.802503:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.802505:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.802506:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.802507:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.802509:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.802512:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.802513:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.802520:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.802523:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:22.0:1619677475.802524:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.802524:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.802525:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.802526:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.802755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:18.0:1619677475.802762:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.802765:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:2.0:1619677475.802766:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:18.0:1619677475.802768:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:18.0:1619677475.802770:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.802772:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:18.0:1619677475.802773:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498104] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.802774:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.802776:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.802777:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:18.0:1619677475.802779:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : ACK try# 0
00000800:00000200:18.0:1619677475.802781:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:18.0:1619677475.802784:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.802785:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:18.0:1619677475.802785:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:18.0:1619677475.802787:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:18.0:1619677475.802788:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.802789:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.802789:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.802793:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.802801:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.802803:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.802804:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.802806:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: ACK: OK
00000800:00000200:13.0:1619677475.802808:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.802809:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:4.2:1619677475.802852:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.802860:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.802862:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.802863:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.802865:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.802867:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.802869:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:26.0:1619677475.802870:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498103] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.802871:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.802872:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.802873:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000400:00000200:26.0:1619677475.802875:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.802877:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.802879:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.802880:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:26.0:1619677475.802880:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:26.0:1619677475.802881:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.802882:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.802882:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.802883:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677475.802887:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.2:1619677475.802889:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:10.0:1619677475.802893:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:10.0:1619677475.802895:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.802895:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.802896:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:31.0:1619677475.802897:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:10.0:1619677475.802897:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.802899:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:10.0:1619677475.802899:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000400:00000200:31.0:1619677475.802901:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.802904:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.802904:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.802906:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.802907:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498102] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.802908:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.802909:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:9.2:1619677475.802909:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (20)++
00000400:00000200:31.0:1619677475.802910:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.802913:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.802915:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.802915:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:31.0:1619677475.802918:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.802918:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.106@o2ib7
00000800:00000200:26.0:1619677475.802918:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:31.0:1619677475.802919:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.802919:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.802919:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677475.802919:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:26.0:1619677475.802920:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:31.0:1619677475.802921:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.802922:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.802922:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.802923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.802927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.802930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc1200] (20)++
00000800:00000200:6.0:1619677475.802935:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:17.0:1619677475.802936:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:22.0:1619677475.802938:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.802938:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:17.0:1619677475.802939:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.56@o2ib3
00000400:00000200:6.0:1619677475.802939:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.802940:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (22)--
00000400:00000200:17.0:1619677475.802942:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.56@o2ib3 : ACK - routed
00000400:00000200:6.0:1619677475.802942:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:17.0:1619677475.802945:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.802945:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:17.0:1619677475.802946:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:17.0:1619677475.802948:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498101] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.802949:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.802950:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.802951:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:17.0:1619677475.802955:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.56@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:4.2:1619677475.802956:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677475.802957:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:17.0:1619677475.802960:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.802960:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:17.0:1619677475.802961:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:17.0:1619677475.802963:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:17.0:1619677475.802963:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc1200] (21)++
00000800:00000200:10.0:1619677475.802963:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677475.802964:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc1200] (22)--
00000800:00000200:17.0:1619677475.802965:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc1200] (21)--
00000800:00000200:10.0:1619677475.802965:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.802966:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.802968:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677475.802969:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000400:00000200:10.0:1619677475.802970:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.802972:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:10.0:1619677475.802973:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498100] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.802974:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.802975:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.802975:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000400:00000200:10.0:1619677475.802976:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000800:00000200:8.0:1619677475.802978:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.802979:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.802979:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.802980:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : PUT try# 0
00000400:00000200:8.0:1619677475.802981:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:10.0:1619677475.802982:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:10.0:1619677475.802984:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.802984:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:10.0:1619677475.802985:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:10.0:1619677475.802985:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:10.0:1619677475.802987:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:10.0:1619677475.802988:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.802989:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.802990:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.802991:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677475.802992:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:4.2:1619677475.802996:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:24.0:1619677475.802998:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:24.0:1619677475.803001:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000400:00000200:24.0:1619677475.803001:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.803002:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:26.0:1619677475.803003:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000400:00000200:24.0:1619677475.803003:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.803005:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:24.0:1619677475.803005:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:26.0:1619677475.803006:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.803006:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677475.803014:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (20)++
00000800:00000200:8.0:1619677475.803020:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.803022:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.101@o2ib7
00000800:00000200:24.0:1619677475.803023:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.803023:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.803024:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:8.0:1619677475.803025:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.803094:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:26.0:1619677475.803101:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.803104:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.803105:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.803106:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.803108:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.803109:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:26.0:1619677475.803110:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498099] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.803111:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:4.2:1619677475.803111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000400:00000200:26.0:1619677475.803112:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.803113:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:26.0:1619677475.803116:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.803117:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:24.0:1619677475.803118:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:26.0:1619677475.803119:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.803120:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.803120:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:24.0:1619677475.803121:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:4.2:1619677475.803121:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.803122:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.0:1619677475.803122:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:26.0:1619677475.803123:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000400:00000200:24.0:1619677475.803123:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.803124:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.803125:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.803126:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000400:00000200:24.0:1619677475.803126:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.803127:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000800:00000200:26.0:1619677475.803128:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:25.2:1619677475.803128:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:10.0:1619677475.803128:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:24.0:1619677475.803129:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:10, s:-96498098] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.803130:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.803131:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.803131:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.803132:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.803132:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677475.803134:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000800:00000200:13.0:1619677475.803134:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000400:00000200:26.0:1619677475.803135:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96498097] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.803135:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.803136:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:24.0:1619677475.803137:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:13.0:1619677475.803137:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:13.0:1619677475.803137:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.803138:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.803139:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:13.0:1619677475.803139:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.803140:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:24.0:1619677475.803141:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.803141:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000400:00000200:26.0:1619677475.803142:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.803142:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677475.803144:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:24.0:1619677475.803144:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.803144:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.803145:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.803145:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:13.0:1619677475.803145:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:26.0:1619677475.803147:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.803147:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:26.0:1619677475.803148:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:26.0:1619677475.803149:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:26.0:1619677475.803150:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.803150:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:8.2:1619677475.803150:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677475.803151:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.803151:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:4.2:1619677475.803152:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:25.2:1619677475.803154:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.803158:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:1.0:1619677475.803158:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:10.0:1619677475.803159:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:12.0:1619677475.803160:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.111@o2ib7
00000800:00000200:1.0:1619677475.803160:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.803161:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:13.0:1619677475.803161:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.803161:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:26.0:1619677475.803162:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:13.0:1619677475.803162:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:12.0:1619677475.803162:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677475.803162:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000400:00000200:1.0:1619677475.803162:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:13.0:1619677475.803163:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:12.0:1619677475.803163:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:12.0:1619677475.803164:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000400:00000200:13.0:1619677475.803165:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.803165:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000400:00000200:10.0:1619677475.803165:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.803165:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.803167:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:13.0:1619677475.803168:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000400:00000200:10.0:1619677475.803169:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:1.0:1619677475.803169:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:10.0:1619677475.803170:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498096] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.803171:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.803172:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.803173:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.803176:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.803178:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.803180:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.803180:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.803181:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:10.0:1619677475.803182:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.803183:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.803183:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.803184:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:9.2:1619677475.803188:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:8.0:1619677475.803194:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:8.0:1619677475.803196:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000800:00000200:4.2:1619677475.803196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:24.0:1619677475.803197:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.803197:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.803199:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:8.0:1619677475.803201:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:26.0:1619677475.803202:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.803205:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.803206:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:26.0:1619677475.803207:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.803209:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.803211:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:26.0:1619677475.803212:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498095] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.803213:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.803214:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.803214:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:26.0:1619677475.803217:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.803219:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:26.0:1619677475.803221:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.803222:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:26.0:1619677475.803223:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:26.0:1619677475.803224:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:26.0:1619677475.803225:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.803226:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.803227:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.803227:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.803230:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.803233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:22.0:1619677475.803238:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.803240:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:10.0:1619677475.803240:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:6.0:1619677475.803241:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.803242:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.803242:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:26.0:1619677475.803243:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.803243:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:22.0:1619677475.803244:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:26.0:1619677475.803246:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:10.0:1619677475.803246:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677475.803247:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000400:00000200:26.0:1619677475.803248:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.803251:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.803253:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:26.0:1619677475.803254:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498094] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.803256:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.803257:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.803257:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:26.0:1619677475.803260:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.803261:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:26.0:1619677475.803264:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.803265:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:26.0:1619677475.803265:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:26.0:1619677475.803267:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:26.0:1619677475.803268:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:26.0:1619677475.803268:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:26.0:1619677475.803269:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:8.2:1619677475.803273:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.803274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:17.0:1619677475.803281:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:10.0:1619677475.803281:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:17.0:1619677475.803284:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:10.0:1619677475.803284:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.803285:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:17.0:1619677475.803285:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.803285:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:10.0:1619677475.803286:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.114@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.803287:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.803289:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:17.0:1619677475.803290:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000400:00000200:10.0:1619677475.803292:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.114@o2ib7 local destination
00000400:00000200:10.0:1619677475.803293:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498093] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.803300:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.803301:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.803301:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.114@o2ib7
00000400:00000200:10.0:1619677475.803304:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.114@o2ib7(10.0.10.114@o2ib7:10.0.10.114@o2ib7) : PUT try# 0
00000800:00000200:4.2:1619677475.803305:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.803306:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.0.10.114@o2ib7
00000800:00000200:10.0:1619677475.803307:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af9cec80] -> 10.0.10.114@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.803308:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (20)++
00000800:00000200:10.0:1619677475.803309:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d200] (21)++
00000800:00000200:10.0:1619677475.803310:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:10.0:1619677475.803311:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.803312:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.803313:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.803314:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.803316:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.803317:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:9.2:1619677475.803317:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d200] (21)++
00000400:00000200:10.0:1619677475.803318:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:26.0:1619677475.803319:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)++
00000800:00000200:26.0:1619677475.803320:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d200] (23)--
00000400:00000200:26.0:1619677475.803320:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.803321:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.803322:0:19941:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.114@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.803323:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.803324:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498092] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.803325:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.803326:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.803327:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.803328:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (22)--
00000800:00000200:26.0:1619677475.803329:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d200] (21)--
00000400:00000200:10.0:1619677475.803331:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.803332:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.803335:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.803335:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.803336:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.803337:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.803338:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.803338:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.803339:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.803344:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.803352:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.803354:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.803355:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.803356:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.803358:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.803360:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.803640:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.803647:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.803651:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.803651:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.803653:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : ACK - routed
00000400:00000200:26.0:1619677475.803656:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.803657:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:26.0:1619677475.803658:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498091] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.803660:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.803661:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.803661:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:26.0:1619677475.803664:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : ACK try# 0
00000800:00000200:26.0:1619677475.803666:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:26.0:1619677475.803668:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.803669:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:26.0:1619677475.803670:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:26.0:1619677475.803671:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:26.0:1619677475.803672:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.803672:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.803673:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:8.2:1619677475.803678:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.803687:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:21.0:1619677475.803689:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:21.0:1619677475.803690:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.2:1619677475.803692:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:20.0:1619677475.803693:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677475.803696:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: ACK: OK
00000800:00000200:22.0:1619677475.803697:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.0:1619677475.803698:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:22.0:1619677475.803700:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.803701:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.803703:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:22.0:1619677475.803705:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.803707:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.803708:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498090] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.803709:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.803710:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:22.0:1619677475.803711:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:22.0:1619677475.803714:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.803716:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.803718:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.803719:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.803720:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.803721:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.803722:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.803722:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.803723:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.803727:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.803734:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.803736:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.803737:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.803739:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.803740:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.803741:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:20.2:1619677475.803926:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000800:00000200:21.0:1619677475.803931:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.803934:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:21.0:1619677475.803936:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.803936:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:21.0:1619677475.803938:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.803940:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.803941:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498089] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.803943:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.803944:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.803945:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:21.0:1619677475.803948:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.803950:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:21.0:1619677475.803953:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.803953:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.803954:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.803956:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.803957:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.803958:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:21.0:1619677475.803958:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.803962:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.803970:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.803972:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.803973:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.803973:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.803976:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.803978:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.803984:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.803992:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.803994:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.803995:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.803996:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.803997:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.803997:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.804232:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.804240:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.804243:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.804246:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.804248:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.804249:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.804250:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.804251:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498088] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.804252:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.804253:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.804254:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.804257:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.804259:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.804261:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.804261:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.804262:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.804263:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.804264:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.804265:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.804265:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.804269:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.804276:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.804278:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.804279:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.804280:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.804282:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.804283:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.804402:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.804410:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.804412:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.804413:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.804415:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.804418:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.804419:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.804420:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498087] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.804421:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.804422:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.804423:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.804426:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.804428:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.804431:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.804431:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.804432:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.804433:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.804434:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.804435:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07179ba00] (22)++
00000800:00000200:10.0:1619677475.804437:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (23)--
00000800:00000200:10.0:1619677475.804437:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:25.2:1619677475.804440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.804442:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (21)++
00000800:00000200:6.0:1619677475.804448:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:26.0:1619677475.804449:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)++
00000800:00000200:26.0:1619677475.804451:0:19941:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07179ba00] (23)--
00000800:00000200:6.0:1619677475.804451:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:26.0:1619677475.804452:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000800:00000200:22.0:1619677475.804452:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000400:00000200:6.0:1619677475.804453:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.804454:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000400:00000200:6.0:1619677475.804455:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.804457:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.804850:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.804859:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.804861:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.804863:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.804865:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.804866:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.804867:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.804868:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498086] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.804869:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.804870:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.804871:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.804874:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.804876:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.804879:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.804880:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.804880:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.804882:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.804883:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.804884:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.804884:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:4.2:1619677475.804885:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:25.2:1619677475.804888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.804889:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.804892:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.804894:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.804895:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:22.0:1619677475.804896:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:20.0:1619677475.804897:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:4.2:1619677475.804897:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000800:00000200:22.0:1619677475.804899:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:20.0:1619677475.804899:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:6.0:1619677475.804899:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.804900:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.804900:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498085] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.804902:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:20.0:1619677475.804902:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.804903:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:2.0:1619677475.804903:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000400:00000200:20.0:1619677475.804904:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000800:00000200:22.0:1619677475.804905:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:2.0:1619677475.804906:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:18.0:1619677475.804907:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:20.0:1619677475.804909:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : ACK try# 0
00000400:00000200:2.0:1619677475.804910:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.106@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.804911:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.804912:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000400:00000200:2.0:1619677475.804912:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.804913:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.804914:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:2.0:1619677475.804914:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.106@o2ib7 local destination
00000400:00000200:2.0:1619677475.804915:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498084] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:20.0:1619677475.804916:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.804916:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000400:00000200:2.0:1619677475.804917:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:20.0:1619677475.804918:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8400] (22)++
00000400:00000200:2.0:1619677475.804918:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.0:1619677475.804919:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (23)--
00000400:00000200:2.0:1619677475.804919:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.106@o2ib7
00000800:00000200:20.0:1619677475.804920:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:8.2:1619677475.804922:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000400:00000200:2.0:1619677475.804922:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.106@o2ib7(10.0.10.106@o2ib7:10.0.10.106@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.804924:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.0.10.106@o2ib7
00000800:00000200:4.2:1619677475.804925:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (21)++
00000800:00000200:2.0:1619677475.804927:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a3af33c600] -> 10.0.10.106@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.804928:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (20)++
00000800:00000200:2.0:1619677475.804929:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:21.0:1619677475.804930:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)++
00000800:00000200:20.0:1619677475.804930:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:2.0:1619677475.804930:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a3aeb2d800] (22)--
00000800:00000200:21.0:1619677475.804931:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8400] (23)--
00000800:00000200:20.0:1619677475.804931:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.804931:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.804931:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:21.0:1619677475.804932:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000800:00000200:2.0:1619677475.804932:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:2.0:1619677475.804932:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000400:00000200:20.0:1619677475.804933:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.804934:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.804935:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:20.0:1619677475.804935:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:9.2:1619677475.804936:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a3aeb2d800] (21)++
00000800:00000200:10.0:1619677475.804943:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (22)++
00000800:00000200:26.0:1619677475.804946:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (23)--
00000800:00000200:10.0:1619677475.804946:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a3aeb2d800] (23)--
00000400:00000200:10.0:1619677475.804947:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.804949:0:19940:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.106@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.804952:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a3aeb2d800] (21)--
00000800:00000200:4.2:1619677475.805235:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000800:00000200:18.0:1619677475.805240:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677475.805243:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:2.0:1619677475.805244:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:18.0:1619677475.805246:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.805249:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.805250:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:18.0:1619677475.805251:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498083] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.805253:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.805254:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.805255:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:18.0:1619677475.805258:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.805260:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:18.0:1619677475.805262:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.805263:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:18.0:1619677475.805263:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:18.0:1619677475.805265:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:18.0:1619677475.805265:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677475.805267:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907a3e8400] (22)++
00000800:00000200:18.0:1619677475.805268:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (23)--
00000800:00000200:18.0:1619677475.805268:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000800:00000200:25.2:1619677475.805271:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:4.2:1619677475.805274:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (21)++
00000800:00000200:6.0:1619677475.805279:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:2.0:1619677475.805279:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)++
00000800:00000200:6.0:1619677475.805281:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:2.0:1619677475.805281:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907a3e8400] (23)--
00000800:00000200:22.0:1619677475.805282:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (23)--
00000800:00000200:2.0:1619677475.805282:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:6.0:1619677475.805283:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.805284:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000400:00000200:6.0:1619677475.805285:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.805287:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.805324:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.805332:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.805335:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.805336:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.805338:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.805341:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.805343:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.2:1619677475.805343:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000400:00000200:24.0:1619677475.805344:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498082] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.805345:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.805347:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.805347:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.805347:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:22.0:1619677475.805349:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000400:00000200:24.0:1619677475.805350:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.805351:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:24.0:1619677475.805352:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.805352:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:22.0:1619677475.805354:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.805355:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:24.0:1619677475.805355:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.805356:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000400:00000200:22.0:1619677475.805356:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.805357:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:24.0:1619677475.805357:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.805358:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.805358:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:24.0:1619677475.805359:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000400:00000200:22.0:1619677475.805359:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498081] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.805360:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.805360:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000400:00000200:30.0:1619677475.805361:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.805362:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:22.0:1619677475.805362:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.805363:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.805364:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.805364:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:25.2:1619677475.805365:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:30.0:1619677475.805366:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.805367:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:510, d:12, s:-96498080] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.805368:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000400:00000200:30.0:1619677475.805369:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.805369:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.805371:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.805371:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:22.0:1619677475.805374:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.805374:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:30.0:1619677475.805375:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:22.0:1619677475.805375:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.805375:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.805376:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:6.0:1619677475.805376:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.805377:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.805378:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.805378:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:6.0:1619677475.805378:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.805379:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.805380:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677475.805380:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.805380:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:30.0:1619677475.805381:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:22.0:1619677475.805381:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.805382:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:6.0:1619677475.805382:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (23)--
00000800:00000200:30.0:1619677475.805383:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:22.0:1619677475.805383:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.805383:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.805384:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.805384:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:22.0:1619677475.805384:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.805385:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:22.0:1619677475.805386:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.805386:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.805388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.0:1619677475.805391:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.805393:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.805394:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.805395:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000400:00000200:21.0:1619677475.805396:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.805397:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:31.0:1619677475.805398:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.805398:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.805398:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000400:00000200:31.0:1619677475.805400:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.805401:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.805402:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.806168:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.806176:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.806179:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.806179:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.806181:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.806183:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.806185:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.806186:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498079] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.806188:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.806188:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.806190:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.806193:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.806194:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.806197:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.806198:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.806199:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.806200:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.806201:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.806202:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.806203:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.806203:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.806207:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.806209:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:6.0:1619677475.806214:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.806216:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:6.0:1619677475.806217:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.806218:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.806218:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000400:00000200:6.0:1619677475.806218:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.806219:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.806220:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:6.0:1619677475.806220:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.806223:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.806440:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.806448:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.806451:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.806452:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.115@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.806455:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.806455:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.806456:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.115@o2ib7 local destination
00000400:00000200:30.0:1619677475.806457:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498078] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.806459:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.806460:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.806461:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.806464:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.806465:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000800:00000200:30.0:1619677475.806469:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.806469:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (20)++
00000800:00000200:30.0:1619677475.806470:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (21)++
00000800:00000200:30.0:1619677475.806472:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (22)--
00000800:00000200:30.0:1619677475.806472:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.806473:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.806473:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:8.2:1619677475.806479:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000800:00000200:20.0:1619677475.806487:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:20.0:1619677475.806489:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000400:00000200:20.0:1619677475.806490:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.806492:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.806493:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.806495:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:21.2:1619677475.806543:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.806551:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.806554:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.806556:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.806557:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.806559:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.806560:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.806562:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498077] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.806563:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.806564:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.806565:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.806568:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.806569:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.806572:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.806573:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.806574:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.806575:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.806576:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.806577:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000800:00000200:31.0:1619677475.806579:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.806579:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:25.2:1619677475.806582:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.806585:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:22.0:1619677475.806591:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:30.0:1619677475.806593:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.806593:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.806594:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.806595:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.806596:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.806596:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.806597:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000400:00000200:22.0:1619677475.806597:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.806600:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.806619:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.806625:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.806628:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.806629:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.806631:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.806633:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.806634:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.806635:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498076] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.806637:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.806638:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.806639:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.806642:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.806644:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.806646:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.806646:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.806647:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.806649:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.806649:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.806650:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.806650:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.806655:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.806662:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.806664:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.806665:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.806666:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.806667:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.806668:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.806717:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:4.2:1619677475.806718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000800:00000200:6.0:1619677475.806722:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:18.0:1619677475.806724:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:6.0:1619677475.806725:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.806726:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:18.0:1619677475.806727:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000400:00000200:6.0:1619677475.806728:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:2.0:1619677475.806728:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:18.0:1619677475.806731:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.806731:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.806732:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.806732:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677475.806734:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.806734:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.806735:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.806736:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498075] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.806737:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.806737:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.806738:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.806739:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.806740:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.806742:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.806744:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.806746:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.806747:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.806748:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677475.806749:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.806749:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.806750:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677475.806751:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.806751:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:18.0:1619677475.806752:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.806752:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.806752:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:18.0:1619677475.806753:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677475.806754:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (23)--
00000800:00000200:18.0:1619677475.806755:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:25.2:1619677475.806756:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677475.806756:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:18.0:1619677475.806757:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:21.0:1619677475.806764:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (23)++
00000800:00000200:21.0:1619677475.806766:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (24)--
00000400:00000200:21.0:1619677475.806767:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.806769:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.806769:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (23)++
00000800:00000200:20.0:1619677475.806770:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (24)--
00000800:00000200:21.0:1619677475.806771:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.806771:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.806773:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.806773:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.806775:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.806778:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.806785:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.806788:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.806789:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.806789:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.806790:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.806791:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.806848:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (20)++
00000800:00000200:1.0:1619677475.806853:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.806856:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.13@o2ib3
00000800:00000200:17.0:1619677475.806857:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:1.0:1619677475.806859:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.13@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.806861:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.806862:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:1.0:1619677475.806864:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498073] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.806865:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.806866:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.806867:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:1.0:1619677475.806870:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.806872:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.806874:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.806875:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.806875:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.806877:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:1.0:1619677475.806877:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:1.0:1619677475.806879:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:1.0:1619677475.806880:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677475.806880:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000800:00000200:8.2:1619677475.806883:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:21.2:1619677475.806886:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918cfd7fcc00] (21)++
00000800:00000200:17.0:1619677475.806891:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.806892:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)++
00000800:00000200:17.0:1619677475.806893:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.806893:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.806893:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918cfd7fcc00] (23)--
00000800:00000200:1.0:1619677475.806894:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (22)--
00000400:00000200:17.0:1619677475.806895:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.806895:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000800:00000200:17.0:1619677475.806896:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:1.0:1619677475.806896:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918cfd7fcc00] (21)--
00000800:00000200:8.2:1619677475.806904:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (20)++
00000800:00000200:1.0:1619677475.806911:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.806913:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.113@o2ib7
00000800:00000200:1.0:1619677475.806914:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.806915:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.806915:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d0600] (23)--
00000800:00000200:1.0:1619677475.806916:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.807296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.807304:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.807307:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.807308:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.807310:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.807313:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.807314:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.807315:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498072] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.807317:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.807318:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.807319:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.807322:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.807323:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.807326:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.807327:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.807327:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.807329:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.807329:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.807330:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.807331:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.807336:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.807344:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.807346:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.807347:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.807348:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.807350:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.807353:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.807644:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:17.0:1619677475.807650:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.807653:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.807654:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000400:00000200:17.0:1619677475.807656:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.807658:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.807659:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:17.0:1619677475.807660:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498071] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.807662:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.807663:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.807664:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:17.0:1619677475.807667:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.807669:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:17.0:1619677475.807672:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.807672:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.807673:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:17.0:1619677475.807675:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.807675:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.807676:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:17.0:1619677475.807677:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:17.0:1619677475.807677:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:8.2:1619677475.807681:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.2:1619677475.807683:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:1.0:1619677475.807688:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)++
00000800:00000200:21.0:1619677475.807690:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:1.0:1619677475.807690:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce4be9c00] (23)--
00000800:00000200:1.0:1619677475.807690:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:1.0:1619677475.807691:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:21.0:1619677475.807692:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.807693:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.807695:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.807696:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.807697:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:8.2:1619677475.807699:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.807707:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.807709:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.807710:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f0c00] (22)++
00000800:00000200:20.0:1619677475.807711:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f0c00] (23)--
00000800:00000200:20.0:1619677475.807712:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.807713:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:21.2:1619677475.807759:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (20)++
00000800:00000200:6.0:1619677475.807765:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.807767:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.7@o2ib3
00000800:00000200:22.0:1619677475.807768:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000400:00000200:6.0:1619677475.807770:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.7@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.807772:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.807774:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:6.0:1619677475.807775:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498070] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.807776:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.807777:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.807778:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:6.0:1619677475.807782:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.807784:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:6.0:1619677475.807786:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.807787:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:6.0:1619677475.807787:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677475.807789:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:6.0:1619677475.807790:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc4e000] (21)++
00000800:00000200:6.0:1619677475.807791:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91987fc4e000] (22)++
00000800:00000200:6.0:1619677475.807792:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc4e000] (23)--
00000800:00000200:6.0:1619677475.807793:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:8.2:1619677475.807795:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.2:1619677475.807798:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc4e000] (21)++
00000800:00000200:22.0:1619677475.807803:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)++
00000800:00000200:17.0:1619677475.807803:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:22.0:1619677475.807805:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91987fc4e000] (23)--
00000800:00000200:22.0:1619677475.807806:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (22)--
00000800:00000200:17.0:1619677475.807806:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:22.0:1619677475.807807:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc4e000] (21)--
00000400:00000200:17.0:1619677475.807807:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.807807:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.807809:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.807812:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.807928:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000800:00000200:2.0:1619677475.807934:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677475.807937:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:18.0:1619677475.807938:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:2.0:1619677475.807940:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.807942:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.807944:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677475.807945:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498069] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.807946:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.807947:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.807948:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677475.807952:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.807954:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677475.807956:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.807957:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677475.807958:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677475.807960:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677475.807960:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677475.807961:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:2.0:1619677475.807962:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:25.2:1619677475.807966:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.807973:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.807975:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.807976:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.807978:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.807979:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.807980:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.808488:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.808496:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.808499:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.808501:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.808503:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.808504:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.808506:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000400:00000200:30.0:1619677475.808507:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498068] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.808508:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.808509:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.808510:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:30.0:1619677475.808514:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.808515:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:30.0:1619677475.808518:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.808519:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:30.0:1619677475.808519:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:30.0:1619677475.808521:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:30.0:1619677475.808522:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.808523:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:21.2:1619677475.808523:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:30.0:1619677475.808524:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:30.0:1619677475.808525:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:9.2:1619677475.808527:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677475.808530:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:21.2:1619677475.808531:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:6.0:1619677475.808532:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.808533:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:8.0:1619677475.808535:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:6.0:1619677475.808535:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.808538:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:8.0:1619677475.808538:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:6.0:1619677475.808538:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.808539:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:8.0:1619677475.808539:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.808539:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:31.0:1619677475.808541:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000400:00000200:6.0:1619677475.808541:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498067] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:31.0:1619677475.808542:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:8.0:1619677475.808542:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.808543:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000400:00000200:6.0:1619677475.808543:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.808544:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:8.0:1619677475.808545:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000400:00000200:6.0:1619677475.808545:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:9.2:1619677475.808546:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (20)++
00000400:00000200:6.0:1619677475.808548:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.808549:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:24.0:1619677475.808552:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677475.808552:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.808553:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.808553:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:24.0:1619677475.808555:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.109@o2ib7
00000800:00000200:8.0:1619677475.808555:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677475.808555:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.808555:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:24.0:1619677475.808556:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)++
00000800:00000200:6.0:1619677475.808556:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000800:00000200:24.0:1619677475.808557:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:6.0:1619677475.808557:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:24.0:1619677475.808558:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:6.0:1619677475.808558:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:25.2:1619677475.808561:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.808562:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.808568:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:22.0:1619677475.808570:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:21.0:1619677475.808570:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:22.0:1619677475.808571:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:22.0:1619677475.808572:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.808572:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.808573:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.808574:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.808576:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.808577:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.808587:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.808595:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.808597:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.808599:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.808600:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.808603:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.808605:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:8.0:1619677475.808606:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498066] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.808608:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.808609:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.808610:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:8.0:1619677475.808614:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.808616:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:8.0:1619677475.808619:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.808620:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:8.0:1619677475.808621:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:8.0:1619677475.808623:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:8.0:1619677475.808623:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.808625:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919feda13a00] (22)++
00000800:00000200:8.0:1619677475.808626:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (23)--
00000800:00000200:8.0:1619677475.808626:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.2:1619677475.808629:0:19939:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:4.2:1619677475.808632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (21)++
00000800:00000200:1.0:1619677475.808637:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:24.0:1619677475.808639:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (22)++
00000800:00000200:1.0:1619677475.808640:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.808641:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.808641:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:24.0:1619677475.808642:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919feda13a00] (23)--
00000800:00000200:24.0:1619677475.808643:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:1.0:1619677475.808643:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:24.0:1619677475.808644:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:1.0:1619677475.808646:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.808808:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.808815:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.808818:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.808820:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.808822:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.808823:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.808824:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.808826:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498065] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.808827:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.808828:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.808829:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.808833:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.808834:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.808837:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.808838:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.808838:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.808840:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.808840:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.808841:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.808842:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.808846:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.808854:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.808856:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.808857:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.808858:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.808860:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.808862:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.808867:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:22.0:1619677475.808874:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.808877:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.808877:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.808878:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.808879:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:22.0:1619677475.808879:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:20.2:1619677475.808930:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907d39de00] (20)++
00000800:00000200:18.0:1619677475.808937:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.808940:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.10@o2ib3
00000800:00000200:2.0:1619677475.808941:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (22)--
00000400:00000200:18.0:1619677475.808943:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.110@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.10@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.808945:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.808946:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.110@o2ib7 local destination
00000400:00000200:18.0:1619677475.808948:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498064] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.808949:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.808950:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.808951:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.110@o2ib7
00000400:00000200:18.0:1619677475.808954:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.10@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.110@o2ib7(10.0.10.110@o2ib7:10.0.10.110@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.808956:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.110@o2ib7
00000800:00000200:18.0:1619677475.808958:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410fb80] -> 10.0.10.110@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.808958:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986000] (20)++
00000800:00000200:18.0:1619677475.808959:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986000] (21)++
00000800:00000200:18.0:1619677475.808961:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986000] (22)--
00000800:00000200:18.0:1619677475.808961:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907d39de00] (21)++
00000800:00000200:18.0:1619677475.808962:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907d39de00] (22)--
00000800:00000200:18.0:1619677475.808963:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907d39de00] (21)--
00000800:00000200:9.2:1619677475.808967:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986000] (21)++
00000800:00000200:2.0:1619677475.808974:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986000] (22)++
00000800:00000200:2.0:1619677475.808976:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986000] (23)--
00000800:00000200:18.0:1619677475.808977:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (22)--
00000400:00000200:2.0:1619677475.808977:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.808979:0:19932:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.110@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.808980:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986000] (21)--
00000800:00000200:4.2:1619677475.809097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000800:00000200:18.0:1619677475.809102:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677475.809104:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:2.0:1619677475.809105:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:18.0:1619677475.809107:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.809109:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.809110:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677475.809111:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498063] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.809112:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.809113:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.809114:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.809117:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.809119:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677475.809122:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.809122:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677475.809123:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.809125:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677475.809125:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677475.809126:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:18.0:1619677475.809126:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:25.2:1619677475.809131:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.809138:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.809140:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.809141:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.809143:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.809144:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.809145:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.809335:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.809343:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.809346:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.809347:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.809349:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000800:00000200:20.2:1619677475.809351:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:10.0:1619677475.809352:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.809354:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.809355:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498062] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:21.0:1619677475.809356:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000400:00000200:10.0:1619677475.809356:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.809357:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:21.0:1619677475.809358:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000400:00000200:10.0:1619677475.809359:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:21.0:1619677475.809361:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.809361:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000400:00000200:10.0:1619677475.809363:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000400:00000200:21.0:1619677475.809364:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.809365:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000400:00000200:21.0:1619677475.809366:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.809367:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498061] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.809368:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.809369:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:10.0:1619677475.809369:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.809370:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:10.0:1619677475.809370:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.809370:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.809372:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.809372:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.809373:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000400:00000200:21.0:1619677475.809374:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.809374:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:21.0:1619677475.809375:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:25.2:1619677475.809378:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:21.0:1619677475.809378:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.809379:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:21.0:1619677475.809379:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:21.0:1619677475.809381:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.809382:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.809383:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197aaee8000] (22)++
00000800:00000200:21.0:1619677475.809384:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (23)--
00000800:00000200:21.0:1619677475.809385:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677475.809386:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:8.2:1619677475.809388:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:6.0:1619677475.809389:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000800:00000200:22.0:1619677475.809390:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:20.2:1619677475.809390:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (21)++
00000400:00000200:6.0:1619677475.809390:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.809392:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)++
00000400:00000200:6.0:1619677475.809393:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.809394:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197aaee8000] (23)--
00000800:00000200:20.0:1619677475.809395:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:6.0:1619677475.809395:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:20.0:1619677475.809396:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:17.0:1619677475.809396:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:25.2:1619677475.809397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (20)++
00000800:00000200:17.0:1619677475.809399:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.809400:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.809400:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.809402:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.809404:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (21)++
00000800:00000200:17.0:1619677475.809404:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:22.0:1619677475.809407:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.112@o2ib7
00000800:00000200:22.0:1619677475.809408:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86953400] (22)++
00000800:00000200:6.0:1619677475.809408:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.809409:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86953400] (22)--
00000800:00000200:22.0:1619677475.809410:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.809647:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fee39d800] (20)++
00000800:00000200:8.0:1619677475.809655:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.809657:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.3.58@o2ib3
00000800:00000200:24.0:1619677475.809658:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (22)--
00000400:00000200:8.0:1619677475.809661:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.3.58@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.809663:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.809665:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:8.0:1619677475.809666:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498060] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.809668:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.809669:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.809670:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:8.0:1619677475.809673:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.3.58@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.809675:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:8.0:1619677475.809678:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.809679:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:8.0:1619677475.809679:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:8.0:1619677475.809681:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:8.0:1619677475.809682:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fee39d800] (21)++
00000800:00000200:8.0:1619677475.809682:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fee39d800] (22)--
00000800:00000200:8.0:1619677475.809683:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fee39d800] (21)--
00000800:00000200:8.2:1619677475.809687:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.809695:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.809698:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.809699:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.809699:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.809701:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.809704:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:8.2:1619677475.809707:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:17.0:1619677475.809715:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.809718:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.809718:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.809719:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.809720:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.809720:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.809803:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:30.0:1619677475.809811:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.809814:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:30.0:1619677475.809817:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.809819:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:30.0:1619677475.809819:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.809821:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:30.0:1619677475.809822:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498059] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.809823:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.809824:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.809825:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:30.0:1619677475.809828:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.809830:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.809832:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.809832:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.809833:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.809834:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.809835:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.809836:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.809836:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.809840:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.809847:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:31.0:1619677475.809849:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:31.0:1619677475.809850:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.809851:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.809853:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.809854:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.809859:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.809866:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.809869:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.809869:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.809870:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.809871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.809871:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:21.2:1619677475.810190:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.810195:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.810198:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.810199:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.810200:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.810202:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.810204:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.810205:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498058] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:4.2:1619677475.810206:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000400:00000200:6.0:1619677475.810207:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.810208:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.810209:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.810212:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.810212:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:6.0:1619677475.810213:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677475.810214:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:18.0:1619677475.810215:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000800:00000200:6.0:1619677475.810216:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.810217:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.810218:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:2.0:1619677475.810218:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.810219:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.810220:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.810220:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:2.0:1619677475.810220:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.810221:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:2.0:1619677475.810222:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677475.810223:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498057] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:25.2:1619677475.810225:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:2.0:1619677475.810225:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.810226:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.810227:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677475.810230:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.810232:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.810233:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677475.810234:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.810235:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:2.0:1619677475.810235:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (23)++
00000400:00000200:20.0:1619677475.810236:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.810236:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (23)++
00000400:00000200:20.0:1619677475.810237:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:2.0:1619677475.810238:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (24)--
00000800:00000200:21.0:1619677475.810239:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:2.0:1619677475.810239:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:20.0:1619677475.810240:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677475.810240:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907a3e8400] (22)++
00000800:00000200:2.0:1619677475.810241:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (23)--
00000800:00000200:2.0:1619677475.810242:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000800:00000200:25.2:1619677475.810244:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677475.810247:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (21)++
00000800:00000200:21.0:1619677475.810251:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677475.810252:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)++
00000800:00000200:21.0:1619677475.810253:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.810254:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.810255:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.810255:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907a3e8400] (23)--
00000800:00000200:18.0:1619677475.810255:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000800:00000200:20.0:1619677475.810256:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677475.810256:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:21.0:1619677475.810258:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.810258:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:20.0:1619677475.810259:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (23)++
00000800:00000200:20.0:1619677475.810259:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.810260:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (23)--
00000800:00000200:20.0:1619677475.810261:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.810612:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.810620:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.810623:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.810626:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.810628:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.810628:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.810629:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.810631:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498056] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.810632:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.810633:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.810634:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.810638:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.810640:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.810643:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.810644:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.810644:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.810646:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.810647:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.810648:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.810648:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.810653:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.810661:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.810664:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.810665:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.810665:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.810667:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.810670:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.810734:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.810742:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.810744:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.810745:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.810747:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.810750:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.810752:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:24.0:1619677475.810753:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498055] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.810754:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.810755:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.810756:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:24.0:1619677475.810760:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.810761:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:24.0:1619677475.810764:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.810765:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:24.0:1619677475.810765:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:24.0:1619677475.810767:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:24.0:1619677475.810768:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.810768:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.810769:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:25.2:1619677475.810773:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.810780:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.810783:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.810784:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.810784:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.810786:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.810789:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.810903:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.810909:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.810912:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.810913:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.810915:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.810917:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.810918:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.810920:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498054] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.810921:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.810922:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.810923:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.810926:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.810928:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.810930:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.810930:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.810931:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.810932:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.810933:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.810934:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (22)--
00000800:00000200:1.0:1619677475.810934:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:8.2:1619677475.810939:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:17.0:1619677475.810946:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.810948:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000400:00000200:17.0:1619677475.810949:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.810949:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:17.0:1619677475.810950:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.810951:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:4.2:1619677475.811007:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.811014:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.811017:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.811018:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.811020:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.811023:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.811024:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.811025:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498053] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.811027:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.811028:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.811029:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.811032:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.811034:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:21.2:1619677475.811035:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:26.0:1619677475.811037:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.811037:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.811038:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.811040:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.811040:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.811041:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.811041:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:31.0:1619677475.811042:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.811045:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000800:00000200:25.2:1619677475.811046:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000400:00000200:31.0:1619677475.811047:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.811049:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.811050:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.811051:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.811052:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498052] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.811053:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.811053:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:31.0:1619677475.811054:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.811055:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000800:00000200:22.0:1619677475.811056:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.811057:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.811057:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:31.0:1619677475.811058:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000400:00000200:22.0:1619677475.811059:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.811060:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.811062:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.811062:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:22.0:1619677475.811062:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:31.0:1619677475.811063:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.811065:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.811065:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.811066:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.811067:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.811070:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.811077:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.811079:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.811080:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.811082:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.811084:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.811085:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:4.2:1619677475.811145:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000800:00000200:2.0:1619677475.811152:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677475.811155:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:18.0:1619677475.811156:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:2.0:1619677475.811158:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.811160:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.811161:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677475.811163:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498051] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.811164:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.811165:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.811166:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677475.811169:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.811171:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677475.811174:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.811174:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677475.811175:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677475.811177:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677475.811178:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677475.811179:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:2.0:1619677475.811179:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:25.2:1619677475.811183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.811191:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.811193:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.811194:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.811196:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.811196:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.811198:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.811355:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919fed9d3400] (20)++
00000800:00000200:8.0:1619677475.811363:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677475.811366:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.15.7@o2ib3
00000800:00000200:24.0:1619677475.811367:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (22)--
00000400:00000200:8.0:1619677475.811369:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.15.7@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.811372:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.811373:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:8.0:1619677475.811375:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498050] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.811376:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.811377:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.811378:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000400:00000200:8.0:1619677475.811381:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.15.7@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.811383:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:8.0:1619677475.811386:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.811386:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:8.0:1619677475.811387:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:8.0:1619677475.811388:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.0:1619677475.811389:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919fed9d3400] (21)++
00000800:00000200:8.0:1619677475.811390:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919fed9d3400] (22)--
00000800:00000200:8.0:1619677475.811390:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919fed9d3400] (21)--
00000800:00000200:25.2:1619677475.811395:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:12.0:1619677475.811402:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:12.0:1619677475.811404:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000400:00000200:12.0:1619677475.811405:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:12.0:1619677475.811407:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:12.0:1619677475.811409:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.811410:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000200:21.2:1619677475.811579:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.811585:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.811587:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:22.0:1619677475.811588:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:6.0:1619677475.811590:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.811592:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.811593:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000800:00000200:20.2:1619677475.811594:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000400:00000200:6.0:1619677475.811595:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498049] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.811596:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.811597:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.811598:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:18.0:1619677475.811600:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000400:00000200:6.0:1619677475.811601:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.811603:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:6.0:1619677475.811603:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677475.811604:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:6.0:1619677475.811605:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:18.0:1619677475.811606:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.811606:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.811607:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677475.811608:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.811608:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.811609:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.811609:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000400:00000200:18.0:1619677475.811610:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000800:00000200:6.0:1619677475.811610:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000400:00000200:18.0:1619677475.811611:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498048] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.811612:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.811613:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.811614:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000400:00000200:18.0:1619677475.811614:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:18.0:1619677475.811618:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.811619:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:20.0:1619677475.811622:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:18.0:1619677475.811623:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.811624:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:18.0:1619677475.811624:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:18.0:1619677475.811624:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000400:00000200:20.0:1619677475.811625:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.811626:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.811626:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:21.0:1619677475.811627:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677475.811627:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:18.0:1619677475.811628:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:18.0:1619677475.811628:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:20.0:1619677475.811629:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.811632:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.811639:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.811642:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:31.0:1619677475.811643:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.811645:0:19945:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.811646:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.811648:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:20.2:1619677475.812051:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919460fda200] (20)++
00000800:00000200:21.0:1619677475.812057:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919460fda200] (21)++
00000800:00000200:21.0:1619677475.812059:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.23@o2ib3
00000400:00000200:21.0:1619677475.812061:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.103@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.23@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.812062:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (22)--
00000400:00000200:21.0:1619677475.812064:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.812066:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.103@o2ib7 local destination
00000400:00000200:21.0:1619677475.812067:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498047] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.812068:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.812069:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.812070:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.103@o2ib7
00000400:00000200:21.0:1619677475.812073:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.103@o2ib7(10.0.10.103@o2ib7:10.0.10.103@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.812075:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.103@o2ib7
00000800:00000200:21.0:1619677475.812079:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537400] -> 10.0.10.103@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.812080:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (20)++
00000800:00000200:21.0:1619677475.812081:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677475.812083:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677475.812083:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919460fda200] (21)++
00000800:00000200:4.2:1619677475.812083:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:21.0:1619677475.812084:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919460fda200] (22)--
00000800:00000200:21.0:1619677475.812085:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919460fda200] (21)--
00000800:00000200:21.0:1619677475.812086:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.812088:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000800:00000200:25.2:1619677475.812089:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000400:00000200:21.0:1619677475.812090:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.812091:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:21.0:1619677475.812092:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.812093:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:21.0:1619677475.812094:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498046] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:21.0:1619677475.812096:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:21.0:1619677475.812096:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.812097:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:12.0:1619677475.812097:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)++
00000400:00000200:21.0:1619677475.812100:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:12.0:1619677475.812100:0:19943:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9600] (23)--
00000800:00000200:21.0:1619677475.812102:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000400:00000200:12.0:1619677475.812102:0:19943:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.812104:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.812105:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:21.0:1619677475.812105:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000400:00000200:12.0:1619677475.812105:0:19943:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.103@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.812106:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:21.0:1619677475.812107:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:25.2:1619677475.812108:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:21.0:1619677475.812108:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:21.0:1619677475.812110:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197a0b87200] (22)++
00000800:00000200:21.0:1619677475.812111:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (23)--
00000800:00000200:21.0:1619677475.812111:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000800:00000200:12.0:1619677475.812111:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:4.2:1619677475.812111:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:25.2:1619677475.812112:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:12.0:1619677475.812113:0:19943:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)++
00000800:00000200:12.0:1619677475.812116:0:19943:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.103@o2ib7
00000800:00000200:4.2:1619677475.812116:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (21)++
00000800:00000200:13.0:1619677475.812117:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:12.0:1619677475.812117:0:19943:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)++
00000800:00000200:12.0:1619677475.812118:0:19943:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a7c17d9600] (22)--
00000800:00000200:10.0:1619677475.812118:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:12.0:1619677475.812119:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9600] (21)--
00000800:00000200:22.0:1619677475.812120:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:20.0:1619677475.812120:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)++
00000800:00000200:10.0:1619677475.812121:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.812122:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:22.0:1619677475.812122:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:20.0:1619677475.812123:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197a0b87200] (23)--
00000800:00000200:20.0:1619677475.812123:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:22.0:1619677475.812124:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:20.0:1619677475.812124:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000400:00000200:10.0:1619677475.812124:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.109@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.812124:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.812126:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:10.0:1619677475.812126:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.812128:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.109@o2ib7 local destination
00000800:00000200:22.0:1619677475.812129:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000400:00000200:10.0:1619677475.812129:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498045] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.812130:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.812131:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:25.2:1619677475.812132:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000400:00000200:10.0:1619677475.812132:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.109@o2ib7
00000400:00000200:10.0:1619677475.812135:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.109@o2ib7(10.0.10.109@o2ib7:10.0.10.109@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.812137:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.109@o2ib7
00000800:00000200:10.0:1619677475.812139:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e58c4f480] -> 10.0.10.109@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.812139:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.812140:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:10.0:1619677475.812140:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (20)++
00000800:00000200:10.0:1619677475.812140:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7cc00] (21)++
00000800:00000200:6.0:1619677475.812141:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.812142:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7cc00] (22)--
00000800:00000200:10.0:1619677475.812142:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.812143:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.812143:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:6.0:1619677475.812143:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.812144:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:10.0:1619677475.812144:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:6.0:1619677475.812144:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.812145:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:6.0:1619677475.812145:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:6.0:1619677475.812146:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:22.0:1619677475.812148:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000800:00000200:9.2:1619677475.812148:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7cc00] (21)++
00000400:00000200:22.0:1619677475.812150:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.812152:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:22.0:1619677475.812153:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498044] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:24.0:1619677475.812154:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)++
00000400:00000200:22.0:1619677475.812154:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.812155:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:24.0:1619677475.812156:0:19938:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7cc00] (23)--
00000400:00000200:22.0:1619677475.812156:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:24.0:1619677475.812157:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:8.0:1619677475.812158:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (22)--
00000400:00000200:24.0:1619677475.812160:0:19938:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.109@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.812161:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.812163:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7cc00] (21)--
00000800:00000200:22.0:1619677475.812163:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:22.0:1619677475.812166:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:22.0:1619677475.812167:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:22.0:1619677475.812168:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:22.0:1619677475.812170:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:22.0:1619677475.812171:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:22.0:1619677475.812171:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:22.0:1619677475.812172:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:25.2:1619677475.812176:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.812183:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.812186:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.812187:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.812189:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.812190:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.812192:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:25.2:1619677475.812194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:31.0:1619677475.812201:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:31.0:1619677475.812204:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.116@o2ib7
00000800:00000200:31.0:1619677475.812205:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:31.0:1619677475.812206:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91abeb71aa00] (23)--
00000800:00000200:31.0:1619677475.812206:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:31.0:1619677475.812208:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.812375:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000800:00000200:2.0:1619677475.812381:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677475.812384:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:18.0:1619677475.812385:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:2.0:1619677475.812387:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.812390:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.812392:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677475.812393:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498043] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.812394:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.812395:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.812396:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677475.812400:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.812402:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677475.812404:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.812405:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677475.812406:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677475.812408:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677475.812408:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677475.812409:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:2.0:1619677475.812410:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:25.2:1619677475.812414:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.812422:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.812424:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.812425:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.812427:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: ACK: OK
00000800:00000200:21.0:1619677475.812428:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.812430:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.812437:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.812444:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.812447:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.812448:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.812449:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.812449:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.812450:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.812851:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:26.0:1619677475.812859:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.812862:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000800:00000200:10.0:1619677475.812863:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:26.0:1619677475.812865:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.102@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.812868:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.812869:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.102@o2ib7 local destination
00000400:00000200:26.0:1619677475.812871:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498042] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.812872:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.812873:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.812874:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.102@o2ib7
00000400:00000200:26.0:1619677475.812877:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.102@o2ib7(10.0.10.102@o2ib7:10.0.10.102@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.812880:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.102@o2ib7
00000800:00000200:26.0:1619677475.812883:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff9193c410f580] -> 10.0.10.102@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.812884:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9193c2986800] (20)++
00000800:00000200:26.0:1619677475.812884:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9193c2986800] (21)++
00000800:00000200:26.0:1619677475.812886:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9193c2986800] (22)--
00000800:00000200:26.0:1619677475.812886:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.812887:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:26.0:1619677475.812888:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:9.2:1619677475.812892:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9193c2986800] (21)++
00000800:00000200:18.0:1619677475.812900:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9193c2986800] (22)++
00000800:00000200:18.0:1619677475.812903:0:19933:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9193c2986800] (23)--
00000800:00000200:2.0:1619677475.812903:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (22)--
00000400:00000200:18.0:1619677475.812904:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.812906:0:19933:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.102@o2ib7: PUT: OK
00000800:00000200:18.0:1619677475.812910:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9193c2986800] (21)--
00000800:00000200:21.2:1619677475.812921:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:30.0:1619677475.812929:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.812932:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:30.0:1619677475.812934:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:31.0:1619677475.812936:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:30.0:1619677475.812936:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.812938:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:30.0:1619677475.812939:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498041] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.812940:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.812941:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.812942:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:30.0:1619677475.812946:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.812947:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.812950:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.812951:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.812952:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:30.0:1619677475.812953:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:30.0:1619677475.812954:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:30.0:1619677475.812955:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:30.0:1619677475.812955:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.812959:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.812968:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.812970:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.812971:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.812972:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.812974:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.812977:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.813330:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000800:00000200:2.0:1619677475.813336:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677475.813339:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:18.0:1619677475.813340:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:2.0:1619677475.813341:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.813344:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.813346:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:2.0:1619677475.813347:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498040] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.813348:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.813349:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.813350:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:2.0:1619677475.813353:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.813355:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 440 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:2.0:1619677475.813357:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.813358:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:2.0:1619677475.813358:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:2.0:1619677475.813360:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:2.0:1619677475.813361:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677475.813362:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:2.0:1619677475.813362:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:25.2:1619677475.813367:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.813375:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.813377:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.813378:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.813379:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.813381:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.813383:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:21.2:1619677475.813397:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:6.0:1619677475.813402:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.813405:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.813405:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000400:00000200:6.0:1619677475.813407:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.813409:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.813411:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:6.0:1619677475.813412:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498039] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.813413:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.813415:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.813415:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:6.0:1619677475.813418:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.813420:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.813422:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.813423:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:6.0:1619677475.813424:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:6.0:1619677475.813425:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:6.0:1619677475.813426:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000800:00000200:6.0:1619677475.813426:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.813427:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:25.2:1619677475.813432:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.813439:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.813441:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.813442:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.813444:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.813445:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.813446:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.813888:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.813896:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.813899:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.813902:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.813904:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.813905:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.813906:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.813908:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498038] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.813909:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.813910:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.813911:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.813914:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.813916:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.813918:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.813919:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.813920:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.813921:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.813922:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.813923:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.813923:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.813927:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.813933:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.813936:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.813937:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.813938:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.813940:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.813940:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:4.2:1619677475.814159:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07179ba00] (20)++
00000800:00000200:10.0:1619677475.814167:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.814170:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.69@o2ib3
00000800:00000200:26.0:1619677475.814171:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (22)--
00000400:00000200:10.0:1619677475.814172:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.112@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.69@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.814175:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.814176:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.112@o2ib7 local destination
00000400:00000200:10.0:1619677475.814177:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498037] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.814179:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.814179:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.814180:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.112@o2ib7
00000400:00000200:10.0:1619677475.814183:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.69@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.112@o2ib7(10.0.10.112@o2ib7:10.0.10.112@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.814185:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.112@o2ib7
00000800:00000200:10.0:1619677475.814187:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b98b49480] -> 10.0.10.112@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.814188:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86953400] (20)++
00000800:00000200:10.0:1619677475.814189:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86953400] (21)++
00000800:00000200:10.0:1619677475.814190:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86953400] (22)--
00000800:00000200:10.0:1619677475.814191:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07179ba00] (21)++
00000800:00000200:10.0:1619677475.814191:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07179ba00] (22)--
00000800:00000200:10.0:1619677475.814192:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07179ba00] (21)--
00000800:00000200:25.2:1619677475.814196:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86953400] (21)++
00000800:00000200:22.0:1619677475.814204:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86953400] (22)++
00000800:00000200:22.0:1619677475.814207:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86953400] (23)--
00000400:00000200:22.0:1619677475.814208:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.814208:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (22)--
00000400:00000200:22.0:1619677475.814210:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.112@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.814213:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86953400] (21)--
00000800:00000200:4.2:1619677475.814262:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8400] (20)++
00000800:00000200:20.0:1619677475.814265:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.814268:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.13.12@o2ib3
00000400:00000200:20.0:1619677475.814270:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.13.12@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.814271:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (22)--
00000400:00000200:20.0:1619677475.814272:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.814274:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000400:00000200:20.0:1619677475.814275:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498036] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.814276:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.814277:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.814278:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:20.0:1619677475.814281:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.13.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000800:00000200:20.0:1619677475.814282:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000800:00000200:20.0:1619677475.814285:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.814286:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:20.0:1619677475.814286:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000800:00000200:20.0:1619677475.814288:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:20.0:1619677475.814289:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8400] (21)++
00000800:00000200:20.0:1619677475.814290:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8400] (22)--
00000800:00000200:20.0:1619677475.814290:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8400] (21)--
00000800:00000200:8.2:1619677475.814296:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:21.0:1619677475.814305:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:21.0:1619677475.814307:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:21.0:1619677475.814307:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.814309:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.814310:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000800:00000200:21.0:1619677475.814311:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:4.2:1619677475.814577:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000800:00000200:18.0:1619677475.814583:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677475.814586:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:2.0:1619677475.814587:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:18.0:1619677475.814589:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.814591:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.814592:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677475.814594:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498035] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.814595:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.814596:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.814597:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.814600:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.814602:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 576 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677475.814605:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.814606:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677475.814606:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.814608:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677475.814609:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677475.814610:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91907a3e8400] (22)++
00000800:00000200:18.0:1619677475.814612:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (23)--
00000800:00000200:18.0:1619677475.814612:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000800:00000200:25.2:1619677475.814615:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:4.2:1619677475.814617:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (21)++
00000800:00000200:20.0:1619677475.814623:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:2.0:1619677475.814623:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)++
00000800:00000200:20.0:1619677475.814625:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:2.0:1619677475.814625:0:19932:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91907a3e8400] (23)--
00000800:00000200:2.0:1619677475.814626:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:20.0:1619677475.814627:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:2.0:1619677475.814627:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:21.0:1619677475.814629:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000400:00000200:20.0:1619677475.814629:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.814631:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:8.2:1619677475.814713:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (20)++
00000800:00000200:21.0:1619677475.814720:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.814722:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.0.10.115@o2ib7
00000400:00000200:21.0:1619677475.814725:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7) <- 10.0.10.115@o2ib7 : GET - for me
00000800:00000200:20.0:1619677475.814725:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677475.814727:0:19935:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.0.10.115@o2ib7 of length 272 into portal 0 MB=0x8000000000000000
00000400:00000200:21.0:1619677475.814729:0:19935:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming get index 0 from 12345-10.0.10.115@o2ib7 of length 64/272 into md 0x61 [1] + 0
00000800:00000200:21.2:1619677475.814731:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:21.0:1619677475.814735:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3200] (21)++
00000800:00000200:21.0:1619677475.814736:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3200] (22)--
00000400:00000200:21.0:1619677475.814738:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source Specified: 10.0.10.237@o2ib7 to MR:  10.0.10.115@o2ib7 local destination
00000800:00000200:30.0:1619677475.814739:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000400:00000200:21.0:1619677475.814741:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.0.10.237@o2ib7(10.0.10.237@o2ib7:10.0.10.237@o2ib7) -> 10.0.10.115@o2ib7(10.0.10.115@o2ib7:10.0.10.115@o2ib7) : REPLY try# 0
00000800:00000200:30.0:1619677475.814742:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000800:00000200:21.0:1619677475.814743:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 64 bytes in 1 frags to 12345-10.0.10.115@o2ib7
00000400:00000200:30.0:1619677475.814744:0:19944:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:21.0:1619677475.814745:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919786445900] -> 10.0.10.115@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.814746:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677475.814746:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3200] (21)++
00000400:00000200:30.0:1619677475.814747:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.814747:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3200] (22)++
00000400:00000200:30.0:1619677475.814748:0:19944:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000800:00000200:21.0:1619677475.814748:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3200] (23)--
00000800:00000200:21.0:1619677475.814749:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000400:00000200:30.0:1619677475.814750:0:19944:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498034] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:30.0:1619677475.814751:0:19944:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:30.0:1619677475.814752:0:19944:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:30.0:1619677475.814753:0:19944:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000800:00000200:8.2:1619677475.814754:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3200] (21)++
00000400:00000200:30.0:1619677475.814757:0:19944:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:30.0:1619677475.814759:0:19944:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:30.0:1619677475.814762:0:19944:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:30.0:1619677475.814763:0:19944:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:30.0:1619677475.814763:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:20.0:1619677475.814763:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3200] (22)++
00000800:00000200:30.0:1619677475.814765:0:19944:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:20.0:1619677475.814765:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3200] (23)--
00000800:00000200:30.0:1619677475.814766:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000400:00000200:20.0:1619677475.814766:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.814767:0:19944:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a86b38b000] (22)++
00000400:00000200:20.0:1619677475.814768:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.115@o2ib7: REPLY: OK
00000800:00000200:30.0:1619677475.814769:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (23)--
00000800:00000200:30.0:1619677475.814769:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:21.0:1619677475.814769:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (22)--
00000800:00000200:20.0:1619677475.814770:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3200] (21)--
00000800:00000200:25.2:1619677475.814771:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:21.2:1619677475.814775:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (21)++
00000800:00000200:6.0:1619677475.814779:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.814782:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:31.0:1619677475.814783:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)++
00000800:00000200:22.0:1619677475.814783:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.814783:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.814785:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.814786:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a86b38b000] (23)--
00000800:00000200:31.0:1619677475.814787:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.814788:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:6.0:1619677475.814788:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.814826:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.814834:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.814837:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.814840:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.814841:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.814842:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.814844:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.814845:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498033] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.814846:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.814847:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.814848:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.814851:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.814853:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.814855:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.814855:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.814856:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.814857:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.814858:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.814859:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a870bf7e00] (22)++
00000800:00000200:31.0:1619677475.814860:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.814860:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:25.2:1619677475.814863:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:21.2:1619677475.814866:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:30.0:1619677475.814870:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.814872:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.814873:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:31.0:1619677475.814874:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)++
00000400:00000200:30.0:1619677475.814874:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:31.0:1619677475.814875:0:19945:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a870bf7e00] (23)--
00000800:00000200:31.0:1619677475.814876:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:30.0:1619677475.814877:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.814878:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:30.0:1619677475.814879:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:4.2:1619677475.815088:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:8.0:1619677475.815096:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.815098:0:19939:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:24.0:1619677475.815099:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:8.0:1619677475.815101:0:19939:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:8.0:1619677475.815103:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.815105:0:19939:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:8.0:1619677475.815106:0:19939:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498032] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:8.0:1619677475.815107:0:19939:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:8.0:1619677475.815108:0:19939:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:8.0:1619677475.815109:0:19939:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:8.0:1619677475.815111:0:19939:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:8.0:1619677475.815113:0:19939:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:8.0:1619677475.815116:0:19939:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:8.0:1619677475.815116:0:19939:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:8.0:1619677475.815117:0:19939:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:8.0:1619677475.815119:0:19939:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:8.0:1619677475.815119:0:19939:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:8.0:1619677475.815120:0:19939:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:8.0:1619677475.815120:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:21.2:1619677475.815123:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (20)++
00000800:00000200:25.2:1619677475.815125:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.815130:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (21)++
00000800:00000200:22.0:1619677475.815132:0:19936:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.23@o2ib3
00000800:00000200:6.0:1619677475.815134:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000400:00000200:22.0:1619677475.815135:0:19936:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.23@o2ib3 : PUT - routed
00000800:00000200:6.0:1619677475.815135:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:6.0:1619677475.815136:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.815138:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.815138:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000400:00000200:22.0:1619677475.815140:0:19936:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:22.0:1619677475.815141:0:19936:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498031] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:22.0:1619677475.815143:0:19936:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:22.0:1619677475.815143:0:19936:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000800:00000200:20.2:1619677475.815143:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197aaee8000] (20)++
00000400:00000200:22.0:1619677475.815145:0:19936:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000800:00000200:6.0:1619677475.815148:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.815149:0:19936:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.23@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.815149:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)++
00000800:00000200:6.0:1619677475.815149:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.815150:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:22.0:1619677475.815151:0:19936:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 960 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.815151:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.71@o2ib3
00000800:00000200:22.0:1619677475.815154:0:19936:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000400:00000200:21.0:1619677475.815154:0:19935:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.71@o2ib3 : PUT - routed
00000800:00000200:20.0:1619677475.815154:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (22)--
00000800:00000200:22.0:1619677475.815155:0:19936:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:22.0:1619677475.815156:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000400:00000200:21.0:1619677475.815156:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:22.0:1619677475.815158:0:19936:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:22.0:1619677475.815158:0:19936:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677475.815158:0:19935:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:21.0:1619677475.815159:0:19935:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:32, s:-96498030] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000800:00000200:22.0:1619677475.815160:0:19936:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9198797ac000] (22)++
00000400:00000200:21.0:1619677475.815160:0:19935:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:22.0:1619677475.815161:0:19936:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9198797ac000] (23)--
00000800:00000200:22.0:1619677475.815161:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000400:00000200:21.0:1619677475.815161:0:19935:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:21.0:1619677475.815162:0:19935:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000800:00000200:25.2:1619677475.815164:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.2:1619677475.815166:0:19935:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9198797ac000] (21)++
00000400:00000200:21.0:1619677475.815169:0:19935:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.71@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:21.0:1619677475.815170:0:19935:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:20.0:1619677475.815171:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:6.0:1619677475.815171:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9198797ac000] (22)++
00000800:00000200:20.0:1619677475.815172:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.815173:0:19935:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:21.0:1619677475.815173:0:19935:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000400:00000200:20.0:1619677475.815173:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.815174:0:19935:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000400:00000200:20.0:1619677475.815174:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.815174:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9198797ac000] (23)--
00000800:00000200:6.0:1619677475.815174:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (22)--
00000800:00000200:6.0:1619677475.815175:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9198797ac000] (21)--
00000800:00000200:21.0:1619677475.815176:0:19935:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:21.0:1619677475.815176:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197aaee8000] (21)++
00000800:00000200:21.0:1619677475.815177:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197aaee8000] (22)--
00000800:00000200:20.0:1619677475.815177:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.815178:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.0:1619677475.815179:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197aaee8000] (21)--
00000800:00000200:8.2:1619677475.815183:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.815191:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:1.0:1619677475.815193:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:17.0:1619677475.815194:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000400:00000200:1.0:1619677475.815194:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.815196:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.815199:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:21.2:1619677475.815472:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce2bc0c00] (20)++
00000800:00000200:17.0:1619677475.815477:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.815480:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.21@o2ib3
00000800:00000200:1.0:1619677475.815481:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (22)--
00000400:00000200:17.0:1619677475.815483:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.21@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.815484:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:17.0:1619677475.815485:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:17.0:1619677475.815486:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498029] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:17.0:1619677475.815488:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.815489:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.815490:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:17.0:1619677475.815493:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.21@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.815494:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:17.0:1619677475.815497:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.815497:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:17.0:1619677475.815498:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:17.0:1619677475.815500:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:17.0:1619677475.815500:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (21)++
00000800:00000200:17.0:1619677475.815501:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce2bc0c00] (22)--
00000800:00000200:17.0:1619677475.815501:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce2bc0c00] (21)--
00000800:00000200:25.2:1619677475.815507:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.815515:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.815517:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.815518:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:6.0:1619677475.815518:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.815521:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.815523:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:25.2:1619677475.815525:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (20)++
00000800:00000200:6.0:1619677475.815532:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.815535:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.815535:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.104@o2ib7
00000800:00000200:6.0:1619677475.815536:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.815537:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919b86950600] (22)--
00000800:00000200:6.0:1619677475.815538:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:5.2:1619677475.815718:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (20)++
00000800:00000200:1.0:1619677475.815724:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.815727:0:19931:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.27@o2ib3
00000800:00000200:17.0:1619677475.815728:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:1.0:1619677475.815730:0:19931:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.105@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.27@o2ib3 : PUT - routed
00000400:00000200:1.0:1619677475.815732:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.815733:0:19931:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.105@o2ib7 local destination
00000400:00000200:1.0:1619677475.815734:0:19931:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498028] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:1.0:1619677475.815735:0:19931:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:1.0:1619677475.815737:0:19931:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:1.0:1619677475.815737:0:19931:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.105@o2ib7
00000400:00000200:1.0:1619677475.815740:0:19931:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.27@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.105@o2ib7(10.0.10.105@o2ib7:10.0.10.105@o2ib7) : PUT try# 0
00000800:00000200:1.0:1619677475.815741:0:19931:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.105@o2ib7
00000800:00000200:1.0:1619677475.815743:0:19931:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918fe6b18100] -> 10.0.10.105@o2ib7 (2) version: 12
00000800:00000200:1.0:1619677475.815743:0:19931:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (20)++
00000800:00000200:1.0:1619677475.815744:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:1.0:1619677475.815746:0:19931:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:1.0:1619677475.815746:0:19931:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce07b6800] (21)++
00000800:00000200:1.0:1619677475.815747:0:19931:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.815748:0:19931:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce07b6800] (23)--
00000800:00000200:1.0:1619677475.815749:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000800:00000200:8.2:1619677475.815752:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d3e00] (21)++
00000800:00000200:5.2:1619677475.815755:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce07b6800] (21)++
00000800:00000200:17.0:1619677475.815761:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)++
00000800:00000200:17.0:1619677475.815762:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d3e00] (23)--
00000800:00000200:1.0:1619677475.815762:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)++
00000800:00000200:1.0:1619677475.815762:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918ce07b6800] (23)--
00000400:00000200:17.0:1619677475.815763:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.815763:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (22)--
00000400:00000200:17.0:1619677475.815764:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.105@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.815764:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (22)--
00000800:00000200:17.0:1619677475.815765:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d3e00] (21)--
00000800:00000200:1.0:1619677475.815766:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce07b6800] (21)--
00000800:00000200:4.2:1619677475.815786:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000800:00000200:18.0:1619677475.815793:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677475.815795:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:2.0:1619677475.815796:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:18.0:1619677475.815799:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.815801:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.815802:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677475.815804:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498027] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.815805:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.815806:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.815807:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.815809:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.815811:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 392 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677475.815814:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.815815:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677475.815815:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.815817:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677475.815817:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677475.815818:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:18.0:1619677475.815819:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:25.2:1619677475.815823:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:20.0:1619677475.815830:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:20.0:1619677475.815833:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:20.0:1619677475.815833:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.815835:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:21.0:1619677475.815836:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.815838:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:25.2:1619677475.815844:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (20)++
00000800:00000200:21.0:1619677475.815851:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.815854:0:19935:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.52@o2ib7
00000800:00000200:21.0:1619677475.815855:0:19935:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.815856:0:19935:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197843f3800] (23)--
00000800:00000200:21.0:1619677475.815856:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:20.0:1619677475.815857:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.815873:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a870bf7e00] (20)++
00000800:00000200:31.0:1619677475.815882:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.815885:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.64@o2ib3
00000400:00000200:31.0:1619677475.815887:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.108@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.64@o2ib3 : PUT - routed
00000400:00000200:31.0:1619677475.815889:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:30.0:1619677475.815889:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (22)--
00000400:00000200:31.0:1619677475.815890:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.108@o2ib7 local destination
00000400:00000200:31.0:1619677475.815892:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498026] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.815893:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.815894:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.815895:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.108@o2ib7
00000400:00000200:31.0:1619677475.815898:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.64@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.108@o2ib7(10.0.10.108@o2ib7:10.0.10.108@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.815900:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.108@o2ib7
00000800:00000200:31.0:1619677475.815902:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61ba00] -> 10.0.10.108@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.815903:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a871382a00] (20)++
00000800:00000200:31.0:1619677475.815903:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a871382a00] (21)++
00000800:00000200:31.0:1619677475.815905:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.815906:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a870bf7e00] (21)++
00000800:00000200:31.0:1619677475.815906:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a870bf7e00] (22)--
00000800:00000200:31.0:1619677475.815907:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a870bf7e00] (21)--
00000800:00000200:25.2:1619677475.815911:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.815917:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.815920:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a871382a00] (23)--
00000400:00000200:30.0:1619677475.815920:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.815922:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.108@o2ib7: PUT: OK
00000800:00000200:30.0:1619677475.815923:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:31.0:1619677475.815925:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:25.2:1619677475.815932:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a871382a00] (20)++
00000800:00000200:30.0:1619677475.815938:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a871382a00] (21)++
00000800:00000200:30.0:1619677475.815941:0:19944:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.0.10.108@o2ib7
00000800:00000200:30.0:1619677475.815942:0:19944:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a871382a00] (22)++
00000800:00000200:30.0:1619677475.815942:0:19944:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a871382a00] (23)--
00000800:00000200:30.0:1619677475.815943:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (22)--
00000800:00000200:30.0:1619677475.815944:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a871382a00] (21)--
00000800:00000200:20.2:1619677475.816061:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91906ffc8a00] (20)++
00000800:00000200:2.0:1619677475.816067:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.816070:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.4.62@o2ib3
00000800:00000200:18.0:1619677475.816071:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (22)--
00000400:00000200:2.0:1619677475.816073:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.116@o2ib7(10.51.0.116@o2ib3) <- 10.51.4.62@o2ib3 : ACK - routed
00000400:00000200:2.0:1619677475.816076:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.816077:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.116@o2ib7 local destination
00000400:00000200:2.0:1619677475.816079:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498025] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.816080:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.816081:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.816082:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.116@o2ib7
00000400:00000200:2.0:1619677475.816085:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.4.62@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.116@o2ib7(10.0.10.116@o2ib7:10.0.10.116@o2ib7) : ACK try# 0
00000800:00000200:2.0:1619677475.816087:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.116@o2ib7
00000800:00000200:2.0:1619677475.816090:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91abeb61be80] -> 10.0.10.116@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.816090:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (20)++
00000800:00000200:2.0:1619677475.816091:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:2.0:1619677475.816093:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:2.0:1619677475.816093:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91906ffc8a00] (21)++
00000800:00000200:2.0:1619677475.816094:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91906ffc8a00] (22)--
00000800:00000200:2.0:1619677475.816095:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91906ffc8a00] (21)--
00000800:00000200:25.2:1619677475.816109:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91abeb71aa00] (21)++
00000800:00000200:30.0:1619677475.816116:0:19944:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)++
00000800:00000200:30.0:1619677475.816119:0:19944:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91abeb71aa00] (23)--
00000400:00000200:30.0:1619677475.816120:0:19944:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:30.0:1619677475.816121:0:19944:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.116@o2ib7: ACK: OK
00000800:00000200:31.0:1619677475.816123:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (22)--
00000800:00000200:30.0:1619677475.816124:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91abeb71aa00] (21)--
00000800:00000200:4.2:1619677475.816290:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919feda13a00] (20)++
00000800:00000200:24.0:1619677475.816298:0:19938:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.816301:0:19938:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.2@o2ib3
00000800:00000200:8.0:1619677475.816302:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (22)--
00000400:00000200:24.0:1619677475.816304:0:19938:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.2@o2ib3 : PUT - routed
00000400:00000200:24.0:1619677475.816306:0:19938:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:24.0:1619677475.816308:0:19938:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:24.0:1619677475.816309:0:19938:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:10, s:-96498024] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:24.0:1619677475.816310:0:19938:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:24.0:1619677475.816311:0:19938:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:24.0:1619677475.816312:0:19938:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:24.0:1619677475.816315:0:19938:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.2@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:24.0:1619677475.816317:0:19938:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:24.0:1619677475.816320:0:19938:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:24.0:1619677475.816321:0:19938:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:24.0:1619677475.816321:0:19938:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:24.0:1619677475.816323:0:19938:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:24.0:1619677475.816323:0:19938:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff919feda13a00] (21)++
00000800:00000200:24.0:1619677475.816324:0:19938:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff919feda13a00] (22)--
00000800:00000200:24.0:1619677475.816324:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919feda13a00] (21)--
00000800:00000200:8.2:1619677475.816329:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:17.0:1619677475.816337:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:17.0:1619677475.816340:0:19930:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000400:00000200:17.0:1619677475.816341:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:1.0:1619677475.816341:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:17.0:1619677475.816343:0:19930:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:17.0:1619677475.816346:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:4.2:1619677475.816392:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:26.0:1619677475.816400:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.816403:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:10.0:1619677475.816404:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:26.0:1619677475.816406:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:26.0:1619677475.816408:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.816410:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:26.0:1619677475.816411:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498023] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.816412:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:26.0:1619677475.816413:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.816414:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:26.0:1619677475.816417:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:26.0:1619677475.816418:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:26.0:1619677475.816421:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.816422:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:26.0:1619677475.816422:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:26.0:1619677475.816424:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:26.0:1619677475.816424:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:26.0:1619677475.816426:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07e37de00] (22)++
00000800:00000200:26.0:1619677475.816427:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (23)--
00000800:00000200:26.0:1619677475.816427:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:25.2:1619677475.816431:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:4.2:1619677475.816433:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (21)++
00000800:00000200:22.0:1619677475.816438:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:10.0:1619677475.816440:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)++
00000800:00000200:22.0:1619677475.816441:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000400:00000200:22.0:1619677475.816442:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.816442:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07e37de00] (23)--
00000800:00000200:6.0:1619677475.816442:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.816443:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000800:00000200:26.0:1619677475.816444:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000400:00000200:22.0:1619677475.816444:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.816447:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:21.2:1619677475.816691:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91987fc6f800] (20)++
00000800:00000200:6.0:1619677475.816696:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.816698:0:19937:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.14.12@o2ib3
00000800:00000200:22.0:1619677475.816699:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (22)--
00000400:00000200:6.0:1619677475.816701:0:19937:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.113@o2ib7(10.51.0.116@o2ib3) <- 10.51.14.12@o2ib3 : PUT - routed
00000400:00000200:6.0:1619677475.816703:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.816704:0:19937:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.113@o2ib7 local destination
00000400:00000200:6.0:1619677475.816705:0:19937:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498022] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:6.0:1619677475.816707:0:19937:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:6.0:1619677475.816708:0:19937:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:6.0:1619677475.816708:0:19937:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.113@o2ib7
00000400:00000200:6.0:1619677475.816711:0:19937:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.14.12@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.113@o2ib7(10.0.10.113@o2ib7:10.0.10.113@o2ib7) : PUT try# 0
00000800:00000200:6.0:1619677475.816713:0:19937:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.113@o2ib7
00000800:00000200:6.0:1619677475.816715:0:19937:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff918feb721980] -> 10.0.10.113@o2ib7 (2) version: 12
00000800:00000200:6.0:1619677475.816716:0:19937:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff918fe95d0600] (20)++
00000800:00000200:6.0:1619677475.816717:0:19937:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff918fe95d0600] (21)++
00000800:00000200:6.0:1619677475.816718:0:19937:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff918fe95d0600] (22)--
00000800:00000200:6.0:1619677475.816719:0:19937:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91987fc6f800] (21)++
00000800:00000200:6.0:1619677475.816719:0:19937:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91987fc6f800] (22)--
00000800:00000200:6.0:1619677475.816720:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91987fc6f800] (21)--
00000800:00000200:8.2:1619677475.816725:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918fe95d0600] (21)++
00000800:00000200:1.0:1619677475.816733:0:19931:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)++
00000800:00000200:1.0:1619677475.816735:0:19931:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff918fe95d0600] (23)--
00000800:00000200:17.0:1619677475.816736:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (22)--
00000400:00000200:1.0:1619677475.816736:0:19931:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:1.0:1619677475.816739:0:19931:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.113@o2ib7: PUT: OK
00000800:00000200:1.0:1619677475.816741:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918fe95d0600] (21)--
00000800:00000200:21.2:1619677475.816829:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a86b38b000] (20)++
00000800:00000200:31.0:1619677475.816837:0:19945:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.816840:0:19945:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.12.20@o2ib3
00000400:00000200:31.0:1619677475.816843:0:19945:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.12.20@o2ib3 : PUT - routed
00000800:00000200:30.0:1619677475.816844:0:19944:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (22)--
00000400:00000200:31.0:1619677475.816845:0:19945:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:31.0:1619677475.816846:0:19945:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:31.0:1619677475.816848:0:19945:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498021] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:31.0:1619677475.816849:0:19945:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:31.0:1619677475.816850:0:19945:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:31.0:1619677475.816851:0:19945:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:31.0:1619677475.816854:0:19945:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.12.20@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:31.0:1619677475.816856:0:19945:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:31.0:1619677475.816859:0:19945:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:31.0:1619677475.816860:0:19945:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:31.0:1619677475.816860:0:19945:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:31.0:1619677475.816862:0:19945:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:31.0:1619677475.816863:0:19945:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a86b38b000] (21)++
00000800:00000200:31.0:1619677475.816864:0:19945:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a86b38b000] (22)--
00000800:00000200:31.0:1619677475.816864:0:19945:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a86b38b000] (21)--
00000800:00000200:25.2:1619677475.816868:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:22.0:1619677475.816876:0:19936:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:22.0:1619677475.816879:0:19936:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:6.0:1619677475.816879:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:22.0:1619677475.816880:0:19936:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:22.0:1619677475.816882:0:19936:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:22.0:1619677475.816885:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.817097:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000800:00000200:18.0:1619677475.817103:0:19933:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677475.817106:0:19933:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:2.0:1619677475.817107:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:18.0:1619677475.817109:0:19933:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:18.0:1619677475.817112:0:19933:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:18.0:1619677475.817113:0:19933:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:18.0:1619677475.817114:0:19933:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498020] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:18.0:1619677475.817115:0:19933:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:18.0:1619677475.817116:0:19933:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:18.0:1619677475.817117:0:19933:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:18.0:1619677475.817120:0:19933:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:18.0:1619677475.817122:0:19933:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:18.0:1619677475.817124:0:19933:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:18.0:1619677475.817125:0:19933:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:18.0:1619677475.817126:0:19933:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:18.0:1619677475.817127:0:19933:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:18.0:1619677475.817128:0:19933:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:18.0:1619677475.817129:0:19933:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:18.0:1619677475.817130:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:25.2:1619677475.817134:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.817141:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.817144:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.817144:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.817146:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.817147:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.817149:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:4.2:1619677475.817173:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07e37de00] (20)++
00000800:00000200:10.0:1619677475.817181:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.817183:0:19940:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.5.22@o2ib3
00000800:00000200:26.0:1619677475.817184:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (22)--
00000400:00000200:10.0:1619677475.817186:0:19940:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.104@o2ib7(10.51.0.116@o2ib3) <- 10.51.5.22@o2ib3 : PUT - routed
00000400:00000200:10.0:1619677475.817189:0:19940:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:10.0:1619677475.817190:0:19940:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.104@o2ib7 local destination
00000400:00000200:10.0:1619677475.817192:0:19940:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:12, s:-96498019] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:10.0:1619677475.817193:0:19940:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:10.0:1619677475.817194:0:19940:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:10.0:1619677475.817195:0:19940:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.104@o2ib7
00000400:00000200:10.0:1619677475.817198:0:19940:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.5.22@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.104@o2ib7(10.0.10.104@o2ib7:10.0.10.104@o2ib7) : PUT try# 0
00000800:00000200:10.0:1619677475.817200:0:19940:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.104@o2ib7
00000800:00000200:10.0:1619677475.817203:0:19940:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919b8e780580] -> 10.0.10.104@o2ib7 (2) version: 12
00000800:00000200:10.0:1619677475.817203:0:19940:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919b86950600] (20)++
00000800:00000200:10.0:1619677475.817204:0:19940:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919b86950600] (21)++
00000800:00000200:10.0:1619677475.817206:0:19940:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919b86950600] (22)--
00000800:00000200:10.0:1619677475.817206:0:19940:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07e37de00] (21)++
00000800:00000200:10.0:1619677475.817207:0:19940:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07e37de00] (22)--
00000800:00000200:10.0:1619677475.817207:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07e37de00] (21)--
00000800:00000200:25.2:1619677475.817211:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919b86950600] (21)++
00000800:00000200:6.0:1619677475.817219:0:19937:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919b86950600] (22)++
00000800:00000200:6.0:1619677475.817221:0:19937:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919b86950600] (23)--
00000800:00000200:22.0:1619677475.817222:0:19936:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (22)--
00000400:00000200:6.0:1619677475.817223:0:19937:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:6.0:1619677475.817225:0:19937:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.104@o2ib7: PUT: OK
00000800:00000200:6.0:1619677475.817227:0:19937:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919b86950600] (21)--
00000800:00000200:4.2:1619677475.817381:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197a0b87200] (20)++
00000800:00000200:20.0:1619677475.817385:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.817388:0:19934:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.6.13@o2ib3
00000400:00000200:20.0:1619677475.817390:0:19934:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.101@o2ib7(10.51.0.116@o2ib3) <- 10.51.6.13@o2ib3 : ACK - routed
00000800:00000200:21.0:1619677475.817391:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (22)--
00000400:00000200:20.0:1619677475.817392:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:20.0:1619677475.817394:0:19934:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.101@o2ib7 local destination
00000400:00000200:20.0:1619677475.817395:0:19934:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498018] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:20.0:1619677475.817397:0:19934:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:20.0:1619677475.817398:0:19934:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:20.0:1619677475.817399:0:19934:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.101@o2ib7
00000400:00000200:20.0:1619677475.817402:0:19934:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.6.13@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.101@o2ib7(10.0.10.101@o2ib7:10.0.10.101@o2ib7) : ACK try# 0
00000800:00000200:20.0:1619677475.817404:0:19934:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.0.10.101@o2ib7
00000800:00000200:20.0:1619677475.817406:0:19934:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919e5f679280] -> 10.0.10.101@o2ib7 (2) version: 12
00000800:00000200:20.0:1619677475.817407:0:19934:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff919e43e7f200] (20)++
00000800:00000200:20.0:1619677475.817408:0:19934:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff919e43e7f200] (21)++
00000800:00000200:20.0:1619677475.817410:0:19934:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff919e43e7f200] (22)--
00000800:00000200:20.0:1619677475.817410:0:19934:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff9197a0b87200] (21)++
00000800:00000200:20.0:1619677475.817411:0:19934:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff9197a0b87200] (22)--
00000800:00000200:20.0:1619677475.817412:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197a0b87200] (21)--
00000800:00000200:9.2:1619677475.817416:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff919e43e7f200] (21)++
00000800:00000200:8.0:1619677475.817424:0:19939:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)++
00000800:00000200:8.0:1619677475.817426:0:19939:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff919e43e7f200] (23)--
00000800:00000200:24.0:1619677475.817427:0:19938:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (22)--
00000400:00000200:8.0:1619677475.817428:0:19939:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:8.0:1619677475.817430:0:19939:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.101@o2ib7: ACK: OK
00000800:00000200:8.0:1619677475.817433:0:19939:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff919e43e7f200] (21)--
00000800:00000200:4.2:1619677475.818194:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91907a3e8400] (20)++
00000800:00000200:2.0:1619677475.818201:0:19932:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677475.818204:0:19932:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.45@o2ib3
00000800:00000200:18.0:1619677475.818206:0:19933:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (22)--
00000400:00000200:2.0:1619677475.818207:0:19932:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.52@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.45@o2ib3 : PUT - routed
00000400:00000200:2.0:1619677475.818210:0:19932:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:2.0:1619677475.818211:0:19932:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.52@o2ib7 local destination
00000400:00000200:2.0:1619677475.818213:0:19932:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498017] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:2.0:1619677475.818214:0:19932:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:2.0:1619677475.818215:0:19932:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:2.0:1619677475.818216:0:19932:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.52@o2ib7
00000400:00000200:2.0:1619677475.818219:0:19932:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.45@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.52@o2ib7(10.0.10.52@o2ib7:10.0.10.52@o2ib7) : PUT try# 0
00000800:00000200:2.0:1619677475.818221:0:19932:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.52@o2ib7
00000800:00000200:2.0:1619677475.818224:0:19932:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919790df0900] -> 10.0.10.52@o2ib7 (2) version: 12
00000800:00000200:2.0:1619677475.818224:0:19932:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f3800] (20)++
00000800:00000200:2.0:1619677475.818225:0:19932:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f3800] (21)++
00000800:00000200:2.0:1619677475.818227:0:19932:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f3800] (22)--
00000800:00000200:2.0:1619677475.818228:0:19932:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91907a3e8400] (21)++
00000800:00000200:2.0:1619677475.818229:0:19932:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91907a3e8400] (22)--
00000800:00000200:2.0:1619677475.818229:0:19932:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91907a3e8400] (21)--
00000800:00000200:25.2:1619677475.818233:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f3800] (21)++
00000800:00000200:21.0:1619677475.818241:0:19935:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f3800] (22)++
00000800:00000200:21.0:1619677475.818243:0:19935:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f3800] (23)--
00000400:00000200:21.0:1619677475.818244:0:19935:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000400:00000200:21.0:1619677475.818246:0:19935:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.52@o2ib7: PUT: OK
00000800:00000200:20.0:1619677475.818246:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (22)--
00000800:00000200:21.0:1619677475.818248:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f3800] (21)--
00000800:00000200:21.2:1619677475.818554:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff918ce4be9c00] (20)++
00000800:00000200:4.2:1619677475.818558:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (20)++
00000800:00000200:17.0:1619677475.818560:0:19930:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.818564:0:19930:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.1.50@o2ib3
00000800:00000200:1.0:1619677475.818565:0:19931:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:26.0:1619677475.818566:0:19941:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)++
00000400:00000200:17.0:1619677475.818567:0:19930:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.107@o2ib7(10.51.0.116@o2ib3) <- 10.51.1.50@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.818569:0:19930:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000800:00000200:26.0:1619677475.818570:0:19941:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.51.2.15@o2ib3
00000400:00000200:17.0:1619677475.818570:0:19930:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.107@o2ib7 local destination
00000800:00000200:10.0:1619677475.818570:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:17.0:1619677475.818572:0:19930:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:512, d:32, s:-96498016] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.818574:0:19941:0:(lib-move.c:4190:lnet_parse()) TRACE: 10.0.10.111@o2ib7(10.51.0.116@o2ib3) <- 10.51.2.15@o2ib3 : PUT - routed
00000400:00000200:17.0:1619677475.818574:0:19930:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000400:00000200:17.0:1619677475.818575:0:19930:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:17.0:1619677475.818576:0:19930:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.107@o2ib7
00000400:00000200:26.0:1619677475.818577:0:19941:0:(lib-msg.c:918:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0
00000400:00000200:26.0:1619677475.818578:0:19941:0:(lib-move.c:2478:lnet_handle_send_case_locked()) Source ANY to MR:  10.0.10.111@o2ib7 local destination
00000400:00000200:17.0:1619677475.818580:0:19930:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.1.50@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.107@o2ib7(10.0.10.107@o2ib7:10.0.10.107@o2ib7) : PUT try# 0
00000400:00000200:26.0:1619677475.818581:0:19941:0:(lib-move.c:1515:lnet_get_best_ni()) compare ni 10.0.10.237@o2ib7 [c:511, d:12, s:-96498015] with best_ni not seleced [c:-2147483648, d:-1, s:0]
00000400:00000200:26.0:1619677475.818582:0:19941:0:(lib-move.c:1558:lnet_get_best_ni()) selected best_ni 10.0.10.237@o2ib7
00000800:00000200:17.0:1619677475.818582:0:19930:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.107@o2ib7
00000400:00000200:26.0:1619677475.818583:0:19941:0:(lib-move.c:1790:lnet_select_peer_ni()) 10.0.10.237@o2ib7 ni_is_pref = 0
00000400:00000200:26.0:1619677475.818584:0:19941:0:(lib-move.c:1846:lnet_select_peer_ni()) sd_best_lpni = 10.0.10.111@o2ib7
00000800:00000200:17.0:1619677475.818585:0:19930:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff919788dc9c80] -> 10.0.10.107@o2ib7 (2) version: 12
00000800:00000200:17.0:1619677475.818586:0:19930:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff9197843f0c00] (20)++
00000800:00000200:17.0:1619677475.818586:0:19930:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff9197843f0c00] (21)++
00000400:00000200:26.0:1619677475.818588:0:19941:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.51.2.15@o2ib3(10.0.10.237@o2ib7:<?>) -> 10.0.10.111@o2ib7(10.0.10.111@o2ib7:10.0.10.111@o2ib7) : PUT try# 0
00000800:00000200:17.0:1619677475.818588:0:19930:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff9197843f0c00] (22)--
00000800:00000200:17.0:1619677475.818589:0:19930:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff918ce4be9c00] (21)++
00000800:00000200:17.0:1619677475.818589:0:19930:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff918ce4be9c00] (22)--
00000800:00000200:26.0:1619677475.818590:0:19941:0:(o2iblnd_cb.c:1667:kiblnd_send()) sending 488 bytes in 1 frags to 12345-10.0.10.111@o2ib7
00000800:00000200:17.0:1619677475.818590:0:19930:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff918ce4be9c00] (21)--
00000800:00000200:26.0:1619677475.818592:0:19941:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff91a7c3537a00] -> 10.0.10.111@o2ib7 (2) version: 12
00000800:00000200:26.0:1619677475.818593:0:19941:0:(o2iblnd_cb.c:1547:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (20)++
00000800:00000200:26.0:1619677475.818594:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:26.0:1619677475.818595:0:19941:0:(o2iblnd_cb.c:1553:kiblnd_launch_tx()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:8.2:1619677475.818595:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff9197843f0c00] (21)++
00000800:00000200:26.0:1619677475.818596:0:19941:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:26.0:1619677475.818597:0:19941:0:(o2iblnd_cb.c:1320:kiblnd_queue_tx_locked()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:26.0:1619677475.818598:0:19941:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff91a07f75fc00] (23)--
00000800:00000200:26.0:1619677475.818599:0:19941:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000800:00000200:25.2:1619677475.818602:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a7c17d9800] (21)++
00000800:00000200:20.0:1619677475.818604:0:19934:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)++
00000800:00000200:4.2:1619677475.818604:0:0:0:(o2iblnd_cb.c:3745:kiblnd_cq_completion()) conn[ffff91a07f75fc00] (21)++
00000800:00000200:20.0:1619677475.818607:0:19934:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff9197843f0c00] (23)--
00000400:00000200:20.0:1619677475.818608:0:19934:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:21.0:1619677475.818610:0:19935:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (22)--
00000400:00000200:20.0:1619677475.818610:0:19934:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.107@o2ib7: PUT: OK
00000800:00000200:13.0:1619677475.818610:0:19942:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)++
00000800:00000200:10.0:1619677475.818611:0:19940:0:(o2iblnd_cb.c:3867:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)++
00000800:00000200:20.0:1619677475.818612:0:19934:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff9197843f0c00] (21)--
00000800:00000200:13.0:1619677475.818613:0:19942:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a7c17d9800] (23)--
00000800:00000200:10.0:1619677475.818613:0:19940:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff91a07f75fc00] (23)--
00000400:00000200:13.0:1619677475.818614:0:19942:0:(lib-msg.c:918:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0
00000800:00000200:10.0:1619677475.818614:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (22)--
00000400:00000200:13.0:1619677475.818615:0:19942:0:(lib-msg.c:803:lnet_health_check()) health check: 10.0.10.237@o2ib7->10.0.10.111@o2ib7: PUT: OK
00000800:00000200:10.0:1619677475.818615:0:19940:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a07f75fc00] (21)--
00000800:00000200:12.0:1619677475.818617:0:19943:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (22)--
00000800:00000200:13.0:1619677475.818618:0:19942:0:(o2iblnd_cb.c:3883:kiblnd_scheduler()) conn[ffff91a7c17d9800] (21)--
00000800:00000100:8.0:1619677476.585571:0:13335:0:(o2iblnd_cb.c:2085:kiblnd_close_conn_locked()) Closing conn to 10.0.10.52@o2ib7: error 0(waiting)
00000400:00000100:21.0:1619677476.586107:0:19935:0:(lib-msg.c:698:lnet_attempt_msg_resend()) msg 10.51.2.45@o2ib3->10.0.10.52@o2ib7 exceeded retry count 0
Debug log: 521136 lines, 521136 kept, 0 dropped, 0 bad.
